| 1 | #!/usr/bin/env python3 |
| 2 | """RSS/Atom-feed: migratie op een bestaande database, aan/uit, sleutels, inhoud en headers.""" |
| 3 | import http.cookiejar |
| 4 | import json |
| 5 | import os |
| 6 | import shutil |
| 7 | import signal |
| 8 | import socket |
| 9 | import sqlite3 |
| 10 | import subprocess |
| 11 | import tempfile |
| 12 | import time |
| 13 | import urllib.error |
| 14 | import urllib.request |
| 15 | import xml.etree.ElementTree as ET |
| 16 | from datetime import date, datetime, timedelta |
| 17 | from pathlib import Path |
| 18 | |
| 19 | WEB = Path(__file__).resolve().parents[1] |
| 20 | ARTIFACTS = WEB / 'tests/artifacts' |
| 21 | ARTIFACTS.mkdir(exist_ok=True) |
| 22 | ATOM = '{http://www.w3.org/2005/Atom}' |
| 23 | checks = 0 |
| 24 | |
| 25 | |
| 26 | def check(value, label): |
| 27 | global checks |
| 28 | checks += 1 |
| 29 | assert value, label |
| 30 | |
| 31 | |
| 32 | # Het schema van vóór de feed: zo staat de productiedatabase er nu bij. |
| 33 | OLD_SCHEMA = ''' |
| 34 | CREATE TABLE users (id TEXT PRIMARY KEY, name TEXT NOT NULL, role TEXT NOT NULL DEFAULT 'user' CHECK(role IN ('admin','user')), |
| 35 | theme TEXT NOT NULL DEFAULT 'system' CHECK(theme IN ('system','light','dark')), font TEXT NOT NULL DEFAULT 'serif' CHECK(font IN ('serif','sans','rounded')), created INTEGER NOT NULL); |
| 36 | CREATE TABLE sessions (id TEXT PRIMARY KEY, payload TEXT NOT NULL, expires INTEGER NOT NULL); |
| 37 | CREATE TABLE days (id INTEGER PRIMARY KEY AUTOINCREMENT, date TEXT NOT NULL UNIQUE, title TEXT NOT NULL, created INTEGER NOT NULL, updated INTEGER NOT NULL); |
| 38 | CREATE TABLE stories (id INTEGER PRIMARY KEY AUTOINCREMENT, day_id INTEGER NOT NULL REFERENCES days(id) ON DELETE CASCADE, position INTEGER NOT NULL CHECK(position BETWEEN 1 AND 3), title TEXT NOT NULL, created INTEGER NOT NULL, updated INTEGER NOT NULL, UNIQUE(day_id, position)); |
| 39 | CREATE TABLE paragraphs (id INTEGER PRIMARY KEY AUTOINCREMENT, story_id INTEGER NOT NULL REFERENCES stories(id) ON DELETE CASCADE, position INTEGER NOT NULL CHECK(position >= 1), body TEXT NOT NULL, UNIQUE(story_id, position)); |
| 40 | CREATE TABLE reads (user_id TEXT NOT NULL REFERENCES users(id), day_id INTEGER NOT NULL REFERENCES days(id) ON DELETE CASCADE, read_at INTEGER NOT NULL, PRIMARY KEY(user_id, day_id)); |
| 41 | CREATE TABLE api_keys (id INTEGER PRIMARY KEY AUTOINCREMENT, user_id TEXT NOT NULL REFERENCES users(id), name TEXT NOT NULL, token_hash TEXT NOT NULL UNIQUE, token_prefix TEXT NOT NULL, token_cipher TEXT NOT NULL DEFAULT '', created INTEGER NOT NULL, revoked INTEGER); |
| 42 | CREATE TABLE wiki_categories (id INTEGER PRIMARY KEY AUTOINCREMENT, slug TEXT NOT NULL UNIQUE, title TEXT NOT NULL, position INTEGER NOT NULL, created INTEGER NOT NULL, updated INTEGER NOT NULL); |
| 43 | CREATE TABLE wiki_pages (id INTEGER PRIMARY KEY AUTOINCREMENT, category_id INTEGER NOT NULL REFERENCES wiki_categories(id), slug TEXT NOT NULL UNIQUE, title TEXT NOT NULL, content TEXT NOT NULL, position INTEGER NOT NULL, created INTEGER NOT NULL, updated INTEGER NOT NULL); |
| 44 | ''' |
| 45 | |
| 46 | # PHP heeft altijd de tijdzonedatabase; de CI-image van Python niet per se. |
| 47 | today = date.fromisoformat(subprocess.check_output(['php', '-r', "echo (new DateTime('now', new DateTimeZone('Europe/Amsterdam')))->format('Y-m-d');"], text=True).strip()) |
| 48 | past = [(today - timedelta(days=n)).isoformat() for n in (2, 1, 0)] |
| 49 | future = (today + timedelta(days=3)).isoformat() |
| 50 | |
| 51 | with tempfile.TemporaryDirectory(prefix='feed-', dir=ARTIFACTS) as tmp: |
| 52 | root = Path(tmp) |
| 53 | for name in ['api.php', 'index.php', 'feed.php', 'router.php']: |
| 54 | shutil.copy(WEB / name, root / name) |
| 55 | shutil.copytree(WEB / 'lib', root / 'lib') |
| 56 | (root / 'assets').mkdir() |
| 57 | shutil.copy(WEB / 'assets/favicon.svg', root / 'assets/favicon.svg') |
| 58 | (root / 'data').mkdir() |
| 59 | dbfile = root / 'data/verhalenboek.sqlite' |
| 60 | with sqlite3.connect(dbfile) as db: |
| 61 | db.executescript(OLD_SCHEMA) |
| 62 | now = int(time.time()) |
| 63 | db.execute("INSERT INTO users VALUES ('local:test_user_1','test_user_1','admin','system','serif',?)", (now,)) |
| 64 | db.execute("INSERT INTO days (date,title,created,updated) VALUES ('2020-01-01','Oude dag',?,?)", (now, now)) |
| 65 | db.execute("INSERT INTO stories (day_id,position,title,created,updated) VALUES (1,1,'Oud verhaal',?,?)", (now, now)) |
| 66 | db.execute("INSERT INTO paragraphs (story_id,position,body) VALUES (1,1,'Bewaard.')") |
| 67 | with socket.socket() as sock: |
| 68 | sock.bind(('127.0.0.1', 0)) |
| 69 | port = sock.getsockname()[1] |
| 70 | base = f'http://127.0.0.1:{port}' |
| 71 | log = (root / 'server.log').open('w') |
| 72 | process = subprocess.Popen(['php', '-d', 'opcache.enable=0', '-S', f'127.0.0.1:{port}', '-t', str(root), str(root / 'router.php')], |
| 73 | stdout=log, stderr=log, start_new_session=True) |
| 74 | |
| 75 | class Client: |
| 76 | def __init__(self, token=None): |
| 77 | self.token = token |
| 78 | self.opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(http.cookiejar.CookieJar())) |
| 79 | self.csrf = None |
| 80 | |
| 81 | def request(self, method, path, body=None, expected=200, headers=None): |
| 82 | h = {'Accept': 'application/json', **(headers or {})} |
| 83 | data = None |
| 84 | if body is not None: |
| 85 | data = json.dumps(body).encode() |
| 86 | h['Content-Type'] = 'application/json' |
| 87 | if self.token: |
| 88 | h['Authorization'] = 'Bearer ' + self.token |
| 89 | elif method not in ('GET', 'HEAD'): |
| 90 | h['X-CSRF-Token'] = self.csrf or '' |
| 91 | try: |
| 92 | response = self.opener.open(urllib.request.Request(base + '/api.php' + path, data=data, headers=h, method=method)) |
| 93 | except urllib.error.HTTPError as error: |
| 94 | response = error |
| 95 | with response: |
| 96 | payload = json.loads(response.read().decode() or 'null') |
| 97 | check(response.status == expected, f'{method} {path}: verwacht {expected}, kreeg {response.status} {payload}') |
| 98 | return payload |
| 99 | |
| 100 | def me(self): |
| 101 | data = self.request('GET', '/me') |
| 102 | self.csrf = data['csrf'] |
| 103 | return data |
| 104 | |
| 105 | def feed(url, expected=200, method='GET'): |
| 106 | try: |
| 107 | response = urllib.request.urlopen(urllib.request.Request(url, method=method)) |
| 108 | except urllib.error.HTTPError as error: |
| 109 | response = error |
| 110 | with response: |
| 111 | body = response.read().decode() |
| 112 | headers = {k.lower(): v for k, v in response.headers.items()} |
| 113 | check(response.status == expected, f'feed {url[-20:]}: verwacht {expected}, kreeg {response.status}') |
| 114 | return body, headers |
| 115 | |
| 116 | try: |
| 117 | for _ in range(100): |
| 118 | try: |
| 119 | urllib.request.urlopen(base + '/index.php').close() |
| 120 | break |
| 121 | except urllib.error.URLError: |
| 122 | time.sleep(0.05) |
| 123 | |
| 124 | # Migratie: de eerste request maakt de nieuwe tabellen en laat bestaande data staan. |
| 125 | admin = Client() |
| 126 | me = admin.me() |
| 127 | with sqlite3.connect(dbfile) as db: |
| 128 | tables = {row[0] for row in db.execute("SELECT name FROM sqlite_master WHERE type='table'")} |
| 129 | old_day = db.execute("SELECT title FROM days WHERE date='2020-01-01'").fetchone() |
| 130 | check({'app_settings', 'feed_tokens'} <= tables, 'Eerste request maakt app_settings en feed_tokens') |
| 131 | check(old_day == ('Oude dag',), 'Bestaande dagen blijven staan na de migratie') |
| 132 | check(me['feed'] is False, 'De feed staat standaard uit') |
| 133 | check(admin.request('GET', '/days/2020-01-01')['stories'][0]['paragraphs'] == ['Bewaard.'], 'Oude dag leest nog') |
| 134 | |
| 135 | key = admin.request('POST', '/keys', {'name': 'Annie'}, expected=201)['key']['token'] |
| 136 | annie = Client(key) |
| 137 | ua = {'User-Agent': 'Annie-Verhalenboek/1.0'} |
| 138 | for d in past + [future]: |
| 139 | annie.request('POST', '/days', {'date': d, 'title': 'Titel ' + d, 'stories': [ |
| 140 | {'title': 'Eerste <b>' + d, 'paragraphs': ['Regel een & twee.\nNieuwe regel.', 'Tweede alinea.']}, |
| 141 | {'title': 'Tweede ' + d, 'paragraphs': ['Midden.']}, |
| 142 | {'title': 'Derde ' + d, 'paragraphs': ['Slot.']}, |
| 143 | ]}, expected=201, headers=ua) |
| 144 | check('missing' in annie.request('GET', '/days/gaps', headers=ua), 'Annie: /days/gaps werkt met eigen User-Agent') |
| 145 | check(annie.request('GET', '/days/' + past[0], headers=ua)['complete'] is True, 'Annie: GET /days/DATUM werkt') |
| 146 | annie.request('POST', '/wiki/categories', {'slug': 'geheim', 'title': 'Wiki'}, expected=201) |
| 147 | annie.request('POST', '/wiki/pages', {'slug': 'wiki-marker', 'title': 'WIKIMARKER', 'category': 'geheim', 'content': 'WIKIMARKER inhoud'}, expected=201) |
| 148 | |
| 149 | # Uit: alles 404 en niets zichtbaar. |
| 150 | reader = Client() |
| 151 | reader.me() |
| 152 | switched = reader.request('POST', '/session/switch', {'user': 'test_user_2'}) |
| 153 | reader.csrf = switched['csrf'] |
| 154 | reader.request('GET', '/me/feed', expected=404) |
| 155 | reader.request('POST', '/me/feed', expected=404) |
| 156 | reader.request('PUT', '/settings/feed', {'enabled': True}, expected=403) |
| 157 | annie.request('PUT', '/settings/feed', {'enabled': True}, expected=403) |
| 158 | annie.request('GET', '/settings/feed', expected=403) |
| 159 | feed(base + '/feed.php?token=vbf_' + 'ab' * 32, expected=404) |
| 160 | check('application/atom+xml' not in urllib.request.urlopen(base + '/index.php').read().decode(), 'Geen autodiscovery als de feed uit staat') |
| 161 | |
| 162 | # Aan. |
| 163 | state = admin.request('PUT', '/settings/feed', {'enabled': True}) |
| 164 | check(state == {'enabled': True, 'tokens': 0}, 'Beheerder zet de feed aan') |
| 165 | admin.request('PUT', '/settings/feed', {'enabled': 'ja'}, expected=422) |
| 166 | check(reader.me()['feed'] is True, '/me meldt dat de feed aan staat') |
| 167 | check(reader.request('GET', '/me/feed')['token'] is None, 'Nog geen feed-sleutel') |
| 168 | created = reader.request('POST', '/me/feed', expected=201)['token'] |
| 169 | url = created['url'] |
| 170 | check(url.startswith(base + '/feed.php?token=vbf_') and created['prefix'] == url.split('token=')[1][:12], 'Feed-adres gebruikt de basis-URL') |
| 171 | check(reader.request('GET', '/me/feed')['token']['url'] == url, 'Feed-adres kan opnieuw worden getoond') |
| 172 | |
| 173 | with sqlite3.connect(dbfile) as db: |
| 174 | reads_before = db.execute('SELECT COUNT(*) FROM reads').fetchone()[0] |
| 175 | sessions_before = db.execute('SELECT COUNT(*) FROM sessions').fetchone()[0] |
| 176 | stored = db.execute("SELECT token_hash, token_cipher FROM feed_tokens WHERE user_id='local:test_user_2'").fetchone() |
| 177 | token = url.split('token=')[1] |
| 178 | check(token not in json.dumps(stored), 'De sleutel staat niet leesbaar in de database') |
| 179 | |
| 180 | body, headers = feed(url) |
| 181 | check(headers.get('content-type', '').startswith('application/atom+xml'), 'Content-Type is Atom') |
| 182 | check('private' in headers.get('cache-control', '') and 'public' not in headers.get('cache-control', ''), 'Cache-Control: private') |
| 183 | check('noindex' in headers.get('x-robots-tag', ''), 'X-Robots-Tag: noindex') |
| 184 | check('set-cookie' not in headers, 'De feed zet geen sessiecookie') |
| 185 | tree = ET.fromstring(body) |
| 186 | entries = tree.findall(ATOM + 'entry') |
| 187 | dates = [e.find(ATOM + 'id').text.rsplit('/', 1)[1] for e in entries] |
| 188 | check(dates == list(reversed(past)) + ['2020-01-01'], f'Eén item per dag, nieuwste eerst, geen toekomst: {dates}') |
| 189 | check(len(set(dates)) == len(dates), 'Geen dubbele dagen') |
| 190 | first = entries[0] |
| 191 | content = first.find(ATOM + 'content') |
| 192 | check(content.get('type') == 'html', 'Inhoud is HTML') |
| 193 | html = content.text |
| 194 | check('<h2>Eerste <b>' + past[2] + '</h2>' in html and '<h2>Derde ' + past[2] + '</h2>' in html, 'Alle drie de verhalen, titels geëscaped') |
| 195 | check('<p>Regel een & twee.<br>\nNieuwe regel.</p>' in html, 'Alinea’s met regeleinden') |
| 196 | check(first.find(ATOM + 'link').get('href') == base + '/#/dag/' + past[2], 'Link naar de dag') |
| 197 | check(tree.find(ATOM + 'link[@rel="self"]').get('href') == url, 'Self-link is het feed-adres') |
| 198 | check('WIKIMARKER' not in body, 'Geen wiki-inhoud in de feed') |
| 199 | check(feed(url, method='HEAD')[0] == '', 'HEAD werkt') |
| 200 | etag = headers.get('etag') |
| 201 | try: |
| 202 | urllib.request.urlopen(urllib.request.Request(url, headers={'If-None-Match': etag})) |
| 203 | check(False, '304 verwacht') |
| 204 | except urllib.error.HTTPError as error: |
| 205 | check(error.code == 304, 'ETag geeft 304') |
| 206 | |
| 207 | with sqlite3.connect(dbfile) as db: |
| 208 | check(db.execute('SELECT COUNT(*) FROM reads').fetchone()[0] == reads_before, 'De feed zet niets op gelezen') |
| 209 | check(db.execute('SELECT COUNT(*) FROM sessions').fetchone()[0] == sessions_before, 'De feed maakt geen sessies') |
| 210 | check(reader.request('GET', '/days/' + past[2])['read'] is False, 'De dag blijft ongelezen') |
| 211 | |
| 212 | page = reader.opener.open(base + '/index.php').read().decode() |
| 213 | check('type="application/atom+xml"' in page and token in page, 'Autodiscovery-link in de HTML-head voor de eigenaar') |
| 214 | |
| 215 | # Ongeldig, vernieuwd en ingetrokken. |
| 216 | feed(base + '/feed.php', expected=404) |
| 217 | feed(base + '/feed.php?token=nee', expected=404) |
| 218 | feed(base + '/feed.php?token=' + key, expected=404) |
| 219 | renewed = reader.request('POST', '/me/feed', expected=201)['token']['url'] |
| 220 | check(renewed != url, 'Vernieuwen geeft een nieuw adres') |
| 221 | feed(url, expected=404) |
| 222 | feed(renewed) |
| 223 | check(admin.request('GET', '/settings/feed')['tokens'] == 1, 'Beheer telt de feed-adressen') |
| 224 | |
| 225 | # Uit zetten: ook een geldige sleutel geeft 404, en de knop verdwijnt. |
| 226 | admin.request('PUT', '/settings/feed', {'enabled': False}) |
| 227 | feed(renewed, expected=404) |
| 228 | check(reader.me()['feed'] is False, '/me meldt uit') |
| 229 | reader.request('GET', '/me/feed', expected=404) |
| 230 | check('application/atom+xml' not in reader.opener.open(base + '/index.php').read().decode(), 'Geen autodiscovery als de feed uit staat') |
| 231 | admin.request('PUT', '/settings/feed', {'enabled': True}) |
| 232 | feed(renewed) |
| 233 | reader.request('DELETE', '/me/feed') |
| 234 | feed(renewed, expected=404) |
| 235 | check(reader.request('GET', '/me/feed')['token'] is None, 'Ingetrokken: geen sleutel meer') |
| 236 | feed(base + '/feed.php?token=' + token, method='POST', expected=405) |
| 237 | |
| 238 | print(f'Feed: {checks} assertions passed.') |
| 239 | except Exception: |
| 240 | log.flush() |
| 241 | print((root / 'server.log').read_text()[-4000:]) |
| 242 | raise |
| 243 | finally: |
| 244 | os.killpg(process.pid, signal.SIGTERM) |
| 245 | process.wait(timeout=5) |
| 246 | log.close() |
projects-timfalken-com verhalenboek