BlueGit

projects-timfalken-com verhalenboek

Files

verhalenboek / web / tests / feed.py

feed.py

feed.py 14.2 KiB Download Raw
1#!/usr/bin/env python3
2"""RSS/Atom-feed: migratie op een bestaande database, aan/uit, sleutels, inhoud en headers."""
3import http.cookiejar
4import json
5import os
6import shutil
7import signal
8import socket
9import sqlite3
10import subprocess
11import tempfile
12import time
13import urllib.error
14import urllib.request
15import xml.etree.ElementTree as ET
16from datetime import date, datetime, timedelta
17from pathlib import Path
18
19WEB = Path(__file__).resolve().parents[1]
20ARTIFACTS = WEB / 'tests/artifacts'
21ARTIFACTS.mkdir(exist_ok=True)
22ATOM = '{http://www.w3.org/2005/Atom}'
23checks = 0
24
25
26def check(value, label):
27 global checks
28 checks += 1
29 assert value, label
30
31
32# Het schema van vóór de feed: zo staat de productiedatabase er nu bij.
33OLD_SCHEMA = '''
34CREATE TABLE users (id TEXT PRIMARY KEY, name TEXT NOT NULL, role TEXT NOT NULL DEFAULT 'user' CHECK(role IN ('admin','user')),
35 theme TEXT NOT NULL DEFAULT 'system' CHECK(theme IN ('system','light','dark')), font TEXT NOT NULL DEFAULT 'serif' CHECK(font IN ('serif','sans','rounded')), created INTEGER NOT NULL);
36CREATE TABLE sessions (id TEXT PRIMARY KEY, payload TEXT NOT NULL, expires INTEGER NOT NULL);
37CREATE TABLE days (id INTEGER PRIMARY KEY AUTOINCREMENT, date TEXT NOT NULL UNIQUE, title TEXT NOT NULL, created INTEGER NOT NULL, updated INTEGER NOT NULL);
38CREATE TABLE stories (id INTEGER PRIMARY KEY AUTOINCREMENT, day_id INTEGER NOT NULL REFERENCES days(id) ON DELETE CASCADE, position INTEGER NOT NULL CHECK(position BETWEEN 1 AND 3), title TEXT NOT NULL, created INTEGER NOT NULL, updated INTEGER NOT NULL, UNIQUE(day_id, position));
39CREATE TABLE paragraphs (id INTEGER PRIMARY KEY AUTOINCREMENT, story_id INTEGER NOT NULL REFERENCES stories(id) ON DELETE CASCADE, position INTEGER NOT NULL CHECK(position >= 1), body TEXT NOT NULL, UNIQUE(story_id, position));
40CREATE TABLE reads (user_id TEXT NOT NULL REFERENCES users(id), day_id INTEGER NOT NULL REFERENCES days(id) ON DELETE CASCADE, read_at INTEGER NOT NULL, PRIMARY KEY(user_id, day_id));
41CREATE TABLE api_keys (id INTEGER PRIMARY KEY AUTOINCREMENT, user_id TEXT NOT NULL REFERENCES users(id), name TEXT NOT NULL, token_hash TEXT NOT NULL UNIQUE, token_prefix TEXT NOT NULL, token_cipher TEXT NOT NULL DEFAULT '', created INTEGER NOT NULL, revoked INTEGER);
42CREATE TABLE wiki_categories (id INTEGER PRIMARY KEY AUTOINCREMENT, slug TEXT NOT NULL UNIQUE, title TEXT NOT NULL, position INTEGER NOT NULL, created INTEGER NOT NULL, updated INTEGER NOT NULL);
43CREATE TABLE wiki_pages (id INTEGER PRIMARY KEY AUTOINCREMENT, category_id INTEGER NOT NULL REFERENCES wiki_categories(id), slug TEXT NOT NULL UNIQUE, title TEXT NOT NULL, content TEXT NOT NULL, position INTEGER NOT NULL, created INTEGER NOT NULL, updated INTEGER NOT NULL);
44'''
45
46# PHP heeft altijd de tijdzonedatabase; de CI-image van Python niet per se.
47today = date.fromisoformat(subprocess.check_output(['php', '-r', "echo (new DateTime('now', new DateTimeZone('Europe/Amsterdam')))->format('Y-m-d');"], text=True).strip())
48past = [(today - timedelta(days=n)).isoformat() for n in (2, 1, 0)]
49future = (today + timedelta(days=3)).isoformat()
50
51with tempfile.TemporaryDirectory(prefix='feed-', dir=ARTIFACTS) as tmp:
52 root = Path(tmp)
53 for name in ['api.php', 'index.php', 'feed.php', 'router.php']:
54 shutil.copy(WEB / name, root / name)
55 shutil.copytree(WEB / 'lib', root / 'lib')
56 (root / 'assets').mkdir()
57 shutil.copy(WEB / 'assets/favicon.svg', root / 'assets/favicon.svg')
58 (root / 'data').mkdir()
59 dbfile = root / 'data/verhalenboek.sqlite'
60 with sqlite3.connect(dbfile) as db:
61 db.executescript(OLD_SCHEMA)
62 now = int(time.time())
63 db.execute("INSERT INTO users VALUES ('local:test_user_1','test_user_1','admin','system','serif',?)", (now,))
64 db.execute("INSERT INTO days (date,title,created,updated) VALUES ('2020-01-01','Oude dag',?,?)", (now, now))
65 db.execute("INSERT INTO stories (day_id,position,title,created,updated) VALUES (1,1,'Oud verhaal',?,?)", (now, now))
66 db.execute("INSERT INTO paragraphs (story_id,position,body) VALUES (1,1,'Bewaard.')")
67 with socket.socket() as sock:
68 sock.bind(('127.0.0.1', 0))
69 port = sock.getsockname()[1]
70 base = f'http://127.0.0.1:{port}'
71 log = (root / 'server.log').open('w')
72 process = subprocess.Popen(['php', '-d', 'opcache.enable=0', '-S', f'127.0.0.1:{port}', '-t', str(root), str(root / 'router.php')],
73 stdout=log, stderr=log, start_new_session=True)
74
75 class Client:
76 def __init__(self, token=None):
77 self.token = token
78 self.opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(http.cookiejar.CookieJar()))
79 self.csrf = None
80
81 def request(self, method, path, body=None, expected=200, headers=None):
82 h = {'Accept': 'application/json', **(headers or {})}
83 data = None
84 if body is not None:
85 data = json.dumps(body).encode()
86 h['Content-Type'] = 'application/json'
87 if self.token:
88 h['Authorization'] = 'Bearer ' + self.token
89 elif method not in ('GET', 'HEAD'):
90 h['X-CSRF-Token'] = self.csrf or ''
91 try:
92 response = self.opener.open(urllib.request.Request(base + '/api.php' + path, data=data, headers=h, method=method))
93 except urllib.error.HTTPError as error:
94 response = error
95 with response:
96 payload = json.loads(response.read().decode() or 'null')
97 check(response.status == expected, f'{method} {path}: verwacht {expected}, kreeg {response.status} {payload}')
98 return payload
99
100 def me(self):
101 data = self.request('GET', '/me')
102 self.csrf = data['csrf']
103 return data
104
105 def feed(url, expected=200, method='GET'):
106 try:
107 response = urllib.request.urlopen(urllib.request.Request(url, method=method))
108 except urllib.error.HTTPError as error:
109 response = error
110 with response:
111 body = response.read().decode()
112 headers = {k.lower(): v for k, v in response.headers.items()}
113 check(response.status == expected, f'feed {url[-20:]}: verwacht {expected}, kreeg {response.status}')
114 return body, headers
115
116 try:
117 for _ in range(100):
118 try:
119 urllib.request.urlopen(base + '/index.php').close()
120 break
121 except urllib.error.URLError:
122 time.sleep(0.05)
123
124 # Migratie: de eerste request maakt de nieuwe tabellen en laat bestaande data staan.
125 admin = Client()
126 me = admin.me()
127 with sqlite3.connect(dbfile) as db:
128 tables = {row[0] for row in db.execute("SELECT name FROM sqlite_master WHERE type='table'")}
129 old_day = db.execute("SELECT title FROM days WHERE date='2020-01-01'").fetchone()
130 check({'app_settings', 'feed_tokens'} <= tables, 'Eerste request maakt app_settings en feed_tokens')
131 check(old_day == ('Oude dag',), 'Bestaande dagen blijven staan na de migratie')
132 check(me['feed'] is False, 'De feed staat standaard uit')
133 check(admin.request('GET', '/days/2020-01-01')['stories'][0]['paragraphs'] == ['Bewaard.'], 'Oude dag leest nog')
134
135 key = admin.request('POST', '/keys', {'name': 'Annie'}, expected=201)['key']['token']
136 annie = Client(key)
137 ua = {'User-Agent': 'Annie-Verhalenboek/1.0'}
138 for d in past + [future]:
139 annie.request('POST', '/days', {'date': d, 'title': 'Titel ' + d, 'stories': [
140 {'title': 'Eerste <b>' + d, 'paragraphs': ['Regel een & twee.\nNieuwe regel.', 'Tweede alinea.']},
141 {'title': 'Tweede ' + d, 'paragraphs': ['Midden.']},
142 {'title': 'Derde ' + d, 'paragraphs': ['Slot.']},
143 ]}, expected=201, headers=ua)
144 check('missing' in annie.request('GET', '/days/gaps', headers=ua), 'Annie: /days/gaps werkt met eigen User-Agent')
145 check(annie.request('GET', '/days/' + past[0], headers=ua)['complete'] is True, 'Annie: GET /days/DATUM werkt')
146 annie.request('POST', '/wiki/categories', {'slug': 'geheim', 'title': 'Wiki'}, expected=201)
147 annie.request('POST', '/wiki/pages', {'slug': 'wiki-marker', 'title': 'WIKIMARKER', 'category': 'geheim', 'content': 'WIKIMARKER inhoud'}, expected=201)
148
149 # Uit: alles 404 en niets zichtbaar.
150 reader = Client()
151 reader.me()
152 switched = reader.request('POST', '/session/switch', {'user': 'test_user_2'})
153 reader.csrf = switched['csrf']
154 reader.request('GET', '/me/feed', expected=404)
155 reader.request('POST', '/me/feed', expected=404)
156 reader.request('PUT', '/settings/feed', {'enabled': True}, expected=403)
157 annie.request('PUT', '/settings/feed', {'enabled': True}, expected=403)
158 annie.request('GET', '/settings/feed', expected=403)
159 feed(base + '/feed.php?token=vbf_' + 'ab' * 32, expected=404)
160 check('application/atom+xml' not in urllib.request.urlopen(base + '/index.php').read().decode(), 'Geen autodiscovery als de feed uit staat')
161
162 # Aan.
163 state = admin.request('PUT', '/settings/feed', {'enabled': True})
164 check(state == {'enabled': True, 'tokens': 0}, 'Beheerder zet de feed aan')
165 admin.request('PUT', '/settings/feed', {'enabled': 'ja'}, expected=422)
166 check(reader.me()['feed'] is True, '/me meldt dat de feed aan staat')
167 check(reader.request('GET', '/me/feed')['token'] is None, 'Nog geen feed-sleutel')
168 created = reader.request('POST', '/me/feed', expected=201)['token']
169 url = created['url']
170 check(url.startswith(base + '/feed.php?token=vbf_') and created['prefix'] == url.split('token=')[1][:12], 'Feed-adres gebruikt de basis-URL')
171 check(reader.request('GET', '/me/feed')['token']['url'] == url, 'Feed-adres kan opnieuw worden getoond')
172
173 with sqlite3.connect(dbfile) as db:
174 reads_before = db.execute('SELECT COUNT(*) FROM reads').fetchone()[0]
175 sessions_before = db.execute('SELECT COUNT(*) FROM sessions').fetchone()[0]
176 stored = db.execute("SELECT token_hash, token_cipher FROM feed_tokens WHERE user_id='local:test_user_2'").fetchone()
177 token = url.split('token=')[1]
178 check(token not in json.dumps(stored), 'De sleutel staat niet leesbaar in de database')
179
180 body, headers = feed(url)
181 check(headers.get('content-type', '').startswith('application/atom+xml'), 'Content-Type is Atom')
182 check('private' in headers.get('cache-control', '') and 'public' not in headers.get('cache-control', ''), 'Cache-Control: private')
183 check('noindex' in headers.get('x-robots-tag', ''), 'X-Robots-Tag: noindex')
184 check('set-cookie' not in headers, 'De feed zet geen sessiecookie')
185 tree = ET.fromstring(body)
186 entries = tree.findall(ATOM + 'entry')
187 dates = [e.find(ATOM + 'id').text.rsplit('/', 1)[1] for e in entries]
188 check(dates == list(reversed(past)) + ['2020-01-01'], f'Eén item per dag, nieuwste eerst, geen toekomst: {dates}')
189 check(len(set(dates)) == len(dates), 'Geen dubbele dagen')
190 first = entries[0]
191 content = first.find(ATOM + 'content')
192 check(content.get('type') == 'html', 'Inhoud is HTML')
193 html = content.text
194 check('<h2>Eerste &lt;b&gt;' + past[2] + '</h2>' in html and '<h2>Derde ' + past[2] + '</h2>' in html, 'Alle drie de verhalen, titels geëscaped')
195 check('<p>Regel een &amp; twee.<br>\nNieuwe regel.</p>' in html, 'Alinea’s met regeleinden')
196 check(first.find(ATOM + 'link').get('href') == base + '/#/dag/' + past[2], 'Link naar de dag')
197 check(tree.find(ATOM + 'link[@rel="self"]').get('href') == url, 'Self-link is het feed-adres')
198 check('WIKIMARKER' not in body, 'Geen wiki-inhoud in de feed')
199 check(feed(url, method='HEAD')[0] == '', 'HEAD werkt')
200 etag = headers.get('etag')
201 try:
202 urllib.request.urlopen(urllib.request.Request(url, headers={'If-None-Match': etag}))
203 check(False, '304 verwacht')
204 except urllib.error.HTTPError as error:
205 check(error.code == 304, 'ETag geeft 304')
206
207 with sqlite3.connect(dbfile) as db:
208 check(db.execute('SELECT COUNT(*) FROM reads').fetchone()[0] == reads_before, 'De feed zet niets op gelezen')
209 check(db.execute('SELECT COUNT(*) FROM sessions').fetchone()[0] == sessions_before, 'De feed maakt geen sessies')
210 check(reader.request('GET', '/days/' + past[2])['read'] is False, 'De dag blijft ongelezen')
211
212 page = reader.opener.open(base + '/index.php').read().decode()
213 check('type="application/atom+xml"' in page and token in page, 'Autodiscovery-link in de HTML-head voor de eigenaar')
214
215 # Ongeldig, vernieuwd en ingetrokken.
216 feed(base + '/feed.php', expected=404)
217 feed(base + '/feed.php?token=nee', expected=404)
218 feed(base + '/feed.php?token=' + key, expected=404)
219 renewed = reader.request('POST', '/me/feed', expected=201)['token']['url']
220 check(renewed != url, 'Vernieuwen geeft een nieuw adres')
221 feed(url, expected=404)
222 feed(renewed)
223 check(admin.request('GET', '/settings/feed')['tokens'] == 1, 'Beheer telt de feed-adressen')
224
225 # Uit zetten: ook een geldige sleutel geeft 404, en de knop verdwijnt.
226 admin.request('PUT', '/settings/feed', {'enabled': False})
227 feed(renewed, expected=404)
228 check(reader.me()['feed'] is False, '/me meldt uit')
229 reader.request('GET', '/me/feed', expected=404)
230 check('application/atom+xml' not in reader.opener.open(base + '/index.php').read().decode(), 'Geen autodiscovery als de feed uit staat')
231 admin.request('PUT', '/settings/feed', {'enabled': True})
232 feed(renewed)
233 reader.request('DELETE', '/me/feed')
234 feed(renewed, expected=404)
235 check(reader.request('GET', '/me/feed')['token'] is None, 'Ingetrokken: geen sleutel meer')
236 feed(base + '/feed.php?token=' + token, method='POST', expected=405)
237
238 print(f'Feed: {checks} assertions passed.')
239 except Exception:
240 log.flush()
241 print((root / 'server.log').read_text()[-4000:])
242 raise
243 finally:
244 os.killpg(process.pid, signal.SIGTERM)
245 process.wait(timeout=5)
246 log.close()