BlueGit

projects-timfalken-com verhalenboek

CI: handmatige .htaccess-probe in tijdelijke mappen

timfa

Authored 2026-10-30 · 2 weeks ago

Hash
54b5d02ae655b388770d2163d87c67f27366bf80
Parent
6f40736d
Pipeline
Passed
2 files changed +69 −0
blue-ci-cd.yml +15 −0
@@ -68,3 +68,18 @@ server_diagnose:
68 68 - apt-get install -y --no-install-recommends python3 lftp ca-certificates openssh-client
69 69 script:
70 70 - python3 ci/diagnose.py
71
72 # Manual: bisect which .htaccess directive the host rejects, in throwaway _vbprobe_* dirs (removed afterwards).
73 htaccess_probe:
74 stage: deploy
75 image: debian:bookworm-slim
76 needs: []
77 rules:
78 - if: '$CI_COMMIT_BRANCH == "master" && $CI_PIPELINE_SOURCE != "merge_request_event"'
79 when: manual
80 allow_failure: true
81 before_script:
82 - apt-get update -qq
83 - apt-get install -y --no-install-recommends python3 lftp ca-certificates openssh-client
84 script:
85 - python3 ci/htaccess_probe.py
@@ -68,3 +68,18 @@ server_diagnose:
68 - apt-get install -y --no-install-recommends python3 lftp ca-certificates openssh-client
68 - apt-get install -y --no-install-recommends python3 lftp ca-certificates openssh-client
69 script:
69 script:
70 - python3 ci/diagnose.py
70 - python3 ci/diagnose.py
71
72# Manual: bisect which .htaccess directive the host rejects, in throwaway _vbprobe_* dirs (removed afterwards).
73htaccess_probe:
74 stage: deploy
75 image: debian:bookworm-slim
76 needs: []
77 rules:
78 - if: '$CI_COMMIT_BRANCH == "master" && $CI_PIPELINE_SOURCE != "merge_request_event"'
79 when: manual
80 allow_failure: true
81 before_script:
82 - apt-get update -qq
83 - apt-get install -y --no-install-recommends python3 lftp ca-certificates openssh-client
84 script:
85 - python3 ci/htaccess_probe.py
ci/htaccess_probe.py added +54 −0
@@ -0,0 +1,54 @@
1 #!/usr/bin/env python3
2 """Manual: upload throwaway probe dirs _vbprobe_N/ (one .htaccess directive each + index.html)
3 next to the project, fetch them over HTTP, then remove ONLY those probe dirs. Never touches the project."""
4 import os, subprocess, sys, tempfile, urllib.request, urllib.error
5 from pathlib import Path
6 from urllib.parse import urlsplit
7 sys.path.insert(0, str(Path(__file__).resolve().parent))
8 from deploy_guard import project_dir # noqa: E402
9 from deploy import quoted # noqa: E402
10
11 PROBES = {
12 'none': '',
13 'options': 'Options -Indexes',
14 'dirindex': 'DirectoryIndex index.php',
15 'rewrite': '<IfModule mod_rewrite.c>\nRewriteEngine On\nRewriteCond %{HTTP:Authorization} .\nRewriteRule .* - [E=HTTP_AUTHORIZATION:%{HTTP:Authorization}]\n</IfModule>',
16 'setenvif': '<IfModule mod_setenvif.c>\nSetEnvIf Authorization "(.+)" HTTP_AUTHORIZATION=$1\n</IfModule>',
17 'require': '<FilesMatch "^(README\\.md|router\\.php)$">\nRequire all denied\n</FilesMatch>',
18 'header': '<IfModule mod_headers.c>\nHeader always set X-Content-Type-Options "nosniff"\n</IfModule>',
19 'addtype': '<IfModule mod_mime.c>\nAddType application/manifest+json .webmanifest\n</IfModule>',
20 'full': (Path(__file__).resolve().parents[1] / 'web/.htaccess').read_text(),
21 }
22
23 def lftp(e, host, cmds):
24 return subprocess.run(['lftp', '--norc'], input='\n'.join(cmds) + '\nbye\n', text=True, capture_output=True,
25 env={**e, 'LFTP_PASSWORD': e['timftppassword']})
26
27 def main():
28 e = os.environ
29 parent = os.path.dirname(project_dir(e).rstrip('/')) or '.'
30 host = e['timftphost'] if '://' in e['timftphost'] else 'ftp://' + e['timftphost']
31 pre = ['set net:timeout 20', 'set ssl:verify-certificate no',
32 'set sftp:connect-program "ssh -a -x -o StrictHostKeyChecking=accept-new"',
33 f'open --user {quoted(e["timftpuser"])} --env-password {quoted(host)}', f'cd {quoted(parent)}']
34 tmp = Path(tempfile.mkdtemp())
35 cmds = list(pre)
36 for name, body in PROBES.items():
37 d = tmp / f'_vbprobe_{name}'; d.mkdir()
38 (d / 'index.html').write_text('ok\n')
39 if body: (d / '.htaccess').write_text(body + '\n')
40 cmds.append(f'mirror --reverse --no-perms {quoted(str(d))} {quoted(d.name)}')
41 r = lftp(e, host, cmds); print('upload rc', r.returncode, r.stderr.replace(e['timftppassword'], '***')[-500:])
42 try:
43 for name in PROBES:
44 url = f'https://projects.timfalken.com/_vbprobe_{name}/index.html'
45 try: code = urllib.request.urlopen(url, timeout=20).status
46 except urllib.error.HTTPError as x: code = x.code
47 print(f'PROBE {name:10s} {code}')
48 finally:
49 r = lftp(e, host, pre + [f'rm -rf {quoted("_vbprobe_" + n)}' for n in PROBES])
50 print('cleanup rc', r.returncode)
51 return 0
52
53 if __name__ == '__main__':
54 sys.exit(main())
@@ -0,0 +1,54 @@
1#!/usr/bin/env python3
2"""Manual: upload throwaway probe dirs _vbprobe_N/ (one .htaccess directive each + index.html)
3next to the project, fetch them over HTTP, then remove ONLY those probe dirs. Never touches the project."""
4import os, subprocess, sys, tempfile, urllib.request, urllib.error
5from pathlib import Path
6from urllib.parse import urlsplit
7sys.path.insert(0, str(Path(__file__).resolve().parent))
8from deploy_guard import project_dir # noqa: E402
9from deploy import quoted # noqa: E402
10
11PROBES = {
12 'none': '',
13 'options': 'Options -Indexes',
14 'dirindex': 'DirectoryIndex index.php',
15 'rewrite': '<IfModule mod_rewrite.c>\nRewriteEngine On\nRewriteCond %{HTTP:Authorization} .\nRewriteRule .* - [E=HTTP_AUTHORIZATION:%{HTTP:Authorization}]\n</IfModule>',
16 'setenvif': '<IfModule mod_setenvif.c>\nSetEnvIf Authorization "(.+)" HTTP_AUTHORIZATION=$1\n</IfModule>',
17 'require': '<FilesMatch "^(README\\.md|router\\.php)$">\nRequire all denied\n</FilesMatch>',
18 'header': '<IfModule mod_headers.c>\nHeader always set X-Content-Type-Options "nosniff"\n</IfModule>',
19 'addtype': '<IfModule mod_mime.c>\nAddType application/manifest+json .webmanifest\n</IfModule>',
20 'full': (Path(__file__).resolve().parents[1] / 'web/.htaccess').read_text(),
21}
22
23def lftp(e, host, cmds):
24 return subprocess.run(['lftp', '--norc'], input='\n'.join(cmds) + '\nbye\n', text=True, capture_output=True,
25 env={**e, 'LFTP_PASSWORD': e['timftppassword']})
26
27def main():
28 e = os.environ
29 parent = os.path.dirname(project_dir(e).rstrip('/')) or '.'
30 host = e['timftphost'] if '://' in e['timftphost'] else 'ftp://' + e['timftphost']
31 pre = ['set net:timeout 20', 'set ssl:verify-certificate no',
32 'set sftp:connect-program "ssh -a -x -o StrictHostKeyChecking=accept-new"',
33 f'open --user {quoted(e["timftpuser"])} --env-password {quoted(host)}', f'cd {quoted(parent)}']
34 tmp = Path(tempfile.mkdtemp())
35 cmds = list(pre)
36 for name, body in PROBES.items():
37 d = tmp / f'_vbprobe_{name}'; d.mkdir()
38 (d / 'index.html').write_text('ok\n')
39 if body: (d / '.htaccess').write_text(body + '\n')
40 cmds.append(f'mirror --reverse --no-perms {quoted(str(d))} {quoted(d.name)}')
41 r = lftp(e, host, cmds); print('upload rc', r.returncode, r.stderr.replace(e['timftppassword'], '***')[-500:])
42 try:
43 for name in PROBES:
44 url = f'https://projects.timfalken.com/_vbprobe_{name}/index.html'
45 try: code = urllib.request.urlopen(url, timeout=20).status
46 except urllib.error.HTTPError as x: code = x.code
47 print(f'PROBE {name:10s} {code}')
48 finally:
49 r = lftp(e, host, pre + [f'rm -rf {quoted("_vbprobe_" + n)}' for n in PROBES])
50 print('cleanup rc', r.returncode)
51 return 0
52
53if __name__ == '__main__':
54 sys.exit(main())