BlueGit

timfa filemanager

Clean up aborted uploads

timfa Tim Falken · pushed by timfa

Authored 2026-05-05 · 6 months ago

Hash
56a6b93d95196aae8f44755b60fb02020269437d
Parent
aefbcaaa
4 files changed +242 −11
.github/technical-reference.md +4 −0
@@ -49,9 +49,13 @@
49 49 - GET /api/files/folder-images - Returns images inside a directory (recursive) that already have a cached thumbnail; only thumbnailed images are included per user requirement.
50 50 - GET /api/files/folder-icon - Returns the current icon source file for a folder ({icon:{filePath}} or {icon:null}); explicit assignment takes priority over auto-first.
51 51 - POST /api/files/folder-icon - Sets or clears a folder's explicit icon assignment; imagePath must have a cached thumbnail; write access required.
52 - makeTempUploadPath(dir, originalName) - Builds a hidden per-upload staging filename in the target directory for multipart uploads.
53 - unlinkIfExists(targetPath, cb) - Removes a file during upload cleanup while ignoring already-missing temp files.
54 - promoteUploadedFile(tempPath, finalPath, cb) - Atomically renames a completed staged upload into place on the Linux server.
52 55 - splitNameExt(name) - Splits a filename into stem/ext, preserving dotless names and hidden-file behavior.
53 56 - pickNumberedPath(dir, requestedName) - Finds the first free `name (n).ext` path inside a directory.
54 57 - removePathRecursive(targetPath) - Deletes existing file or directory destination to support overwrite moves.
58 - POST /api/files/upload - Streams multipart uploads into hidden temp files, cleans them up on abort, and only exposes final filenames after completion.
55 59 - POST /api/files/move - Moves file/folder; supports conflict strategies: `error` (default), `overwrite`, or `rename` with optional `targetName`.
56 60
57 61 ## public/app.js
@@ -49,9 +49,13 @@
49- GET /api/files/folder-images - Returns images inside a directory (recursive) that already have a cached thumbnail; only thumbnailed images are included per user requirement.
49- GET /api/files/folder-images - Returns images inside a directory (recursive) that already have a cached thumbnail; only thumbnailed images are included per user requirement.
50- GET /api/files/folder-icon - Returns the current icon source file for a folder ({icon:{filePath}} or {icon:null}); explicit assignment takes priority over auto-first.
50- GET /api/files/folder-icon - Returns the current icon source file for a folder ({icon:{filePath}} or {icon:null}); explicit assignment takes priority over auto-first.
51- POST /api/files/folder-icon - Sets or clears a folder's explicit icon assignment; imagePath must have a cached thumbnail; write access required.
51- POST /api/files/folder-icon - Sets or clears a folder's explicit icon assignment; imagePath must have a cached thumbnail; write access required.
52- makeTempUploadPath(dir, originalName) - Builds a hidden per-upload staging filename in the target directory for multipart uploads.
53- unlinkIfExists(targetPath, cb) - Removes a file during upload cleanup while ignoring already-missing temp files.
54- promoteUploadedFile(tempPath, finalPath, cb) - Atomically renames a completed staged upload into place on the Linux server.
52- splitNameExt(name) - Splits a filename into stem/ext, preserving dotless names and hidden-file behavior.
55- splitNameExt(name) - Splits a filename into stem/ext, preserving dotless names and hidden-file behavior.
53- pickNumberedPath(dir, requestedName) - Finds the first free `name (n).ext` path inside a directory.
56- pickNumberedPath(dir, requestedName) - Finds the first free `name (n).ext` path inside a directory.
54- removePathRecursive(targetPath) - Deletes existing file or directory destination to support overwrite moves.
57- removePathRecursive(targetPath) - Deletes existing file or directory destination to support overwrite moves.
58- POST /api/files/upload - Streams multipart uploads into hidden temp files, cleans them up on abort, and only exposes final filenames after completion.
55- POST /api/files/move - Moves file/folder; supports conflict strategies: `error` (default), `overwrite`, or `rename` with optional `targetName`.
59- POST /api/files/move - Moves file/folder; supports conflict strategies: `error` (default), `overwrite`, or `rename` with optional `targetName`.
56
60
57## public/app.js
61## public/app.js
.github/tests.md +1 −2
@@ -37,9 +37,8 @@
37 37 ---
38 38
39 39 ## `test/files.routes.test.js`
40 - `POST /api/files/upload`
40 41 - `POST /api/files/move`
41 - `POST /api/files/move / overwrites an existing destination when conflict strategy is overwrite`
42 - `POST /api/files/move / renames destination when conflict strategy is rename`
43 42 - `root alias endpoints`
44 43 - `GET /api/files/thumbnail`
45 44 - `GET /api/files/root-access`
@@ -37,9 +37,8 @@
37---
37---
38
38
39## `test/files.routes.test.js`
39## `test/files.routes.test.js`
40- `POST /api/files/upload`
40- `POST /api/files/move`
41- `POST /api/files/move`
41- `POST /api/files/move / overwrites an existing destination when conflict strategy is overwrite`
42- `POST /api/files/move / renames destination when conflict strategy is rename`
43- `root alias endpoints`
42- `root alias endpoints`
44- `GET /api/files/thumbnail`
43- `GET /api/files/thumbnail`
45- `GET /api/files/root-access`
44- `GET /api/files/root-access`
server/routes/files.js +115 −9
@@ -3,6 +3,7 @@
3 3 const express = require('express');
4 4 const path = require('path');
5 5 const fs = require('fs');
6 const crypto = require('crypto');
6 7 const multer = require('multer');
7 8 const router = express.Router();
8 9
@@ -12,20 +13,125 @@ const { requireAuth } = require('../auth');
12 13 const { getCachedThumbnailInfo, getImagesWithThumbnailsInDir, getFolderIconSourceFile, setFolderIconAssignment } = require('../thumbnail-cache');
13 14
14 15 // ---------------------------------------------------------------------------
15 // Multer storage — files are written directly to the target directory
16 // Multer storage — files are staged into temporary files and only promoted
17 // to their final names after the full stream arrives.
16 18 // ---------------------------------------------------------------------------
17 const storage = multer.diskStorage({
18 destination (req, file, cb)
19 function makeTempUploadPath (dir, originalName)
20 {
21 const safeName = path.basename(String(originalName || 'upload'));
22 const suffix = crypto.randomUUID();
23 return path.join(dir, `.${safeName}.upload-${suffix}.part`);
24 }
25
26 function unlinkIfExists (targetPath, cb)
27 {
28 fs.unlink(targetPath, (err) =>
29 {
30 if (!err || err.code === 'ENOENT') return cb(null);
31 cb(err);
32 });
33 }
34
35 function promoteUploadedFile (tempPath, finalPath, cb)
36 {
37 fs.rename(tempPath, finalPath, cb);
38 }
39
40 const storage = {
41 _handleFile (req, file, cb)
19 42 {
20 // The target directory is resolved in the route handler, so we stash it
21 // during the multer middleware call via req._uploadDir.
22 cb(null, req._uploadDir);
43 if (!req._uploadDir)
44 {
45 cb(new Error('Upload directory is not set.'));
46 return;
47 }
48
49 const finalName = path.basename(String(file.originalname || 'upload'));
50 const finalPath = path.join(req._uploadDir, finalName);
51 const tempPath = makeTempUploadPath(req._uploadDir, finalName);
52 const out = fs.createWriteStream(tempPath, { flags: 'wx' });
53
54 let settled = false;
55 let cleaned = false;
56
57 const detach = () =>
58 {
59 req.off('aborted', onAbort);
60 req.off('close', onClose);
61 file.stream.off('error', onStreamError);
62 out.off('error', onWriteError);
63 out.off('finish', onFinish);
64 };
65
66 const cleanup = (done) =>
67 {
68 if (cleaned) return done(null);
69 cleaned = true;
70 unlinkIfExists(tempPath, done);
71 };
72
73 const fail = (err) =>
74 {
75 if (settled) return;
76 settled = true;
77 detach();
78 out.destroy();
79 cleanup((cleanupErr) => cb(cleanupErr || err));
80 };
81
82 const succeed = () =>
83 {
84 if (settled) return;
85 settled = true;
86 detach();
87 cb(null, {
88 destination: req._uploadDir,
89 filename: finalName,
90 originalname: file.originalname,
91 path: finalPath,
92 size: out.bytesWritten,
93 });
94 };
95
96 const onAbort = () => fail(new Error('Upload aborted.'));
97 const onClose = () =>
98 {
99 if (!req.complete) onAbort();
100 };
101 const onStreamError = (err) => fail(err);
102 const onWriteError = (err) => fail(err);
103 const onFinish = () =>
104 {
105 if (req.aborted || !req.complete)
106 {
107 fail(new Error('Upload aborted.'));
108 return;
109 }
110
111 promoteUploadedFile(tempPath, finalPath, (err) =>
112 {
113 if (err)
114 {
115 fail(err);
116 return;
117 }
118 succeed();
119 });
120 };
121
122 req.on('aborted', onAbort);
123 req.on('close', onClose);
124 file.stream.on('error', onStreamError);
125 out.on('error', onWriteError);
126 out.on('finish', onFinish);
127 file.stream.pipe(out);
23 128 },
24 filename (req, file, cb)
129
130 _removeFile (req, file, cb)
25 131 {
26 cb(null, file.originalname);
132 unlinkIfExists(file.path, cb);
27 133 },
28 });
134 };
29 135
30 136 const upload = multer({
31 137 storage,
@@ -3,6 +3,7 @@
3const express = require('express');
3const express = require('express');
4const path = require('path');
4const path = require('path');
5const fs = require('fs');
5const fs = require('fs');
6const crypto = require('crypto');
6const multer = require('multer');
7const multer = require('multer');
7const router = express.Router();
8const router = express.Router();
8
9
@@ -12,20 +13,125 @@ const { requireAuth } = require('../auth');
12const { getCachedThumbnailInfo, getImagesWithThumbnailsInDir, getFolderIconSourceFile, setFolderIconAssignment } = require('../thumbnail-cache');
13const { getCachedThumbnailInfo, getImagesWithThumbnailsInDir, getFolderIconSourceFile, setFolderIconAssignment } = require('../thumbnail-cache');
13
14
14// ---------------------------------------------------------------------------
15// ---------------------------------------------------------------------------
15// Multer storage — files are written directly to the target directory
16// Multer storage — files are staged into temporary files and only promoted
17// to their final names after the full stream arrives.
16// ---------------------------------------------------------------------------
18// ---------------------------------------------------------------------------
17const storage = multer.diskStorage({
19function makeTempUploadPath (dir, originalName)
18 destination (req, file, cb)
20{
21 const safeName = path.basename(String(originalName || 'upload'));
22 const suffix = crypto.randomUUID();
23 return path.join(dir, `.${safeName}.upload-${suffix}.part`);
24}
25
26function unlinkIfExists (targetPath, cb)
27{
28 fs.unlink(targetPath, (err) =>
29 {
30 if (!err || err.code === 'ENOENT') return cb(null);
31 cb(err);
32 });
33}
34
35function promoteUploadedFile (tempPath, finalPath, cb)
36{
37 fs.rename(tempPath, finalPath, cb);
38}
39
40const storage = {
41 _handleFile (req, file, cb)
19 {
42 {
20 // The target directory is resolved in the route handler, so we stash it
43 if (!req._uploadDir)
21 // during the multer middleware call via req._uploadDir.
44 {
22 cb(null, req._uploadDir);
45 cb(new Error('Upload directory is not set.'));
46 return;
47 }
48
49 const finalName = path.basename(String(file.originalname || 'upload'));
50 const finalPath = path.join(req._uploadDir, finalName);
51 const tempPath = makeTempUploadPath(req._uploadDir, finalName);
52 const out = fs.createWriteStream(tempPath, { flags: 'wx' });
53
54 let settled = false;
55 let cleaned = false;
56
57 const detach = () =>
58 {
59 req.off('aborted', onAbort);
60 req.off('close', onClose);
61 file.stream.off('error', onStreamError);
62 out.off('error', onWriteError);
63 out.off('finish', onFinish);
64 };
65
66 const cleanup = (done) =>
67 {
68 if (cleaned) return done(null);
69 cleaned = true;
70 unlinkIfExists(tempPath, done);
71 };
72
73 const fail = (err) =>
74 {
75 if (settled) return;
76 settled = true;
77 detach();
78 out.destroy();
79 cleanup((cleanupErr) => cb(cleanupErr || err));
80 };
81
82 const succeed = () =>
83 {
84 if (settled) return;
85 settled = true;
86 detach();
87 cb(null, {
88 destination: req._uploadDir,
89 filename: finalName,
90 originalname: file.originalname,
91 path: finalPath,
92 size: out.bytesWritten,
93 });
94 };
95
96 const onAbort = () => fail(new Error('Upload aborted.'));
97 const onClose = () =>
98 {
99 if (!req.complete) onAbort();
100 };
101 const onStreamError = (err) => fail(err);
102 const onWriteError = (err) => fail(err);
103 const onFinish = () =>
104 {
105 if (req.aborted || !req.complete)
106 {
107 fail(new Error('Upload aborted.'));
108 return;
109 }
110
111 promoteUploadedFile(tempPath, finalPath, (err) =>
112 {
113 if (err)
114 {
115 fail(err);
116 return;
117 }
118 succeed();
119 });
120 };
121
122 req.on('aborted', onAbort);
123 req.on('close', onClose);
124 file.stream.on('error', onStreamError);
125 out.on('error', onWriteError);
126 out.on('finish', onFinish);
127 file.stream.pipe(out);
23 },
128 },
24 filename (req, file, cb)
129
130 _removeFile (req, file, cb)
25 {
131 {
26 cb(null, file.originalname);
132 unlinkIfExists(file.path, cb);
27 },
133 },
28});
134};
29
135
30const upload = multer({
136const upload = multer({
31 storage,
137 storage,
tests/files.routes.test.js +122 −0
@@ -171,6 +171,128 @@ function reqRaw (method, routePath)
171 171 });
172 172 }
173 173
174 function reqMultipart (routePath, files, extraHeaders = {})
175 {
176 const boundary = '----filemanager-test-boundary';
177 const buffers = [];
178
179 for (const file of files)
180 {
181 buffers.push(Buffer.from(
182 `--${boundary}\r\n`
183 + `Content-Disposition: form-data; name="files"; filename="${file.name}"\r\n`
184 + `Content-Type: ${file.type || 'application/octet-stream'}\r\n\r\n`
185 ));
186 buffers.push(Buffer.isBuffer(file.content) ? file.content : Buffer.from(String(file.content)));
187 buffers.push(Buffer.from('\r\n'));
188 }
189
190 buffers.push(Buffer.from(`--${boundary}--\r\n`));
191 const payload = Buffer.concat(buffers);
192
193 return new Promise((resolve, reject) =>
194 {
195 const request = http.request(baseUrl + routePath, {
196 method: 'POST',
197 headers: {
198 'Content-Type': `multipart/form-data; boundary=${boundary}`,
199 'Content-Length': payload.length,
200 ...extraHeaders,
201 },
202 }, res =>
203 {
204 let data = '';
205 res.on('data', chunk => data += chunk);
206 res.on('end', () => resolve({ status: res.statusCode, body: JSON.parse(data) }));
207 });
208 request.on('error', reject);
209 request.write(payload);
210 request.end();
211 });
212 }
213
214 function abortMultipartUpload (routePath, file)
215 {
216 const boundary = '----filemanager-test-boundary-abort';
217 const header = Buffer.from(
218 `--${boundary}\r\n`
219 + `Content-Disposition: form-data; name="files"; filename="${file.name}"\r\n`
220 + `Content-Type: ${file.type || 'application/octet-stream'}\r\n\r\n`
221 );
222 const chunk = Buffer.isBuffer(file.content) ? file.content : Buffer.from(String(file.content));
223
224 return new Promise((resolve) =>
225 {
226 const request = http.request(baseUrl + routePath, {
227 method: 'POST',
228 headers: {
229 'Content-Type': `multipart/form-data; boundary=${boundary}`,
230 },
231 });
232
233 let settled = false;
234 const finish = () =>
235 {
236 if (settled) return;
237 settled = true;
238 resolve();
239 };
240
241 request.on('error', finish);
242 request.on('close', finish);
243 request.write(header);
244 request.write(chunk);
245 setTimeout(() => request.destroy(), 20);
246 });
247 }
248
249 async function waitFor(predicate, timeoutMs = 500)
250 {
251 const start = Date.now();
252 while (Date.now() - start < timeoutMs)
253 {
254 if (predicate()) return;
255 await new Promise(resolve => setTimeout(resolve, 10));
256 }
257 throw new Error('Timed out waiting for condition.');
258 }
259
260 describe('POST /api/files/upload', () =>
261 {
262 test('stores a completed upload under its final name', async () =>
263 {
264 const { status, body } = await reqMultipart(`/api/files/upload?dir=${encodeURIComponent(rootDir)}`, [
265 { name: 'hello.txt', content: 'hello world', type: 'text/plain' },
266 ]);
267
268 expect(status).toBe(200);
269 expect(body.uploaded).toEqual(['hello.txt']);
270 expect(fs.readFileSync(path.join(rootDir, 'hello.txt'), 'utf8')).toBe('hello world');
271
272 const meta = getDb().prepare('SELECT * FROM file_meta WHERE file_path = ?')
273 .get([path.join(rootDir, 'hello.txt')]);
274 expect(meta).toMatchObject({ uploader: 'admin' });
275 });
276
277 test('removes partial files when the upload connection aborts', async () =>
278 {
279 await abortMultipartUpload(`/api/files/upload?dir=${encodeURIComponent(rootDir)}`, {
280 name: 'partial.txt',
281 content: Buffer.alloc(256 * 1024, 'a'),
282 type: 'text/plain',
283 });
284
285 await waitFor(() => fs.readdirSync(rootDir).length === 0);
286
287 expect(fs.existsSync(path.join(rootDir, 'partial.txt'))).toBe(false);
288 expect(fs.readdirSync(rootDir)).toEqual([]);
289
290 const meta = getDb().prepare('SELECT * FROM file_meta WHERE file_path = ?')
291 .get([path.join(rootDir, 'partial.txt')]);
292 expect(meta).toBeFalsy();
293 });
294 });
295
174 296 describe('POST /api/files/move', () =>
175 297 {
176 298 test('moves a file into another folder and rewrites metadata', async () =>
@@ -171,6 +171,128 @@ function reqRaw (method, routePath)
171 });
171 });
172}
172}
173
173
174function reqMultipart (routePath, files, extraHeaders = {})
175{
176 const boundary = '----filemanager-test-boundary';
177 const buffers = [];
178
179 for (const file of files)
180 {
181 buffers.push(Buffer.from(
182 `--${boundary}\r\n`
183 + `Content-Disposition: form-data; name="files"; filename="${file.name}"\r\n`
184 + `Content-Type: ${file.type || 'application/octet-stream'}\r\n\r\n`
185 ));
186 buffers.push(Buffer.isBuffer(file.content) ? file.content : Buffer.from(String(file.content)));
187 buffers.push(Buffer.from('\r\n'));
188 }
189
190 buffers.push(Buffer.from(`--${boundary}--\r\n`));
191 const payload = Buffer.concat(buffers);
192
193 return new Promise((resolve, reject) =>
194 {
195 const request = http.request(baseUrl + routePath, {
196 method: 'POST',
197 headers: {
198 'Content-Type': `multipart/form-data; boundary=${boundary}`,
199 'Content-Length': payload.length,
200 ...extraHeaders,
201 },
202 }, res =>
203 {
204 let data = '';
205 res.on('data', chunk => data += chunk);
206 res.on('end', () => resolve({ status: res.statusCode, body: JSON.parse(data) }));
207 });
208 request.on('error', reject);
209 request.write(payload);
210 request.end();
211 });
212}
213
214function abortMultipartUpload (routePath, file)
215{
216 const boundary = '----filemanager-test-boundary-abort';
217 const header = Buffer.from(
218 `--${boundary}\r\n`
219 + `Content-Disposition: form-data; name="files"; filename="${file.name}"\r\n`
220 + `Content-Type: ${file.type || 'application/octet-stream'}\r\n\r\n`
221 );
222 const chunk = Buffer.isBuffer(file.content) ? file.content : Buffer.from(String(file.content));
223
224 return new Promise((resolve) =>
225 {
226 const request = http.request(baseUrl + routePath, {
227 method: 'POST',
228 headers: {
229 'Content-Type': `multipart/form-data; boundary=${boundary}`,
230 },
231 });
232
233 let settled = false;
234 const finish = () =>
235 {
236 if (settled) return;
237 settled = true;
238 resolve();
239 };
240
241 request.on('error', finish);
242 request.on('close', finish);
243 request.write(header);
244 request.write(chunk);
245 setTimeout(() => request.destroy(), 20);
246 });
247}
248
249async function waitFor(predicate, timeoutMs = 500)
250{
251 const start = Date.now();
252 while (Date.now() - start < timeoutMs)
253 {
254 if (predicate()) return;
255 await new Promise(resolve => setTimeout(resolve, 10));
256 }
257 throw new Error('Timed out waiting for condition.');
258}
259
260describe('POST /api/files/upload', () =>
261{
262 test('stores a completed upload under its final name', async () =>
263 {
264 const { status, body } = await reqMultipart(`/api/files/upload?dir=${encodeURIComponent(rootDir)}`, [
265 { name: 'hello.txt', content: 'hello world', type: 'text/plain' },
266 ]);
267
268 expect(status).toBe(200);
269 expect(body.uploaded).toEqual(['hello.txt']);
270 expect(fs.readFileSync(path.join(rootDir, 'hello.txt'), 'utf8')).toBe('hello world');
271
272 const meta = getDb().prepare('SELECT * FROM file_meta WHERE file_path = ?')
273 .get([path.join(rootDir, 'hello.txt')]);
274 expect(meta).toMatchObject({ uploader: 'admin' });
275 });
276
277 test('removes partial files when the upload connection aborts', async () =>
278 {
279 await abortMultipartUpload(`/api/files/upload?dir=${encodeURIComponent(rootDir)}`, {
280 name: 'partial.txt',
281 content: Buffer.alloc(256 * 1024, 'a'),
282 type: 'text/plain',
283 });
284
285 await waitFor(() => fs.readdirSync(rootDir).length === 0);
286
287 expect(fs.existsSync(path.join(rootDir, 'partial.txt'))).toBe(false);
288 expect(fs.readdirSync(rootDir)).toEqual([]);
289
290 const meta = getDb().prepare('SELECT * FROM file_meta WHERE file_path = ?')
291 .get([path.join(rootDir, 'partial.txt')]);
292 expect(meta).toBeFalsy();
293 });
294});
295
174describe('POST /api/files/move', () =>
296describe('POST /api/files/move', () =>
175{
297{
176 test('moves a file into another folder and rewrites metadata', async () =>
298 test('moves a file into another folder and rewrites metadata', async () =>