BlueGit

timfa filemanager

WIP: Duplicate detection

timfa Tim Falken · pushed by timfa

Authored 2026-05-13 · 6 months ago

Hash
3fcdc0cad31be6de9f597642fa54da82d7b94eeb
Parent
d5740d7c
6 files changed +382 −16
public/app.js +91 −0
@@ -1585,6 +1585,88 @@ function selectItem (el, item)
1585 1585 // SIDEBAR
1586 1586 // ───────────────────────────────────────────────────────────────────────────
1587 1587
1588 /**
1589 * Renders (or clears) the duplicates panel at the bottom of the sidebar.
1590 * @param {object|null} item — the item whose duplicates are shown; null = clear
1591 * @param {Array|null} duplicates — array of {path, name, dir}; null = loading state
1592 */
1593 function renderSidebarDuplicates (item, duplicates)
1594 {
1595 const panel = document.getElementById('sidebar-duplicates');
1596 if (!panel) return;
1597
1598 // Clear / hide when no item or no duplicates.
1599 if (!item || (Array.isArray(duplicates) && duplicates.length === 0))
1600 {
1601 panel.classList.add('hidden');
1602 panel.innerHTML = '';
1603 return;
1604 }
1605
1606 // Loading state.
1607 if (duplicates === null)
1608 {
1609 panel.classList.add('hidden');
1610 panel.innerHTML = '';
1611 return;
1612 }
1613
1614 panel.innerHTML = '';
1615 const heading = document.createElement('p');
1616 heading.className = 'sidebar-duplicates-heading';
1617 heading.textContent = `Duplicates (${duplicates.length})`;
1618 panel.appendChild(heading);
1619
1620 const list = document.createElement('ul');
1621 list.className = 'sidebar-duplicates-list';
1622
1623 for (const dup of duplicates)
1624 {
1625 const li = document.createElement('li');
1626 const btn = document.createElement('button');
1627 btn.className = 'sidebar-duplicate-entry';
1628 btn.title = dup.path;
1629 // Show the full path; let CSS truncate with ellipsis if it does not fit.
1630 btn.textContent = dup.path;
1631 btn.addEventListener('click', () => navigateToDuplicate(dup));
1632 li.appendChild(btn);
1633 list.appendChild(li);
1634 }
1635
1636 panel.appendChild(list);
1637 panel.classList.remove('hidden');
1638 }
1639
1640 /**
1641 * Navigates to the folder containing a duplicate file and then auto-opens it.
1642 * @param {{ path: string, name: string, dir: string }} dup
1643 */
1644 async function navigateToDuplicate (dup)
1645 {
1646 // Find the root that contains the duplicate.
1647 const root = STATE.roots.find(
1648 (r) => dup.dir === r.path || dup.dir.startsWith(r.path + '/') || dup.dir.startsWith(r.path + '\\'),
1649 );
1650 if (!root)
1651 {
1652 toast('Cannot navigate to duplicate: root not accessible.', 'err');
1653 return;
1654 }
1655
1656 await navigateTo(dup.dir, root);
1657
1658 // After navigation, find and select the file tile to open it.
1659 const tile = Array.from(document.querySelectorAll('.file-item'))
1660 .find((el) => el.dataset.path === dup.path);
1661 const dupItem = STATE.items.find((i) => i.path === dup.path);
1662
1663 if (tile && dupItem)
1664 {
1665 selectItem(tile, dupItem);
1666 }
1667 }
1668 // ───────────────────────────────────────────────────────────────────────────
1669
1588 1670 async function openSidebar (item)
1589 1671 {
1590 1672 const sidebar = document.getElementById('sidebar');
@@ -1608,6 +1690,9 @@ async function openSidebar (item)
1608 1690 const actions = document.getElementById('sidebar-actions');
1609 1691 actions.innerHTML = '';
1610 1692
1693 // Reset duplicates panel whenever a new item is opened.
1694 renderSidebarDuplicates(null, null);
1695
1611 1696 if (!item.isDir)
1612 1697 {
1613 1698 const meta = await api('GET', `/files/meta?file=${encodeURIComponent(item.path)}`);
@@ -1628,6 +1713,12 @@ async function openSidebar (item)
1628 1713 `;
1629 1714 }
1630 1715
1716 // Fetch and render duplicate files.
1717 renderSidebarDuplicates(item, null); // show loading state
1718 api('GET', `/files/duplicates?file=${encodeURIComponent(item.path)}`)
1719 .then((data) => renderSidebarDuplicates(item, data ? data.duplicates : []))
1720 .catch(() => renderSidebarDuplicates(item, []));
1721
1631 1722 // View / download button
1632 1723 const btnDl = document.createElement('button');
1633 1724 btnDl.textContent = 'View / Download';
@@ -1585,6 +1585,88 @@ function selectItem (el, item)
1585// SIDEBAR
1585// SIDEBAR
1586// ───────────────────────────────────────────────────────────────────────────
1586// ───────────────────────────────────────────────────────────────────────────
1587
1587
1588/**
1589 * Renders (or clears) the duplicates panel at the bottom of the sidebar.
1590 * @param {object|null} item — the item whose duplicates are shown; null = clear
1591 * @param {Array|null} duplicates — array of {path, name, dir}; null = loading state
1592 */
1593function renderSidebarDuplicates (item, duplicates)
1594{
1595 const panel = document.getElementById('sidebar-duplicates');
1596 if (!panel) return;
1597
1598 // Clear / hide when no item or no duplicates.
1599 if (!item || (Array.isArray(duplicates) && duplicates.length === 0))
1600 {
1601 panel.classList.add('hidden');
1602 panel.innerHTML = '';
1603 return;
1604 }
1605
1606 // Loading state.
1607 if (duplicates === null)
1608 {
1609 panel.classList.add('hidden');
1610 panel.innerHTML = '';
1611 return;
1612 }
1613
1614 panel.innerHTML = '';
1615 const heading = document.createElement('p');
1616 heading.className = 'sidebar-duplicates-heading';
1617 heading.textContent = `Duplicates (${duplicates.length})`;
1618 panel.appendChild(heading);
1619
1620 const list = document.createElement('ul');
1621 list.className = 'sidebar-duplicates-list';
1622
1623 for (const dup of duplicates)
1624 {
1625 const li = document.createElement('li');
1626 const btn = document.createElement('button');
1627 btn.className = 'sidebar-duplicate-entry';
1628 btn.title = dup.path;
1629 // Show the full path; let CSS truncate with ellipsis if it does not fit.
1630 btn.textContent = dup.path;
1631 btn.addEventListener('click', () => navigateToDuplicate(dup));
1632 li.appendChild(btn);
1633 list.appendChild(li);
1634 }
1635
1636 panel.appendChild(list);
1637 panel.classList.remove('hidden');
1638}
1639
1640/**
1641 * Navigates to the folder containing a duplicate file and then auto-opens it.
1642 * @param {{ path: string, name: string, dir: string }} dup
1643 */
1644async function navigateToDuplicate (dup)
1645{
1646 // Find the root that contains the duplicate.
1647 const root = STATE.roots.find(
1648 (r) => dup.dir === r.path || dup.dir.startsWith(r.path + '/') || dup.dir.startsWith(r.path + '\\'),
1649 );
1650 if (!root)
1651 {
1652 toast('Cannot navigate to duplicate: root not accessible.', 'err');
1653 return;
1654 }
1655
1656 await navigateTo(dup.dir, root);
1657
1658 // After navigation, find and select the file tile to open it.
1659 const tile = Array.from(document.querySelectorAll('.file-item'))
1660 .find((el) => el.dataset.path === dup.path);
1661 const dupItem = STATE.items.find((i) => i.path === dup.path);
1662
1663 if (tile && dupItem)
1664 {
1665 selectItem(tile, dupItem);
1666 }
1667}
1668// ───────────────────────────────────────────────────────────────────────────
1669
1588async function openSidebar (item)
1670async function openSidebar (item)
1589{
1671{
1590 const sidebar = document.getElementById('sidebar');
1672 const sidebar = document.getElementById('sidebar');
@@ -1608,6 +1690,9 @@ async function openSidebar (item)
1608 const actions = document.getElementById('sidebar-actions');
1690 const actions = document.getElementById('sidebar-actions');
1609 actions.innerHTML = '';
1691 actions.innerHTML = '';
1610
1692
1693 // Reset duplicates panel whenever a new item is opened.
1694 renderSidebarDuplicates(null, null);
1695
1611 if (!item.isDir)
1696 if (!item.isDir)
1612 {
1697 {
1613 const meta = await api('GET', `/files/meta?file=${encodeURIComponent(item.path)}`);
1698 const meta = await api('GET', `/files/meta?file=${encodeURIComponent(item.path)}`);
@@ -1628,6 +1713,12 @@ async function openSidebar (item)
1628 `;
1713 `;
1629 }
1714 }
1630
1715
1716 // Fetch and render duplicate files.
1717 renderSidebarDuplicates(item, null); // show loading state
1718 api('GET', `/files/duplicates?file=${encodeURIComponent(item.path)}`)
1719 .then((data) => renderSidebarDuplicates(item, data ? data.duplicates : []))
1720 .catch(() => renderSidebarDuplicates(item, []));
1721
1631 // View / download button
1722 // View / download button
1632 const btnDl = document.createElement('button');
1723 const btnDl = document.createElement('button');
1633 btnDl.textContent = 'View / Download';
1724 btnDl.textContent = 'View / Download';
public/index.html +1 −0
@@ -132,6 +132,7 @@
132 132 <h2 id="sidebar-name"></h2>
133 133 <dl id="sidebar-meta"></dl>
134 134 <div id="sidebar-actions"></div>
135 <div id="sidebar-duplicates" class="hidden"></div>
135 136 </aside>
136 137
137 138 </div>
@@ -132,6 +132,7 @@
132 <h2 id="sidebar-name"></h2>
132 <h2 id="sidebar-name"></h2>
133 <dl id="sidebar-meta"></dl>
133 <dl id="sidebar-meta"></dl>
134 <div id="sidebar-actions"></div>
134 <div id="sidebar-actions"></div>
135 <div id="sidebar-duplicates" class="hidden"></div>
135 </aside>
136 </aside>
136
137
137 </div>
138 </div>
public/style.css +44 −0
@@ -446,6 +446,50 @@ button.danger:hover {
446 446 color: var(--danger);
447 447 }
448 448
449 /* ─── Duplicates panel ─────────────────────────────────── */
450 #sidebar-duplicates {
451 margin-top: 1rem;
452 border-top: 1px solid var(--border);
453 padding-top: 0.75rem;
454 }
455
456 .sidebar-duplicates-heading {
457 font-size: 0.75rem;
458 font-weight: 600;
459 text-transform: uppercase;
460 letter-spacing: 0.05em;
461 color: var(--text-muted, #888);
462 margin: 0 0 0.4rem;
463 }
464
465 .sidebar-duplicates-list {
466 list-style: none;
467 margin: 0;
468 padding: 0;
469 display: flex;
470 flex-direction: column;
471 gap: 0.25rem;
472 }
473
474 .sidebar-duplicate-entry {
475 display: block;
476 width: 100%;
477 background: var(--surface2);
478 color: var(--text);
479 padding: 0.35rem 0.6rem;
480 text-align: left;
481 font-size: 0.8rem;
482 white-space: nowrap;
483 overflow: hidden;
484 text-overflow: ellipsis;
485 border-radius: var(--radius);
486 cursor: pointer;
487 }
488
489 .sidebar-duplicate-entry:hover {
490 background: var(--border);
491 }
492
449 493 /* =========================================================
450 494 MODALS
451 495 ========================================================= */
@@ -446,6 +446,50 @@ button.danger:hover {
446 color: var(--danger);
446 color: var(--danger);
447}
447}
448
448
449/* ─── Duplicates panel ─────────────────────────────────── */
450#sidebar-duplicates {
451 margin-top: 1rem;
452 border-top: 1px solid var(--border);
453 padding-top: 0.75rem;
454}
455
456.sidebar-duplicates-heading {
457 font-size: 0.75rem;
458 font-weight: 600;
459 text-transform: uppercase;
460 letter-spacing: 0.05em;
461 color: var(--text-muted, #888);
462 margin: 0 0 0.4rem;
463}
464
465.sidebar-duplicates-list {
466 list-style: none;
467 margin: 0;
468 padding: 0;
469 display: flex;
470 flex-direction: column;
471 gap: 0.25rem;
472}
473
474.sidebar-duplicate-entry {
475 display: block;
476 width: 100%;
477 background: var(--surface2);
478 color: var(--text);
479 padding: 0.35rem 0.6rem;
480 text-align: left;
481 font-size: 0.8rem;
482 white-space: nowrap;
483 overflow: hidden;
484 text-overflow: ellipsis;
485 border-radius: var(--radius);
486 cursor: pointer;
487}
488
489.sidebar-duplicate-entry:hover {
490 background: var(--border);
491}
492
449/* =========================================================
493/* =========================================================
450 MODALS
494 MODALS
451 ========================================================= */
495 ========================================================= */
server/db.js +12 −2
@@ -64,11 +64,21 @@ function initDb ()
64 64 `);
65 65
66 66 const fileMetaCols = d.prepare('PRAGMA table_info(file_meta)').all();
67 const hasWatchedBy = fileMetaCols.some((col) => String(col.name || '') === 'watched_by');
68 if (!hasWatchedBy)
67 const colNames = fileMetaCols.map((col) => String(col.name || ''));
68
69 if (!colNames.includes('watched_by'))
69 70 {
70 71 d.exec("ALTER TABLE file_meta ADD COLUMN watched_by TEXT NOT NULL DEFAULT '{}'");
71 72 }
73
74 if (!colNames.includes('sha256'))
75 {
76 d.exec('ALTER TABLE file_meta ADD COLUMN sha256 TEXT');
77 d.exec('CREATE INDEX IF NOT EXISTS idx_file_meta_sha256 ON file_meta(sha256)');
78 } else
79 {
80 d.exec('CREATE INDEX IF NOT EXISTS idx_file_meta_sha256 ON file_meta(sha256)');
81 }
72 82 }
73 83
74 84 function closeDb ()
@@ -64,11 +64,21 @@ function initDb ()
64 `);
64 `);
65
65
66 const fileMetaCols = d.prepare('PRAGMA table_info(file_meta)').all();
66 const fileMetaCols = d.prepare('PRAGMA table_info(file_meta)').all();
67 const hasWatchedBy = fileMetaCols.some((col) => String(col.name || '') === 'watched_by');
67 const colNames = fileMetaCols.map((col) => String(col.name || ''));
68 if (!hasWatchedBy)
68
69 if (!colNames.includes('watched_by'))
69 {
70 {
70 d.exec("ALTER TABLE file_meta ADD COLUMN watched_by TEXT NOT NULL DEFAULT '{}'");
71 d.exec("ALTER TABLE file_meta ADD COLUMN watched_by TEXT NOT NULL DEFAULT '{}'");
71 }
72 }
73
74 if (!colNames.includes('sha256'))
75 {
76 d.exec('ALTER TABLE file_meta ADD COLUMN sha256 TEXT');
77 d.exec('CREATE INDEX IF NOT EXISTS idx_file_meta_sha256 ON file_meta(sha256)');
78 } else
79 {
80 d.exec('CREATE INDEX IF NOT EXISTS idx_file_meta_sha256 ON file_meta(sha256)');
81 }
72}
82}
73
83
74function closeDb ()
84function closeDb ()
server/routes/files.js +119 −13
@@ -14,6 +14,28 @@ const { getCachedThumbnailInfo, getImagesWithThumbnailsInDir, getFolderIconSourc
14 14 const { buildZip } = require('./share');
15 15 const { createVideoStreamSession, sendVideoStreamAsset, releaseVideoStreamSession } = require('../video-stream');
16 16
17 // ---------------------------------------------------------------------------
18 // SHA-256 hashing helper
19 // ---------------------------------------------------------------------------
20
21 /**
22 * Computes the SHA-256 hex digest of the file at `filePath`.
23 * Streams the file so it works for large files without loading them into memory.
24 * @param {string} filePath
25 * @returns {Promise<string>} hex digest
26 */
27 function computeFileSha256 (filePath)
28 {
29 return new Promise((resolve, reject) =>
30 {
31 const hash = crypto.createHash('sha256');
32 const stream = fs.createReadStream(filePath);
33 stream.on('error', reject);
34 stream.on('data', (chunk) => hash.update(chunk));
35 stream.on('end', () => resolve(hash.digest('hex')));
36 });
37 }
38
17 39 // ---------------------------------------------------------------------------
18 40 // Multer storage — files are staged into temporary files and only promoted
19 41 // to their final names after the full stream arrives.
@@ -271,19 +293,20 @@ function pathInsideResolved (child, root)
271 293 function rewriteMetaPaths (fromPath, toPath)
272 294 {
273 295 const db = getDb();
274 const rows = db.prepare('SELECT file_path, uploader, downloaders, watched_by FROM file_meta WHERE file_path = ? OR file_path LIKE ?')
296 const rows = db.prepare('SELECT file_path, uploader, downloaders, watched_by, sha256 FROM file_meta WHERE file_path = ? OR file_path LIKE ?')
275 297 .all([fromPath, fromPath + path.sep + '%']);
276 298
277 299 if (!rows.length) return;
278 300
279 301 const del = db.prepare('DELETE FROM file_meta WHERE file_path = ?');
280 302 const upsert = db.prepare(`
281 INSERT INTO file_meta (file_path, uploader, downloaders, watched_by)
282 VALUES (?, ?, ?, ?)
303 INSERT INTO file_meta (file_path, uploader, downloaders, watched_by, sha256)
304 VALUES (?, ?, ?, ?, ?)
283 305 ON CONFLICT(file_path) DO UPDATE SET
284 306 uploader = excluded.uploader,
285 307 downloaders = excluded.downloaders,
286 watched_by = excluded.watched_by
308 watched_by = excluded.watched_by,
309 sha256 = excluded.sha256
287 310 `);
288 311
289 312 for (const row of rows)
@@ -292,7 +315,7 @@ function rewriteMetaPaths (fromPath, toPath)
292 315 ? toPath
293 316 : toPath + row.file_path.slice(fromPath.length);
294 317 del.run([row.file_path]);
295 upsert.run([nextPath, row.uploader, row.downloaders, row.watched_by || '{}']);
318 upsert.run([nextPath, row.uploader, row.downloaders, row.watched_by || '{}', row.sha256 || null]);
296 319 }
297 320 }
298 321
@@ -569,9 +592,9 @@ router.get('/list', requireAuth, (req, res) =>
569 592
570 593 // ---------------------------------------------------------------------------
571 594 // GET /api/files/meta?file=<absolute-path>
572 // Returns file metadata from the DB.
595 // Returns file metadata from the DB. Lazily computes SHA-256 when missing.
573 596 // ---------------------------------------------------------------------------
574 router.get('/meta', requireAuth, (req, res) =>
597 router.get('/meta', requireAuth, async (req, res) =>
575 598 {
576 599 const file = req.query.file;
577 600 if (!file) return res.status(400).json({ error: 'file is required.' });
@@ -593,7 +616,29 @@ router.get('/meta', requireAuth, (req, res) =>
593 616 }
594 617
595 618 const db = getDb();
596 const meta = db.prepare('SELECT * FROM file_meta WHERE file_path = ?').get([file]);
619 let meta = db.prepare('SELECT * FROM file_meta WHERE file_path = ?').get([file]);
620
621 // Lazily compute and persist the SHA-256 hash when it is absent.
622 let sha256 = meta ? (meta.sha256 || null) : null;
623 if (!sha256 && !stat.isDirectory())
624 {
625 try
626 {
627 sha256 = await computeFileSha256(file);
628 if (meta)
629 {
630 db.prepare('UPDATE file_meta SET sha256 = ? WHERE file_path = ?').run([sha256, file]);
631 } else
632 {
633 db.prepare('INSERT INTO file_meta (file_path, uploader, downloaders, sha256) VALUES (?, ?, ?, ?)')
634 .run([file, '', '[]', sha256]);
635 meta = db.prepare('SELECT * FROM file_meta WHERE file_path = ?').get([file]);
636 }
637 } catch
638 {
639 // Non-fatal — hash will be retried next time.
640 }
641 }
597 642
598 643 res.json({
599 644 name: path.basename(file),
@@ -602,9 +647,60 @@ router.get('/meta', requireAuth, (req, res) =>
602 647 uploader: meta ? meta.uploader : '',
603 648 downloaders: meta ? parseJsonArray(meta.downloaders) : [],
604 649 watchedBy: meta ? parseJsonObject(meta.watched_by) : {},
650 sha256: sha256 || null,
605 651 });
606 652 });
607 653
654 // ---------------------------------------------------------------------------
655 // GET /api/files/duplicates?file=<absolute-path>
656 // Returns other files with the same SHA-256 hash that the user can access.
657 // ---------------------------------------------------------------------------
658 router.get('/duplicates', requireAuth, (req, res) =>
659 {
660 const file = req.query.file;
661 if (!file) return res.status(400).json({ error: 'file is required.' });
662
663 const roots = effectivePaths();
664 const info = resolveRoot(file, roots);
665 if (!info || !canRead(req.user, info.label))
666 {
667 return res.status(403).json({ error: 'Access denied.' });
668 }
669
670 const db = getDb();
671 const row = db.prepare('SELECT sha256 FROM file_meta WHERE file_path = ?').get([file]);
672 const hash = row ? (row.sha256 || null) : null;
673 if (!hash)
674 {
675 return res.json({ duplicates: [] });
676 }
677
678 // All other files sharing the same hash.
679 const candidates = db.prepare(
680 'SELECT file_path FROM file_meta WHERE sha256 = ? AND file_path != ?',
681 ).all([hash, file]);
682
683 const readable = roots.filter((r) => canRead(req.user, path.basename(r)));
684
685 const duplicates = [];
686 for (const { file_path } of candidates)
687 {
688 // Only return files inside a root the user can access and that still exist on disk.
689 const allowedRoot = readable.find(
690 (r) => file_path === r || (file_path.startsWith(r) && (file_path[r.length] === path.sep || file_path[r.length] === '/')),
691 );
692 if (!allowedRoot) continue;
693 if (!fs.existsSync(file_path)) continue;
694 duplicates.push({
695 path: file_path,
696 name: path.basename(file_path),
697 dir: path.dirname(file_path),
698 });
699 }
700
701 res.json({ duplicates });
702 });
703
608 704 // ---------------------------------------------------------------------------
609 705 // GET /api/files/space?dir=<absolute-path>
610 706 // Returns recursive used bytes for dir and available bytes on its root fs.
@@ -904,7 +1000,7 @@ router.post('/upload', requireAuth, (req, res, next) =>
904 1000
905 1001 req._uploadDir = dir;
906 1002
907 upload.array('files')(req, res, (err) =>
1003 upload.array('files')(req, res, async (err) =>
908 1004 {
909 1005 if (err) return res.status(500).json({ error: err.message });
910 1006
@@ -912,15 +1008,25 @@ router.post('/upload', requireAuth, (req, res, next) =>
912 1008 for (const f of (req.files || []))
913 1009 {
914 1010 const filePath = path.join(dir, f.originalname);
1011
1012 let sha256 = null;
1013 try
1014 {
1015 sha256 = await computeFileSha256(filePath);
1016 } catch
1017 {
1018 // Non-fatal — hash can be computed lazily when the file is accessed.
1019 }
1020
915 1021 const existing = db.prepare('SELECT * FROM file_meta WHERE file_path = ?').get([filePath]);
916 1022 if (existing)
917 1023 {
918 db.prepare('UPDATE file_meta SET uploader = ? WHERE file_path = ?')
919 .run([req.user.username, filePath]);
1024 db.prepare('UPDATE file_meta SET uploader = ?, sha256 = ? WHERE file_path = ?')
1025 .run([req.user.username, sha256, filePath]);
920 1026 } else
921 1027 {
922 db.prepare('INSERT INTO file_meta (file_path, uploader, downloaders) VALUES (?, ?, ?)')
923 .run([filePath, req.user.username, '[]']);
1028 db.prepare('INSERT INTO file_meta (file_path, uploader, downloaders, sha256) VALUES (?, ?, ?, ?)')
1029 .run([filePath, req.user.username, '[]', sha256]);
924 1030 }
925 1031 }
926 1032 res.json({ uploaded: (req.files || []).map(f => f.originalname) });
@@ -14,6 +14,28 @@ const { getCachedThumbnailInfo, getImagesWithThumbnailsInDir, getFolderIconSourc
14const { buildZip } = require('./share');
14const { buildZip } = require('./share');
15const { createVideoStreamSession, sendVideoStreamAsset, releaseVideoStreamSession } = require('../video-stream');
15const { createVideoStreamSession, sendVideoStreamAsset, releaseVideoStreamSession } = require('../video-stream');
16
16
17// ---------------------------------------------------------------------------
18// SHA-256 hashing helper
19// ---------------------------------------------------------------------------
20
21/**
22 * Computes the SHA-256 hex digest of the file at `filePath`.
23 * Streams the file so it works for large files without loading them into memory.
24 * @param {string} filePath
25 * @returns {Promise<string>} hex digest
26 */
27function computeFileSha256 (filePath)
28{
29 return new Promise((resolve, reject) =>
30 {
31 const hash = crypto.createHash('sha256');
32 const stream = fs.createReadStream(filePath);
33 stream.on('error', reject);
34 stream.on('data', (chunk) => hash.update(chunk));
35 stream.on('end', () => resolve(hash.digest('hex')));
36 });
37}
38
17// ---------------------------------------------------------------------------
39// ---------------------------------------------------------------------------
18// Multer storage — files are staged into temporary files and only promoted
40// Multer storage — files are staged into temporary files and only promoted
19// to their final names after the full stream arrives.
41// to their final names after the full stream arrives.
@@ -271,19 +293,20 @@ function pathInsideResolved (child, root)
271function rewriteMetaPaths (fromPath, toPath)
293function rewriteMetaPaths (fromPath, toPath)
272{
294{
273 const db = getDb();
295 const db = getDb();
274 const rows = db.prepare('SELECT file_path, uploader, downloaders, watched_by FROM file_meta WHERE file_path = ? OR file_path LIKE ?')
296 const rows = db.prepare('SELECT file_path, uploader, downloaders, watched_by, sha256 FROM file_meta WHERE file_path = ? OR file_path LIKE ?')
275 .all([fromPath, fromPath + path.sep + '%']);
297 .all([fromPath, fromPath + path.sep + '%']);
276
298
277 if (!rows.length) return;
299 if (!rows.length) return;
278
300
279 const del = db.prepare('DELETE FROM file_meta WHERE file_path = ?');
301 const del = db.prepare('DELETE FROM file_meta WHERE file_path = ?');
280 const upsert = db.prepare(`
302 const upsert = db.prepare(`
281 INSERT INTO file_meta (file_path, uploader, downloaders, watched_by)
303 INSERT INTO file_meta (file_path, uploader, downloaders, watched_by, sha256)
282 VALUES (?, ?, ?, ?)
304 VALUES (?, ?, ?, ?, ?)
283 ON CONFLICT(file_path) DO UPDATE SET
305 ON CONFLICT(file_path) DO UPDATE SET
284 uploader = excluded.uploader,
306 uploader = excluded.uploader,
285 downloaders = excluded.downloaders,
307 downloaders = excluded.downloaders,
286 watched_by = excluded.watched_by
308 watched_by = excluded.watched_by,
309 sha256 = excluded.sha256
287 `);
310 `);
288
311
289 for (const row of rows)
312 for (const row of rows)
@@ -292,7 +315,7 @@ function rewriteMetaPaths (fromPath, toPath)
292 ? toPath
315 ? toPath
293 : toPath + row.file_path.slice(fromPath.length);
316 : toPath + row.file_path.slice(fromPath.length);
294 del.run([row.file_path]);
317 del.run([row.file_path]);
295 upsert.run([nextPath, row.uploader, row.downloaders, row.watched_by || '{}']);
318 upsert.run([nextPath, row.uploader, row.downloaders, row.watched_by || '{}', row.sha256 || null]);
296 }
319 }
297}
320}
298
321
@@ -569,9 +592,9 @@ router.get('/list', requireAuth, (req, res) =>
569
592
570// ---------------------------------------------------------------------------
593// ---------------------------------------------------------------------------
571// GET /api/files/meta?file=<absolute-path>
594// GET /api/files/meta?file=<absolute-path>
572// Returns file metadata from the DB.
595// Returns file metadata from the DB. Lazily computes SHA-256 when missing.
573// ---------------------------------------------------------------------------
596// ---------------------------------------------------------------------------
574router.get('/meta', requireAuth, (req, res) =>
597router.get('/meta', requireAuth, async (req, res) =>
575{
598{
576 const file = req.query.file;
599 const file = req.query.file;
577 if (!file) return res.status(400).json({ error: 'file is required.' });
600 if (!file) return res.status(400).json({ error: 'file is required.' });
@@ -593,7 +616,29 @@ router.get('/meta', requireAuth, (req, res) =>
593 }
616 }
594
617
595 const db = getDb();
618 const db = getDb();
596 const meta = db.prepare('SELECT * FROM file_meta WHERE file_path = ?').get([file]);
619 let meta = db.prepare('SELECT * FROM file_meta WHERE file_path = ?').get([file]);
620
621 // Lazily compute and persist the SHA-256 hash when it is absent.
622 let sha256 = meta ? (meta.sha256 || null) : null;
623 if (!sha256 && !stat.isDirectory())
624 {
625 try
626 {
627 sha256 = await computeFileSha256(file);
628 if (meta)
629 {
630 db.prepare('UPDATE file_meta SET sha256 = ? WHERE file_path = ?').run([sha256, file]);
631 } else
632 {
633 db.prepare('INSERT INTO file_meta (file_path, uploader, downloaders, sha256) VALUES (?, ?, ?, ?)')
634 .run([file, '', '[]', sha256]);
635 meta = db.prepare('SELECT * FROM file_meta WHERE file_path = ?').get([file]);
636 }
637 } catch
638 {
639 // Non-fatal — hash will be retried next time.
640 }
641 }
597
642
598 res.json({
643 res.json({
599 name: path.basename(file),
644 name: path.basename(file),
@@ -602,9 +647,60 @@ router.get('/meta', requireAuth, (req, res) =>
602 uploader: meta ? meta.uploader : '',
647 uploader: meta ? meta.uploader : '',
603 downloaders: meta ? parseJsonArray(meta.downloaders) : [],
648 downloaders: meta ? parseJsonArray(meta.downloaders) : [],
604 watchedBy: meta ? parseJsonObject(meta.watched_by) : {},
649 watchedBy: meta ? parseJsonObject(meta.watched_by) : {},
650 sha256: sha256 || null,
605 });
651 });
606});
652});
607
653
654// ---------------------------------------------------------------------------
655// GET /api/files/duplicates?file=<absolute-path>
656// Returns other files with the same SHA-256 hash that the user can access.
657// ---------------------------------------------------------------------------
658router.get('/duplicates', requireAuth, (req, res) =>
659{
660 const file = req.query.file;
661 if (!file) return res.status(400).json({ error: 'file is required.' });
662
663 const roots = effectivePaths();
664 const info = resolveRoot(file, roots);
665 if (!info || !canRead(req.user, info.label))
666 {
667 return res.status(403).json({ error: 'Access denied.' });
668 }
669
670 const db = getDb();
671 const row = db.prepare('SELECT sha256 FROM file_meta WHERE file_path = ?').get([file]);
672 const hash = row ? (row.sha256 || null) : null;
673 if (!hash)
674 {
675 return res.json({ duplicates: [] });
676 }
677
678 // All other files sharing the same hash.
679 const candidates = db.prepare(
680 'SELECT file_path FROM file_meta WHERE sha256 = ? AND file_path != ?',
681 ).all([hash, file]);
682
683 const readable = roots.filter((r) => canRead(req.user, path.basename(r)));
684
685 const duplicates = [];
686 for (const { file_path } of candidates)
687 {
688 // Only return files inside a root the user can access and that still exist on disk.
689 const allowedRoot = readable.find(
690 (r) => file_path === r || (file_path.startsWith(r) && (file_path[r.length] === path.sep || file_path[r.length] === '/')),
691 );
692 if (!allowedRoot) continue;
693 if (!fs.existsSync(file_path)) continue;
694 duplicates.push({
695 path: file_path,
696 name: path.basename(file_path),
697 dir: path.dirname(file_path),
698 });
699 }
700
701 res.json({ duplicates });
702});
703
608// ---------------------------------------------------------------------------
704// ---------------------------------------------------------------------------
609// GET /api/files/space?dir=<absolute-path>
705// GET /api/files/space?dir=<absolute-path>
610// Returns recursive used bytes for dir and available bytes on its root fs.
706// Returns recursive used bytes for dir and available bytes on its root fs.
@@ -904,7 +1000,7 @@ router.post('/upload', requireAuth, (req, res, next) =>
904
1000
905 req._uploadDir = dir;
1001 req._uploadDir = dir;
906
1002
907 upload.array('files')(req, res, (err) =>
1003 upload.array('files')(req, res, async (err) =>
908 {
1004 {
909 if (err) return res.status(500).json({ error: err.message });
1005 if (err) return res.status(500).json({ error: err.message });
910
1006
@@ -912,15 +1008,25 @@ router.post('/upload', requireAuth, (req, res, next) =>
912 for (const f of (req.files || []))
1008 for (const f of (req.files || []))
913 {
1009 {
914 const filePath = path.join(dir, f.originalname);
1010 const filePath = path.join(dir, f.originalname);
1011
1012 let sha256 = null;
1013 try
1014 {
1015 sha256 = await computeFileSha256(filePath);
1016 } catch
1017 {
1018 // Non-fatal — hash can be computed lazily when the file is accessed.
1019 }
1020
915 const existing = db.prepare('SELECT * FROM file_meta WHERE file_path = ?').get([filePath]);
1021 const existing = db.prepare('SELECT * FROM file_meta WHERE file_path = ?').get([filePath]);
916 if (existing)
1022 if (existing)
917 {
1023 {
918 db.prepare('UPDATE file_meta SET uploader = ? WHERE file_path = ?')
1024 db.prepare('UPDATE file_meta SET uploader = ?, sha256 = ? WHERE file_path = ?')
919 .run([req.user.username, filePath]);
1025 .run([req.user.username, sha256, filePath]);
920 } else
1026 } else
921 {
1027 {
922 db.prepare('INSERT INTO file_meta (file_path, uploader, downloaders) VALUES (?, ?, ?)')
1028 db.prepare('INSERT INTO file_meta (file_path, uploader, downloaders, sha256) VALUES (?, ?, ?, ?)')
923 .run([filePath, req.user.username, '[]']);
1029 .run([filePath, req.user.username, '[]', sha256]);
924 }
1030 }
925 }
1031 }
926 res.json({ uploaded: (req.files || []).map(f => f.originalname) });
1032 res.json({ uploaded: (req.files || []).map(f => f.originalname) });
tests/files.routes.test.js +115 −1
@@ -39,8 +39,10 @@ jest.mock('../server/db', () =>
39 39 file_path TEXT PRIMARY KEY,
40 40 uploader TEXT NOT NULL DEFAULT '',
41 41 downloaders TEXT NOT NULL DEFAULT '[]',
42 watched_by TEXT NOT NULL DEFAULT '{}'
42 watched_by TEXT NOT NULL DEFAULT '{}',
43 sha256 TEXT
43 44 );
45 CREATE INDEX IF NOT EXISTS idx_file_meta_sha256 ON file_meta(sha256);
44 46 `);
45 47 }
46 48 return db;
@@ -773,3 +775,115 @@ describe('video streaming routes', () =>
773 775 expect(result.body.watchedBy).toEqual({ admin: 'fully' });
774 776 });
775 777 });
778
779 describe('SHA-256 hashing and duplicate detection', () =>
780 {
781 test('POST /api/files/upload stores sha256 hash for uploaded files', async () =>
782 {
783 const { status } = await reqMultipart(`/api/files/upload?dir=${encodeURIComponent(rootDir)}`, [
784 { name: 'hashme.txt', content: 'hello hash', type: 'text/plain' },
785 ]);
786 expect(status).toBe(200);
787
788 const meta = getDb().prepare('SELECT sha256 FROM file_meta WHERE file_path = ?')
789 .get([path.join(rootDir, 'hashme.txt')]);
790 expect(meta).toBeTruthy();
791 // SHA-256 is a 64-char hex string.
792 expect(meta.sha256).toMatch(/^[0-9a-f]{64}$/);
793 });
794
795 test('GET /api/files/meta returns sha256 and computes it lazily when missing', async () =>
796 {
797 const filePath = path.join(rootDir, 'lazy.txt');
798 fs.writeFileSync(filePath, 'lazy content');
799
800 // Insert a row without a hash to simulate a pre-existing file.
801 getDb().prepare('INSERT INTO file_meta (file_path, uploader, downloaders) VALUES (?, ?, ?)')
802 .run([filePath, 'admin', '[]']);
803
804 const { status, body } = await req('GET', `/api/files/meta?file=${encodeURIComponent(filePath)}`);
805 expect(status).toBe(200);
806 expect(body.sha256).toMatch(/^[0-9a-f]{64}$/);
807
808 // Verify it was persisted.
809 const row = getDb().prepare('SELECT sha256 FROM file_meta WHERE file_path = ?').get([filePath]);
810 expect(row.sha256).toBe(body.sha256);
811 });
812
813 test('GET /api/files/duplicates returns other files with the same hash', async () =>
814 {
815 const content = 'duplicate content';
816 const fileA = path.join(rootDir, 'dup-a.txt');
817 const fileB = path.join(rootDir, 'dup-b.txt');
818 fs.writeFileSync(fileA, content);
819 fs.writeFileSync(fileB, content);
820
821 // Compute the expected hash.
822 const crypto = require('crypto');
823 const expectedHash = crypto.createHash('sha256').update(content).digest('hex');
824
825 getDb().prepare('INSERT INTO file_meta (file_path, uploader, downloaders, sha256) VALUES (?, ?, ?, ?)')
826 .run([fileA, 'admin', '[]', expectedHash]);
827 getDb().prepare('INSERT INTO file_meta (file_path, uploader, downloaders, sha256) VALUES (?, ?, ?, ?)')
828 .run([fileB, 'admin', '[]', expectedHash]);
829
830 const { status, body } = await req('GET', `/api/files/duplicates?file=${encodeURIComponent(fileA)}`);
831 expect(status).toBe(200);
832 expect(body.duplicates).toHaveLength(1);
833 expect(body.duplicates[0].path).toBe(fileB);
834 expect(body.duplicates[0].name).toBe('dup-b.txt');
835 });
836
837 test('GET /api/files/duplicates returns empty list when no hash is stored', async () =>
838 {
839 const filePath = path.join(rootDir, 'nohash.txt');
840 fs.writeFileSync(filePath, 'no hash yet');
841
842 getDb().prepare('INSERT INTO file_meta (file_path, uploader, downloaders) VALUES (?, ?, ?)')
843 .run([filePath, 'admin', '[]']);
844
845 const { status, body } = await req('GET', `/api/files/duplicates?file=${encodeURIComponent(filePath)}`);
846 expect(status).toBe(200);
847 expect(body.duplicates).toEqual([]);
848 });
849
850 test('GET /api/files/duplicates excludes files outside accessible roots', async () =>
851 {
852 const content = 'secret dupe';
853 const accessible = path.join(rootDir, 'visible.txt');
854 const outsideRoot = path.join(os.tmpdir(), 'outside.txt');
855 fs.writeFileSync(accessible, content);
856
857 const crypto = require('crypto');
858 const hash = crypto.createHash('sha256').update(content).digest('hex');
859
860 getDb().prepare('INSERT INTO file_meta (file_path, uploader, downloaders, sha256) VALUES (?, ?, ?, ?)')
861 .run([accessible, 'admin', '[]', hash]);
862 // Insert a row for a path that is NOT inside any configured root.
863 getDb().prepare('INSERT INTO file_meta (file_path, uploader, downloaders, sha256) VALUES (?, ?, ?, ?)')
864 .run([outsideRoot, 'admin', '[]', hash]);
865
866 const { status, body } = await req('GET', `/api/files/duplicates?file=${encodeURIComponent(accessible)}`);
867 expect(status).toBe(200);
868 expect(body.duplicates.every(d => d.path !== outsideRoot)).toBe(true);
869 });
870
871 test('POST /api/files/move preserves sha256 after rename', async () =>
872 {
873 const from = path.join(rootDir, 'withHash.txt');
874 const toDir = path.join(rootDir, 'subfolder');
875 fs.writeFileSync(from, 'content');
876 fs.mkdirSync(toDir);
877
878 const hash = 'a'.repeat(64);
879 getDb().prepare('INSERT INTO file_meta (file_path, uploader, downloaders, sha256) VALUES (?, ?, ?, ?)')
880 .run([from, 'admin', '[]', hash]);
881
882 const { status } = await req('POST', '/api/files/move', { from, toDir });
883 expect(status).toBe(200);
884
885 const meta = getDb().prepare('SELECT sha256 FROM file_meta WHERE file_path = ?')
886 .get([path.join(toDir, 'withHash.txt')]);
887 expect(meta.sha256).toBe(hash);
888 });
889 });
@@ -39,8 +39,10 @@ jest.mock('../server/db', () =>
39 file_path TEXT PRIMARY KEY,
39 file_path TEXT PRIMARY KEY,
40 uploader TEXT NOT NULL DEFAULT '',
40 uploader TEXT NOT NULL DEFAULT '',
41 downloaders TEXT NOT NULL DEFAULT '[]',
41 downloaders TEXT NOT NULL DEFAULT '[]',
42 watched_by TEXT NOT NULL DEFAULT '{}'
42 watched_by TEXT NOT NULL DEFAULT '{}',
43 sha256 TEXT
43 );
44 );
45 CREATE INDEX IF NOT EXISTS idx_file_meta_sha256 ON file_meta(sha256);
44 `);
46 `);
45 }
47 }
46 return db;
48 return db;
@@ -773,3 +775,115 @@ describe('video streaming routes', () =>
773 expect(result.body.watchedBy).toEqual({ admin: 'fully' });
775 expect(result.body.watchedBy).toEqual({ admin: 'fully' });
774 });
776 });
775});
777});
778
779describe('SHA-256 hashing and duplicate detection', () =>
780{
781 test('POST /api/files/upload stores sha256 hash for uploaded files', async () =>
782 {
783 const { status } = await reqMultipart(`/api/files/upload?dir=${encodeURIComponent(rootDir)}`, [
784 { name: 'hashme.txt', content: 'hello hash', type: 'text/plain' },
785 ]);
786 expect(status).toBe(200);
787
788 const meta = getDb().prepare('SELECT sha256 FROM file_meta WHERE file_path = ?')
789 .get([path.join(rootDir, 'hashme.txt')]);
790 expect(meta).toBeTruthy();
791 // SHA-256 is a 64-char hex string.
792 expect(meta.sha256).toMatch(/^[0-9a-f]{64}$/);
793 });
794
795 test('GET /api/files/meta returns sha256 and computes it lazily when missing', async () =>
796 {
797 const filePath = path.join(rootDir, 'lazy.txt');
798 fs.writeFileSync(filePath, 'lazy content');
799
800 // Insert a row without a hash to simulate a pre-existing file.
801 getDb().prepare('INSERT INTO file_meta (file_path, uploader, downloaders) VALUES (?, ?, ?)')
802 .run([filePath, 'admin', '[]']);
803
804 const { status, body } = await req('GET', `/api/files/meta?file=${encodeURIComponent(filePath)}`);
805 expect(status).toBe(200);
806 expect(body.sha256).toMatch(/^[0-9a-f]{64}$/);
807
808 // Verify it was persisted.
809 const row = getDb().prepare('SELECT sha256 FROM file_meta WHERE file_path = ?').get([filePath]);
810 expect(row.sha256).toBe(body.sha256);
811 });
812
813 test('GET /api/files/duplicates returns other files with the same hash', async () =>
814 {
815 const content = 'duplicate content';
816 const fileA = path.join(rootDir, 'dup-a.txt');
817 const fileB = path.join(rootDir, 'dup-b.txt');
818 fs.writeFileSync(fileA, content);
819 fs.writeFileSync(fileB, content);
820
821 // Compute the expected hash.
822 const crypto = require('crypto');
823 const expectedHash = crypto.createHash('sha256').update(content).digest('hex');
824
825 getDb().prepare('INSERT INTO file_meta (file_path, uploader, downloaders, sha256) VALUES (?, ?, ?, ?)')
826 .run([fileA, 'admin', '[]', expectedHash]);
827 getDb().prepare('INSERT INTO file_meta (file_path, uploader, downloaders, sha256) VALUES (?, ?, ?, ?)')
828 .run([fileB, 'admin', '[]', expectedHash]);
829
830 const { status, body } = await req('GET', `/api/files/duplicates?file=${encodeURIComponent(fileA)}`);
831 expect(status).toBe(200);
832 expect(body.duplicates).toHaveLength(1);
833 expect(body.duplicates[0].path).toBe(fileB);
834 expect(body.duplicates[0].name).toBe('dup-b.txt');
835 });
836
837 test('GET /api/files/duplicates returns empty list when no hash is stored', async () =>
838 {
839 const filePath = path.join(rootDir, 'nohash.txt');
840 fs.writeFileSync(filePath, 'no hash yet');
841
842 getDb().prepare('INSERT INTO file_meta (file_path, uploader, downloaders) VALUES (?, ?, ?)')
843 .run([filePath, 'admin', '[]']);
844
845 const { status, body } = await req('GET', `/api/files/duplicates?file=${encodeURIComponent(filePath)}`);
846 expect(status).toBe(200);
847 expect(body.duplicates).toEqual([]);
848 });
849
850 test('GET /api/files/duplicates excludes files outside accessible roots', async () =>
851 {
852 const content = 'secret dupe';
853 const accessible = path.join(rootDir, 'visible.txt');
854 const outsideRoot = path.join(os.tmpdir(), 'outside.txt');
855 fs.writeFileSync(accessible, content);
856
857 const crypto = require('crypto');
858 const hash = crypto.createHash('sha256').update(content).digest('hex');
859
860 getDb().prepare('INSERT INTO file_meta (file_path, uploader, downloaders, sha256) VALUES (?, ?, ?, ?)')
861 .run([accessible, 'admin', '[]', hash]);
862 // Insert a row for a path that is NOT inside any configured root.
863 getDb().prepare('INSERT INTO file_meta (file_path, uploader, downloaders, sha256) VALUES (?, ?, ?, ?)')
864 .run([outsideRoot, 'admin', '[]', hash]);
865
866 const { status, body } = await req('GET', `/api/files/duplicates?file=${encodeURIComponent(accessible)}`);
867 expect(status).toBe(200);
868 expect(body.duplicates.every(d => d.path !== outsideRoot)).toBe(true);
869 });
870
871 test('POST /api/files/move preserves sha256 after rename', async () =>
872 {
873 const from = path.join(rootDir, 'withHash.txt');
874 const toDir = path.join(rootDir, 'subfolder');
875 fs.writeFileSync(from, 'content');
876 fs.mkdirSync(toDir);
877
878 const hash = 'a'.repeat(64);
879 getDb().prepare('INSERT INTO file_meta (file_path, uploader, downloaders, sha256) VALUES (?, ?, ?, ?)')
880 .run([from, 'admin', '[]', hash]);
881
882 const { status } = await req('POST', '/api/files/move', { from, toDir });
883 expect(status).toBe(200);
884
885 const meta = getDb().prepare('SELECT sha256 FROM file_meta WHERE file_path = ?')
886 .get([path.join(toDir, 'withHash.txt')]);
887 expect(meta.sha256).toBe(hash);
888 });
889});