WIP: Duplicate detection
@@ -1585,6 +1585,88 @@ function selectItem (el, item)
1585
1585
// SIDEBAR
1586
1586
// ───────────────────────────────────────────────────────────────────────────
1587
1587
1588
/**
1589
* Renders (or clears) the duplicates panel at the bottom of the sidebar.
1590
* @param {object|null} item — the item whose duplicates are shown; null = clear
1591
* @param {Array|null} duplicates — array of {path, name, dir}; null = loading state
1592
*/
1593
function renderSidebarDuplicates (item, duplicates)
1594
{
1595
const panel = document.getElementById('sidebar-duplicates');
1596
if (!panel) return;
1597
1598
// Clear / hide when no item or no duplicates.
1599
if (!item || (Array.isArray(duplicates) && duplicates.length === 0))
1600
{
1601
panel.classList.add('hidden');
1602
panel.innerHTML = '';
1603
return;
1604
}
1605
1606
// Loading state.
1607
if (duplicates === null)
1608
{
1609
panel.classList.add('hidden');
1610
panel.innerHTML = '';
1611
return;
1612
}
1613
1614
panel.innerHTML = '';
1615
const heading = document.createElement('p');
1616
heading.className = 'sidebar-duplicates-heading';
1617
heading.textContent = `Duplicates (${duplicates.length})`;
1618
panel.appendChild(heading);
1619
1620
const list = document.createElement('ul');
1621
list.className = 'sidebar-duplicates-list';
1622
1623
for (const dup of duplicates)
1624
{
1625
const li = document.createElement('li');
1626
const btn = document.createElement('button');
1627
btn.className = 'sidebar-duplicate-entry';
1628
btn.title = dup.path;
1629
// Show the full path; let CSS truncate with ellipsis if it does not fit.
1630
btn.textContent = dup.path;
1631
btn.addEventListener('click', () => navigateToDuplicate(dup));
1632
li.appendChild(btn);
1633
list.appendChild(li);
1634
}
1635
1636
panel.appendChild(list);
1637
panel.classList.remove('hidden');
1638
}
1639
1640
/**
1641
* Navigates to the folder containing a duplicate file and then auto-opens it.
1642
* @param {{ path: string, name: string, dir: string }} dup
1643
*/
1644
async function navigateToDuplicate (dup)
1645
{
1646
// Find the root that contains the duplicate.
1647
const root = STATE.roots.find(
1648
(r) => dup.dir === r.path || dup.dir.startsWith(r.path + '/') || dup.dir.startsWith(r.path + '\\'),
1649
);
1650
if (!root)
1651
{
1652
toast('Cannot navigate to duplicate: root not accessible.', 'err');
1653
return;
1654
}
1655
1656
await navigateTo(dup.dir, root);
1657
1658
// After navigation, find and select the file tile to open it.
1659
const tile = Array.from(document.querySelectorAll('.file-item'))
1660
.find((el) => el.dataset.path === dup.path);
1661
const dupItem = STATE.items.find((i) => i.path === dup.path);
1662
1663
if (tile && dupItem)
1664
{
1665
selectItem(tile, dupItem);
1666
}
1667
}
1668
// ───────────────────────────────────────────────────────────────────────────
1669
1588
1670
async function openSidebar (item)
1589
1671
{
1590
1672
const sidebar = document.getElementById('sidebar');
@@ -1608,6 +1690,9 @@ async function openSidebar (item)
1608
1690
const actions = document.getElementById('sidebar-actions');
1609
1691
actions.innerHTML = '';
1610
1692
1693
// Reset duplicates panel whenever a new item is opened.
1694
renderSidebarDuplicates(null, null);
1695
1611
1696
if (!item.isDir)
1612
1697
{
1613
1698
const meta = await api('GET', `/files/meta?file=${encodeURIComponent(item.path)}`);
@@ -1628,6 +1713,12 @@ async function openSidebar (item)
1628
1713
`;
1629
1714
}
1630
1715
1716
// Fetch and render duplicate files.
1717
renderSidebarDuplicates(item, null); // show loading state
1718
api('GET', `/files/duplicates?file=${encodeURIComponent(item.path)}`)
1719
.then((data) => renderSidebarDuplicates(item, data ? data.duplicates : []))
1720
.catch(() => renderSidebarDuplicates(item, []));
1721
1631
1722
// View / download button
1632
1723
const btnDl = document.createElement('button');
1633
1724
btnDl.textContent = 'View / Download';
@@ -1585,6 +1585,88 @@ function selectItem (el, item)1585
// SIDEBAR1585
// SIDEBAR1586
// ───────────────────────────────────────────────────────────────────────────1586
// ───────────────────────────────────────────────────────────────────────────1587
1587
1588
/**1589
* Renders (or clears) the duplicates panel at the bottom of the sidebar.1590
* @param {object|null} item — the item whose duplicates are shown; null = clear1591
* @param {Array|null} duplicates — array of {path, name, dir}; null = loading state1592
*/1593
function renderSidebarDuplicates (item, duplicates)1594
{1595
const panel = document.getElementById('sidebar-duplicates');1596
if (!panel) return;1597
1598
// Clear / hide when no item or no duplicates.1599
if (!item || (Array.isArray(duplicates) && duplicates.length === 0))1600
{1601
panel.classList.add('hidden');1602
panel.innerHTML = '';1603
return;1604
}1605
1606
// Loading state.1607
if (duplicates === null)1608
{1609
panel.classList.add('hidden');1610
panel.innerHTML = '';1611
return;1612
}1613
1614
panel.innerHTML = '';1615
const heading = document.createElement('p');1616
heading.className = 'sidebar-duplicates-heading';1617
heading.textContent = `Duplicates (${duplicates.length})`;1618
panel.appendChild(heading);1619
1620
const list = document.createElement('ul');1621
list.className = 'sidebar-duplicates-list';1622
1623
for (const dup of duplicates)1624
{1625
const li = document.createElement('li');1626
const btn = document.createElement('button');1627
btn.className = 'sidebar-duplicate-entry';1628
btn.title = dup.path;1629
// Show the full path; let CSS truncate with ellipsis if it does not fit.1630
btn.textContent = dup.path;1631
btn.addEventListener('click', () => navigateToDuplicate(dup));1632
li.appendChild(btn);1633
list.appendChild(li);1634
}1635
1636
panel.appendChild(list);1637
panel.classList.remove('hidden');1638
}1639
1640
/**1641
* Navigates to the folder containing a duplicate file and then auto-opens it.1642
* @param {{ path: string, name: string, dir: string }} dup1643
*/1644
async function navigateToDuplicate (dup)1645
{1646
// Find the root that contains the duplicate.1647
const root = STATE.roots.find(1648
(r) => dup.dir === r.path || dup.dir.startsWith(r.path + '/') || dup.dir.startsWith(r.path + '\\'),1649
);1650
if (!root)1651
{1652
toast('Cannot navigate to duplicate: root not accessible.', 'err');1653
return;1654
}1655
1656
await navigateTo(dup.dir, root);1657
1658
// After navigation, find and select the file tile to open it.1659
const tile = Array.from(document.querySelectorAll('.file-item'))1660
.find((el) => el.dataset.path === dup.path);1661
const dupItem = STATE.items.find((i) => i.path === dup.path);1662
1663
if (tile && dupItem)1664
{1665
selectItem(tile, dupItem);1666
}1667
}1668
// ───────────────────────────────────────────────────────────────────────────1669
1588
async function openSidebar (item)1670
async function openSidebar (item)1589
{1671
{1590
const sidebar = document.getElementById('sidebar');1672
const sidebar = document.getElementById('sidebar');@@ -1608,6 +1690,9 @@ async function openSidebar (item)1608
const actions = document.getElementById('sidebar-actions');1690
const actions = document.getElementById('sidebar-actions');1609
actions.innerHTML = '';1691
actions.innerHTML = '';1610
1692
1693
// Reset duplicates panel whenever a new item is opened.1694
renderSidebarDuplicates(null, null);1695
1611
if (!item.isDir)1696
if (!item.isDir)1612
{1697
{1613
const meta = await api('GET', `/files/meta?file=${encodeURIComponent(item.path)}`);1698
const meta = await api('GET', `/files/meta?file=${encodeURIComponent(item.path)}`);@@ -1628,6 +1713,12 @@ async function openSidebar (item)1628
`;1713
`;1629
}1714
}1630
1715
1716
// Fetch and render duplicate files.1717
renderSidebarDuplicates(item, null); // show loading state1718
api('GET', `/files/duplicates?file=${encodeURIComponent(item.path)}`)1719
.then((data) => renderSidebarDuplicates(item, data ? data.duplicates : []))1720
.catch(() => renderSidebarDuplicates(item, []));1721
1631
// View / download button1722
// View / download button1632
const btnDl = document.createElement('button');1723
const btnDl = document.createElement('button');1633
btnDl.textContent = 'View / Download';1724
btnDl.textContent = 'View / Download';@@ -132,6 +132,7 @@
132
132
<h2 id="sidebar-name"></h2>
133
133
<dl id="sidebar-meta"></dl>
134
134
<div id="sidebar-actions"></div>
135
<div id="sidebar-duplicates" class="hidden"></div>
135
136
</aside>
136
137
137
138
</div>
@@ -132,6 +132,7 @@132
<h2 id="sidebar-name"></h2>132
<h2 id="sidebar-name"></h2>133
<dl id="sidebar-meta"></dl>133
<dl id="sidebar-meta"></dl>134
<div id="sidebar-actions"></div>134
<div id="sidebar-actions"></div>135
<div id="sidebar-duplicates" class="hidden"></div>135
</aside>136
</aside>136
137
137
</div>138
</div>@@ -446,6 +446,50 @@ button.danger:hover {
446
446
color: var(--danger);
447
447
}
448
448
449
/* ─── Duplicates panel ─────────────────────────────────── */
450
#sidebar-duplicates {
451
margin-top: 1rem;
452
border-top: 1px solid var(--border);
453
padding-top: 0.75rem;
454
}
455
456
.sidebar-duplicates-heading {
457
font-size: 0.75rem;
458
font-weight: 600;
459
text-transform: uppercase;
460
letter-spacing: 0.05em;
461
color: var(--text-muted, #888);
462
margin: 0 0 0.4rem;
463
}
464
465
.sidebar-duplicates-list {
466
list-style: none;
467
margin: 0;
468
padding: 0;
469
display: flex;
470
flex-direction: column;
471
gap: 0.25rem;
472
}
473
474
.sidebar-duplicate-entry {
475
display: block;
476
width: 100%;
477
background: var(--surface2);
478
color: var(--text);
479
padding: 0.35rem 0.6rem;
480
text-align: left;
481
font-size: 0.8rem;
482
white-space: nowrap;
483
overflow: hidden;
484
text-overflow: ellipsis;
485
border-radius: var(--radius);
486
cursor: pointer;
487
}
488
489
.sidebar-duplicate-entry:hover {
490
background: var(--border);
491
}
492
449
493
/* =========================================================
450
494
MODALS
451
495
========================================================= */
@@ -446,6 +446,50 @@ button.danger:hover {446
color: var(--danger);
446
color: var(--danger);
447
}
447
}
448
448
449
/* ─── Duplicates panel ─────────────────────────────────── */
450
#sidebar-duplicates {
451
margin-top: 1rem;
452
border-top: 1px solid var(--border);
453
padding-top: 0.75rem;
454
}
455
456
.sidebar-duplicates-heading {
457
font-size: 0.75rem;
458
font-weight: 600;
459
text-transform: uppercase;
460
letter-spacing: 0.05em;
461
color: var(--text-muted, #888);
462
margin: 0 0 0.4rem;
463
}
464
465
.sidebar-duplicates-list {
466
list-style: none;
467
margin: 0;
468
padding: 0;
469
display: flex;
470
flex-direction: column;
471
gap: 0.25rem;
472
}
473
474
.sidebar-duplicate-entry {
475
display: block;
476
width: 100%;
477
background: var(--surface2);
478
color: var(--text);
479
padding: 0.35rem 0.6rem;
480
text-align: left;
481
font-size: 0.8rem;
482
white-space: nowrap;
483
overflow: hidden;
484
text-overflow: ellipsis;
485
border-radius: var(--radius);
486
cursor: pointer;
487
}
488
489
.sidebar-duplicate-entry:hover {
490
background: var(--border);
491
}
492
449
/* =========================================================
493
/* =========================================================
450
MODALS
494
MODALS
451
========================================================= */
495
========================================================= */
@@ -64,11 +64,21 @@ function initDb ()
64
64
`);
65
65
66
66
const fileMetaCols = d.prepare('PRAGMA table_info(file_meta)').all();
67
const hasWatchedBy = fileMetaCols.some((col) => String(col.name || '') === 'watched_by');
68
if (!hasWatchedBy)
67
const colNames = fileMetaCols.map((col) => String(col.name || ''));
68
69
if (!colNames.includes('watched_by'))
69
70
{
70
71
d.exec("ALTER TABLE file_meta ADD COLUMN watched_by TEXT NOT NULL DEFAULT '{}'");
71
72
}
73
74
if (!colNames.includes('sha256'))
75
{
76
d.exec('ALTER TABLE file_meta ADD COLUMN sha256 TEXT');
77
d.exec('CREATE INDEX IF NOT EXISTS idx_file_meta_sha256 ON file_meta(sha256)');
78
} else
79
{
80
d.exec('CREATE INDEX IF NOT EXISTS idx_file_meta_sha256 ON file_meta(sha256)');
81
}
72
82
}
73
83
74
84
function closeDb ()
@@ -64,11 +64,21 @@ function initDb ()64
`);64
`);65
65
66
const fileMetaCols = d.prepare('PRAGMA table_info(file_meta)').all();66
const fileMetaCols = d.prepare('PRAGMA table_info(file_meta)').all();67
const hasWatchedBy = fileMetaCols.some((col) => String(col.name || '') === 'watched_by');67
const colNames = fileMetaCols.map((col) => String(col.name || ''));68
if (!hasWatchedBy)68
69
if (!colNames.includes('watched_by'))69
{70
{70
d.exec("ALTER TABLE file_meta ADD COLUMN watched_by TEXT NOT NULL DEFAULT '{}'");71
d.exec("ALTER TABLE file_meta ADD COLUMN watched_by TEXT NOT NULL DEFAULT '{}'");71
}72
}73
74
if (!colNames.includes('sha256'))75
{76
d.exec('ALTER TABLE file_meta ADD COLUMN sha256 TEXT');77
d.exec('CREATE INDEX IF NOT EXISTS idx_file_meta_sha256 ON file_meta(sha256)');78
} else79
{80
d.exec('CREATE INDEX IF NOT EXISTS idx_file_meta_sha256 ON file_meta(sha256)');81
}72
}82
}73
83
74
function closeDb ()84
function closeDb ()@@ -14,6 +14,28 @@ const { getCachedThumbnailInfo, getImagesWithThumbnailsInDir, getFolderIconSourc
14
14
const { buildZip } = require('./share');
15
15
const { createVideoStreamSession, sendVideoStreamAsset, releaseVideoStreamSession } = require('../video-stream');
16
16
17
// ---------------------------------------------------------------------------
18
// SHA-256 hashing helper
19
// ---------------------------------------------------------------------------
20
21
/**
22
* Computes the SHA-256 hex digest of the file at `filePath`.
23
* Streams the file so it works for large files without loading them into memory.
24
* @param {string} filePath
25
* @returns {Promise<string>} hex digest
26
*/
27
function computeFileSha256 (filePath)
28
{
29
return new Promise((resolve, reject) =>
30
{
31
const hash = crypto.createHash('sha256');
32
const stream = fs.createReadStream(filePath);
33
stream.on('error', reject);
34
stream.on('data', (chunk) => hash.update(chunk));
35
stream.on('end', () => resolve(hash.digest('hex')));
36
});
37
}
38
17
39
// ---------------------------------------------------------------------------
18
40
// Multer storage — files are staged into temporary files and only promoted
19
41
// to their final names after the full stream arrives.
@@ -271,19 +293,20 @@ function pathInsideResolved (child, root)
271
293
function rewriteMetaPaths (fromPath, toPath)
272
294
{
273
295
const db = getDb();
274
const rows = db.prepare('SELECT file_path, uploader, downloaders, watched_by FROM file_meta WHERE file_path = ? OR file_path LIKE ?')
296
const rows = db.prepare('SELECT file_path, uploader, downloaders, watched_by, sha256 FROM file_meta WHERE file_path = ? OR file_path LIKE ?')
275
297
.all([fromPath, fromPath + path.sep + '%']);
276
298
277
299
if (!rows.length) return;
278
300
279
301
const del = db.prepare('DELETE FROM file_meta WHERE file_path = ?');
280
302
const upsert = db.prepare(`
281
INSERT INTO file_meta (file_path, uploader, downloaders, watched_by)
282
VALUES (?, ?, ?, ?)
303
INSERT INTO file_meta (file_path, uploader, downloaders, watched_by, sha256)
304
VALUES (?, ?, ?, ?, ?)
283
305
ON CONFLICT(file_path) DO UPDATE SET
284
306
uploader = excluded.uploader,
285
307
downloaders = excluded.downloaders,
286
watched_by = excluded.watched_by
308
watched_by = excluded.watched_by,
309
sha256 = excluded.sha256
287
310
`);
288
311
289
312
for (const row of rows)
@@ -292,7 +315,7 @@ function rewriteMetaPaths (fromPath, toPath)
292
315
? toPath
293
316
: toPath + row.file_path.slice(fromPath.length);
294
317
del.run([row.file_path]);
295
upsert.run([nextPath, row.uploader, row.downloaders, row.watched_by || '{}']);
318
upsert.run([nextPath, row.uploader, row.downloaders, row.watched_by || '{}', row.sha256 || null]);
296
319
}
297
320
}
298
321
@@ -569,9 +592,9 @@ router.get('/list', requireAuth, (req, res) =>
569
592
570
593
// ---------------------------------------------------------------------------
571
594
// GET /api/files/meta?file=<absolute-path>
572
// Returns file metadata from the DB.
595
// Returns file metadata from the DB. Lazily computes SHA-256 when missing.
573
596
// ---------------------------------------------------------------------------
574
router.get('/meta', requireAuth, (req, res) =>
597
router.get('/meta', requireAuth, async (req, res) =>
575
598
{
576
599
const file = req.query.file;
577
600
if (!file) return res.status(400).json({ error: 'file is required.' });
@@ -593,7 +616,29 @@ router.get('/meta', requireAuth, (req, res) =>
593
616
}
594
617
595
618
const db = getDb();
596
const meta = db.prepare('SELECT * FROM file_meta WHERE file_path = ?').get([file]);
619
let meta = db.prepare('SELECT * FROM file_meta WHERE file_path = ?').get([file]);
620
621
// Lazily compute and persist the SHA-256 hash when it is absent.
622
let sha256 = meta ? (meta.sha256 || null) : null;
623
if (!sha256 && !stat.isDirectory())
624
{
625
try
626
{
627
sha256 = await computeFileSha256(file);
628
if (meta)
629
{
630
db.prepare('UPDATE file_meta SET sha256 = ? WHERE file_path = ?').run([sha256, file]);
631
} else
632
{
633
db.prepare('INSERT INTO file_meta (file_path, uploader, downloaders, sha256) VALUES (?, ?, ?, ?)')
634
.run([file, '', '[]', sha256]);
635
meta = db.prepare('SELECT * FROM file_meta WHERE file_path = ?').get([file]);
636
}
637
} catch
638
{
639
// Non-fatal — hash will be retried next time.
640
}
641
}
597
642
598
643
res.json({
599
644
name: path.basename(file),
@@ -602,9 +647,60 @@ router.get('/meta', requireAuth, (req, res) =>
602
647
uploader: meta ? meta.uploader : '',
603
648
downloaders: meta ? parseJsonArray(meta.downloaders) : [],
604
649
watchedBy: meta ? parseJsonObject(meta.watched_by) : {},
650
sha256: sha256 || null,
605
651
});
606
652
});
607
653
654
// ---------------------------------------------------------------------------
655
// GET /api/files/duplicates?file=<absolute-path>
656
// Returns other files with the same SHA-256 hash that the user can access.
657
// ---------------------------------------------------------------------------
658
router.get('/duplicates', requireAuth, (req, res) =>
659
{
660
const file = req.query.file;
661
if (!file) return res.status(400).json({ error: 'file is required.' });
662
663
const roots = effectivePaths();
664
const info = resolveRoot(file, roots);
665
if (!info || !canRead(req.user, info.label))
666
{
667
return res.status(403).json({ error: 'Access denied.' });
668
}
669
670
const db = getDb();
671
const row = db.prepare('SELECT sha256 FROM file_meta WHERE file_path = ?').get([file]);
672
const hash = row ? (row.sha256 || null) : null;
673
if (!hash)
674
{
675
return res.json({ duplicates: [] });
676
}
677
678
// All other files sharing the same hash.
679
const candidates = db.prepare(
680
'SELECT file_path FROM file_meta WHERE sha256 = ? AND file_path != ?',
681
).all([hash, file]);
682
683
const readable = roots.filter((r) => canRead(req.user, path.basename(r)));
684
685
const duplicates = [];
686
for (const { file_path } of candidates)
687
{
688
// Only return files inside a root the user can access and that still exist on disk.
689
const allowedRoot = readable.find(
690
(r) => file_path === r || (file_path.startsWith(r) && (file_path[r.length] === path.sep || file_path[r.length] === '/')),
691
);
692
if (!allowedRoot) continue;
693
if (!fs.existsSync(file_path)) continue;
694
duplicates.push({
695
path: file_path,
696
name: path.basename(file_path),
697
dir: path.dirname(file_path),
698
});
699
}
700
701
res.json({ duplicates });
702
});
703
608
704
// ---------------------------------------------------------------------------
609
705
// GET /api/files/space?dir=<absolute-path>
610
706
// Returns recursive used bytes for dir and available bytes on its root fs.
@@ -904,7 +1000,7 @@ router.post('/upload', requireAuth, (req, res, next) =>
904
1000
905
1001
req._uploadDir = dir;
906
1002
907
upload.array('files')(req, res, (err) =>
1003
upload.array('files')(req, res, async (err) =>
908
1004
{
909
1005
if (err) return res.status(500).json({ error: err.message });
910
1006
@@ -912,15 +1008,25 @@ router.post('/upload', requireAuth, (req, res, next) =>
912
1008
for (const f of (req.files || []))
913
1009
{
914
1010
const filePath = path.join(dir, f.originalname);
1011
1012
let sha256 = null;
1013
try
1014
{
1015
sha256 = await computeFileSha256(filePath);
1016
} catch
1017
{
1018
// Non-fatal — hash can be computed lazily when the file is accessed.
1019
}
1020
915
1021
const existing = db.prepare('SELECT * FROM file_meta WHERE file_path = ?').get([filePath]);
916
1022
if (existing)
917
1023
{
918
db.prepare('UPDATE file_meta SET uploader = ? WHERE file_path = ?')
919
.run([req.user.username, filePath]);
1024
db.prepare('UPDATE file_meta SET uploader = ?, sha256 = ? WHERE file_path = ?')
1025
.run([req.user.username, sha256, filePath]);
920
1026
} else
921
1027
{
922
db.prepare('INSERT INTO file_meta (file_path, uploader, downloaders) VALUES (?, ?, ?)')
923
.run([filePath, req.user.username, '[]']);
1028
db.prepare('INSERT INTO file_meta (file_path, uploader, downloaders, sha256) VALUES (?, ?, ?, ?)')
1029
.run([filePath, req.user.username, '[]', sha256]);
924
1030
}
925
1031
}
926
1032
res.json({ uploaded: (req.files || []).map(f => f.originalname) });
@@ -14,6 +14,28 @@ const { getCachedThumbnailInfo, getImagesWithThumbnailsInDir, getFolderIconSourc14
const { buildZip } = require('./share');14
const { buildZip } = require('./share');15
const { createVideoStreamSession, sendVideoStreamAsset, releaseVideoStreamSession } = require('../video-stream');15
const { createVideoStreamSession, sendVideoStreamAsset, releaseVideoStreamSession } = require('../video-stream');16
16
17
// ---------------------------------------------------------------------------18
// SHA-256 hashing helper19
// ---------------------------------------------------------------------------20
21
/**22
* Computes the SHA-256 hex digest of the file at `filePath`.23
* Streams the file so it works for large files without loading them into memory.24
* @param {string} filePath25
* @returns {Promise<string>} hex digest26
*/27
function computeFileSha256 (filePath)28
{29
return new Promise((resolve, reject) =>30
{31
const hash = crypto.createHash('sha256');32
const stream = fs.createReadStream(filePath);33
stream.on('error', reject);34
stream.on('data', (chunk) => hash.update(chunk));35
stream.on('end', () => resolve(hash.digest('hex')));36
});37
}38
17
// ---------------------------------------------------------------------------39
// ---------------------------------------------------------------------------18
// Multer storage — files are staged into temporary files and only promoted40
// Multer storage — files are staged into temporary files and only promoted19
// to their final names after the full stream arrives.41
// to their final names after the full stream arrives.@@ -271,19 +293,20 @@ function pathInsideResolved (child, root)271
function rewriteMetaPaths (fromPath, toPath)293
function rewriteMetaPaths (fromPath, toPath)272
{294
{273
const db = getDb();295
const db = getDb();274
const rows = db.prepare('SELECT file_path, uploader, downloaders, watched_by FROM file_meta WHERE file_path = ? OR file_path LIKE ?')296
const rows = db.prepare('SELECT file_path, uploader, downloaders, watched_by, sha256 FROM file_meta WHERE file_path = ? OR file_path LIKE ?')275
.all([fromPath, fromPath + path.sep + '%']);297
.all([fromPath, fromPath + path.sep + '%']);276
298
277
if (!rows.length) return;299
if (!rows.length) return;278
300
279
const del = db.prepare('DELETE FROM file_meta WHERE file_path = ?');301
const del = db.prepare('DELETE FROM file_meta WHERE file_path = ?');280
const upsert = db.prepare(`302
const upsert = db.prepare(`281
INSERT INTO file_meta (file_path, uploader, downloaders, watched_by)303
INSERT INTO file_meta (file_path, uploader, downloaders, watched_by, sha256)282
VALUES (?, ?, ?, ?)304
VALUES (?, ?, ?, ?, ?)283
ON CONFLICT(file_path) DO UPDATE SET305
ON CONFLICT(file_path) DO UPDATE SET284
uploader = excluded.uploader,306
uploader = excluded.uploader,285
downloaders = excluded.downloaders,307
downloaders = excluded.downloaders,286
watched_by = excluded.watched_by308
watched_by = excluded.watched_by,309
sha256 = excluded.sha256287
`);310
`);288
311
289
for (const row of rows)312
for (const row of rows)@@ -292,7 +315,7 @@ function rewriteMetaPaths (fromPath, toPath)292
? toPath315
? toPath293
: toPath + row.file_path.slice(fromPath.length);316
: toPath + row.file_path.slice(fromPath.length);294
del.run([row.file_path]);317
del.run([row.file_path]);295
upsert.run([nextPath, row.uploader, row.downloaders, row.watched_by || '{}']);318
upsert.run([nextPath, row.uploader, row.downloaders, row.watched_by || '{}', row.sha256 || null]);296
}319
}297
}320
}298
321
@@ -569,9 +592,9 @@ router.get('/list', requireAuth, (req, res) =>569
592
570
// ---------------------------------------------------------------------------593
// ---------------------------------------------------------------------------571
// GET /api/files/meta?file=<absolute-path>594
// GET /api/files/meta?file=<absolute-path>572
// Returns file metadata from the DB.595
// Returns file metadata from the DB. Lazily computes SHA-256 when missing.573
// ---------------------------------------------------------------------------596
// ---------------------------------------------------------------------------574
router.get('/meta', requireAuth, (req, res) =>597
router.get('/meta', requireAuth, async (req, res) =>575
{598
{576
const file = req.query.file;599
const file = req.query.file;577
if (!file) return res.status(400).json({ error: 'file is required.' });600
if (!file) return res.status(400).json({ error: 'file is required.' });@@ -593,7 +616,29 @@ router.get('/meta', requireAuth, (req, res) =>593
}616
}594
617
595
const db = getDb();618
const db = getDb();596
const meta = db.prepare('SELECT * FROM file_meta WHERE file_path = ?').get([file]);619
let meta = db.prepare('SELECT * FROM file_meta WHERE file_path = ?').get([file]);620
621
// Lazily compute and persist the SHA-256 hash when it is absent.622
let sha256 = meta ? (meta.sha256 || null) : null;623
if (!sha256 && !stat.isDirectory())624
{625
try626
{627
sha256 = await computeFileSha256(file);628
if (meta)629
{630
db.prepare('UPDATE file_meta SET sha256 = ? WHERE file_path = ?').run([sha256, file]);631
} else632
{633
db.prepare('INSERT INTO file_meta (file_path, uploader, downloaders, sha256) VALUES (?, ?, ?, ?)')634
.run([file, '', '[]', sha256]);635
meta = db.prepare('SELECT * FROM file_meta WHERE file_path = ?').get([file]);636
}637
} catch638
{639
// Non-fatal — hash will be retried next time.640
}641
}597
642
598
res.json({643
res.json({599
name: path.basename(file),644
name: path.basename(file),@@ -602,9 +647,60 @@ router.get('/meta', requireAuth, (req, res) =>602
uploader: meta ? meta.uploader : '',647
uploader: meta ? meta.uploader : '',603
downloaders: meta ? parseJsonArray(meta.downloaders) : [],648
downloaders: meta ? parseJsonArray(meta.downloaders) : [],604
watchedBy: meta ? parseJsonObject(meta.watched_by) : {},649
watchedBy: meta ? parseJsonObject(meta.watched_by) : {},650
sha256: sha256 || null,605
});651
});606
});652
});607
653
654
// ---------------------------------------------------------------------------655
// GET /api/files/duplicates?file=<absolute-path>656
// Returns other files with the same SHA-256 hash that the user can access.657
// ---------------------------------------------------------------------------658
router.get('/duplicates', requireAuth, (req, res) =>659
{660
const file = req.query.file;661
if (!file) return res.status(400).json({ error: 'file is required.' });662
663
const roots = effectivePaths();664
const info = resolveRoot(file, roots);665
if (!info || !canRead(req.user, info.label))666
{667
return res.status(403).json({ error: 'Access denied.' });668
}669
670
const db = getDb();671
const row = db.prepare('SELECT sha256 FROM file_meta WHERE file_path = ?').get([file]);672
const hash = row ? (row.sha256 || null) : null;673
if (!hash)674
{675
return res.json({ duplicates: [] });676
}677
678
// All other files sharing the same hash.679
const candidates = db.prepare(680
'SELECT file_path FROM file_meta WHERE sha256 = ? AND file_path != ?',681
).all([hash, file]);682
683
const readable = roots.filter((r) => canRead(req.user, path.basename(r)));684
685
const duplicates = [];686
for (const { file_path } of candidates)687
{688
// Only return files inside a root the user can access and that still exist on disk.689
const allowedRoot = readable.find(690
(r) => file_path === r || (file_path.startsWith(r) && (file_path[r.length] === path.sep || file_path[r.length] === '/')),691
);692
if (!allowedRoot) continue;693
if (!fs.existsSync(file_path)) continue;694
duplicates.push({695
path: file_path,696
name: path.basename(file_path),697
dir: path.dirname(file_path),698
});699
}700
701
res.json({ duplicates });702
});703
608
// ---------------------------------------------------------------------------704
// ---------------------------------------------------------------------------609
// GET /api/files/space?dir=<absolute-path>705
// GET /api/files/space?dir=<absolute-path>610
// Returns recursive used bytes for dir and available bytes on its root fs.706
// Returns recursive used bytes for dir and available bytes on its root fs.@@ -904,7 +1000,7 @@ router.post('/upload', requireAuth, (req, res, next) =>904
1000
905
req._uploadDir = dir;1001
req._uploadDir = dir;906
1002
907
upload.array('files')(req, res, (err) =>1003
upload.array('files')(req, res, async (err) =>908
{1004
{909
if (err) return res.status(500).json({ error: err.message });1005
if (err) return res.status(500).json({ error: err.message });910
1006
@@ -912,15 +1008,25 @@ router.post('/upload', requireAuth, (req, res, next) =>912
for (const f of (req.files || []))1008
for (const f of (req.files || []))913
{1009
{914
const filePath = path.join(dir, f.originalname);1010
const filePath = path.join(dir, f.originalname);1011
1012
let sha256 = null;1013
try1014
{1015
sha256 = await computeFileSha256(filePath);1016
} catch1017
{1018
// Non-fatal — hash can be computed lazily when the file is accessed.1019
}1020
915
const existing = db.prepare('SELECT * FROM file_meta WHERE file_path = ?').get([filePath]);1021
const existing = db.prepare('SELECT * FROM file_meta WHERE file_path = ?').get([filePath]);916
if (existing)1022
if (existing)917
{1023
{918
db.prepare('UPDATE file_meta SET uploader = ? WHERE file_path = ?')1024
db.prepare('UPDATE file_meta SET uploader = ?, sha256 = ? WHERE file_path = ?')919
.run([req.user.username, filePath]);1025
.run([req.user.username, sha256, filePath]);920
} else1026
} else921
{1027
{922
db.prepare('INSERT INTO file_meta (file_path, uploader, downloaders) VALUES (?, ?, ?)')1028
db.prepare('INSERT INTO file_meta (file_path, uploader, downloaders, sha256) VALUES (?, ?, ?, ?)')923
.run([filePath, req.user.username, '[]']);1029
.run([filePath, req.user.username, '[]', sha256]);924
}1030
}925
}1031
}926
res.json({ uploaded: (req.files || []).map(f => f.originalname) });1032
res.json({ uploaded: (req.files || []).map(f => f.originalname) });@@ -39,8 +39,10 @@ jest.mock('../server/db', () =>
39
39
file_path TEXT PRIMARY KEY,
40
40
uploader TEXT NOT NULL DEFAULT '',
41
41
downloaders TEXT NOT NULL DEFAULT '[]',
42
watched_by TEXT NOT NULL DEFAULT '{}'
42
watched_by TEXT NOT NULL DEFAULT '{}',
43
sha256 TEXT
43
44
);
45
CREATE INDEX IF NOT EXISTS idx_file_meta_sha256 ON file_meta(sha256);
44
46
`);
45
47
}
46
48
return db;
@@ -773,3 +775,115 @@ describe('video streaming routes', () =>
773
775
expect(result.body.watchedBy).toEqual({ admin: 'fully' });
774
776
});
775
777
});
778
779
describe('SHA-256 hashing and duplicate detection', () =>
780
{
781
test('POST /api/files/upload stores sha256 hash for uploaded files', async () =>
782
{
783
const { status } = await reqMultipart(`/api/files/upload?dir=${encodeURIComponent(rootDir)}`, [
784
{ name: 'hashme.txt', content: 'hello hash', type: 'text/plain' },
785
]);
786
expect(status).toBe(200);
787
788
const meta = getDb().prepare('SELECT sha256 FROM file_meta WHERE file_path = ?')
789
.get([path.join(rootDir, 'hashme.txt')]);
790
expect(meta).toBeTruthy();
791
// SHA-256 is a 64-char hex string.
792
expect(meta.sha256).toMatch(/^[0-9a-f]{64}$/);
793
});
794
795
test('GET /api/files/meta returns sha256 and computes it lazily when missing', async () =>
796
{
797
const filePath = path.join(rootDir, 'lazy.txt');
798
fs.writeFileSync(filePath, 'lazy content');
799
800
// Insert a row without a hash to simulate a pre-existing file.
801
getDb().prepare('INSERT INTO file_meta (file_path, uploader, downloaders) VALUES (?, ?, ?)')
802
.run([filePath, 'admin', '[]']);
803
804
const { status, body } = await req('GET', `/api/files/meta?file=${encodeURIComponent(filePath)}`);
805
expect(status).toBe(200);
806
expect(body.sha256).toMatch(/^[0-9a-f]{64}$/);
807
808
// Verify it was persisted.
809
const row = getDb().prepare('SELECT sha256 FROM file_meta WHERE file_path = ?').get([filePath]);
810
expect(row.sha256).toBe(body.sha256);
811
});
812
813
test('GET /api/files/duplicates returns other files with the same hash', async () =>
814
{
815
const content = 'duplicate content';
816
const fileA = path.join(rootDir, 'dup-a.txt');
817
const fileB = path.join(rootDir, 'dup-b.txt');
818
fs.writeFileSync(fileA, content);
819
fs.writeFileSync(fileB, content);
820
821
// Compute the expected hash.
822
const crypto = require('crypto');
823
const expectedHash = crypto.createHash('sha256').update(content).digest('hex');
824
825
getDb().prepare('INSERT INTO file_meta (file_path, uploader, downloaders, sha256) VALUES (?, ?, ?, ?)')
826
.run([fileA, 'admin', '[]', expectedHash]);
827
getDb().prepare('INSERT INTO file_meta (file_path, uploader, downloaders, sha256) VALUES (?, ?, ?, ?)')
828
.run([fileB, 'admin', '[]', expectedHash]);
829
830
const { status, body } = await req('GET', `/api/files/duplicates?file=${encodeURIComponent(fileA)}`);
831
expect(status).toBe(200);
832
expect(body.duplicates).toHaveLength(1);
833
expect(body.duplicates[0].path).toBe(fileB);
834
expect(body.duplicates[0].name).toBe('dup-b.txt');
835
});
836
837
test('GET /api/files/duplicates returns empty list when no hash is stored', async () =>
838
{
839
const filePath = path.join(rootDir, 'nohash.txt');
840
fs.writeFileSync(filePath, 'no hash yet');
841
842
getDb().prepare('INSERT INTO file_meta (file_path, uploader, downloaders) VALUES (?, ?, ?)')
843
.run([filePath, 'admin', '[]']);
844
845
const { status, body } = await req('GET', `/api/files/duplicates?file=${encodeURIComponent(filePath)}`);
846
expect(status).toBe(200);
847
expect(body.duplicates).toEqual([]);
848
});
849
850
test('GET /api/files/duplicates excludes files outside accessible roots', async () =>
851
{
852
const content = 'secret dupe';
853
const accessible = path.join(rootDir, 'visible.txt');
854
const outsideRoot = path.join(os.tmpdir(), 'outside.txt');
855
fs.writeFileSync(accessible, content);
856
857
const crypto = require('crypto');
858
const hash = crypto.createHash('sha256').update(content).digest('hex');
859
860
getDb().prepare('INSERT INTO file_meta (file_path, uploader, downloaders, sha256) VALUES (?, ?, ?, ?)')
861
.run([accessible, 'admin', '[]', hash]);
862
// Insert a row for a path that is NOT inside any configured root.
863
getDb().prepare('INSERT INTO file_meta (file_path, uploader, downloaders, sha256) VALUES (?, ?, ?, ?)')
864
.run([outsideRoot, 'admin', '[]', hash]);
865
866
const { status, body } = await req('GET', `/api/files/duplicates?file=${encodeURIComponent(accessible)}`);
867
expect(status).toBe(200);
868
expect(body.duplicates.every(d => d.path !== outsideRoot)).toBe(true);
869
});
870
871
test('POST /api/files/move preserves sha256 after rename', async () =>
872
{
873
const from = path.join(rootDir, 'withHash.txt');
874
const toDir = path.join(rootDir, 'subfolder');
875
fs.writeFileSync(from, 'content');
876
fs.mkdirSync(toDir);
877
878
const hash = 'a'.repeat(64);
879
getDb().prepare('INSERT INTO file_meta (file_path, uploader, downloaders, sha256) VALUES (?, ?, ?, ?)')
880
.run([from, 'admin', '[]', hash]);
881
882
const { status } = await req('POST', '/api/files/move', { from, toDir });
883
expect(status).toBe(200);
884
885
const meta = getDb().prepare('SELECT sha256 FROM file_meta WHERE file_path = ?')
886
.get([path.join(toDir, 'withHash.txt')]);
887
expect(meta.sha256).toBe(hash);
888
});
889
});
@@ -39,8 +39,10 @@ jest.mock('../server/db', () =>39
file_path TEXT PRIMARY KEY,39
file_path TEXT PRIMARY KEY,40
uploader TEXT NOT NULL DEFAULT '',40
uploader TEXT NOT NULL DEFAULT '',41
downloaders TEXT NOT NULL DEFAULT '[]',41
downloaders TEXT NOT NULL DEFAULT '[]',42
watched_by TEXT NOT NULL DEFAULT '{}'42
watched_by TEXT NOT NULL DEFAULT '{}',43
sha256 TEXT43
);44
);45
CREATE INDEX IF NOT EXISTS idx_file_meta_sha256 ON file_meta(sha256);44
`);46
`);45
}47
}46
return db;48
return db;@@ -773,3 +775,115 @@ describe('video streaming routes', () =>773
expect(result.body.watchedBy).toEqual({ admin: 'fully' });775
expect(result.body.watchedBy).toEqual({ admin: 'fully' });774
});776
});775
});777
});778
779
describe('SHA-256 hashing and duplicate detection', () =>780
{781
test('POST /api/files/upload stores sha256 hash for uploaded files', async () =>782
{783
const { status } = await reqMultipart(`/api/files/upload?dir=${encodeURIComponent(rootDir)}`, [784
{ name: 'hashme.txt', content: 'hello hash', type: 'text/plain' },785
]);786
expect(status).toBe(200);787
788
const meta = getDb().prepare('SELECT sha256 FROM file_meta WHERE file_path = ?')789
.get([path.join(rootDir, 'hashme.txt')]);790
expect(meta).toBeTruthy();791
// SHA-256 is a 64-char hex string.792
expect(meta.sha256).toMatch(/^[0-9a-f]{64}$/);793
});794
795
test('GET /api/files/meta returns sha256 and computes it lazily when missing', async () =>796
{797
const filePath = path.join(rootDir, 'lazy.txt');798
fs.writeFileSync(filePath, 'lazy content');799
800
// Insert a row without a hash to simulate a pre-existing file.801
getDb().prepare('INSERT INTO file_meta (file_path, uploader, downloaders) VALUES (?, ?, ?)')802
.run([filePath, 'admin', '[]']);803
804
const { status, body } = await req('GET', `/api/files/meta?file=${encodeURIComponent(filePath)}`);805
expect(status).toBe(200);806
expect(body.sha256).toMatch(/^[0-9a-f]{64}$/);807
808
// Verify it was persisted.809
const row = getDb().prepare('SELECT sha256 FROM file_meta WHERE file_path = ?').get([filePath]);810
expect(row.sha256).toBe(body.sha256);811
});812
813
test('GET /api/files/duplicates returns other files with the same hash', async () =>814
{815
const content = 'duplicate content';816
const fileA = path.join(rootDir, 'dup-a.txt');817
const fileB = path.join(rootDir, 'dup-b.txt');818
fs.writeFileSync(fileA, content);819
fs.writeFileSync(fileB, content);820
821
// Compute the expected hash.822
const crypto = require('crypto');823
const expectedHash = crypto.createHash('sha256').update(content).digest('hex');824
825
getDb().prepare('INSERT INTO file_meta (file_path, uploader, downloaders, sha256) VALUES (?, ?, ?, ?)')826
.run([fileA, 'admin', '[]', expectedHash]);827
getDb().prepare('INSERT INTO file_meta (file_path, uploader, downloaders, sha256) VALUES (?, ?, ?, ?)')828
.run([fileB, 'admin', '[]', expectedHash]);829
830
const { status, body } = await req('GET', `/api/files/duplicates?file=${encodeURIComponent(fileA)}`);831
expect(status).toBe(200);832
expect(body.duplicates).toHaveLength(1);833
expect(body.duplicates[0].path).toBe(fileB);834
expect(body.duplicates[0].name).toBe('dup-b.txt');835
});836
837
test('GET /api/files/duplicates returns empty list when no hash is stored', async () =>838
{839
const filePath = path.join(rootDir, 'nohash.txt');840
fs.writeFileSync(filePath, 'no hash yet');841
842
getDb().prepare('INSERT INTO file_meta (file_path, uploader, downloaders) VALUES (?, ?, ?)')843
.run([filePath, 'admin', '[]']);844
845
const { status, body } = await req('GET', `/api/files/duplicates?file=${encodeURIComponent(filePath)}`);846
expect(status).toBe(200);847
expect(body.duplicates).toEqual([]);848
});849
850
test('GET /api/files/duplicates excludes files outside accessible roots', async () =>851
{852
const content = 'secret dupe';853
const accessible = path.join(rootDir, 'visible.txt');854
const outsideRoot = path.join(os.tmpdir(), 'outside.txt');855
fs.writeFileSync(accessible, content);856
857
const crypto = require('crypto');858
const hash = crypto.createHash('sha256').update(content).digest('hex');859
860
getDb().prepare('INSERT INTO file_meta (file_path, uploader, downloaders, sha256) VALUES (?, ?, ?, ?)')861
.run([accessible, 'admin', '[]', hash]);862
// Insert a row for a path that is NOT inside any configured root.863
getDb().prepare('INSERT INTO file_meta (file_path, uploader, downloaders, sha256) VALUES (?, ?, ?, ?)')864
.run([outsideRoot, 'admin', '[]', hash]);865
866
const { status, body } = await req('GET', `/api/files/duplicates?file=${encodeURIComponent(accessible)}`);867
expect(status).toBe(200);868
expect(body.duplicates.every(d => d.path !== outsideRoot)).toBe(true);869
});870
871
test('POST /api/files/move preserves sha256 after rename', async () =>872
{873
const from = path.join(rootDir, 'withHash.txt');874
const toDir = path.join(rootDir, 'subfolder');875
fs.writeFileSync(from, 'content');876
fs.mkdirSync(toDir);877
878
const hash = 'a'.repeat(64);879
getDb().prepare('INSERT INTO file_meta (file_path, uploader, downloaders, sha256) VALUES (?, ?, ?, ?)')880
.run([from, 'admin', '[]', hash]);881
882
const { status } = await req('POST', '/api/files/move', { from, toDir });883
expect(status).toBe(200);884
885
const meta = getDb().prepare('SELECT sha256 FROM file_meta WHERE file_path = ?')886
.get([path.join(toDir, 'withHash.txt')]);887
expect(meta.sha256).toBe(hash);888
});889
});