From 688094cd9a6764612afe177f5eac5ab3b3f6397f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Omar=20Gonz=C3=A1les=20D=C3=ADaz?= Date: Wed, 18 Nov 2026 13:28:14 -0500 Subject: [PATCH 1/2] Find duplicate files with dupes: in a search Like Everything's duplicate finder: "dupes:" lists the files here and below that have an identical twin, compared by size, then the first 64 KB, then every byte, each set of twins together and the biggest first. Co-Authored-By: Claude Sonnet 5.5 --- README.md | 2 +- src/foldermodel.cpp | 108 ++++++++++++++++++++++++++++++++++++++++-- src/foldermodel.h | 3 ++ tests/folio_tests.cpp | 30 ++++++++++++ 4 files changed, 137 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index aa126641..43359521 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Files (`folio`) is Omarchy's file manager. It's one quiet window that follows yo | Space | Preview | | Backspace, Alt+← / Alt+→ | Up, back, forward | | Ctrl+L | Type a path | -| / | Search here and below | +| / | Search here and below; add `ext:jpg,png` to narrow by extension, or `dupes:` to list files with an identical twin (same bytes), set by set | | Ctrl+C / X / V | Copy, cut, paste | | F2, Delete | Rename, move to the trash | | Ctrl+Shift+N | New folder | diff --git a/src/foldermodel.cpp b/src/foldermodel.cpp index 465ea17b..2ba4fc0f 100644 --- a/src/foldermodel.cpp +++ b/src/foldermodel.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -390,10 +391,13 @@ FolderModel::Entry FolderModel::entryFor(const QFileInfo &info, const QMimeDatab } // What a search query asks for: a name containing some text, with one of -// some extensions if it names them ("beach ext:jpg,png", "ext:pdf"). +// some extensions if it names them ("beach ext:jpg,png", "ext:pdf"). With +// "dupes:" it lists only the files that have an identical twin among the +// matches, Everything's "find duplicates". struct SearchFilter { QString text; QStringList extensions; + bool dupes = false; explicit SearchFilter(const QString &query) { const QString q = query.trimmed(); @@ -401,7 +405,9 @@ struct SearchFilter { // left once the ext: words are taken out is as it was typed. QStringList words; for (const QString &word : q.split(QLatin1Char(' '))) { - if (word.startsWith(QLatin1String("ext:"), Qt::CaseInsensitive)) { + if (word.compare(QLatin1String("dupes:"), Qt::CaseInsensitive) == 0) { + dupes = true; + } else if (word.startsWith(QLatin1String("ext:"), Qt::CaseInsensitive)) { for (QString ext : word.mid(4).split(QLatin1Char(','), Qt::SkipEmptyParts)) { if (ext.startsWith(QLatin1Char('.'))) ext.remove(0, 1); @@ -417,12 +423,93 @@ struct SearchFilter { bool matches(const QFileInfo &info) const { const QString name = info.fileName(); + // Folders, links and empty files are never each other's duplicates. + if (dupes && (info.isDir() || info.isSymLink() || info.size() == 0)) + return false; if (!extensions.isEmpty() && (info.isDir() || !extensions.contains(info.suffix().toLower()))) return false; return name.contains(text, Qt::CaseInsensitive); } }; +// The SHA-1 of a file's first `bytes` bytes (all of it for -1); empty if it +// can't be read. +static QByteArray hashOf(const QString &path, qint64 bytes) { + QFile file(path); + if (!file.open(QIODevice::ReadOnly)) + return {}; + QCryptographicHash hash(QCryptographicHash::Sha1); + qint64 left = bytes; + while (left != 0) { + const QByteArray chunk = file.read(left < 0 ? 1 << 20 : qMin(left, 1 << 20)); + if (chunk.isEmpty()) + break; + hash.addData(chunk); + if (left > 0) + left -= chunk.size(); + } + return hash.result(); +} + +// The files among `files` that have another with the same bytes, in sets +// (Entry::group) with the biggest first. Only files of the same size are +// compared, by their first 64 KB and then, if those agree, all of them. +static QVector duplicatesAmong(const QVector &files, int limit, + const std::function &stop) { + QHash> bySize; + for (int i = 0; i < files.size(); ++i) + bySize[files[i].size] << i; + QList sizes = bySize.keys(); + std::sort(sizes.begin(), sizes.end(), std::greater()); + + // `rows` split into the sets that share a hash of their first `bytes`. + const auto split = [&](const QVector &rows, qint64 bytes) { + QHash> sets; + for (int row : rows) { + const QByteArray hash = hashOf(files[row].path, bytes); + if (!hash.isEmpty()) + sets[hash] << row; + } + QVector> twins; + for (const QVector &set : std::as_const(sets)) + if (set.size() > 1) + twins << set; + return twins; + }; + + constexpr qint64 head = 64 * 1024; + QVector found; + int group = 0; + for (qint64 size : std::as_const(sizes)) { + if (stop && stop()) + return {}; + const QVector &same = bySize[size]; + if (same.size() < 2) + continue; + QVector> sets = split(same, size <= head ? -1 : head); + if (size > head) { + QVector> whole; + for (const QVector &set : std::as_const(sets)) { + if (stop && stop()) + return {}; + whole += split(set, -1); + } + sets = whole; + } + for (const QVector &set : std::as_const(sets)) { + ++group; + for (int row : set) { + if (found.size() >= limit) + return found; + FolderModel::Entry e = files[row]; + e.group = group; + found.push_back(e); + } + } + } + return found; +} + QVector FolderModel::search(const QString &root, const QString &query, bool showHidden, const QString &sort, const std::function &stop, int limit, const QLocale &locale, const QStringList &filter, @@ -450,7 +537,9 @@ QVector FolderModel::search(const QString &root, const QStri QMimeDatabase mimes; const QDir base(root); QStringList folders{root}; - while (!folders.isEmpty() && entries.size() < limit) { + // A search for duplicates gathers every candidate and keeps the twins + // at the end; the limit is on what it shows. + while (!folders.isEmpty() && (wanted.dupes || entries.size() < limit)) { if (stop && stop()) return {}; QDirIterator it(folders.takeFirst(), filters); @@ -458,7 +547,7 @@ QVector FolderModel::search(const QString &root, const QStri if (stop && stop()) return {}; const QFileInfo info = it.nextFileInfo(); - if (entries.size() < limit && wanted.matches(info)) { + if ((wanted.dupes || entries.size() < limit) && wanted.matches(info)) { Entry e = entryFor(info, mimes); e.name = base.relativeFilePath(e.path); // Only what fits a file chooser's filter uses up the limit. @@ -467,7 +556,8 @@ QVector FolderModel::search(const QString &root, const QStri } // At every entry, not just at matches: a match found early isn't // held back by a long stretch of folders with none. - tell(false); + if (!wanted.dupes) + tell(false); // Other filesystems (a mounted drive, another machine, /proc // from /) and symlinked folders aren't walked into. struct stat st; @@ -477,6 +567,11 @@ QVector FolderModel::search(const QString &root, const QStri folders << info.absoluteFilePath(); } } + if (wanted.dupes) { + entries = duplicatesAmong(entries, limit, stop); + if (stop && stop()) + return {}; + } tell(true); sortEntries(entries, sort, locale); return entries; @@ -500,6 +595,9 @@ std::function Fold // Folders first in every order, as everywhere else on the desktop. if (a.isDir != b.isDir) return a.isDir; + // A search for duplicates keeps each set together. + if (a.group != b.group) + return a.group < b.group; if (sort == "modified" && a.modified != b.modified) return a.modified > b.modified; if (sort == "oldest" && a.modified != b.modified) diff --git a/src/foldermodel.h b/src/foldermodel.h index 020f1ba2..0820f1b9 100644 --- a/src/foldermodel.h +++ b/src/foldermodel.h @@ -76,6 +76,9 @@ class FolderModel : public QAbstractListModel { // How deep in the tree: 0 for the folder's own items, 1 for those of // a folder opened in place under them, and so on. int depth = 0; + // In a search for duplicates ("dupes:"), which set of identical + // files this is one of, the biggest set first; 0 otherwise. + int group = 0; }; explicit FolderModel(QObject *parent = nullptr); diff --git a/tests/folio_tests.cpp b/tests/folio_tests.cpp index 1de00c72..037add01 100644 --- a/tests/folio_tests.cpp +++ b/tests/folio_tests.cpp @@ -1096,6 +1096,36 @@ private slots: QCOMPARE(names("my notes"), QString("my notes.txt")); } + void folderSearchFindsDuplicates() + { + QTemporaryDir dir; + QDir(dir.path()).mkpath("sub"); + write(dir.filePath("a.txt"), "same"); + write(dir.filePath("sub/b.txt"), "same"); + // Same size, different bytes; and one alone; and empty twins. + write(dir.filePath("c.txt"), "diff"); + write(dir.filePath("only.txt"), "unique file"); + write(dir.filePath("e1"), ""); + write(dir.filePath("e2"), ""); + // Bigger than the 64 KB compared first, differing only at the end. + QByteArray big(200 * 1024, 'x'); + write(dir.filePath("big1"), big); + write(dir.filePath("sub/big2"), big); + big.back() = 'y'; + write(dir.filePath("big3"), big); + const auto found = FolderModel::search(dir.path(), "dupes:", false, "name"); + QStringList names; + for (const auto &e : found) + names << e.name; + // The biggest set first, its files together. + QCOMPARE(names.join(", "), QString("big1, sub/big2, a.txt, sub/b.txt")); + QCOMPARE(found.at(0).group, found.at(1).group); + QVERIFY(found.at(1).group != found.at(2).group); + // The rest of the query still narrows what's compared. + QCOMPARE(FolderModel::search(dir.path(), "dupes: big", false, "name").size(), 2); + QVERIFY(FolderModel::search(dir.path(), "dupes: only", false, "name").isEmpty()); + } + void folderSearchSkipsSymlinkedFolders() { // A link back up the tree would otherwise be walked round and round. From 5b20dc028d55e775d86299205226770123b07c08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Omar=20Gonz=C3=A1les=20D=C3=ADaz?= Date: Wed, 18 Nov 2026 13:39:13 -0500 Subject: [PATCH 2/2] Narrow a search from what it already found Typing on ("beac" to "beach") or adding an ext: keeps the matches that still fit instead of walking the folder again, when the last search wasn't cut short by the limit. Widening, or a change of folder, looks again. Co-Authored-By: Claude Sonnet 5.5 --- src/foldermodel.cpp | 59 ++++++++++++++++++++++++++++++++++++++++--- src/foldermodel.h | 11 ++++++++ tests/folio_tests.cpp | 59 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 125 insertions(+), 4 deletions(-) diff --git a/src/foldermodel.cpp b/src/foldermodel.cpp index 465ea17b..7799e8ee 100644 --- a/src/foldermodel.cpp +++ b/src/foldermodel.cpp @@ -416,13 +416,36 @@ struct SearchFilter { } bool matches(const QFileInfo &info) const { - const QString name = info.fileName(); - if (!extensions.isEmpty() && (info.isDir() || !extensions.contains(info.suffix().toLower()))) + return matches(info.fileName(), info.isDir(), info.suffix()); + } + + bool matches(const QString &name, bool isDir, const QString &suffix) const { + if (!extensions.isEmpty() && (isDir || !extensions.contains(suffix.toLower()))) return false; return name.contains(text, Qt::CaseInsensitive); } + + // Whether what this matches, `other` matches too: its text holds ours, + // and it takes no extension we don't (or we take any). + bool narrowedBy(const SearchFilter &other) const { + if (!other.text.contains(text, Qt::CaseInsensitive)) + return false; + if (extensions.isEmpty()) + return true; + if (other.extensions.isEmpty()) + return false; + return std::all_of(other.extensions.begin(), other.extensions.end(), + [&](const QString &ext) { return extensions.contains(ext); }); + } }; +// How many matches a search of a folder shows before it stops looking. +static constexpr int searchLimit = 1000; + +bool FolderModel::narrows(const QString &from, const QString &to) { + return SearchFilter(from).narrowedBy(SearchFilter(to)); +} + QVector FolderModel::search(const QString &root, const QString &query, bool showHidden, const QString &sort, const std::function &stop, int limit, const QLocale &locale, const QStringList &filter, @@ -628,6 +651,7 @@ void FolderModel::reload() { return; } m_shownQuery.clear(); + m_complete = false; const auto latest = m_latest; auto *watcher = new QFutureWatcher, QString, QStringList, QHash>, QHash>>(this); connect(watcher, &QFutureWatcherBase::finished, this, [this, watcher, generation] { @@ -854,6 +878,28 @@ void FolderModel::searchHere(quint64 generation) { const bool fresh = query != m_shownQuery; m_shownQuery = query; m_staleClear.stop(); + // What's shown is all the previous search found, and this one only asks + // for less of it: keep what still matches, with no walk. + const QString key = path + QLatin1Char('\n') + (showHidden ? "h" : "") + QLatin1Char('\n') + filter.join(QLatin1Char('\n')); + if (fresh && m_complete && key == m_completeKey && narrows(m_completeQuery, query)) { + const SearchFilter wanted(query); + QVector kept; + for (const Entry &e : std::as_const(m_entries)) + if (wanted.matches(QFileInfo(e.path).fileName(), e.isDir, e.extension)) + kept.push_back(e); + m_completeQuery = query; + setError({}); + setEntries(kept); + QSet still; + for (const Entry &e : std::as_const(m_entries)) + if (m_selected.contains(e.path)) + still.insert(e.path); + setSelected(still); + setLoading(false); + emit loaded(); + return; + } + m_complete = false; if (fresh) clearEntries(); setError({}); @@ -870,7 +916,7 @@ void FolderModel::searchHere(quint64 generation) { emit loaded(); } }); - connect(watcher, &QFutureWatcherBase::finished, this, [this, watcher, generation, fresh, announced] { + connect(watcher, &QFutureWatcherBase::finished, this, [this, watcher, generation, fresh, announced, query, key] { watcher->deleteLater(); if (generation != m_generation) return; @@ -888,12 +934,17 @@ void FolderModel::searchHere(quint64 generation) { still.insert(e.path); setSelected(still); } + // Unless it was cut short by the limit, this is all there is to + // narrow by a longer query. + m_complete = m_entries.size() < searchLimit; + m_completeQuery = query; + m_completeKey = key; setLoading(false); emit loaded(); countFolders(generation); }); watcher->setFuture(QtConcurrent::run(&FolderModel::listingPool(), [path, query, showHidden, sort, filter, locale, latest, generation](QPromise> &promise) { - search(path, query, showHidden, sort, [&] { return *latest != generation; }, 1000, locale, filter, + search(path, query, showHidden, sort, [&] { return *latest != generation; }, searchLimit, locale, filter, [&](const QVector &found) { promise.addResult(found); }); })); } diff --git a/src/foldermodel.h b/src/foldermodel.h index 020f1ba2..9e0d1c84 100644 --- a/src/foldermodel.h +++ b/src/foldermodel.h @@ -197,6 +197,11 @@ class FolderModel : public QAbstractListModel { const QLocale &locale = QLocale(), const QStringList &filter = {}, const std::function &)> &found = {}); + // Whether every match of the search `from` is also a match of `to`, so + // typing on ("beac" to "beach", or adding an ext:) can narrow what's + // already found instead of looking again. + static bool narrows(const QString &from, const QString &to); + signals: void pathChanged(); void countChanged(); @@ -222,6 +227,12 @@ class FolderModel : public QAbstractListModel { void insertFound(const QVector &found); // The search whose matches are shown (or being found). QString m_shownQuery; + // Set, m_entries holds every match of m_completeQuery (not cut short by + // the limit) in the folder that m_completeKey names, so a search that + // narrows it is answered from them. + bool m_complete = false; + QString m_completeQuery; + QString m_completeKey; void setEntries(const QVector &entries); void clearEntries(); QString cacheKey() const; diff --git a/tests/folio_tests.cpp b/tests/folio_tests.cpp index 1de00c72..736f1ba9 100644 --- a/tests/folio_tests.cpp +++ b/tests/folio_tests.cpp @@ -1096,6 +1096,65 @@ private slots: QCOMPARE(names("my notes"), QString("my notes.txt")); } + void searchNarrowsWhatsAlreadyFound() + { + QVERIFY(FolderModel::narrows("beac", "beach")); + QVERIFY(FolderModel::narrows("BEAC", "beach")); + QVERIFY(FolderModel::narrows("beach", "beach")); + QVERIFY(!FolderModel::narrows("beach", "beac")); + QVERIFY(!FolderModel::narrows("beach", "sea")); + // An extension narrows; dropping it or taking another widens. + QVERIFY(FolderModel::narrows("beach", "beach ext:jpg")); + QVERIFY(FolderModel::narrows("ext:jpg,png", "ext:jpg")); + QVERIFY(FolderModel::narrows("ext:jpg", "img ext:jpg")); + QVERIFY(!FolderModel::narrows("ext:jpg", "ext:jpg,png")); + QVERIFY(!FolderModel::narrows("ext:jpg", "ext:png")); + QVERIFY(!FolderModel::narrows("ext:jpg", "beach")); + // Plain text is matched as typed, spaces and all. + QVERIFY(!FolderModel::narrows("my notes", "my notes")); + QVERIFY(FolderModel::narrows("my", "my notes")); + + // The model answers a longer query from what it found, and the + // same as a fresh search would. + QTemporaryDir dir; + QDir(dir.path()).mkpath("sub"); + write(dir.filePath("beach.jpg"), "x"); + write(dir.filePath("beach.png"), "x"); + write(dir.filePath("sub/beach-2.jpg"), "x"); + write(dir.filePath("sub/beam.txt"), "x"); + write(dir.filePath("sub/beach.d"), "x"); + QDir(dir.filePath("sub")).mkpath("beach-folder"); + FolderModel model; + QSignalSpy loaded(&model, &FolderModel::loaded); + model.setPath(dir.path()); + QVERIFY(loaded.wait(3000)); + auto names = [&] { + QStringList out; + for (int row = 0; row < model.count(); ++row) + out << model.get(row).value("name").toString(); + return out; + }; + model.setQuery("bea"); + QTRY_VERIFY(!model.loading()); + QCOMPARE(model.count(), 6); + // Written after the search, in a folder the model doesn't watch: + // only a new walk would find it. + write(dir.filePath("sub/beach-new.jpg"), "x"); + model.setQuery("beach"); + QVERIFY(!model.loading()); + QCOMPARE(model.count(), 5); + QVERIFY(!names().contains("sub/beach-new.jpg")); + model.selectOnly(model.indexOfPath(dir.filePath("beach.png"))); + model.setQuery("beach ext:jpg"); + QVERIFY(!model.loading()); + QCOMPARE(names().join(", "), QString("beach.jpg, sub/beach-2.jpg")); + QCOMPARE(model.selectedCount(), 0); + // Widening looks again, and finds what was written meanwhile. + model.setQuery("beach"); + QTRY_VERIFY(!model.loading()); + QVERIFY(names().contains("sub/beach-new.jpg")); + } + void folderSearchSkipsSymlinkedFolders() { // A link back up the tree would otherwise be walked round and round.