diff --git a/README.md b/README.md index aa126641..43359521 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Files (`folio`) is Omarchy's file manager. It's one quiet window that follows yo | Space | Preview | | Backspace, Alt+← / Alt+→ | Up, back, forward | | Ctrl+L | Type a path | -| / | Search here and below | +| / | Search here and below; add `ext:jpg,png` to narrow by extension, or `dupes:` to list files with an identical twin (same bytes), set by set | | Ctrl+C / X / V | Copy, cut, paste | | F2, Delete | Rename, move to the trash | | Ctrl+Shift+N | New folder | diff --git a/src/foldermodel.cpp b/src/foldermodel.cpp index 465ea17b..69ad0b51 100644 --- a/src/foldermodel.cpp +++ b/src/foldermodel.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -390,10 +391,13 @@ FolderModel::Entry FolderModel::entryFor(const QFileInfo &info, const QMimeDatab } // What a search query asks for: a name containing some text, with one of -// some extensions if it names them ("beach ext:jpg,png", "ext:pdf"). +// some extensions if it names them ("beach ext:jpg,png", "ext:pdf"). With +// "dupes:" it lists only the files that have an identical twin among the +// matches, Everything's "find duplicates". struct SearchFilter { QString text; QStringList extensions; + bool dupes = false; explicit SearchFilter(const QString &query) { const QString q = query.trimmed(); @@ -401,7 +405,9 @@ struct SearchFilter { // left once the ext: words are taken out is as it was typed. QStringList words; for (const QString &word : q.split(QLatin1Char(' '))) { - if (word.startsWith(QLatin1String("ext:"), Qt::CaseInsensitive)) { + if (word.compare(QLatin1String("dupes:"), Qt::CaseInsensitive) == 0) { + dupes = true; + } else if (word.startsWith(QLatin1String("ext:"), Qt::CaseInsensitive)) { for (QString ext : word.mid(4).split(QLatin1Char(','), Qt::SkipEmptyParts)) { if (ext.startsWith(QLatin1Char('.'))) ext.remove(0, 1); @@ -416,13 +422,121 @@ struct SearchFilter { } bool matches(const QFileInfo &info) const { - const QString name = info.fileName(); - if (!extensions.isEmpty() && (info.isDir() || !extensions.contains(info.suffix().toLower()))) + // Folders, links and empty files are never each other's duplicates. + if (dupes && (info.isDir() || info.isSymLink() || info.size() == 0)) + return false; + return matches(info.fileName(), info.isDir(), info.suffix()); + } + + bool matches(const QString &name, bool isDir, const QString &suffix) const { + if (!extensions.isEmpty() && (isDir || !extensions.contains(suffix.toLower()))) return false; return name.contains(text, Qt::CaseInsensitive); } + + // Whether what this matches, `other` matches too: its text holds ours, + // and it takes no extension we don't (or we take any). + bool narrowedBy(const SearchFilter &other) const { + // What has a twin depends on all the files compared, not just the + // ones kept: a duplicates search is always made again. + if (dupes || other.dupes) + return false; + if (!other.text.contains(text, Qt::CaseInsensitive)) + return false; + if (extensions.isEmpty()) + return true; + if (other.extensions.isEmpty()) + return false; + return std::all_of(other.extensions.begin(), other.extensions.end(), + [&](const QString &ext) { return extensions.contains(ext); }); + } }; +// The SHA-1 of a file's first `bytes` bytes (all of it for -1); empty if it +// can't be read. +static QByteArray hashOf(const QString &path, qint64 bytes) { + QFile file(path); + if (!file.open(QIODevice::ReadOnly)) + return {}; + QCryptographicHash hash(QCryptographicHash::Sha1); + qint64 left = bytes; + while (left != 0) { + const QByteArray chunk = file.read(left < 0 ? 1 << 20 : qMin(left, 1 << 20)); + if (chunk.isEmpty()) + break; + hash.addData(chunk); + if (left > 0) + left -= chunk.size(); + } + return hash.result(); +} + +// The files among `files` that have another with the same bytes, in sets +// (Entry::group) with the biggest first. Only files of the same size are +// compared, by their first 64 KB and then, if those agree, all of them. +static QVector duplicatesAmong(const QVector &files, int limit, + const std::function &stop) { + QHash> bySize; + for (int i = 0; i < files.size(); ++i) + bySize[files[i].size] << i; + QList sizes = bySize.keys(); + std::sort(sizes.begin(), sizes.end(), std::greater()); + + // `rows` split into the sets that share a hash of their first `bytes`. + const auto split = [&](const QVector &rows, qint64 bytes) { + QHash> sets; + for (int row : rows) { + const QByteArray hash = hashOf(files[row].path, bytes); + if (!hash.isEmpty()) + sets[hash] << row; + } + QVector> twins; + for (const QVector &set : std::as_const(sets)) + if (set.size() > 1) + twins << set; + return twins; + }; + + constexpr qint64 head = 64 * 1024; + QVector found; + int group = 0; + for (qint64 size : std::as_const(sizes)) { + if (stop && stop()) + return {}; + const QVector &same = bySize[size]; + if (same.size() < 2) + continue; + QVector> sets = split(same, size <= head ? -1 : head); + if (size > head) { + QVector> whole; + for (const QVector &set : std::as_const(sets)) { + if (stop && stop()) + return {}; + whole += split(set, -1); + } + sets = whole; + } + for (const QVector &set : std::as_const(sets)) { + ++group; + for (int row : set) { + if (found.size() >= limit) + return found; + FolderModel::Entry e = files[row]; + e.group = group; + found.push_back(e); + } + } + } + return found; +} + +// How many matches a search of a folder shows before it stops looking. +static constexpr int searchLimit = 1000; + +bool FolderModel::narrows(const QString &from, const QString &to) { + return SearchFilter(from).narrowedBy(SearchFilter(to)); +} + QVector FolderModel::search(const QString &root, const QString &query, bool showHidden, const QString &sort, const std::function &stop, int limit, const QLocale &locale, const QStringList &filter, @@ -450,7 +564,9 @@ QVector FolderModel::search(const QString &root, const QStri QMimeDatabase mimes; const QDir base(root); QStringList folders{root}; - while (!folders.isEmpty() && entries.size() < limit) { + // A search for duplicates gathers every candidate and keeps the twins + // at the end; the limit is on what it shows. + while (!folders.isEmpty() && (wanted.dupes || entries.size() < limit)) { if (stop && stop()) return {}; QDirIterator it(folders.takeFirst(), filters); @@ -458,7 +574,7 @@ QVector FolderModel::search(const QString &root, const QStri if (stop && stop()) return {}; const QFileInfo info = it.nextFileInfo(); - if (entries.size() < limit && wanted.matches(info)) { + if ((wanted.dupes || entries.size() < limit) && wanted.matches(info)) { Entry e = entryFor(info, mimes); e.name = base.relativeFilePath(e.path); // Only what fits a file chooser's filter uses up the limit. @@ -467,7 +583,8 @@ QVector FolderModel::search(const QString &root, const QStri } // At every entry, not just at matches: a match found early isn't // held back by a long stretch of folders with none. - tell(false); + if (!wanted.dupes) + tell(false); // Other filesystems (a mounted drive, another machine, /proc // from /) and symlinked folders aren't walked into. struct stat st; @@ -477,6 +594,11 @@ QVector FolderModel::search(const QString &root, const QStri folders << info.absoluteFilePath(); } } + if (wanted.dupes) { + entries = duplicatesAmong(entries, limit, stop); + if (stop && stop()) + return {}; + } tell(true); sortEntries(entries, sort, locale); return entries; @@ -500,6 +622,9 @@ std::function Fold // Folders first in every order, as everywhere else on the desktop. if (a.isDir != b.isDir) return a.isDir; + // A search for duplicates keeps each set together. + if (a.group != b.group) + return a.group < b.group; if (sort == "modified" && a.modified != b.modified) return a.modified > b.modified; if (sort == "oldest" && a.modified != b.modified) @@ -628,6 +753,7 @@ void FolderModel::reload() { return; } m_shownQuery.clear(); + m_complete = false; const auto latest = m_latest; auto *watcher = new QFutureWatcher, QString, QStringList, QHash>, QHash>>(this); connect(watcher, &QFutureWatcherBase::finished, this, [this, watcher, generation] { @@ -854,6 +980,28 @@ void FolderModel::searchHere(quint64 generation) { const bool fresh = query != m_shownQuery; m_shownQuery = query; m_staleClear.stop(); + // What's shown is all the previous search found, and this one only asks + // for less of it: keep what still matches, with no walk. + const QString key = path + QLatin1Char('\n') + (showHidden ? "h" : "") + QLatin1Char('\n') + filter.join(QLatin1Char('\n')); + if (fresh && m_complete && key == m_completeKey && narrows(m_completeQuery, query)) { + const SearchFilter wanted(query); + QVector kept; + for (const Entry &e : std::as_const(m_entries)) + if (wanted.matches(QFileInfo(e.path).fileName(), e.isDir, e.extension)) + kept.push_back(e); + m_completeQuery = query; + setError({}); + setEntries(kept); + QSet still; + for (const Entry &e : std::as_const(m_entries)) + if (m_selected.contains(e.path)) + still.insert(e.path); + setSelected(still); + setLoading(false); + emit loaded(); + return; + } + m_complete = false; if (fresh) clearEntries(); setError({}); @@ -870,7 +1018,7 @@ void FolderModel::searchHere(quint64 generation) { emit loaded(); } }); - connect(watcher, &QFutureWatcherBase::finished, this, [this, watcher, generation, fresh, announced] { + connect(watcher, &QFutureWatcherBase::finished, this, [this, watcher, generation, fresh, announced, query, key] { watcher->deleteLater(); if (generation != m_generation) return; @@ -888,12 +1036,17 @@ void FolderModel::searchHere(quint64 generation) { still.insert(e.path); setSelected(still); } + // Unless it was cut short by the limit, this is all there is to + // narrow by a longer query. + m_complete = m_entries.size() < searchLimit; + m_completeQuery = query; + m_completeKey = key; setLoading(false); emit loaded(); countFolders(generation); }); watcher->setFuture(QtConcurrent::run(&FolderModel::listingPool(), [path, query, showHidden, sort, filter, locale, latest, generation](QPromise> &promise) { - search(path, query, showHidden, sort, [&] { return *latest != generation; }, 1000, locale, filter, + search(path, query, showHidden, sort, [&] { return *latest != generation; }, searchLimit, locale, filter, [&](const QVector &found) { promise.addResult(found); }); })); } diff --git a/src/foldermodel.h b/src/foldermodel.h index 020f1ba2..79983adf 100644 --- a/src/foldermodel.h +++ b/src/foldermodel.h @@ -76,6 +76,9 @@ class FolderModel : public QAbstractListModel { // How deep in the tree: 0 for the folder's own items, 1 for those of // a folder opened in place under them, and so on. int depth = 0; + // In a search for duplicates ("dupes:"), which set of identical + // files this is one of, the biggest set first; 0 otherwise. + int group = 0; }; explicit FolderModel(QObject *parent = nullptr); @@ -197,6 +200,11 @@ class FolderModel : public QAbstractListModel { const QLocale &locale = QLocale(), const QStringList &filter = {}, const std::function &)> &found = {}); + // Whether every match of the search `from` is also a match of `to`, so + // typing on ("beac" to "beach", or adding an ext:) can narrow what's + // already found instead of looking again. + static bool narrows(const QString &from, const QString &to); + signals: void pathChanged(); void countChanged(); @@ -222,6 +230,12 @@ class FolderModel : public QAbstractListModel { void insertFound(const QVector &found); // The search whose matches are shown (or being found). QString m_shownQuery; + // Set, m_entries holds every match of m_completeQuery (not cut short by + // the limit) in the folder that m_completeKey names, so a search that + // narrows it is answered from them. + bool m_complete = false; + QString m_completeQuery; + QString m_completeKey; void setEntries(const QVector &entries); void clearEntries(); QString cacheKey() const; diff --git a/tests/folio_tests.cpp b/tests/folio_tests.cpp index 1de00c72..b88a5bd6 100644 --- a/tests/folio_tests.cpp +++ b/tests/folio_tests.cpp @@ -1096,6 +1096,100 @@ private slots: QCOMPARE(names("my notes"), QString("my notes.txt")); } + void folderSearchFindsDuplicates() + { + QTemporaryDir dir; + QDir(dir.path()).mkpath("sub"); + write(dir.filePath("a.txt"), "same"); + write(dir.filePath("sub/b.txt"), "same"); + // Same size, different bytes; and one alone; and empty twins. + write(dir.filePath("c.txt"), "diff"); + write(dir.filePath("only.txt"), "unique file"); + write(dir.filePath("e1"), ""); + write(dir.filePath("e2"), ""); + // Bigger than the 64 KB compared first, differing only at the end. + QByteArray big(200 * 1024, 'x'); + write(dir.filePath("big1"), big); + write(dir.filePath("sub/big2"), big); + big.back() = 'y'; + write(dir.filePath("big3"), big); + const auto found = FolderModel::search(dir.path(), "dupes:", false, "name"); + QStringList names; + for (const auto &e : found) + names << e.name; + // The biggest set first, its files together. + QCOMPARE(names.join(", "), QString("big1, sub/big2, a.txt, sub/b.txt")); + QCOMPARE(found.at(0).group, found.at(1).group); + QVERIFY(found.at(1).group != found.at(2).group); + // The rest of the query still narrows what's compared. + QCOMPARE(FolderModel::search(dir.path(), "dupes: big", false, "name").size(), 2); + QVERIFY(FolderModel::search(dir.path(), "dupes: only", false, "name").isEmpty()); + } + + void searchNarrowsWhatsAlreadyFound() + { + QVERIFY(FolderModel::narrows("beac", "beach")); + QVERIFY(FolderModel::narrows("BEAC", "beach")); + QVERIFY(FolderModel::narrows("beach", "beach")); + QVERIFY(!FolderModel::narrows("beach", "beac")); + QVERIFY(!FolderModel::narrows("beach", "sea")); + // An extension narrows; dropping it or taking another widens. + QVERIFY(FolderModel::narrows("beach", "beach ext:jpg")); + QVERIFY(FolderModel::narrows("ext:jpg,png", "ext:jpg")); + QVERIFY(FolderModel::narrows("ext:jpg", "img ext:jpg")); + QVERIFY(!FolderModel::narrows("ext:jpg", "ext:jpg,png")); + QVERIFY(!FolderModel::narrows("ext:jpg", "ext:png")); + QVERIFY(!FolderModel::narrows("ext:jpg", "beach")); + // A duplicates search is made again: whether a file has a twin + // depends on the files kept, not just on its own name. + QVERIFY(!FolderModel::narrows("beach", "beach dupes:")); + QVERIFY(!FolderModel::narrows("dupes: beach", "dupes: beach.jpg")); + QVERIFY(!FolderModel::narrows("dupes:", "beach")); + // Plain text is matched as typed, spaces and all. + QVERIFY(!FolderModel::narrows("my notes", "my notes")); + QVERIFY(FolderModel::narrows("my", "my notes")); + + // The model answers a longer query from what it found, and the + // same as a fresh search would. + QTemporaryDir dir; + QDir(dir.path()).mkpath("sub"); + write(dir.filePath("beach.jpg"), "x"); + write(dir.filePath("beach.png"), "x"); + write(dir.filePath("sub/beach-2.jpg"), "x"); + write(dir.filePath("sub/beam.txt"), "x"); + write(dir.filePath("sub/beach.d"), "x"); + QDir(dir.filePath("sub")).mkpath("beach-folder"); + FolderModel model; + QSignalSpy loaded(&model, &FolderModel::loaded); + model.setPath(dir.path()); + QVERIFY(loaded.wait(3000)); + auto names = [&] { + QStringList out; + for (int row = 0; row < model.count(); ++row) + out << model.get(row).value("name").toString(); + return out; + }; + model.setQuery("bea"); + QTRY_VERIFY(!model.loading()); + QCOMPARE(model.count(), 6); + // Written after the search, in a folder the model doesn't watch: + // only a new walk would find it. + write(dir.filePath("sub/beach-new.jpg"), "x"); + model.setQuery("beach"); + QVERIFY(!model.loading()); + QCOMPARE(model.count(), 5); + QVERIFY(!names().contains("sub/beach-new.jpg")); + model.selectOnly(model.indexOfPath(dir.filePath("beach.png"))); + model.setQuery("beach ext:jpg"); + QVERIFY(!model.loading()); + QCOMPARE(names().join(", "), QString("beach.jpg, sub/beach-2.jpg")); + QCOMPARE(model.selectedCount(), 0); + // Widening looks again, and finds what was written meanwhile. + model.setQuery("beach"); + QTRY_VERIFY(!model.loading()); + QVERIFY(names().contains("sub/beach-new.jpg")); + } + void folderSearchSkipsSymlinkedFolders() { // A link back up the tree would otherwise be walked round and round.