From cbc176b9bb02c2c136695a7d2321d840781decfd Mon Sep 17 00:00:00 2001 From: Wander Nauta Date: Tue, 18 Aug 2026 21:12:00 +0200 Subject: [PATCH 1/2] adapt LockedSqw to upstream changes in sqlwriter See https://github.com/berthubert/sqlitewrite/commit/5e291af887d57a0541d6777d3dcc0d1915db4d11 --- sws.hh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sws.hh b/sws.hh index ca22f30..1272d6c 100644 --- a/sws.hh +++ b/sws.hh @@ -33,12 +33,12 @@ struct LockedSqw return packResultsJson(result); } - void addValue(const std::initializer_list>& values, const std::string& table="data") + void addValue(const std::initializer_list>& values, const std::string& table="data") { std::lock_guard l(sqwlock); sqw.addValue(values, table); } - void addValue(const std::vector>& values, const std::string& table="data") + void addValue(const std::vector>& values, const std::string& table="data") { std::lock_guard l(sqwlock); sqw.addValue(values, table); @@ -117,10 +117,10 @@ struct SimpleWebSystem { return sws.getIP(req); } - void log(const std::initializer_list>& fields) + void log(const std::initializer_list>& fields) { // add agent? - std::vector> values{{"user", user}, {"ip", getIP()}, {"tstamp", time(0)}}; + std::vector> values{{"user", user}, {"ip", getIP()}, {"tstamp", time(0)}}; for(const auto& f : fields) values.push_back(f); lsqw.addValue(values, "log"); From 63138dacf242e1f87e7caa836913bf0ca420d362 Mon Sep 17 00:00:00 2001 From: Wander Nauta Date: Tue, 18 Aug 2026 22:11:43 +0200 Subject: [PATCH 2/2] implement soorten filter inside sql query This moves checking the 'soort' of search results inside the search SQL query, as suggested in #141, so before the SQLite LIMIT rather than after. SearchHelper is modified to support the 'soort' parameter. Care is taken to closely follow the behavior of the existing 'categories' parameter (for example, default value = empty set = all soorten). As before, we look in the "meta" database for the value. The advantage of this is that the index database does not need to change. An alternative approach would be to add the 'soort' value to the docsearch table. --- scanmon.cc | 2 +- scanmon.hh | 2 +- search.cc | 16 +++++++++++++--- search.hh | 1 + testrunner.cc | 2 +- tkserv.cc | 25 +++++++++++++------------ 6 files changed, 30 insertions(+), 18 deletions(-) diff --git a/scanmon.cc b/scanmon.cc index 2309ebf..9af562d 100644 --- a/scanmon.cc +++ b/scanmon.cc @@ -20,7 +20,7 @@ std::vector ZoekScanner::get(SQLiteWriter& sqlw) { SearchHelper sh(sqlw); - auto matches = sh.search(d_query, {}, d_cutoff); + auto matches = sh.search(d_query, {}, {}, d_cutoff); // std::cout<<"Got "< ret; diff --git a/scanmon.hh b/scanmon.hh index 1d5e2cd..ec4d533 100644 --- a/scanmon.hh +++ b/scanmon.hh @@ -208,7 +208,7 @@ struct ZaakScanner : Scanner for(auto& h : hits) already.insert(eget(h,"nummer")); - auto sresults = sh.search(d_nummer, {"Document"}, d_cutoff); + auto sresults = sh.search(d_nummer, {"Document"}, {}, d_cutoff); for(const auto& sr : sresults) { if(already.count(sr.nummer)) continue; diff --git a/search.cc b/search.cc index d93e42c..07a81e1 100644 --- a/search.cc +++ b/search.cc @@ -3,7 +3,7 @@ using namespace std; -std::vector SearchHelper::search(const std::string& query, const std::set& categories, const std::string& cutoff, unsigned int mseclimit, unsigned int itemlimit) +std::vector SearchHelper::search(const std::string& query, const std::set& categories, const std::set& sorts, const std::string& cutoff, unsigned int mseclimit, unsigned int itemlimit) { std::vector ret; // nummer, category, relurl, score, date, snippet, title @@ -16,9 +16,19 @@ std::vector SearchHelper::search(const std::string& query, continue; categoriesstr+= "'" + c + "'"; } - auto matches = d_sqw.queryT("SELECT uuid, datum, snippet(docsearch,-1, '', '', '...', 20) as snip, category, bm25(docsearch) as score FROM docsearch WHERE docsearch match ? and (datum >= ? or datum='') and (? or category in ("+categoriesstr+")) order by rowid desc" + + string sortsstr; + for(const auto& s : sorts) { + if(!sortsstr.empty()) + sortsstr += ", "; + if(s.find_first_of("'\"") != string::npos) + continue; + sortsstr += "'" + s + "'"; + } + + auto matches = d_sqw.queryT("SELECT uuid, datum, snippet(docsearch,-1, '', '', '...', 20) as snip, category, bm25(docsearch) as score FROM docsearch WHERE docsearch match ? and (datum >= ? or datum='') and (? or category in ("+categoriesstr+")) and (? or (case category when 'Document' then (select soort from meta.Document where id=uuid) when 'Activiteit' then (select soort from meta.Activiteit where id=uuid) end) in ("+sortsstr+")) order by rowid desc" + (itemlimit ? " limit "+to_string(itemlimit) : ""), - {query, cutoff, categories.empty()}, mseclimit); + {query, cutoff, categories.empty(), sorts.empty()}, mseclimit); for(auto& m : matches) { Result r; diff --git a/search.hh b/search.hh index 22d1250..1750fbb 100644 --- a/search.hh +++ b/search.hh @@ -39,6 +39,7 @@ struct SearchHelper std::vector search(const std::string& query, const std::set& categories={}, + const std::set& sorts={}, const std::string& cutoff="", unsigned int mseclimit=10000, unsigned int itemlimit = 0); diff --git a/testrunner.cc b/testrunner.cc index 8368c98..407efa9 100644 --- a/testrunner.cc +++ b/testrunner.cc @@ -66,7 +66,7 @@ TEST_CASE("Search" * doctest::skip()) sqw.query("ATTACH DATABASE 'tk.sqlite3' as meta"); SearchHelper sh(sqw); - auto ret = sh.search("Werkbezoek Knooppunten internationaal", {"Activiteit", "Toezegging"}, "2024-11-01"); + auto ret = sh.search("Werkbezoek Knooppunten internationaal", {"Activiteit", "Toezegging"}, {}, "2024-11-01"); for(const auto& r : ret) { cout< categories; - if(soorten=="activiteiten") + set sorts; + if(soorten=="activiteiten") { categories.insert("Activiteit"); + } else if(soorten=="moties") { + categories.insert("Document"); + sorts.insert("Motie"); + } else if(soorten=="vragenantwoorden") { + categories.insert("Document"); + sorts.insert("Schriftelijke vragen"); + sorts.insert("Antwoord schriftelijke vragen"); + sorts.insert("Antwoord schriftelijke vragen (nader)"); + } fmt::print("Categories: {}\n", categories); + fmt::print("Sorts: {}\n", sorts); - auto sres = sh.search(term, categories, limit, mseclimit, 280); + auto sres = sh.search(term, categories, sorts, limit, mseclimit, 280); nlohmann::json results = nlohmann::json::array(); for(const auto& r : sres) { - - if(soorten=="moties" && r.soort != "Motie") - continue; - else if(soorten=="vragenantwoorden" && - (r.soort != "Schriftelijke vragen" && - r.soort != "Antwoord schriftelijke vragen" && - r.soort != "Antwoord schriftelijke vragen (nader)")) - continue; - results.push_back(nlohmann::json({ {"nummer", r.nummer}, {"datum", r.datum}, @@ -2281,7 +2283,6 @@ int main(int argc, char** argv) {"bijgewerkt", r.bijgewerkt}, {"persoonnummer", r.persoonnummer} })); - } // soorten!