Skip to content

Implement soorten filter inside sql query #150

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wandernauta wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Implement soorten filter inside sql query #150

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
cbc176b
adapt LockedSqw to upstream changes in sqlwriter
wandernauta Aug 18, 2026
63138da
implement soorten filter inside sql query
wandernauta Aug 18, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion scanmon.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ std::vector<ScannerHit> ZoekScanner::get(SQLiteWriter& sqlw)
{
SearchHelper sh(sqlw);

auto matches = sh.search(d_query, {}, d_cutoff);
auto matches = sh.search(d_query, {}, {}, d_cutoff);
// std::cout<<"Got "<<matches.size()<<" matches for '"<<d_query<<"' with d_cutoff "<<d_cutoff<"\n";
// nummer/identifier, datum, category
std::vector<ScannerHit> ret;
Expand Down
2 changes: 1 addition & 1 deletion scanmon.hh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -208,7 +208,7 @@ struct ZaakScanner : Scanner
for(auto& h : hits)
already.insert(eget(h,"nummer"));

auto sresults = sh.search(d_nummer, {"Document"}, d_cutoff);
auto sresults = sh.search(d_nummer, {"Document"}, {}, d_cutoff);
for(const auto& sr : sresults) {
if(already.count(sr.nummer))
continue;
Expand Down
16 changes: 13 additions & 3 deletions search.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
using namespace std;


std::vector<SearchHelper::Result> SearchHelper::search(const std::string& query, const std::set<string>& categories, const std::string& cutoff, unsigned int mseclimit, unsigned int itemlimit)
std::vector<SearchHelper::Result> SearchHelper::search(const std::string& query, const std::set<string>& categories, const std::set<string>& sorts, const std::string& cutoff, unsigned int mseclimit, unsigned int itemlimit)
{
std::vector<SearchHelper::Result> ret;
// nummer, category, relurl, score, date, snippet, title
Expand All @@ -16,9 +16,19 @@ std::vector<SearchHelper::Result> SearchHelper::search(const std::string& query,
continue;
categoriesstr+= "'" + c + "'";
}
auto matches = d_sqw.queryT("SELECT uuid, datum, snippet(docsearch,-1, '<b>', '</b>', '...', 20) as snip, category, bm25(docsearch) as score FROM docsearch WHERE docsearch match ? and (datum >= ? or datum='') and (? or category in ("+categoriesstr+")) order by rowid desc"

string sortsstr;
for(const auto& s : sorts) {
if(!sortsstr.empty())
sortsstr += ", ";
if(s.find_first_of("'\"") != string::npos)
continue;
sortsstr += "'" + s + "'";
}

auto matches = d_sqw.queryT("SELECT uuid, datum, snippet(docsearch,-1, '<b>', '</b>', '...', 20) as snip, category, bm25(docsearch) as score FROM docsearch WHERE docsearch match ? and (datum >= ? or datum='') and (? or category in ("+categoriesstr+")) and (? or (case category when 'Document' then (select soort from meta.Document where id=uuid) when 'Activiteit' then (select soort from meta.Activiteit where id=uuid) end) in ("+sortsstr+")) order by rowid desc"
+ (itemlimit ? " limit "+to_string(itemlimit) : ""),
{query, cutoff, categories.empty()}, mseclimit);
{query, cutoff, categories.empty(), sorts.empty()}, mseclimit);

for(auto& m : matches) {
Result r;
Expand Down
1 change: 1 addition & 0 deletions search.hh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ struct SearchHelper

std::vector<Result> search(const std::string& query,
const std::set<std::string>& categories={},
const std::set<std::string>& sorts={},
const std::string& cutoff="", unsigned int mseclimit=10000,
unsigned int itemlimit = 0);

Expand Down
8 changes: 4 additions & 4 deletions sws.hh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,12 +33,12 @@ struct LockedSqw
return packResultsJson(result);
}

void addValue(const std::initializer_list<std::pair<const char*, SQLiteWriter::var_t>>& values, const std::string& table="data")
void addValue(const std::initializer_list<std::pair<std::string, SQLiteWriter::var_t>>& values, const std::string& table="data")
{
std::lock_guard<std::mutex> l(sqwlock);
sqw.addValue(values, table);
}
void addValue(const std::vector<std::pair<const char*, SQLiteWriter::var_t>>& values, const std::string& table="data")
void addValue(const std::vector<std::pair<std::string, SQLiteWriter::var_t>>& values, const std::string& table="data")
{
std::lock_guard<std::mutex> l(sqwlock);
sqw.addValue(values, table);
Expand Down Expand Up @@ -117,10 +117,10 @@ struct SimpleWebSystem
{
return sws.getIP(req);
}
void log(const std::initializer_list<std::pair<const char*, SQLiteWriter::var_t>>& fields)
void log(const std::initializer_list<std::pair<std::string, SQLiteWriter::var_t>>& fields)
{
// add agent?
std::vector<std::pair<const char*, SQLiteWriter::var_t>> values{{"user", user}, {"ip", getIP()}, {"tstamp", time(0)}};
std::vector<std::pair<std::string, SQLiteWriter::var_t>> values{{"user", user}, {"ip", getIP()}, {"tstamp", time(0)}};
for(const auto& f : fields)
values.push_back(f);
lsqw.addValue(values, "log");
Expand Down
2 changes: 1 addition & 1 deletion testrunner.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ TEST_CASE("Search" * doctest::skip())
sqw.query("ATTACH DATABASE 'tk.sqlite3' as meta");

SearchHelper sh(sqw);
auto ret = sh.search("Werkbezoek Knooppunten internationaal", {"Activiteit", "Toezegging"}, "2024-11-01");
auto ret = sh.search("Werkbezoek Knooppunten internationaal", {"Activiteit", "Toezegging"}, {}, "2024-11-01");
for(const auto& r : ret) {
cout<<r.nummer<<" ("<<r.datum<<"): "<<r.onderwerp<<" / " << r.titel<<"\n";
}
Expand Down
25 changes: 13 additions & 12 deletions tkserv.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2253,23 +2253,25 @@ int main(int argc, char** argv)
SearchHelper sh(idx);

set<string> categories;
if(soorten=="activiteiten")
set<string> sorts;
if(soorten=="activiteiten") {
categories.insert("Activiteit");
} else if(soorten=="moties") {
categories.insert("Document");
sorts.insert("Motie");

@wandernauta wandernauta Aug 18, 2026 •
edited
Loading

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hier moet misschien ook bij: Motie (gewijzigd/nader). Of anders gedocumenteerd waarom die er niet bij moeten.

} else if(soorten=="vragenantwoorden") {
categories.insert("Document");
sorts.insert("Schriftelijke vragen");
sorts.insert("Antwoord schriftelijke vragen");
sorts.insert("Antwoord schriftelijke vragen (nader)");
}

fmt::print("Categories: {}\n", categories);
fmt::print("Sorts: {}\n", sorts);

auto sres = sh.search(term, categories, limit, mseclimit, 280);
auto sres = sh.search(term, categories, sorts, limit, mseclimit, 280);
nlohmann::json results = nlohmann::json::array();
for(const auto& r : sres) {

if(soorten=="moties" && r.soort != "Motie")
continue;
else if(soorten=="vragenantwoorden" &&
(r.soort != "Schriftelijke vragen" &&
r.soort != "Antwoord schriftelijke vragen" &&
r.soort != "Antwoord schriftelijke vragen (nader)"))
continue;

results.push_back(nlohmann::json({
{"nummer", r.nummer},
{"datum", r.datum},
Expand All @@ -2281,7 +2283,6 @@ int main(int argc, char** argv)
{"bijgewerkt", r.bijgewerkt},
{"persoonnummer", r.persoonnummer}
}));

}

// soorten!
Expand Down
Loading