Add spell check feature for queries that give few results

html/logic.js

@@ -306,6 +306,18 @@ async function getSearchResults(f)
 	rssurl.searchParams.set("q", f.searchQuery);
 	f.rssurl = rssurl.href;
 
+	if (f.alternatief == '' && data.suggest) {
+            const link = document.createElement('a');
+            const href = new URL(location.href);
+            href.searchParams.set('q', data.suggest);
+            href.searchParams.set('twomonths', f.twomonths);
+            href.searchParams.set('soorten', f.soorten);
+            link.href = href;
+            link.innerText = data.suggest;
+
+            f.alternatief = `<p><em>Bedoelt u mogelijk ${link.outerHTML}?</em></p>`;
+	}
+
 	f.message = `< ${Math.ceil(data["milliseconds"])} milliseconden`;
 	f.busy=false;
 	orderByDate(f, false);

meson.build

@@ -95,7 +95,7 @@ executable('oppull', 'oppull.cc', 'support.cc', 'siphash.cc',
 
 
 executable('tkserv', 'tkserv.cc', 'support.cc', 'siphash.cc', 'sws.cc', 'users.cc', 'scanmon.cc', 'search.cc',
-		     'enrich.cc', 'ical.cc', 'sitemaps.cc',
+		     'enrich.cc', 'ical.cc', 'sitemaps.cc', 'suggest.cc', 'qparser.cc',
 	dependencies: [sqlitedep, json_dep,
 	simplesockets_dep, fmt_dep, cpphttplib, sqlitewriter_dep, pugi_dep,
 	argparse_dep, vcs_dep, bcryptcpp_dep])
@@ -115,6 +115,6 @@ executable('playground', 'playground.cc', 'support.cc', 'siphash.cc',
 #	argparse_dep, vcs_dep])
 
 
-executable('testrunner', 'testrunner.cc', 'ical.cc', 'icaltest.cc', 'support.cc',  'siphash.cc', 'search.cc', 'meta.cc',
+executable('testrunner', 'testrunner.cc', 'ical.cc', 'icaltest.cc', 'support.cc',  'siphash.cc', 'search.cc', 'meta.cc', 'qparser.cc',
 dependencies: [sqlitedep, json_dep, fmt_dep, sqlitedep, sqlitewriter_dep, doctest_dep, cpphttplib, simplesockets_dep])
 

qparser.cc

@@ -0,0 +1,86 @@
+#include "qparser.hh"
+
+#include "peglib.h"
+#include <fmt/core.h>
+#include <vector>
+
+using std::string;
+
+static string quote(const string& in) {
+  return "\"" + in + "\"";
+}
+
+string transformQuery(const string& in,
+  std::function<string(const string&)> bareWord,
+  std::function<string(const string&)> quotedWord)
+{
+  peg::parser p;
+  p.set_logger([](size_t line, size_t col, const string& msg, const string &rule) {
+    fmt::print("line {}, col {}: {}\n", line, col,msg, rule);
+  }); // gets us some helpful errors if the grammar is wrong
+
+  // the BareWord is like that because of UTF-8
+  auto ret = p.load_grammar(R"a(
+    Root <- (Paren / BareWord / QuotedWord)+
+    Paren <- ('(' / ')')
+    BareWord      <- < [^" ()]+ >
+    QuotedWord <-  '"' < [^"]* > '"'
+    %whitespace <- [\t ]*
+    )a");
+  if(!ret)
+    throw std::runtime_error("cpp-peglib grammar did not compile");
+
+  p["BareWord"] = [bareWord](const peg::SemanticValues &vs) {
+    return bareWord(vs.token_to_string());
+  };
+
+  p["QuotedWord"] = [quotedWord](const peg::SemanticValues &vs) {
+    return quote(quotedWord(vs.token_to_string()));
+  };
+
+  p["Paren"] = [](const peg::SemanticValues &vs) {
+    return vs.token_to_string();
+  };
+
+  p["Root"] = [](const peg::SemanticValues &vs) {
+    return vs.transform<string>();
+  };
+
+  std::vector<string> result;
+  int rc = p.parse(in, result);
+
+  if(!rc)
+    return in; // we tried
+
+  string retval;
+  for(const auto& r : result) {
+    if(!retval.empty())
+      retval.append(1, ' ');
+    retval += r;
+  }
+  return retval;
+}
+
+/*
+  SQLite FTS5 has some oddities where you can't search for Fox-IT as a bare word,
+  because of the dash you must do "Fox-IT".
+*/
+string convertToSQLiteFTS5(const string& in)
+{
+  return transformQuery(in, [](const string& s) {
+    /*
+      As an FTS5 bareword that is not "AND", "OR" or "NOT" (case sensitive). An FTS5 bareword is a string of one or more consecutive characters that are all either:
+
+    Non-ASCII range characters (i.e. unicode codepoints greater than 127), or
+    One of the 52 upper and lower case ASCII characters, or
+    One of the 10 decimal digit ASCII characters, or
+    The underscore character (unicode codepoint 95).
+    The substitute character (unicode codepoint 26).
+    XXX this is NOT quite what we do!
+     */
+    if(auto pos = s.find_first_of(",.-[];"); pos != string::npos)
+      return quote(s);
+    else
+      return s;
+  });
+}

qparser.hh

@@ -0,0 +1,12 @@
+#pragma once
+
+#include <functional>
+#include <string>
+
+// Parse and transform the given query, or return it unmodified on error.
+std::string transformQuery(const std::string& in,
+  std::function<std::string(const std::string&)> bareWord = std::identity(),
+  std::function<std::string(const std::string&)> quotedWord = std::identity());
+
+// Parse the query, adding quotes to fit FTS5 syntax, or return unmodified.
+std::string convertToSQLiteFTS5(const std::string& in);

suggest.cc

@@ -0,0 +1,66 @@
+#include "suggest.hh"
+
+#include "qparser.hh"
+
+using std::nullptr_t;
+using std::string;
+
+#include "fstlib.h"
+#include "sqlwriter.hh"
+
+string Suggester::spell(string q) {
+  string best_term = q;
+  size_t best_docs = 0;
+
+  if(!q.empty() && !bytes.empty()) {
+    fst::map<uint64_t> dictionary(bytes.data(), bytes.size());
+
+    if(!dictionary.contains(q)) {
+      // of words within edit distance, return the one that appears in most
+      // documents (and thus the one most likely to give results)
+      for(const auto &[term, docs] : dictionary.edit_distance_search(q, 2)) {
+        if(docs >= best_docs) {
+          best_term = term;
+          best_docs = docs;
+        }
+      }
+    }
+  }
+
+  return best_term;
+}
+
+string Suggester::correct_query(string in) {
+  auto fn = [this](const string& s) {
+    return spell(s);
+  };
+
+  return transformQuery(in, fn, fn);
+}
+
+Suggester suggester_from_pairs(const std::vector<std::pair<string, uint64_t>> &pairs) {
+  std::stringstream out;
+
+  auto [result, _] = fst::compile<uint64_t>(pairs, out, true);
+
+  if(result != fst::Result::Success)
+    throw std::runtime_error("Suggester could not build FST (not sorted?)");
+
+  return {out.str()};
+}
+
+Suggester suggester_from_table(SQLiteWriter *sql) {
+  std::vector<std::pair<string, uint64_t>> pairs;
+
+  auto rows = sql->queryT("select term as term, doc as doc from lexicon order by term");
+
+  pairs.reserve(rows.size());
+
+  for(const auto& row : rows) {
+    string term = std::get<string>(row.at("term"));
+    uint64_t doc = std::get<int64_t>(row.at("doc"));
+    pairs.push_back({term, doc});
+  }
+
+  return suggester_from_pairs(pairs);
+}

suggest.hh

@@ -0,0 +1,21 @@
+#pragma once
+
+#include <cstdint>
+#include <string>
+#include <vector>
+
+class SQLiteWriter;
+
+// Spelling corrections using the search index as a dictionary, preferring common terms. 
+struct Suggester {
+  std::string bytes;
+
+  // Return the given word, spell-corrected or unmodified.
+  std::string spell(std::string word);
+
+  // Return the given search query, spell-corrected or unmodified.
+  std::string correct_query(std::string query);
+};
+
+Suggester suggester_from_table(SQLiteWriter *tkindex_sqlw);
+Suggester suggester_from_pairs(const std::vector<std::pair<std::string, uint64_t>> &term_score_pairs);

support.cc

@@ -10,7 +10,6 @@
 #include <sclasses.hh>
 #include "httplib.h"
 #include "base64.hpp"
-#include "peglib.h"
 #include <locale>
 using namespace std;
 
@@ -555,76 +554,6 @@ std::string getTodayDBFormat()
   return getDateDBFormat(time(0));
 }
 
-/*
-  SQLite FTS5 has some oddities where you can't search for Fox-IT as a bare word,
-  because of the dash you must do "Fox-IT".
-*/
-string convertToSQLiteFTS5(const std::string& in)
-{
-  peg::parser p;
-  p.set_logger([](size_t line, size_t col, const string& msg, const string &rule) {
-    fmt::print("line {}, col {}: {}\n", line, col,msg, rule);
-  }); // gets us some helpful errors if the grammar is wrong
-
-  // sequence of words AND "words" - add quotes to everything not quoted with a . or - in there
-
-  // the BareWord is like that because of UTF-8
-  auto ret = p.load_grammar(R"a(
-Root <- (Paren / BareWord / QuotedWord)+
-Paren <- ('(' / ')')
-BareWord      <- < [^" ()]+ > 
-QuotedWord <-  < '"'  [^"]*  '"' > 
-%whitespace <- [\t ]*
-)a");
-  if(!ret)
-    throw runtime_error("cpp-peglib grammar did not compile");
-
-  p["BareWord"] = [](const peg::SemanticValues &vs) {
-    /*
-      As an FTS5 bareword that is not "AND", "OR" or "NOT" (case sensitive). An FTS5 bareword is a string of one or more consecutive characters that are all either:
-
-    Non-ASCII range characters (i.e. unicode codepoints greater than 127), or
-    One of the 52 upper and lower case ASCII characters, or
-    One of the 10 decimal digit ASCII characters, or
-    The underscore character (unicode codepoint 95).
-    The substitute character (unicode codepoint 26).
-    XXX this is NOT quite what we do!
-     */
-
-    if(auto pos = vs.token_to_string().find_first_of(",.-[];"); pos != string::npos) {
-      return "\"" + vs.token_to_string() +"\"";
-    }
-    else
-      return vs.token_to_string();
-  };
-
-  p["QuotedWord"] = [](const peg::SemanticValues &vs) {
-    return vs.token_to_string();
-  };
-
-  p["Paren"] = [](const peg::SemanticValues &vs) {
-    return vs.token_to_string();
-  };
-
-
-  p["Root"] = [](const peg::SemanticValues &vs) {
-    return vs.transform<string>();
-  };
-  vector<string> result;
-  int rc = p.parse(in, result);
-
-  if(!rc)
-    return in; // we tried
-
-  string retval;
-  for(const auto& r : result) {
-    if(!retval.empty())
-      retval.append(1, ' ');
-    retval += r;
-  }
-  return retval;
-}
-
 std::string deHTML(const std::string& html, const std::string& rep)
 {
   std::regex html_re("<[^>]*>");

support.hh

@@ -135,7 +135,6 @@ int64_t iget(const T& cont, const std::string& fname)
 }
 
 
-std::string convertToSQLiteFTS5(const std::string& in);
 std::string enrichHTML(const std::string& html, SQLiteWriter& sqlw);
 std::string getContentsOfFile(const std::string& fname);
 

sws.hh

@@ -33,12 +33,12 @@ struct LockedSqw
     return packResultsJson(result);
   }
 
-  void addValue(const std::initializer_list<std::pair<const char*, SQLiteWriter::var_t>>& values, const std::string& table="data")
+  void addValue(const std::initializer_list<std::pair<std::string, SQLiteWriter::var_t>>& values, const std::string& table="data")
   {
     std::lock_guard<std::mutex> l(sqwlock);
     sqw.addValue(values, table);
   }
-  void addValue(const std::vector<std::pair<const char*, SQLiteWriter::var_t>>& values, const std::string& table="data")
+  void addValue(const std::vector<std::pair<std::string, SQLiteWriter::var_t>>& values, const std::string& table="data")
   {
     std::lock_guard<std::mutex> l(sqwlock);
     sqw.addValue(values, table);
@@ -117,10 +117,10 @@ struct SimpleWebSystem
     {
       return sws.getIP(req);
     }
-    void log(const std::initializer_list<std::pair<const char*, SQLiteWriter::var_t>>& fields)
+    void log(const std::initializer_list<std::pair<std::string, SQLiteWriter::var_t>>& fields)
     {
       // add agent?
-      std::vector<std::pair<const char*, SQLiteWriter::var_t>> values{{"user", user}, {"ip", getIP()}, {"tstamp", time(0)}};
+      std::vector<std::pair<std::string, SQLiteWriter::var_t>> values{{"user", user}, {"ip", getIP()}, {"tstamp", time(0)}};
       for(const auto& f : fields)
         values.push_back(f);
       lsqw.addValue(values, "log");

testrunner.cc

@@ -13,6 +13,7 @@
 #include "nlohmann/json.hpp"
 #include "meta.hh"
 #include "support.hh"
+#include "qparser.hh"
 
 using namespace std;
 

tkindex.cc

@@ -190,6 +190,12 @@ CREATE VIRTUAL TABLE IF NOT EXISTS docsearch USING fts5(onderwerp, titel, tekst,
   sqlw.queryT("create table if not exists indexed as select datum, uuid,contentLength,bijgewerkt, category from docsearch");
   sqlw.queryT("create unique index if not exists uuididx on indexed(uuid)");
 
+  // for spell checking, see suggest.cc
+  sqlw.queryT(R"(
+CREATE VIRTUAL TABLE IF NOT EXISTS vocab USING fts5vocab(docsearch, row); 
+  )");
+  sqlw.queryT("create table if not exists lexicon (term text primary key not null, doc integer not null)");
+
   if (args["--cleanup"] == true) {
     fmt::print("Cleaning up documents that are older than {}\n", limit);
     sqlw.queryT("delete from indexed where datum < ? and category != 'PersoonGeschenk'", {limit});
@@ -582,4 +588,11 @@ CREATE VIRTUAL TABLE IF NOT EXISTS docsearch USING fts5(onderwerp, titel, tekst,
 
   fmt::print("Indexed {} new documents, of which {} were reindexes. {} weren't present, {} of unsupported type, {} were indexed already\n",
 	     (int)indexed, reindex.size(), (int)notpresent, (int)wrong, (int)skipped);
+
+  // see suggest.cc; must happen after docsearch table is updated. terms
+  // that appear in very few documents (typos, OCR mistakes...) are ignored.
+  fmt::println("Rebuilding lexicon table for spelling suggestions...");
+  sqlw.queryT("delete from lexicon");
+  sqlw.queryT("insert into lexicon select term, doc from vocab where doc >= 20");
+  fmt::println("tkindex complete.");
 }