update routes search + logic index

Author: Lucaterre

api/admin/views.py

@@ -3,6 +3,7 @@
 
 Model views for the admin interface.
 """
+from unidecode import unidecode
 
 from flask import (url_for,
                    jsonify,
@@ -124,6 +125,10 @@ def is_accessible(self):
         return True
 
 
+
+
+
+
 class UserView(ModelView):
     edit_template = 'admin/edit.user.html'
     create_template = 'admin/edit.user.html'
@@ -571,6 +576,41 @@ def render(self, template, **kwargs):
                 return super(PrinterView, self).render(template, **kwargs)
         return super(PrinterView, self).render(template, **kwargs)
 
+    def get_list(self, page, sort_field, sort_desc, search, filters, page_size=None):
+        query = self.session.query(self.model)
+        all_rows = query.all()
+
+        if search:
+            normalized_search = unidecode(search).lower().strip()
+            search_tokens = normalized_search.split()
+
+            def match(row):
+                combined = f"{row.lastname or ''} {row.firstnames or ''}"
+                normalized_combined = unidecode(combined).lower()
+                return all(token in normalized_combined for token in search_tokens)
+
+            filtered_rows = list(filter(match, all_rows))
+        else:
+            filtered_rows = all_rows
+
+        count = len(filtered_rows)
+
+        # Tri optionnel
+        if sort_field:
+            reverse = sort_desc
+            filtered_rows.sort(
+                key=lambda x: getattr(x, sort_field, '').lower() if getattr(x, sort_field) else '',
+                reverse=reverse
+            )
+
+        # Pagination Python
+        if page_size:
+            start = page * page_size
+            end = start + page_size
+            filtered_rows = filtered_rows[start:end]
+
+        return count, filtered_rows
+
     # Expose custom routes for printer view
     # for Ajax requests
 
@@ -1101,6 +1141,41 @@ def on_model_change(self, form, model, is_created):
         model.last_editor = current_user.username
         session.commit()
 
+    def get_list(self, page, sort_field, sort_desc, search, filters, page_size=None):
+        query = self.session.query(self.model)
+        all_rows = query.all()
+
+        if search:
+            normalized_search = unidecode(search).lower().strip()
+            search_tokens = normalized_search.split()
+
+            def match(row):
+                combined = f"{row.label or ''}"
+                normalized_combined = unidecode(combined).lower()
+                return all(token in normalized_combined for token in search_tokens)
+
+            filtered_rows = list(filter(match, all_rows))
+        else:
+            filtered_rows = all_rows
+
+        count = len(filtered_rows)
+
+        # Tri optionnel
+        if sort_field:
+            reverse = sort_desc
+            filtered_rows.sort(
+                key=lambda x: getattr(x, sort_field, '').lower() if getattr(x, sort_field) else '',
+                reverse=reverse
+            )
+
+        # Pagination Python
+        if page_size:
+            start = page * page_size
+            end = start + page_size
+            filtered_rows = filtered_rows[start:end]
+
+        return count, filtered_rows
+
 
 class AboutView(BaseView):
     """Custom view for database documentation."""

api/admin/views_dir/utils.py

@@ -4,7 +4,14 @@
 """
 import os
 from werkzeug.utils import secure_filename
+from sqlalchemy import func
+from unidecode import unidecode
+import unicodedata
 
 def prefix_name(_, file_data):
     parts = os.path.splitext(file_data.filename)
-    return secure_filename('file-%s%s' % parts)
+    return secure_filename('file-%s%s' % parts)
+
+def get_search_filter(column, search, dialect_name):
+    normalized_search = unidecode(search).lower()
+    return func.lower(column).like(f"%{normalized_search}%")

api/index_fts/index_utils.py

@@ -4,8 +4,10 @@
 """
 from shutil import rmtree
 import bleach
-
+from unidecode import unidecode
+from tqdm import tqdm
 from whoosh import index
+from joblib import Parallel, delayed
 
 
 def create_store(store, path) -> None:
@@ -37,19 +39,46 @@ def prepare_content(obj):
     return clean_text
 
 
-def populate_index(session, index_, model):
+def extract_data(printer):
+    """Extraction safe des données sans session SQLAlchemy"""
+    content = bleach.clean(
+                " ".join([
+                    printer.personal_information or "",
+                    printer.professional_information or ""
+                ] + [p.references or "" for p in printer.patents]),
+                tags=[], attributes=[], strip=True
+            )
+
+    return dict(
+        id_dil=str(printer._id_dil),
+        lastname=unidecode(printer.lastname or "").lower(),
+        firstnames=unidecode(printer.firstnames or "").lower(),
+        clean_text=unidecode(
+            content
+        ).lower(),
+        text=content,
+    )
+
+def populate_index(session, index_, model, n_jobs=-1):
     persons = session.query(model).all()
-    writer = index_.writer()
-    for printer in persons:
-        clean_text = prepare_content(printer)
 
-        lastname = printer.lastname or ""
-        firstnames = printer.firstnames or ""
+    serialized_data = [extract_data(printer) for printer in tqdm(persons, desc="Serializing data for index")]
+
+    def process(doc):
+        return dict(
+            id_dil=doc['id_dil'],
+            lastname=doc['lastname'],
+            firstnames=doc['firstnames'],
+            firstnames_lastname=f"{' '.join(doc['firstnames'].split(','))} {doc['lastname']}",
+            content=doc['text'],
+            content_ngram=doc['clean_text'],
+        )
 
-        writer.add_document(id_dil=str(printer._id_dil),
-                            lastname=lastname,
-                            firstnames=firstnames,
-                            content=clean_text)
+    documents = Parallel(n_jobs=n_jobs)(
+        delayed(process)(doc) for doc in tqdm(serialized_data, desc="Prepare documents for index")
+    )
 
-    writer.commit()
+    with index_.writer() as writer:
+        for doc in tqdm(documents, desc="Index documents in Whoosh backend"):
+            writer.add_document(**doc)
     return index_

api/index_fts/schemas.py

@@ -5,12 +5,15 @@
 
 from whoosh.fields import (SchemaClass,
                            ID,
-                           TEXT)
-
+                           TEXT,
+                           NGRAM,
+                           NGRAMWORDS)
 
 class PersonIdxSchema(SchemaClass):
     """Schema for the PersonIdx index."""
     id_dil = ID(stored=True, unique=True)
-    lastname = TEXT(stored=True)
-    firstnames = TEXT(stored=True)
-    content = TEXT(stored=True)
+    lastname = NGRAM(minsize=2, maxsize=15, stored=True)
+    firstnames = NGRAM(minsize=2, maxsize=15, stored=True)
+    firstnames_lastname = NGRAM(minsize=2, maxsize=30, stored=True)
+    content = TEXT(stored=True)
+    content_ngram = NGRAMWORDS(minsize=2, maxsize=10, stored=True, field_boost=1.0, tokenizer=None, at='start', queryor=False, sortable=False)

api/index_fts/search_utils.py

@@ -1,35 +1,55 @@
-from whoosh.qparser import MultifieldParser
+from unidecode import unidecode
+from whoosh.qparser import MultifieldParser, FuzzyTermPlugin
+from whoosh.query import Or, Term
 from whoosh import index
 from api.index_fts.index_conf import st
 
-def search_whoosh(keyword, fields=["content"], limit=50):
-    """
-    Recherche plein texte dans un ou plusieurs champs (Whoosh).
+from whoosh.qparser import QueryParser, OrGroup, AndGroup
+from whoosh.query import Or, Term, And
+from unidecode import unidecode
 
-    :param keyword: le terme recherché
-    :param fields: liste de champs sur lesquels faire la recherche (par défaut: ["content"])
-    :param limit: nombre maximum de résultats
-    :return: liste de dicts avec id_dil et highlights
-    """
+
+
+
+def search_whoosh(query_firstnames_lastname: str = "",
+                  query_content: str = "",
+                  limit=10000000):
     ix = st.open_index()
+    hits = {}
+
+    # Normalisation
+    def remove_first_joker(query):
+        """Remove the first joker character if present."""
+        if query.strip().startswith("*"):
+            return query[1:]
+        return query
+
+    query_firstnames_lastname = remove_first_joker(unidecode(query_firstnames_lastname.lower().strip())) if query_firstnames_lastname else ""
+    query_content = remove_first_joker(query_content.strip()) if query_content else ""
+
+    if not query_firstnames_lastname and not query_content:
+        return {}
+
     with ix.searcher() as searcher:
-        parser = MultifieldParser(fields, ix.schema)
-        q = parser.parse(keyword)
-        results = searcher.search(q, limit=limit)
+        # Choix de la requête
+        if query_firstnames_lastname and query_content:
+            # Requête AND
+            parser1 = QueryParser('firstnames_lastname', schema=ix.schema)
+            parser2 = QueryParser('content', schema=ix.schema)
+            query = And([
+                parser1.parse(query_firstnames_lastname),
+                parser2.parse(query_content)
+            ])
+        elif query_firstnames_lastname:
+            parser = QueryParser('firstnames_lastname', schema=ix.schema, group=AndGroup)
+            query = parser.parse(query_firstnames_lastname)
+        elif query_content:
+            parser = MultifieldParser(['content'], schema=ix.schema, group=AndGroup)
+            query = parser.parse(query_content)
 
-        hits = []
+        results = searcher.search(query, limit=limit)
         for r in results:
-            # Try to highlight the first matching field (fallback on 'content')
-            highlight = None
-            for field in fields:
-                if field in r:
-                    highlight = r.highlights(field)
-                    if highlight:
-                        break
-
-            hits.append({
-                "id_dil": r["id_dil"],
-                "highlight": highlight or ""  # fallback to empty if nothing
-            })
+            highlight = r.highlights("content") if "content" in r else None
+            hits[r["id_dil"]] = {"highlight": highlight}
 
         return hits

api/models/models.py

@@ -9,6 +9,7 @@
 import uuid
 import time
 from functools import wraps
+from unidecode import unidecode
 
 from sqlalchemy import (Column,
                         Integer,
@@ -222,12 +223,17 @@ def update_person_fts_index_after_update(cls, mapper, connection, target, ix):
             clean_text = prepare_content(target)
             lastname = target.lastname or ""
             firstnames = target.firstnames or ""
-            writer.update_document(
+            lastname = unidecode(lastname or "").lower().encode('utf-8').decode('utf-8')
+            firstnames = unidecode(firstnames or "").lower().encode('utf-8').decode('utf-8')
+            writer.add_document(
                 id_dil=str(target._id_dil).encode('utf-8').decode('utf-8'),
-                lastname=lastname.encode('utf-8').decode('utf-8'),
-                firstnames=firstnames.encode('utf-8').decode('utf-8'),
-                content=clean_text.encode('utf-8').decode('utf-8')
+                lastname=lastname,
+                firstnames=firstnames,
+                content=clean_text.encode('utf-8').decode('utf-8'),
+                content_ngram=unidecode(clean_text).lower().encode('utf-8').decode('utf-8'),
+                firstnames_lastname=f"{' '.join(firstnames.split(','))} {lastname}"
             )
+
             writer.commit()
 
     @classmethod
@@ -239,11 +245,15 @@ def insert_person_fts_index_after_insert(cls, mapper, connection, target, ix):
             clean_text = prepare_content(target)
             lastname = target.lastname or ""
             firstnames = target.firstnames or ""
+            lastname = unidecode(lastname or "").lower().encode('utf-8').decode('utf-8')
+            firstnames = unidecode(firstnames or "").lower().encode('utf-8').decode('utf-8')
             writer.add_document(
                 id_dil=str(target._id_dil).encode('utf-8').decode('utf-8'),
-                lastname=lastname.encode('utf-8').decode('utf-8'),
-                firstnames=firstnames.encode('utf-8').decode('utf-8'),
-                content=clean_text.encode('utf-8').decode('utf-8')
+                lastname=lastname,
+                firstnames=firstnames,
+                content=clean_text.encode('utf-8').decode('utf-8'),
+                content_ngram=unidecode(clean_text).lower().encode('utf-8').decode('utf-8'),
+                firstnames_lastname=f"{' '.join(firstnames.split(','))} {lastname}"
             )
 
             writer.commit()