diff --git a/src/app/api/api_v1/endpoints/search.py b/src/app/api/api_v1/endpoints/search.py index 19c326a..dd5a085 100644 --- a/src/app/api/api_v1/endpoints/search.py +++ b/src/app/api/api_v1/endpoints/search.py @@ -6,7 +6,7 @@ from src.app.models.db_models import CorpusEmbedding from src.app.models.documents import Collection_schema, Document from src.app.models.search import EnhancedSearchQuery, SearchFilter, SearchQuery -from src.app.services.exceptions import EmptyQueryError, NoResultsError, bad_request +from src.app.services.exceptions import EmptyQueryError, bad_request from src.app.services.search import SearchService from src.app.services.search_helpers import ( search_all_base, @@ -107,12 +107,19 @@ async def search_all_slices_by_lang( response: Response, qp: EnhancedSearchQuery = Depends(get_params), ): - return await search_all_base( + res = await search_all_base( response=response, qp=qp, search_func=sp.search, ) + if not res: + logger.error("No results found") + response.status_code = 404 + return None + + return res + @router.post( "/multiple_by_slices", @@ -127,18 +134,20 @@ async def multi_search_all_slices_by_lang( if isinstance(qp.query, str): qp.query = [qp.query] - try: - results = await search_multi_inputs( - response=response, - nb_results=qp.nb_results, - sdg_filter=qp.sdg_filter, - collections=qp.corpora, - inputs=qp.query, - callback_function=sp.search, - ) - return results - except NoResultsError: + results = await search_multi_inputs( + response=response, + nb_results=qp.nb_results, + sdg_filter=qp.sdg_filter, + collections=qp.corpora, + inputs=qp.query, + callback_function=sp.search, + ) + if not results: + logger.error("No results found") response.status_code = 404 + return None + + return results @router.post( @@ -151,8 +160,15 @@ async def search_all( response: Response, qp: EnhancedSearchQuery = Depends(get_params), ): - return await search_all_base( + res = await search_all_base( response=response, qp=qp, search_func=sp.search_group_by_document, ) + + if not res: + logger.error("No results found") + response.status_code = 404 + return None + + return res diff --git a/src/app/services/search.py b/src/app/services/search.py index 0993542..16e5914 100644 --- a/src/app/services/search.py +++ b/src/app/services/search.py @@ -237,7 +237,6 @@ async def search( filters: Optional[List[int]] = None, nb_results: int = 100, ) -> List[http_models.ScoredPoint]: - logger.debug("method=search collection=%s", collection_info) try: resp = await self.client.search( query_filter=self.build_filters(filters), @@ -248,6 +247,7 @@ async def search( with_payload=self.payload_keys, score_threshold=0.5, ) + logger.debug("method=search nb_results=%s", collection_info, len(resp)) except qdrant_exceptions.ResponseHandlingException: return [] return resp diff --git a/src/app/services/search_helpers.py b/src/app/services/search_helpers.py index 898e7c3..bf65723 100644 --- a/src/app/services/search_helpers.py +++ b/src/app/services/search_helpers.py @@ -98,7 +98,7 @@ async def search_all_base( ) if not data: - raise NoResultsError() + return [] sorted_data = sorted(data, key=lambda x: x.score, reverse=True) sorted_data = sort_slices_using_mmr(sorted_data, theta=qp.relevance_factor) diff --git a/src/app/tests/api/api_v1/test_search.py b/src/app/tests/api/api_v1/test_search.py index 1f00b73..8b6c55f 100644 --- a/src/app/tests/api/api_v1/test_search.py +++ b/src/app/tests/api/api_v1/test_search.py @@ -10,7 +10,6 @@ CollectionNotFoundError, LanguageNotSupportedError, ModelNotFoundError, - NoResultsError, ) from src.app.services.search import sort_slices_using_mmr from src.main import app @@ -228,22 +227,46 @@ async def test_search_all_slices_no_collections(self, *mocks): ) @patch( - f"{search_pipeline_path}.get_collections_aliases_by_language", - return_value=("collection_fr_model"), - ) - @patch(f"{search_pipeline_path}.get_collection_dict_with_embed") - @patch( - "src.app.services.search_helpers.parallel_search", - return_value=mocked_scored_points, + "src.app.api.api_v1.endpoints.search.search_all_base", + return_value=[ + { + "score": 0.7513504, + "payload": { + "document_corpus": "conversation", + "document_desc": "More and more evidence has accumulated which shows that changes in global and regional climate over the last 50 years are almost entirely due to human influence.", + "document_details": { + "authors": [ + {"misc": "", "name": "Mark New"}, + {"misc": "", "name": "University of Cape Town"}, + ], + "duration": "316", + "readability": "56.49", + "source": "africa", + }, + "document_id": "af2ca7b2-1011-4b4d-828e-9678597fd255", + "document_lang": "en", + "document_sdg": [13], + "document_title": "Climate explained: how much of climate change is natural? How much is man-made?", + "document_url": "https://theconversation.com/climate-explained-how-much-of-climate-change-is-natural-how-much-is-man-made-123604", + "slice_content": "The Intergovernmental Panel on Climate Change defines climate change as: a change in the state of the climate that can be identified by changes in the mean and/or the variability of its properties and that persists for an extended period, typically decades or longer. The causes of climate change can be any combination of: Internal variability in the climate system, when various components of the climate system – like the atmosphere and ocean – vary on their own to cause fluctuations in climatic conditions, such as temperature or rainfall. These internally-driven changes generally happen over decades or longer; shorter variations such as those related to El Niño fall in the bracket of climate variability, not climate change. Natural external causes such as increases or decreases in volcanic activity or solar radiation. For example, every 11 years or so, the Sun’s magnetic field completely flips and this can cause small fluctuations in global temperature, up to about 0.2 degrees. On longer time scales – tens to hundreds of millions of years – geological processes can drive changes in the climate, due to shifting continents and mountain building. Human influence through greenhouse gases (gases that trap heat in the atmosphere such as carbon dioxide and methane), other particles released into the air (which absorb or reflect sunlight such as soot and aerosols) and land-use change (which affects how much sunlight is absorbed on land surfaces and also how much carbon dioxide and methane is absorbed and released by vegetation and soils). What changes have been detected?\n\nThe Intergovernmental Panel on Climate Change’s recent report showed that, on average, the global surface air temperature has risen by 1°C since the beginning of significant industrialisation (which roughly started in the 1850s). And it is increasing at ever faster rates, currently 0.2°C per decade, because the concentrations of greenhouse gases in the atmosphere have themselves been increasing ever faster. The oceans are warming as well. In fact, about 90% of the extra heat trapped in the atmosphere by greenhouse gases is being absorbed by the oceans. A warmer atmosphere and oceans are causing dramatic changes, including steep decreases in Arctic summer sea ice which is profoundly impacting arctic marine ecosystems, increasing sea level rise which is inundating low lying coastal areas such as Pacific island atolls, and an increasing frequency of many climate extremes such as drought and heavy rain, as well as disasters where climate is an important driver, such as wildfire, flooding and landslides. Multiple lines of evidence, using different methods, show that human influence is the only plausible explanation for the patterns and magnitude of changes that have been detected. This human influence is largely due to our activities that release greenhouse gases, such as carbon dioxide and methane, as well sunlight absorbing soot. The main sources of these warming gases and particles are fossil fuel burning, cement production, land cover change (especially deforestation) and agriculture. Weather attribution Most of us will struggle to pick up slow changes in the climate.", + "slice_sdg": 13, + }, + } + ], ) async def test_search_all_slices_ok(self, *mocks): response = client.post( - f"{settings.API_V1_STR}/search/by_slices?nb_results=10", + f"{settings.API_V1_STR}/search/by_slices", json={ - "query": "une phrase plus longue pour tester la recherche en français. et voir ce que cela donne" + "query": "Comment est-ce que les gouvernements font pour suivre ces conseils et les mettre en place ?", + "relevance_factor": 0.75, + "sdg_filter": [], + "corpora": [], }, headers={"X-API-Key": "test"}, ) + print(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>", response.json()) + self.assertEqual(response.status_code, 200) async def test_search_all_slices_no_query(self, *mocks): @@ -268,15 +291,14 @@ async def test_search_all_slices_no_query(self, *mocks): return_value=[], ) async def test_search_all_slices_no_result(self, *mocks): - with self.assertRaises(NoResultsError): - response = client.post( - f"{settings.API_V1_STR}/search/by_slices?nb_results=10", - json={ - "query": "une phrase plus longue pour tester la recherche en français. et voir ce que cela donne" - }, - headers={"X-API-Key": "test"}, - ) - self.assertEqual(response.status_code, 204) + response = client.post( + f"{settings.API_V1_STR}/search/by_slices?nb_results=10", + json={ + "query": "une phrase plus longue pour tester la recherche en français. et voir ce que cela donne" + }, + headers={"X-API-Key": "test"}, + ) + self.assertEqual(response.status_code, 404) @patch("src.app.services.sql_db.session_maker") @@ -343,15 +365,14 @@ async def test_search_all_no_collections(self, *mocks): return_value=[], ) async def test_search_all_no_result(self, *mocks): - with self.assertRaises(NoResultsError): - response = client.post( - f"{settings.API_V1_STR}/search/by_document?nb_results=10", - json={ - "query": "une phrase plus longue pour tester la recherche en français. et voir ce que cela donne" - }, - headers={"X-API-Key": "test"}, - ) - self.assertEqual(response.status_code, 204) + response = client.post( + f"{settings.API_V1_STR}/search/by_document?nb_results=10", + json={ + "query": "une phrase plus longue pour tester la recherche en français. et voir ce que cela donne" + }, + headers={"X-API-Key": "test"}, + ) + self.assertEqual(response.status_code, 404) async def test_search_all_no_query(self, *mocks): response = client.post(