added crowdflower

Signed-off-by: Lejin Varghese <lejinsnests@gmail.com>

Author: lejinvarghese

representation_learning/product_search/moe/adapters/__init__.py

@@ -1,6 +1,7 @@
 from adapters.core import BaseDataset
-from adapters.home_depot import HomeDepotDataset
 from adapters.amazon import AmazonDataset
+from adapters.crowdflower import CrowdFlowerDataset
 from adapters.google import GoogleDataset
+from adapters.home_depot import HomeDepotDataset
 from adapters.wayfair import WayfairDataset
 from adapters.aggregator import DatasetAggregator

representation_learning/product_search/moe/adapters/aggregator.py

@@ -1,8 +1,7 @@
 from typing import Optional
-from datasets import concatenate_datasets, DatasetDict, Dataset
 from click import secho
-
-from adapters import BaseDataset, AmazonDataset, HomeDepotDataset, GoogleDataset, WayfairDataset
+from datasets import Dataset, DatasetDict, concatenate_datasets
+from adapters import AmazonDataset, BaseDataset, CrowdFlowerDataset, GoogleDataset, HomeDepotDataset, WayfairDataset
 
 DATASET_NAME = "lv12/ProductSearchDataset"
 
@@ -13,8 +12,8 @@ def __init__(
         sample_size: Optional[int] = None,
         splits: list[str] = ["train", "test"],
     ):
-        self.sources = [HomeDepotDataset, AmazonDataset, WayfairDataset, GoogleDataset]
-        self.sources = [GoogleDataset]
+        self.sources = [HomeDepotDataset, AmazonDataset, WayfairDataset, GoogleDataset, CrowdFlowerDataset]
+        # self.sources = [GoogleDataset]
         self.sample_size = sample_size
         self.splits = splits
         self.datasets = self.generate_datasets()
@@ -74,7 +73,7 @@ def push_to_hub(
         self,
         repo_id: str = DATASET_NAME,
         private: bool = False,
-    ):
+    ) -> None:
         """Push the dataset to HuggingFace Hub."""
         secho(f"Pushing the dataset to {repo_id}", fg=(229, 192, 123))
 

representation_learning/product_search/moe/adapters/core.py

@@ -1,9 +1,10 @@
-from abc import ABC
-from multiprocessing import cpu_count
 import json
 import re
+from abc import ABC
+from multiprocessing import cpu_count
+
 from click import secho
-from datasets import load_dataset, Dataset
+from datasets import Dataset, load_dataset
 
 RANDOM_STATE = 42
 

representation_learning/product_search/moe/adapters/crowdflower.py

@@ -0,0 +1,44 @@
+from adapters.core import BaseDataset
+
+
+FEATURE_COLUMNS = [
+    "query",
+    "product_title",
+    "product_description",
+    "median_relevance",
+]
+
+
+class CrowdFlowerDataset(BaseDataset):
+    def __init__(
+        self,
+        repo_id="napsternxg/kaggle_crowdflower_ecommerce_search_relevance",
+        sample_size=None,
+        split="train",
+        cols=FEATURE_COLUMNS,
+    ):
+        super().__init__(repo_id, sample_size, split, cols)
+        self.name = "crowdflower"
+        self.generate_query()
+        self.generate_document()
+        self._map_relevance()
+
+    def _map_relevance(self):
+        self._data = self._data.map(
+            lambda x: {"relevance": x.get("median_relevance", 1.0) - 1.0},
+            num_proc=self._num_procs,
+            remove_columns=["median_relevance"],
+        )
+
+    def generate_document(self):
+        self._data = self._data.map(
+            lambda row: {
+                "document": self.format_document(
+                    title=row.get("product_title"),
+                    description=row.get("product_description"),
+                )
+            },
+            remove_columns=["product_title", "product_description"],
+            num_proc=self._num_procs,
+        )
+        self._n_documents = len(set(self._data.unique("document")))

representation_learning/product_search/moe/adapters/google.py

@@ -1,7 +1,8 @@
 from click import secho
-from datasets import load_dataset, Dataset
-from adapters.core import BaseDataset, RANDOM_STATE
-from adapters.negative_miner import HardNegativeMiner
+from datasets import Dataset, load_dataset
+
+from adapters.core import RANDOM_STATE, BaseDataset
+from adapters.miners import HardNegativeMiner
 
 FEATURE_COLUMNS = [
     "query",

…ct_search/moe/adapters/negative_miner.py …ng/product_search/moe/adapters/miners.pyrepresentation_learning/product_search/moe/adapters/negative_miner.py renamed to representation_learning/product_search/moe/adapters/miners.py representation_learning/product_search/moe/adapters/negative_miner.py renamed to representation_learning/product_search/moe/adapters/miners.py

@@ -1,9 +1,9 @@
-from sentence_transformers.util import mine_hard_negatives
-from sentence_transformers import SentenceTransformer, CrossEncoder
-import torch
 from multiprocessing import cpu_count
+import torch
+from sentence_transformers import CrossEncoder, SentenceTransformer
+from sentence_transformers.util import mine_hard_negatives
 
-
+DATASET_NAME = "lv12/ProductSearchDataset"
 DEVICE = "mps" if torch.backends.mps.is_available() else "cpu"
 
 
@@ -14,12 +14,14 @@ def __init__(
         bi_encoder_name="thenlper/gte-base",
         cross_encoder_name="Alibaba-NLP/gte-reranker-modernbert-base",
         max_score=0.8,
+        min_score=0.6,
     ):
 
         self.dataset = dataset
         self.bi_encoder = SentenceTransformer(bi_encoder_name, device=DEVICE)
         self.cross_encoder = CrossEncoder(cross_encoder_name, device=DEVICE, model_kwargs={"torch_dtype": "auto"})
         self.max_score = max_score
+        self.min_score = min_score
         self.num_procs = cpu_count() - 1
 
     def run(self):
@@ -30,17 +32,17 @@ def run(self):
             anchor_column_name="anchor",
             positive_column_name="document",
             range_min=5,
-            range_max=30,
+            range_max=20,
             max_score=self.max_score,
-            min_score=0.5,
+            min_score=self.min_score,
             margin=0,
             num_negatives=10,
             sampling_strategy="random",
             batch_size=32,
             use_faiss=False,
         )
         dataset = dataset.map(
-            {"relevance": 0.9},
+            lambda x: {"relevance": 0.6},
             num_proc=self.num_procs,
             remove_columns=["document"],
         )

representation_learning/product_search/moe/processor.py

@@ -1,5 +1,5 @@
 import click
-from adapters import DatasetAggregator, HomeDepotDataset
+from adapters import DatasetAggregator
 
 
 @click.command()