add html stripper

Signed-off-by: Lejin Varghese <lejinsnests@gmail.com>

Author: lejinvarghese

deep_learning/moe/README.md

@@ -7,8 +7,11 @@
 
  ## Datasets
 - [x] Home Depot
-- [ ] Amazon
+- [x] Amazon
 - [ ] Google
 - [ ] Wayfair
 - [ ] Crowdflower
-- [ ] Walmart
+- [ ] Walmart
+
+## Improvements
+- [ ] Add more Amazon metadata

deep_learning/moe/adapters/aggregator.py

@@ -13,7 +13,7 @@ class DatasetAggregator:
     def __init__(
         self,
         sample_size: Optional[int] = None,
-        splits: list[str] = ["train", "valid", "test"],
+        splits: list[str] = ["train", "test"],
     ):
         self.sources = [AmazonDataset, HomeDepotDataset]
         self.sample_size = sample_size

deep_learning/moe/adapters/core.py

@@ -1,6 +1,7 @@
 from abc import ABC
 from multiprocessing import cpu_count
 import json
+import re
 from click import secho
 from datasets import load_dataset, Dataset
 
@@ -38,6 +39,13 @@ def generate_query(self):
     def generate_document(self):
         pass
 
+    @staticmethod
+    def strip_html(text):
+        if not isinstance(text, str):
+            return ""
+        clean = re.compile("<.*?>")
+        return re.sub(clean, "", text)
+
     @staticmethod
     def format_document(**kwargs):
         if kwargs.get("title"):
@@ -53,7 +61,7 @@ def format_document(**kwargs):
 
         if kwargs.get("description"):
             template += f"""**product description**: {kwargs.get('description')}"""
-        return template.strip().lower()
+        return BaseDataset.strip_html(template.strip().lower())
 
     def load(self, split: str, cols: list[str] = None):
         secho(