style: format code with Autopep8, isort and Ruff Formatter

src/everyai/classifier/huggingface_classifier.py

@@ -4,14 +4,13 @@
 import evaluate
 import torch
 from transformers import (
+    BertForSequenceClassification,
+    BertTokenizer,
     DataCollatorWithPadding,
     Trainer,
     TrainingArguments,
-    BertForSequenceClassification,
-    BertTokenizer,
 )
 
-
 from everyai.classifier.classify import TextClassifer, label_encode, split_data
 from everyai.utils.everyai_path import MODEL_PATH
 
@@ -34,9 +33,7 @@ def __init__(
         )
         if self.model_name != "bert-base-uncased":
             raise ValueError(f"Model name {self.model_name} not supported yet")
-        self.model = BertForSequenceClassification.from_pretrained(
-            self.model_name
-        )
+        self.model = BertForSequenceClassification.from_pretrained(self.model_name)
         self.tokenizer = BertTokenizer.from_pretrained(
             self.tokenizer_name, **self.tokenizer_config
         )
@@ -51,9 +48,7 @@ def __init__(
     def _tokenize(self, texts: list[str], labels: list[str]):
         self.label_encoder, tokenzied_labels = label_encode(labels)
         tokenzied_labels = torch.tensor(tokenzied_labels)
-        dataset = datasets.Dataset.from_dict(
-            {"text": texts, "label": tokenzied_labels}
-        )
+        dataset = datasets.Dataset.from_dict({"text": texts, "label": tokenzied_labels})
 
         def _tokenizer_fn(example):
             return self.tokenizer(example["text"], **self.tokenizer_config)
@@ -77,12 +72,8 @@ def train(self):
             self.data.valid_indices,
             self.data.test_indices,
         ) = split_data(self.texts, self.labels)
-        self.train_dataset = self._tokenize(
-            self.data.x_train, self.data.y_train
-        )
-        self.valid_dataset = self._tokenize(
-            self.data.x_valid, self.data.y_valid
-        )
+        self.train_dataset = self._tokenize(self.data.x_train, self.data.y_train)
+        self.valid_dataset = self._tokenize(self.data.x_valid, self.data.y_valid)
         self.test_dataset = self._tokenize(self.data.x_test, self.data.y_test)
         train_args = TrainingArguments(**self.train_args)
         data_collator = DataCollatorWithPadding(tokenizer=self.tokenizer)
@@ -98,9 +89,7 @@ def train(self):
     def test(self):
         trainer = Trainer(model=self.model)
         predictions = trainer.predict(self.test_dataset)
-        self.data.y_pred = torch.argmax(
-            torch.tensor(predictions.predictions), axis=1
-        )
+        self.data.y_pred = torch.argmax(torch.tensor(predictions.predictions), axis=1)
 
         self.data.y_test = self.label_encoder.transform(self.data.y_test)
 

src/everyai/classifier/multi_feature_model/fusionBert.py

@@ -33,9 +33,7 @@ def __init__(self, semantic_tokenizer: PreTrainedTokenizer, **kwargs):
         self.all_tags = self.nlp.get_pipe("tagger").labels
         self.sentiment_analyzer = SentimentIntensityAnalyzer()
 
-    def analyze_word_level_sentiment(
-        self, text: str, max_length=512
-    ) -> torch.tensor:
+    def analyze_word_level_sentiment(self, text: str, max_length=512) -> torch.tensor:
         # 使用SpaCy进行词汇级分析
         doc = self.nlp(text)
 
@@ -45,9 +43,9 @@ def analyze_word_level_sentiment(
             if token.is_stop or token.is_punct:
                 sentiment = 0.0
             else:
-                sentiment = self.sentiment_analyzer.polarity_scores(
-                    token.text
-                )["compound"]
+                sentiment = self.sentiment_analyzer.polarity_scores(token.text)[
+                    "compound"
+                ]
             word_sentiment.append(sentiment)
         sentiment = torch.tensor(word_sentiment, dtype=torch.float)
         sentiment = truncate_and_pad_single_sequence(sentiment, max_length)
@@ -90,12 +88,8 @@ def batch_encode_plus(self, batch_text: list[str], **kwargs):
         批量编码函数
         :param batch_text: 输入文本列表
         """
-        batch_encoding = self.semantic_tokenizer.batch_encode_plus(
-            batch_text, **kwargs
-        )
-        batch_pos = torch.cat(
-            [self.pos_feature(text) for text in batch_text], dim=0
-        )
+        batch_encoding = self.semantic_tokenizer.batch_encode_plus(batch_text, **kwargs)
+        batch_pos = torch.cat([self.pos_feature(text) for text in batch_text], dim=0)
         batch_sentiments = torch.cat(
             [
                 self.analyze_word_level_sentiment(
@@ -146,9 +140,7 @@ def forward(self, features: list[torch.tensor]):
         outputs = []
         for i in range(self.feature_num):
             if len(self.projections) <= i:
-                self.projections.append(
-                    nn.Linear(features[i].size(-1), self.proj_dim)
-                )
+                self.projections.append(nn.Linear(features[i].size(-1), self.proj_dim))
                 projected_features.append(self.projections[i](features[i]))
 
         for i in range(self.feature_num):
@@ -163,9 +155,7 @@ def forward(self, features: list[torch.tensor]):
 
 
 class FeatureFusionBertClassfier(nn.Module):
-    def __init__(
-        self, feature_num=3, proj_dim=64, bert_input_dim=768, num_labels=2
-    ):
+    def __init__(self, feature_num=3, proj_dim=64, bert_input_dim=768, num_labels=2):
         super(FeatureFusionBertClassfier, self).__init__()
         bert_classifier = BertForSequenceClassification.from_pretrained(
             "bert-base-uncased", num_labels=num_labels
@@ -238,7 +228,7 @@ def forward(self, features: list[torch.tensor], input_ids: torch.tensor):
         num_warmup_steps=0,
         num_training_steps=num_training_steps,
     )
-    # TODO Fix error: KeyError: 'Invalid key. Only three types of key are available: 
+    # TODO Fix error: KeyError: 'Invalid key. Only three types of key are available:
     # (1) string, (2) integers for backend Encoding, and (3) slices for data subsetting.'
     model.train()
     for epoch in range(num_epochs):

src/everyai/data_loader/data_load.py

@@ -1,5 +1,5 @@
-from collections.abc import Callable
 import logging
+from collections.abc import Callable
 from pathlib import Path
 
 import pandas as pd
@@ -35,26 +35,19 @@ def __init__(
     def load_data(
         self, max_count: int = None, return_type: str = "list"
     ) -> list[dict] | pd.DataFrame:
-        if (
-            Path(self.file_name_or_path).exists()
-            or self.file_type == "huggingface"
-        ):
+        if Path(self.file_name_or_path).exists() or self.file_type == "huggingface":
             logging.info("Loading data from %s", self.file_name_or_path)
             match self.file_type:
                 case "csv":
                     loaded_data = pd.read_csv(self.file_name_or_path)
                 case "xlsx":
                     loaded_data = pd.read_excel(self.file_name_or_path)
                 case "jsonl":
-                    loaded_data = pd.read_json(
-                        self.file_name_or_path, lines=True
-                    )
+                    loaded_data = pd.read_json(self.file_name_or_path, lines=True)
                 case "json":
                     loaded_data = pd.read_json(self.file_name_or_path)
                 case "huggingface":
-                    loaded_data = load_dataset(
-                        self.file_name_or_path
-                    )
+                    loaded_data = load_dataset(self.file_name_or_path)
                     if "train" in loaded_data.keys():
                         loaded_data = loaded_data["train"].to_pandas()
                     elif "validation" in loaded_data.keys():
@@ -78,9 +71,7 @@ def load_data(
         if max_count is not None and loaded_data is not None:
             loaded_data = loaded_data.head(max_count)
         else:
-            logging.info(
-                "Max count is None and all the records will be loaded"
-            )
+            logging.info("Max count is None and all the records will be loaded")
         loaded_data.rename(
             columns={
                 self.question_column: "question",
@@ -91,9 +82,7 @@ def load_data(
         if return_type == "pandas":
             result = loaded_data
         elif return_type == "list":
-            result = loaded_data[["question", "answer"]].to_dict(
-                orient="records"
-            )
+            result = loaded_data[["question", "answer"]].to_dict(orient="records")
         else:
             logging.error("Invalid return type")
         return result

src/everyai/data_loader/everyai_dataset.py

@@ -1,6 +1,6 @@
 import logging
-from pathlib import Path
 import re
+from pathlib import Path
 
 import pandas as pd
 import pymongo
@@ -32,9 +32,7 @@ def __init__(
         if datas is not None:
             self.datas: pd.DataFrame = datas
         else:
-            self.datas: pd.DataFrame = pd.DataFrame(
-                columns=["question", "human"]
-            )
+            self.datas: pd.DataFrame = pd.DataFrame(columns=["question", "human"])
         if ai_list is not None:
             for ai_name in ai_list:
                 self.datas[ai_name] = None
@@ -87,33 +85,25 @@ def insert_ai_response(self, question, ai_name: str, ai_response: str):
         else:
             logging.info("AI %s exists in the dataset", ai_name)
         question_exists = not self.datas[
-            (self.datas["question"] == question)
-            & (self.datas[ai_name] == ai_name)
+            (self.datas["question"] == question) & (self.datas[ai_name] == ai_name)
         ].empty
         if question_exists:
             self._update_new_row(question, ai_name, ai_response)
         else:
-            self.datas.loc[self.datas["question"] == question, ai_name] = (
-                ai_response
-            )
+            self.datas.loc[self.datas["question"] == question, ai_name] = ai_response
 
     def insert_human_response(self, question, human_response: str):
         if self.datas[self.datas["question"] == question].empty:
             self._update_new_row(question, "human", human_response)
         else:
-            self.datas.loc[self.datas["question"] == question, "human"] = (
-                human_response
-            )
+            self.datas.loc[self.datas["question"] == question, "human"] = human_response
 
-    def record_exist(
-        self, question: str, label: str, file_type: str = "mongodb"
-    ):
+    def record_exist(self, question: str, label: str, file_type: str = "mongodb"):
         if file_type != "mongodb":
             raise ValueError("Unsupported file type: %s", file_type)
         query = {"question": question, label: {"$exists": True}}
         record = self.mongo_collection.find_one(query)
         return False if record is None else isinstance(record[label], str)
-
 
     def upsert2mongo(self, question: str, label: str, answer: str):
         query = {"question": question}
@@ -128,9 +118,7 @@ def _update_new_row(self, question, arg1, arg2):
     def output_question(self):  # -> Iterator:
         return iter(self.datas["question"])
 
-    def _save2mongodb(
-        self, database: pymongo.database.Database, insert_mode="insert"
-    ):
+    def _save2mongodb(self, database: pymongo.database.Database, insert_mode="insert"):
         self.datas = self.datas.dropna()
         logging.info("Saving dataset to mongodb: %s", database)
         collection = database[self.data_name]
@@ -160,9 +148,7 @@ def _read_from_mongodb(self, database: pymongo.database.Database):
         data = data.drop(columns=["timestamp"])
         return data
 
-    def read(
-        self, path_or_database: str | Path = None, file_format: str = "csv"
-    ):
+    def read(self, path_or_database: str | Path = None, file_format: str = "csv"):
         if file_format == "mongodb":
             if path_or_database is None:
                 path_or_database = _initialize_mongo_connection()
@@ -171,17 +157,13 @@ def read(
             loaded_data = self._read_from_mongodb(path_or_database)
         else:
             if path_or_database is None:
-                path_or_database = (
-                    DATA_PATH / f"{self.data_name}.{file_format}"
-                )
+                path_or_database = DATA_PATH / f"{self.data_name}.{file_format}"
             logging.info("Load dataset from %s", path_or_database)
             if isinstance(path_or_database, str):
                 path_or_database = Path(path_or_database)
             if path_or_database.suffix != f".{file_format}":
                 logging.warning("Change file format to %s", file_format)
-                path_or_database = path_or_database.with_suffix(
-                    f".{file_format}"
-                )
+                path_or_database = path_or_database.with_suffix(f".{file_format}")
             match path_or_database.suffix:
                 case ".csv":
                     loaded_data = pd.read_csv(path_or_database)
@@ -204,27 +186,21 @@ def read(
         self.datas.dropna(inplace=True)
         self.ai_list = list(set(self.datas.columns) - {"question", "human"})
 
-    def save(
-        self, path_or_database: str | Path = None, file_format: str = "csv"
-    ):
+    def save(self, path_or_database: str | Path = None, file_format: str = "csv"):
         self.datas = self.datas.dropna()
         if file_format == "mongodb":
             if path_or_database is None:
                 path_or_database = self._initialize_mongo_connection()
             self._save2mongodb(path_or_database)
         else:
             if path_or_database is None:
-                path_or_database = (
-                    DATA_PATH / f"{self.data_name}.{file_format}"
-                )
+                path_or_database = DATA_PATH / f"{self.data_name}.{file_format}"
             logging.info("Save dataset to %s", path_or_database)
             if isinstance(path_or_database, str):
                 path_or_database = Path(path_or_database)
             if path_or_database.suffix != f".{file_format}":
                 logging.warning("Change file format to %s", file_format)
-                path_or_database = path_or_database.with_suffix(
-                    f".{file_format}"
-                )
+                path_or_database = path_or_database.with_suffix(f".{file_format}")
             match path_or_database.suffix:
                 case ".csv":
                     self.datas.to_csv(path_or_database, index=False)

src/everyai/explanation/explain.py

@@ -36,7 +36,9 @@ def explain(self, output_path: Path = None):
                 exp.save_to_file(output_path / f"text{i}.html")
             logging.info("Lime explanation saved to %s", output_path)
         else:
-            logging.error("Lime explanation not supported for %s", self.classifier.classifier_type)
+            logging.error(
+                "Lime explanation not supported for %s", self.classifier.classifier_type
+            )
 
 
 class ShapExplanation(Explanation):
@@ -57,4 +59,6 @@ def explain(self, output_path: Path = None):
                 shap.plots.text(shap_values, output_path / f"text{i}")
             logging.info("Shap explanation saved to %s", output_path)
         else:
-            logging.error("Shap explanation not supported for %s", self.classifier.classifier_type)
+            logging.error(
+                "Shap explanation not supported for %s", self.classifier.classifier_type
+            )

src/everyai/generator/generate.py

@@ -2,8 +2,8 @@
 from collections.abc import Callable
 from pathlib import Path
 
-from everyai.generator.openai_generate import openai_generate
 from everyai.generator.huggingface_generate import glm4, llama, qwen2_5
+from everyai.generator.openai_generate import openai_generate
 from everyai.utils.everyai_path import GENERATE_CONFIG_PATH
 from everyai.utils.load_args import set_attrs_2class
 from everyai.utils.load_config import get_config
@@ -37,9 +37,7 @@ def __init__(self, config: dict, formatter: Callable[[str], str] = None):
             ]
             default_params = ["model_name", "base_url", "api_key"]
         else:
-            raise ValueError(
-                "Unsupported generator type: %s", self.generator_type
-            )
+            raise ValueError("Unsupported generator type: %s", self.generator_type)
         self.formatter = formatter
         set_attrs_2class(self, config, allowed_keys, default_params)
 

src/everyai/generator/huggingface_generate.py

@@ -2,8 +2,8 @@
 from pathlib import Path
 
 import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer
 import transformers
+from transformers import AutoModelForCausalLM, AutoTokenizer
 
 
 def glm4(

src/everyai/generator/openai_generate.py

@@ -1,6 +1,7 @@
 import logging
 
 from openai import OpenAI
+
 from everyai.utils.proxy import TempProxy
 
 
@@ -17,9 +18,7 @@ def openai_generate(
         logging.info("Proxy started: %s", proxy)
     client = OpenAI(api_key=api_key, base_url=base_url)
     messages = [{"role": "user", "content": user_input}]
-    result = client.chat.completions.create(
-        messages=messages, model=model_name
-    )
+    result = client.chat.completions.create(messages=messages, model=model_name)
     if proxy is not None:
         temp_proxy.reset_proxy()
         logging.info("Proxy reset")