style: format code with Autopep8, isort and Ruff Formatter

src/everyai/classifier/classify.py

@@ -41,24 +41,20 @@ def split_data(
     original_indices = pd.DataFrame(x).index
 
     # 第一次划分: 训练集和测试集
-    x_train, x_test, y_train, y_test, train_indices, test_indices = (
-        train_test_split(
-            x,
-            y,
-            original_indices,
-            test_size=test_size,
-            random_state=42,
-        )
+    x_train, x_test, y_train, y_test, train_indices, test_indices = train_test_split(
+        x,
+        y,
+        original_indices,
+        test_size=test_size,
+        random_state=42,
     )
     # 第二次划分: 训练集和验证集
-    x_train, x_valid, y_train, y_valid, train_indices, valid_indices = (
-        train_test_split(
-            x_train,
-            y_train,
-            train_indices,
-            test_size=valid_size / (train_size + valid_size),
-            random_state=42,
-        )
+    x_train, x_valid, y_train, y_valid, train_indices, valid_indices = train_test_split(
+        x_train,
+        y_train,
+        train_indices,
+        test_size=valid_size / (train_size + valid_size),
+        random_state=42,
     )
     return (
         x_train,
@@ -104,8 +100,7 @@ def __init__(
             f"{self.model_name}_{self.tokenizer_name}_{self.data_name}"
         )
         self.model_path = (
-            MODEL_PATH
-            / f"{self.model_name}_{self.tokenizer_name}_{self.data_name}.pkl"
+            MODEL_PATH / f"{self.model_name}_{self.tokenizer_name}_{self.data_name}.pkl"
         )
         if self.split_size is not None:
             self.train_size = self.split_size.get("train_size", 0.8)
@@ -118,9 +113,7 @@ def load_data(self, texts, labels, data_name):
             raise ValueError("Length of texts and labels should be same")
         self.texts = texts
         self.labels = labels
-        logging.info(
-            "Loading data: %s to classifier %s", data_name, self.model_name
-        )
+        logging.info("Loading data: %s to classifier %s", data_name, self.model_name)
         self.data_name = data_name
         self.classifier_name = (
             f"{self.model_name}_{self.tokenizer_name}_{self.data_name}"
@@ -135,4 +128,4 @@ def process_data(self):
                 ),
                 self.texts,
             )
-        )
+        )

src/everyai/classifier/fusion_classifer.py

@@ -11,10 +11,8 @@
 )
 
 from everyai.classifier.classify import TextClassifer, label_encode, split_data
+from everyai.classifier.multi_feature_model.fusionBert import FeatureFusionBertClassfier
 from everyai.utils.everyai_path import MODEL_PATH
-from everyai.classifier.multi_feature_model.fusionBert import (
-    FeatureFusionBertClassfier,
-)
 
 
 class HuggingfaceClassifer(TextClassifer):
@@ -32,7 +30,7 @@ def __init__(
         fusion_model_dict = {
             "FeatureFusionBertClassfier": FeatureFusionBertClassfier(),
         }
-        
+
         self.model = fusion_model_dict[self.model_name]
         if "bert" in self.model_name.lower():
             self.tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
@@ -47,9 +45,7 @@ def __init__(
     def _tokenize(self, texts: list[str], labels: list[str]):
         self.label_encoder, tokenzied_labels = label_encode(labels)
         tokenzied_labels = torch.tensor(tokenzied_labels)
-        dataset = datasets.Dataset.from_dict(
-            {"text": texts, "label": tokenzied_labels}
-        )
+        dataset = datasets.Dataset.from_dict({"text": texts, "label": tokenzied_labels})
 
         def _tokenizer_fn(example):
             return self.tokenizer(example["text"], **self.tokenizer_config)
@@ -73,12 +69,8 @@ def train(self):
             self.data.valid_indices,
             self.data.test_indices,
         ) = split_data(self.texts, self.labels)
-        self.train_dataset = self._tokenize(
-            self.data.x_train, self.data.y_train
-        )
-        self.valid_dataset = self._tokenize(
-            self.data.x_valid, self.data.y_valid
-        )
+        self.train_dataset = self._tokenize(self.data.x_train, self.data.y_train)
+        self.valid_dataset = self._tokenize(self.data.x_valid, self.data.y_valid)
         self.test_dataset = self._tokenize(self.data.x_test, self.data.y_test)
         train_args = TrainingArguments(**self.train_args)
         data_collator = DataCollatorWithPadding(tokenizer=self.tokenizer)
@@ -94,15 +86,11 @@ def train(self):
     def test(self):
         trainer = Trainer(model=self.model)
         predictions = trainer.predict(self.test_dataset)
-        self.data.y_pred = torch.argmax(
-            torch.tensor(predictions.predictions), axis=1
-        )
+        self.data.y_pred = torch.argmax(torch.tensor(predictions.predictions), axis=1)
 
         self.data.y_test = self.label_encoder.transform(self.data.y_test)
 
     def show_score(self):
         metric = evaluate.load("accuracy")
-        metric.compute(
-            predictions=self.data.y_pred, references=self.data.y_test
-        )
+        metric.compute(predictions=self.data.y_pred, references=self.data.y_test)
         logging.info("Accuracy: %s", metric)

src/everyai/classifier/huggingface_classifier.py

@@ -32,9 +32,7 @@ def __init__(
             **classfiy_config,
         )
         self.tokenizer = AutoTokenizer.from_pretrained(self.tokenizer_name)
-        self.model = AutoModelForSequenceClassification.from_pretrained(
-            self.model_name
-        )
+        self.model = AutoModelForSequenceClassification.from_pretrained(self.model_name)
         self.label_encoder = None
         self.train_dataset, self.valid_dataset, self.test_dataset = (
             None,
@@ -46,9 +44,7 @@ def __init__(
     def _tokenize(self, texts: list[str], labels: list[str]):
         self.label_encoder, tokenzied_labels = label_encode(labels)
         tokenzied_labels = torch.tensor(tokenzied_labels)
-        dataset = datasets.Dataset.from_dict(
-            {"text": texts, "label": tokenzied_labels}
-        )
+        dataset = datasets.Dataset.from_dict({"text": texts, "label": tokenzied_labels})
 
         def _tokenizer_fn(example):
             return self.tokenizer(example["text"], **self.tokenizer_config)
@@ -72,12 +68,8 @@ def train(self):
             self.data.valid_indices,
             self.data.test_indices,
         ) = split_data(self.texts, self.labels)
-        self.train_dataset = self._tokenize(
-            self.data.x_train, self.data.y_train
-        )
-        self.valid_dataset = self._tokenize(
-            self.data.x_valid, self.data.y_valid
-        )
+        self.train_dataset = self._tokenize(self.data.x_train, self.data.y_train)
+        self.valid_dataset = self._tokenize(self.data.x_valid, self.data.y_valid)
         self.test_dataset = self._tokenize(self.data.x_test, self.data.y_test)
         train_args = TrainingArguments(**self.train_args)
         data_collator = DataCollatorWithPadding(tokenizer=self.tokenizer)
@@ -93,15 +85,11 @@ def train(self):
     def test(self):
         trainer = Trainer(model=self.model)
         predictions = trainer.predict(self.test_dataset)
-        self.data.y_pred = torch.argmax(
-            torch.tensor(predictions.predictions), axis=1
-        )
+        self.data.y_pred = torch.argmax(torch.tensor(predictions.predictions), axis=1)
 
         self.data.y_test = self.label_encoder.transform(self.data.y_test)
 
     def show_score(self):
         metric = evaluate.load("accuracy")
-        metric.compute(
-            predictions=self.data.y_pred, references=self.data.y_test
-        )
+        metric.compute(predictions=self.data.y_pred, references=self.data.y_test)
         logging.info("Accuracy: %s", metric)

src/everyai/classifier/multi_feature_model/fusionBert.py

@@ -4,9 +4,12 @@
 import torch
 import torch.nn as nn
 import torch.nn.functional as F
-from datasets import load_dataset,Dataset
-from transformers import (BertForSequenceClassification, BertTokenizer,
-                          PreTrainedTokenizer)
+from datasets import Dataset, load_dataset
+from transformers import (
+    BertForSequenceClassification,
+    BertTokenizer,
+    PreTrainedTokenizer,
+)
 from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
 
 
@@ -26,9 +29,7 @@ def __init__(self, semantic_tokenizer: PreTrainedTokenizer, **kwargs):
         self.all_tags = self.nlp.get_pipe("tagger").labels
         self.sentiment_analyzer = SentimentIntensityAnalyzer()
 
-    def analyze_word_level_sentiment(
-        self, text: str, max_length=512
-    ) -> torch.tensor:
+    def analyze_word_level_sentiment(self, text: str, max_length=512) -> torch.tensor:
         # 使用SpaCy进行词汇级分析
         doc = self.nlp(text)
 
@@ -38,9 +39,9 @@ def analyze_word_level_sentiment(
             if token.is_stop or token.is_punct:
                 sentiment = 0.0
             else:
-                sentiment = self.sentiment_analyzer.polarity_scores(
-                    token.text
-                )["compound"]
+                sentiment = self.sentiment_analyzer.polarity_scores(token.text)[
+                    "compound"
+                ]
             word_sentiment.append(sentiment)
         sentiment = torch.tensor(word_sentiment, dtype=torch.float)
         sentiment = truncate_and_pad_single_sequence(sentiment, max_length)
@@ -116,9 +117,7 @@ def forward(self, features: list[torch.tensor]):
         outputs = []
         for i in range(self.feature_num):
             if len(self.projections) <= i:
-                self.projections.append(
-                    nn.Linear(features[i].size(-1), self.proj_dim)
-                )
+                self.projections.append(nn.Linear(features[i].size(-1), self.proj_dim))
                 projected_features.append(self.projections[i](features[i]))
 
         for i in range(self.feature_num):
@@ -133,9 +132,7 @@ def forward(self, features: list[torch.tensor]):
 
 
 class FeatureFusionBertClassfier(nn.Module):
-    def __init__(
-        self, feature_num=3, proj_dim=64, bert_input_dim=768, num_labels=2
-    ):
+    def __init__(self, feature_num=3, proj_dim=64, bert_input_dim=768, num_labels=2):
         super(FeatureFusionBertClassfier, self).__init__()
         bert_classifier = BertForSequenceClassification.from_pretrained(
             "bert-base-uncased", num_labels=num_labels
@@ -177,8 +174,8 @@ def forward(self, features: list[torch.tensor], input_ids: torch.tensor):
         semantic_tokenzier, sentiment_max_length=20
     )
 
-    def tokenzier_funtion(examples:Dataset):
-        featuress_input_ids =tokenizer(
+    def tokenzier_funtion(examples: Dataset):
+        featuress_input_ids = tokenizer(
             examples["text"],
             padding=True,
             truncation=True,

src/everyai/classifier/sklearn_classifier.py

@@ -10,9 +10,16 @@
 from sklearn.ensemble import RandomForestClassifier
 from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
 from sklearn.linear_model import LogisticRegression
-from sklearn.metrics import (accuracy_score, auc, confusion_matrix, f1_score,
-                             precision_recall_curve, precision_score,
-                             recall_score, roc_curve)
+from sklearn.metrics import (
+    accuracy_score,
+    auc,
+    confusion_matrix,
+    f1_score,
+    precision_recall_curve,
+    precision_score,
+    recall_score,
+    roc_curve,
+)
 from sklearn.naive_bayes import GaussianNB
 from sklearn.neighbors import KNeighborsClassifier
 from sklearn.pipeline import make_pipeline
@@ -141,9 +148,7 @@ def _init_sklearn_pipeline(pipeline_config: list[dict]):
             step_params = pipeline_config[step_name]
             steps.append((step_name, step_dict[step_name](**step_params)))
         else:
-            logging.warning(
-                "Step %s not recognized and will be skipped", step_name
-            )
+            logging.warning("Step %s not recognized and will be skipped", step_name)
     return make_pipeline(*[step[1] for step in steps])
 
 
@@ -234,7 +239,11 @@ def predict(self, x):
         return self.model.predict(x)
 
     def show_score(self):
-        output_path = RESULT_PATH/ "classfiy_result"/ f"{self.model_name}_{self.tokenizer_name}_{self.data_name}"
+        output_path = (
+            RESULT_PATH
+            / "classfiy_result"
+            / f"{self.model_name}_{self.tokenizer_name}_{self.data_name}"
+        )
         self.score = evaluate_classification_model(
             self.data.y_test,
             self.data.y_pred,

src/everyai/data_loader/data_load.py

@@ -1,5 +1,5 @@
-from collections.abc import Callable
 import logging
+from collections.abc import Callable
 from pathlib import Path
 
 import pandas as pd
@@ -35,20 +35,15 @@ def __init__(
     def load_data(
         self, max_count: int = None, return_type: str = "list"
     ) -> list[dict] | pd.DataFrame:
-        if (
-            Path(self.file_name_or_path).exists()
-            or self.file_type == "huggingface"
-        ):
+        if Path(self.file_name_or_path).exists() or self.file_type == "huggingface":
             logging.info("Loading data from %s", self.file_name_or_path)
             match self.file_type:
                 case "csv":
                     loaded_data = pd.read_csv(self.file_name_or_path)
                 case "xlsx":
                     loaded_data = pd.read_excel(self.file_name_or_path)
                 case "jsonl":
-                    loaded_data = pd.read_json(
-                        self.file_name_or_path, lines=True
-                    )
+                    loaded_data = pd.read_json(self.file_name_or_path, lines=True)
                 case "json":
                     loaded_data = pd.read_json(self.file_name_or_path)
                 case "huggingface":
@@ -72,9 +67,7 @@ def load_data(
         if max_count is not None and loaded_data is not None:
             loaded_data = loaded_data.head(max_count)
         else:
-            logging.info(
-                "Max count is None and all the records will be loaded"
-            )
+            logging.info("Max count is None and all the records will be loaded")
         loaded_data.rename(
             columns={
                 self.question_column: "question",
@@ -85,9 +78,7 @@ def load_data(
         if return_type == "pandas":
             result = loaded_data
         elif return_type == "list":
-            result = loaded_data[["question", "answer"]].to_dict(
-                orient="records"
-            )
+            result = loaded_data[["question", "answer"]].to_dict(orient="records")
         else:
             logging.error("Invalid return type")
         return result