From 565193f3e7d5f3bec9b9b429a643e6a1b114cf47 Mon Sep 17 00:00:00 2001 From: "deepsource-autofix[bot]" <62050782+deepsource-autofix[bot]@users.noreply.github.com> Date: Thu, 9 Jan 2025 11:40:00 +0000 Subject: [PATCH] style: format code with Autopep8, isort and Ruff Formatter This commit fixes the style issues introduced in ad10446 according to the output from Autopep8, isort and Ruff Formatter. Details: None --- .../multi_feature_model/fusionBert.py | 32 +++++--------- src/everyai/classifier/pytorch_classier.py | 44 +++++++++---------- 2 files changed, 32 insertions(+), 44 deletions(-) diff --git a/src/everyai/classifier/multi_feature_model/fusionBert.py b/src/everyai/classifier/multi_feature_model/fusionBert.py index bbb99fe..9793ada 100644 --- a/src/everyai/classifier/multi_feature_model/fusionBert.py +++ b/src/everyai/classifier/multi_feature_model/fusionBert.py @@ -1,11 +1,11 @@ import math +import test import spacy -import test import torch import torch.nn as nn import torch.nn.functional as F -from datasets import load_dataset, Dataset +from datasets import Dataset, load_dataset from transformers import ( BertForSequenceClassification, BertTokenizer, @@ -30,9 +30,7 @@ def __init__(self, semantic_tokenizer: PreTrainedTokenizer, **kwargs): self.all_tags = self.nlp.get_pipe("tagger").labels self.sentiment_analyzer = SentimentIntensityAnalyzer() - def analyze_word_level_sentiment( - self, text: str, max_length=512 - ) -> torch.tensor: + def analyze_word_level_sentiment(self, text: str, max_length=512) -> torch.tensor: # 使用SpaCy进行词汇级分析 doc = self.nlp(text) @@ -42,9 +40,9 @@ def analyze_word_level_sentiment( if token.is_stop or token.is_punct: sentiment = 0.0 else: - sentiment = self.sentiment_analyzer.polarity_scores( - token.text - )["compound"] + sentiment = self.sentiment_analyzer.polarity_scores(token.text)[ + "compound" + ] word_sentiment.append(sentiment) sentiment = torch.tensor(word_sentiment, dtype=torch.float) sentiment = truncate_and_pad_single_sequence(sentiment, max_length) @@ -87,12 +85,8 @@ def batch_encode_plus(self, batch_text: list[str], **kwargs): 批量编码函数 :param batch_text: 输入文本列表 """ - batch_encoding = self.semantic_tokenizer.batch_encode_plus( - batch_text, **kwargs - ) - batch_pos = torch.cat( - [self.pos_feature(text) for text in batch_text], dim=0 - ) + batch_encoding = self.semantic_tokenizer.batch_encode_plus(batch_text, **kwargs) + batch_pos = torch.cat([self.pos_feature(text) for text in batch_text], dim=0) batch_sentiments = torch.cat( [ self.analyze_word_level_sentiment( @@ -143,9 +137,7 @@ def forward(self, features: list[torch.tensor]): outputs = [] for i in range(self.feature_num): if len(self.projections) <= i: - self.projections.append( - nn.Linear(features[i].size(-1), self.proj_dim) - ) + self.projections.append(nn.Linear(features[i].size(-1), self.proj_dim)) projected_features.append(self.projections[i](features[i])) for i in range(self.feature_num): @@ -160,9 +152,7 @@ def forward(self, features: list[torch.tensor]): class FeatureFusionBertClassfier(nn.Module): - def __init__( - self, feature_num=3, proj_dim=64, bert_input_dim=768, num_labels=2 - ): + def __init__(self, feature_num=3, proj_dim=64, bert_input_dim=768, num_labels=2): super(FeatureFusionBertClassfier, self).__init__() bert_classifier = BertForSequenceClassification.from_pretrained( "bert-base-uncased", num_labels=num_labels @@ -228,4 +218,4 @@ def tokenzier_funtion(examples: Dataset): print(tokenized_input["input_ids"].shape) for feature in tokenized_input["features"]: print(feature.shape) - print(model(tokenized_input["features"], tokenized_input["input_ids"])) \ No newline at end of file + print(model(tokenized_input["features"], tokenized_input["input_ids"])) diff --git a/src/everyai/classifier/pytorch_classier.py b/src/everyai/classifier/pytorch_classier.py index 671092f..6084765 100644 --- a/src/everyai/classifier/pytorch_classier.py +++ b/src/everyai/classifier/pytorch_classier.py @@ -3,11 +3,19 @@ import datasets import evaluate import torch -from transformers import (AutoModelForSequenceClassification, AutoTokenizer, - DataCollatorWithPadding, Trainer, TrainingArguments) +from transformers import ( + AutoModelForSequenceClassification, + AutoTokenizer, + DataCollatorWithPadding, + Trainer, + TrainingArguments, +) from everyai.classifier.classify import TextClassifer, label_encode, split_data -from everyai.classifier.multi_feature_model.fusionBert import FeatureFusionBertClassfier, FeatureFusionBertTokenizer +from everyai.classifier.multi_feature_model.fusionBert import ( + FeatureFusionBertClassfier, + FeatureFusionBertTokenizer, +) from everyai.utils.everyai_path import MODEL_PATH @@ -27,13 +35,13 @@ def __init__( language=language, **classfiy_config, ) - tokenzier_dict = { - "fusion_bert": FeatureFusionBertTokenizer(AutoTokenizer("bert-base-uncased")) + tokenzier_dict = { + "fusion_bert": FeatureFusionBertTokenizer( + AutoTokenizer("bert-base-uncased") + ) } self.tokenizer = tokenzier_dict[self.tokenizer_name] - model_dict = { - "fusion_bert": FeatureFusionBertClassfier(feature_num=3) - } + model_dict = {"fusion_bert": FeatureFusionBertClassfier(feature_num=3)} self.model = model_dict[self.model_name] self.label_encoder = None self.train_dataset, self.valid_dataset, self.test_dataset = ( @@ -46,9 +54,7 @@ def __init__( def _tokenize(self, texts: list[str], labels: list[str]): self.label_encoder, tokenzied_labels = label_encode(labels) tokenzied_labels = torch.tensor(tokenzied_labels) - dataset = datasets.Dataset.from_dict( - {"text": texts, "label": tokenzied_labels} - ) + dataset = datasets.Dataset.from_dict({"text": texts, "label": tokenzied_labels}) def _tokenizer_fn(example): return self.tokenizer(example["text"], **self.tokenizer_config) @@ -72,12 +78,8 @@ def train(self): self.data.valid_indices, self.data.test_indices, ) = split_data(self.texts, self.labels) - self.train_dataset = self._tokenize( - self.data.x_train, self.data.y_train - ) - self.valid_dataset = self._tokenize( - self.data.x_valid, self.data.y_valid - ) + self.train_dataset = self._tokenize(self.data.x_train, self.data.y_train) + self.valid_dataset = self._tokenize(self.data.x_valid, self.data.y_valid) self.test_dataset = self._tokenize(self.data.x_test, self.data.y_test) train_args = TrainingArguments(**self.train_args) data_collator = DataCollatorWithPadding(tokenizer=self.tokenizer) @@ -93,15 +95,11 @@ def train(self): def test(self): trainer = Trainer(model=self.model) predictions = trainer.predict(self.test_dataset) - self.data.y_pred = torch.argmax( - torch.tensor(predictions.predictions), axis=1 - ) + self.data.y_pred = torch.argmax(torch.tensor(predictions.predictions), axis=1) self.data.y_test = self.label_encoder.transform(self.data.y_test) def show_score(self): metric = evaluate.load("accuracy") - metric.compute( - predictions=self.data.y_pred, references=self.data.y_test - ) + metric.compute(predictions=self.data.y_pred, references=self.data.y_test) logging.info("Accuracy: %s", metric)