diff --git a/src/everyai/classifier/huggingface_classifier.py b/src/everyai/classifier/huggingface_classifier.py index e031e01..012f0fe 100644 --- a/src/everyai/classifier/huggingface_classifier.py +++ b/src/everyai/classifier/huggingface_classifier.py @@ -4,14 +4,13 @@ import evaluate import torch from transformers import ( + BertForSequenceClassification, + BertTokenizer, DataCollatorWithPadding, Trainer, TrainingArguments, - BertForSequenceClassification, - BertTokenizer, ) - from everyai.classifier.classify import TextClassifer, label_encode, split_data from everyai.utils.everyai_path import MODEL_PATH @@ -34,9 +33,7 @@ def __init__( ) if self.model_name != "bert-base-uncased": raise ValueError(f"Model name {self.model_name} not supported yet") - self.model = BertForSequenceClassification.from_pretrained( - self.model_name - ) + self.model = BertForSequenceClassification.from_pretrained(self.model_name) self.tokenizer = BertTokenizer.from_pretrained( self.tokenizer_name, **self.tokenizer_config ) @@ -51,9 +48,7 @@ def __init__( def _tokenize(self, texts: list[str], labels: list[str]): self.label_encoder, tokenzied_labels = label_encode(labels) tokenzied_labels = torch.tensor(tokenzied_labels) - dataset = datasets.Dataset.from_dict( - {"text": texts, "label": tokenzied_labels} - ) + dataset = datasets.Dataset.from_dict({"text": texts, "label": tokenzied_labels}) def _tokenizer_fn(example): return self.tokenizer(example["text"], **self.tokenizer_config) @@ -77,12 +72,8 @@ def train(self): self.data.valid_indices, self.data.test_indices, ) = split_data(self.texts, self.labels) - self.train_dataset = self._tokenize( - self.data.x_train, self.data.y_train - ) - self.valid_dataset = self._tokenize( - self.data.x_valid, self.data.y_valid - ) + self.train_dataset = self._tokenize(self.data.x_train, self.data.y_train) + self.valid_dataset = self._tokenize(self.data.x_valid, self.data.y_valid) self.test_dataset = self._tokenize(self.data.x_test, self.data.y_test) train_args = TrainingArguments(**self.train_args) data_collator = DataCollatorWithPadding(tokenizer=self.tokenizer) @@ -98,9 +89,7 @@ def train(self): def test(self): trainer = Trainer(model=self.model) predictions = trainer.predict(self.test_dataset) - self.data.y_pred = torch.argmax( - torch.tensor(predictions.predictions), axis=1 - ) + self.data.y_pred = torch.argmax(torch.tensor(predictions.predictions), axis=1) self.data.y_test = self.label_encoder.transform(self.data.y_test) diff --git a/src/everyai/classifier/multi_feature_model/fusionBert.py b/src/everyai/classifier/multi_feature_model/fusionBert.py index 932122d..e5cfa3f 100644 --- a/src/everyai/classifier/multi_feature_model/fusionBert.py +++ b/src/everyai/classifier/multi_feature_model/fusionBert.py @@ -33,9 +33,7 @@ def __init__(self, semantic_tokenizer: PreTrainedTokenizer, **kwargs): self.all_tags = self.nlp.get_pipe("tagger").labels self.sentiment_analyzer = SentimentIntensityAnalyzer() - def analyze_word_level_sentiment( - self, text: str, max_length=512 - ) -> torch.tensor: + def analyze_word_level_sentiment(self, text: str, max_length=512) -> torch.tensor: # 使用SpaCy进行词汇级分析 doc = self.nlp(text) @@ -45,9 +43,9 @@ def analyze_word_level_sentiment( if token.is_stop or token.is_punct: sentiment = 0.0 else: - sentiment = self.sentiment_analyzer.polarity_scores( - token.text - )["compound"] + sentiment = self.sentiment_analyzer.polarity_scores(token.text)[ + "compound" + ] word_sentiment.append(sentiment) sentiment = torch.tensor(word_sentiment, dtype=torch.float) sentiment = truncate_and_pad_single_sequence(sentiment, max_length) @@ -90,12 +88,8 @@ def batch_encode_plus(self, batch_text: list[str], **kwargs): 批量编码函数 :param batch_text: 输入文本列表 """ - batch_encoding = self.semantic_tokenizer.batch_encode_plus( - batch_text, **kwargs - ) - batch_pos = torch.cat( - [self.pos_feature(text) for text in batch_text], dim=0 - ) + batch_encoding = self.semantic_tokenizer.batch_encode_plus(batch_text, **kwargs) + batch_pos = torch.cat([self.pos_feature(text) for text in batch_text], dim=0) batch_sentiments = torch.cat( [ self.analyze_word_level_sentiment( @@ -146,9 +140,7 @@ def forward(self, features: list[torch.tensor]): outputs = [] for i in range(self.feature_num): if len(self.projections) <= i: - self.projections.append( - nn.Linear(features[i].size(-1), self.proj_dim) - ) + self.projections.append(nn.Linear(features[i].size(-1), self.proj_dim)) projected_features.append(self.projections[i](features[i])) for i in range(self.feature_num): @@ -163,9 +155,7 @@ def forward(self, features: list[torch.tensor]): class FeatureFusionBertClassfier(nn.Module): - def __init__( - self, feature_num=3, proj_dim=64, bert_input_dim=768, num_labels=2 - ): + def __init__(self, feature_num=3, proj_dim=64, bert_input_dim=768, num_labels=2): super(FeatureFusionBertClassfier, self).__init__() bert_classifier = BertForSequenceClassification.from_pretrained( "bert-base-uncased", num_labels=num_labels @@ -238,7 +228,7 @@ def forward(self, features: list[torch.tensor], input_ids: torch.tensor): num_warmup_steps=0, num_training_steps=num_training_steps, ) - # TODO Fix error: KeyError: 'Invalid key. Only three types of key are available: + # TODO Fix error: KeyError: 'Invalid key. Only three types of key are available: # (1) string, (2) integers for backend Encoding, and (3) slices for data subsetting.' model.train() for epoch in range(num_epochs): diff --git a/src/everyai/data_loader/data_load.py b/src/everyai/data_loader/data_load.py index ef3f4c7..cb7c9b2 100644 --- a/src/everyai/data_loader/data_load.py +++ b/src/everyai/data_loader/data_load.py @@ -1,5 +1,5 @@ -from collections.abc import Callable import logging +from collections.abc import Callable from pathlib import Path import pandas as pd @@ -35,10 +35,7 @@ def __init__( def load_data( self, max_count: int = None, return_type: str = "list" ) -> list[dict] | pd.DataFrame: - if ( - Path(self.file_name_or_path).exists() - or self.file_type == "huggingface" - ): + if Path(self.file_name_or_path).exists() or self.file_type == "huggingface": logging.info("Loading data from %s", self.file_name_or_path) match self.file_type: case "csv": @@ -46,15 +43,11 @@ def load_data( case "xlsx": loaded_data = pd.read_excel(self.file_name_or_path) case "jsonl": - loaded_data = pd.read_json( - self.file_name_or_path, lines=True - ) + loaded_data = pd.read_json(self.file_name_or_path, lines=True) case "json": loaded_data = pd.read_json(self.file_name_or_path) case "huggingface": - loaded_data = load_dataset( - self.file_name_or_path - ) + loaded_data = load_dataset(self.file_name_or_path) if "train" in loaded_data.keys(): loaded_data = loaded_data["train"].to_pandas() elif "validation" in loaded_data.keys(): @@ -78,9 +71,7 @@ def load_data( if max_count is not None and loaded_data is not None: loaded_data = loaded_data.head(max_count) else: - logging.info( - "Max count is None and all the records will be loaded" - ) + logging.info("Max count is None and all the records will be loaded") loaded_data.rename( columns={ self.question_column: "question", @@ -91,9 +82,7 @@ def load_data( if return_type == "pandas": result = loaded_data elif return_type == "list": - result = loaded_data[["question", "answer"]].to_dict( - orient="records" - ) + result = loaded_data[["question", "answer"]].to_dict(orient="records") else: logging.error("Invalid return type") return result diff --git a/src/everyai/data_loader/everyai_dataset.py b/src/everyai/data_loader/everyai_dataset.py index 1087a61..1e0f14d 100644 --- a/src/everyai/data_loader/everyai_dataset.py +++ b/src/everyai/data_loader/everyai_dataset.py @@ -1,6 +1,6 @@ import logging -from pathlib import Path import re +from pathlib import Path import pandas as pd import pymongo @@ -32,9 +32,7 @@ def __init__( if datas is not None: self.datas: pd.DataFrame = datas else: - self.datas: pd.DataFrame = pd.DataFrame( - columns=["question", "human"] - ) + self.datas: pd.DataFrame = pd.DataFrame(columns=["question", "human"]) if ai_list is not None: for ai_name in ai_list: self.datas[ai_name] = None @@ -87,33 +85,25 @@ def insert_ai_response(self, question, ai_name: str, ai_response: str): else: logging.info("AI %s exists in the dataset", ai_name) question_exists = not self.datas[ - (self.datas["question"] == question) - & (self.datas[ai_name] == ai_name) + (self.datas["question"] == question) & (self.datas[ai_name] == ai_name) ].empty if question_exists: self._update_new_row(question, ai_name, ai_response) else: - self.datas.loc[self.datas["question"] == question, ai_name] = ( - ai_response - ) + self.datas.loc[self.datas["question"] == question, ai_name] = ai_response def insert_human_response(self, question, human_response: str): if self.datas[self.datas["question"] == question].empty: self._update_new_row(question, "human", human_response) else: - self.datas.loc[self.datas["question"] == question, "human"] = ( - human_response - ) + self.datas.loc[self.datas["question"] == question, "human"] = human_response - def record_exist( - self, question: str, label: str, file_type: str = "mongodb" - ): + def record_exist(self, question: str, label: str, file_type: str = "mongodb"): if file_type != "mongodb": raise ValueError("Unsupported file type: %s", file_type) query = {"question": question, label: {"$exists": True}} record = self.mongo_collection.find_one(query) return False if record is None else isinstance(record[label], str) - def upsert2mongo(self, question: str, label: str, answer: str): query = {"question": question} @@ -128,9 +118,7 @@ def _update_new_row(self, question, arg1, arg2): def output_question(self): # -> Iterator: return iter(self.datas["question"]) - def _save2mongodb( - self, database: pymongo.database.Database, insert_mode="insert" - ): + def _save2mongodb(self, database: pymongo.database.Database, insert_mode="insert"): self.datas = self.datas.dropna() logging.info("Saving dataset to mongodb: %s", database) collection = database[self.data_name] @@ -160,9 +148,7 @@ def _read_from_mongodb(self, database: pymongo.database.Database): data = data.drop(columns=["timestamp"]) return data - def read( - self, path_or_database: str | Path = None, file_format: str = "csv" - ): + def read(self, path_or_database: str | Path = None, file_format: str = "csv"): if file_format == "mongodb": if path_or_database is None: path_or_database = _initialize_mongo_connection() @@ -171,17 +157,13 @@ def read( loaded_data = self._read_from_mongodb(path_or_database) else: if path_or_database is None: - path_or_database = ( - DATA_PATH / f"{self.data_name}.{file_format}" - ) + path_or_database = DATA_PATH / f"{self.data_name}.{file_format}" logging.info("Load dataset from %s", path_or_database) if isinstance(path_or_database, str): path_or_database = Path(path_or_database) if path_or_database.suffix != f".{file_format}": logging.warning("Change file format to %s", file_format) - path_or_database = path_or_database.with_suffix( - f".{file_format}" - ) + path_or_database = path_or_database.with_suffix(f".{file_format}") match path_or_database.suffix: case ".csv": loaded_data = pd.read_csv(path_or_database) @@ -204,9 +186,7 @@ def read( self.datas.dropna(inplace=True) self.ai_list = list(set(self.datas.columns) - {"question", "human"}) - def save( - self, path_or_database: str | Path = None, file_format: str = "csv" - ): + def save(self, path_or_database: str | Path = None, file_format: str = "csv"): self.datas = self.datas.dropna() if file_format == "mongodb": if path_or_database is None: @@ -214,17 +194,13 @@ def save( self._save2mongodb(path_or_database) else: if path_or_database is None: - path_or_database = ( - DATA_PATH / f"{self.data_name}.{file_format}" - ) + path_or_database = DATA_PATH / f"{self.data_name}.{file_format}" logging.info("Save dataset to %s", path_or_database) if isinstance(path_or_database, str): path_or_database = Path(path_or_database) if path_or_database.suffix != f".{file_format}": logging.warning("Change file format to %s", file_format) - path_or_database = path_or_database.with_suffix( - f".{file_format}" - ) + path_or_database = path_or_database.with_suffix(f".{file_format}") match path_or_database.suffix: case ".csv": self.datas.to_csv(path_or_database, index=False) diff --git a/src/everyai/explanation/explain.py b/src/everyai/explanation/explain.py index 244fe79..a9b98f0 100644 --- a/src/everyai/explanation/explain.py +++ b/src/everyai/explanation/explain.py @@ -36,7 +36,9 @@ def explain(self, output_path: Path = None): exp.save_to_file(output_path / f"text{i}.html") logging.info("Lime explanation saved to %s", output_path) else: - logging.error("Lime explanation not supported for %s", self.classifier.classifier_type) + logging.error( + "Lime explanation not supported for %s", self.classifier.classifier_type + ) class ShapExplanation(Explanation): @@ -57,4 +59,6 @@ def explain(self, output_path: Path = None): shap.plots.text(shap_values, output_path / f"text{i}") logging.info("Shap explanation saved to %s", output_path) else: - logging.error("Shap explanation not supported for %s", self.classifier.classifier_type) \ No newline at end of file + logging.error( + "Shap explanation not supported for %s", self.classifier.classifier_type + ) diff --git a/src/everyai/generator/generate.py b/src/everyai/generator/generate.py index 594abd5..dfc73da 100644 --- a/src/everyai/generator/generate.py +++ b/src/everyai/generator/generate.py @@ -2,8 +2,8 @@ from collections.abc import Callable from pathlib import Path -from everyai.generator.openai_generate import openai_generate from everyai.generator.huggingface_generate import glm4, llama, qwen2_5 +from everyai.generator.openai_generate import openai_generate from everyai.utils.everyai_path import GENERATE_CONFIG_PATH from everyai.utils.load_args import set_attrs_2class from everyai.utils.load_config import get_config @@ -37,9 +37,7 @@ def __init__(self, config: dict, formatter: Callable[[str], str] = None): ] default_params = ["model_name", "base_url", "api_key"] else: - raise ValueError( - "Unsupported generator type: %s", self.generator_type - ) + raise ValueError("Unsupported generator type: %s", self.generator_type) self.formatter = formatter set_attrs_2class(self, config, allowed_keys, default_params) diff --git a/src/everyai/generator/huggingface_generate.py b/src/everyai/generator/huggingface_generate.py index 42b6499..ef2bd0e 100644 --- a/src/everyai/generator/huggingface_generate.py +++ b/src/everyai/generator/huggingface_generate.py @@ -2,8 +2,8 @@ from pathlib import Path import torch -from transformers import AutoModelForCausalLM, AutoTokenizer import transformers +from transformers import AutoModelForCausalLM, AutoTokenizer def glm4( diff --git a/src/everyai/generator/openai_generate.py b/src/everyai/generator/openai_generate.py index a4f64a6..51c0005 100644 --- a/src/everyai/generator/openai_generate.py +++ b/src/everyai/generator/openai_generate.py @@ -1,6 +1,7 @@ import logging from openai import OpenAI + from everyai.utils.proxy import TempProxy @@ -17,9 +18,7 @@ def openai_generate( logging.info("Proxy started: %s", proxy) client = OpenAI(api_key=api_key, base_url=base_url) messages = [{"role": "user", "content": user_input}] - result = client.chat.completions.create( - messages=messages, model=model_name - ) + result = client.chat.completions.create(messages=messages, model=model_name) if proxy is not None: temp_proxy.reset_proxy() logging.info("Proxy reset") diff --git a/src/everyai/main.py b/src/everyai/main.py index adfca4a..40f4336 100644 --- a/src/everyai/main.py +++ b/src/everyai/main.py @@ -12,10 +12,14 @@ from everyai.explanation.explain import LimeExplanation, ShapExplanation from everyai.generator.generate import Generator from everyai.topic.my_bertopic import create_topic -from everyai.utils.everyai_path import (BERT_TOPIC_CONFIG_PATH, - CLASSIFY_CONFIG_PATH, - DATA_LOAD_CONFIG_PATH, DATA_PATH, - FIG_PATH, GENERATE_CONFIG_PATH) +from everyai.utils.everyai_path import ( + BERT_TOPIC_CONFIG_PATH, + CLASSIFY_CONFIG_PATH, + DATA_LOAD_CONFIG_PATH, + DATA_PATH, + FIG_PATH, + GENERATE_CONFIG_PATH, +) from everyai.utils.load_config import get_config @@ -45,9 +49,7 @@ def generate(): for generate_config in generate_list_configs["generate_list"]: logging.info("Generate config: %s", generate_config) generator = Generator(config=generate_config) - for data in tqdm( - qa_datas, desc="Generating data", total=len(qa_datas) - ): + for data in tqdm(qa_datas, desc="Generating data", total=len(qa_datas)): if everyai_dataset.record_exist( data["question"], generate_config["model_name"] ): @@ -131,9 +133,7 @@ def classify(): **classify_config, ) case "huggingface": - text_classifier = HuggingfaceClassifer( - **classify_config - ) + text_classifier = HuggingfaceClassifer(**classify_config) case _: raise ValueError("Classifier type not supported") text_classifier.load_data( diff --git a/src/everyai/test.py b/src/everyai/test.py index 744b8ac..4cbc4a6 100644 --- a/src/everyai/test.py +++ b/src/everyai/test.py @@ -1,7 +1,6 @@ import pandas as pd -from everyai.data_loader.mongo_connection import get_mongo_connection - +from everyai.data_loader.mongo_connection import get_mongo_connection if __name__ == "__main__": database = get_mongo_connection(None, "everyai") @@ -9,6 +8,6 @@ # Load data from MongoDB collection into a DataFrame data = pd.DataFrame(list(collection.find())) print(data["question"].nunique()) - result = data.groupby("question",as_index=False).last() + result = data.groupby("question", as_index=False).last() print(result.shape) - result.to_csv("output.csv", index=False) \ No newline at end of file + result.to_csv("output.csv", index=False) diff --git a/src/everyai/utils/load_args.py b/src/everyai/utils/load_args.py index bd110ee..3ace55c 100644 --- a/src/everyai/utils/load_args.py +++ b/src/everyai/utils/load_args.py @@ -11,7 +11,7 @@ ] -def set_attrs_2class(self, classify_config, allowed_keys,necessary_keys): +def set_attrs_2class(self, classify_config, allowed_keys, necessary_keys): for key, value in classify_config.items(): if key in allowed_keys: setattr(self, key, value) @@ -21,4 +21,3 @@ def set_attrs_2class(self, classify_config, allowed_keys,necessary_keys): if key not in classify_config: setattr(self, key, None) logging.warning("Necessary key not provided: %s", key) -