diff --git a/src/everyai/classifier/classify.py b/src/everyai/classifier/classify.py index 097434f..9501e80 100644 --- a/src/everyai/classifier/classify.py +++ b/src/everyai/classifier/classify.py @@ -41,24 +41,20 @@ def split_data( original_indices = pd.DataFrame(x).index # 第一次划分: 训练集和测试集 - x_train, x_test, y_train, y_test, train_indices, test_indices = ( - train_test_split( - x, - y, - original_indices, - test_size=test_size, - random_state=42, - ) + x_train, x_test, y_train, y_test, train_indices, test_indices = train_test_split( + x, + y, + original_indices, + test_size=test_size, + random_state=42, ) # 第二次划分: 训练集和验证集 - x_train, x_valid, y_train, y_valid, train_indices, valid_indices = ( - train_test_split( - x_train, - y_train, - train_indices, - test_size=valid_size / (train_size + valid_size), - random_state=42, - ) + x_train, x_valid, y_train, y_valid, train_indices, valid_indices = train_test_split( + x_train, + y_train, + train_indices, + test_size=valid_size / (train_size + valid_size), + random_state=42, ) return ( x_train, @@ -104,8 +100,7 @@ def __init__( f"{self.model_name}_{self.tokenizer_name}_{self.data_name}" ) self.model_path = ( - MODEL_PATH - / f"{self.model_name}_{self.tokenizer_name}_{self.data_name}.pkl" + MODEL_PATH / f"{self.model_name}_{self.tokenizer_name}_{self.data_name}.pkl" ) if self.split_size is not None: self.train_size = self.split_size.get("train_size", 0.8) @@ -118,9 +113,7 @@ def load_data(self, texts, labels, data_name): raise ValueError("Length of texts and labels should be same") self.texts = texts self.labels = labels - logging.info( - "Loading data: %s to classifier %s", data_name, self.model_name - ) + logging.info("Loading data: %s to classifier %s", data_name, self.model_name) self.data_name = data_name self.classifier_name = ( f"{self.model_name}_{self.tokenizer_name}_{self.data_name}" @@ -135,4 +128,4 @@ def process_data(self): ), self.texts, ) - ) \ No newline at end of file + ) diff --git a/src/everyai/classifier/fusion_classifer.py b/src/everyai/classifier/fusion_classifer.py index 6f18e0d..55c0745 100644 --- a/src/everyai/classifier/fusion_classifer.py +++ b/src/everyai/classifier/fusion_classifer.py @@ -11,10 +11,8 @@ ) from everyai.classifier.classify import TextClassifer, label_encode, split_data +from everyai.classifier.multi_feature_model.fusionBert import FeatureFusionBertClassfier from everyai.utils.everyai_path import MODEL_PATH -from everyai.classifier.multi_feature_model.fusionBert import ( - FeatureFusionBertClassfier, -) class HuggingfaceClassifer(TextClassifer): @@ -32,7 +30,7 @@ def __init__( fusion_model_dict = { "FeatureFusionBertClassfier": FeatureFusionBertClassfier(), } - + self.model = fusion_model_dict[self.model_name] if "bert" in self.model_name.lower(): self.tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased") @@ -47,9 +45,7 @@ def __init__( def _tokenize(self, texts: list[str], labels: list[str]): self.label_encoder, tokenzied_labels = label_encode(labels) tokenzied_labels = torch.tensor(tokenzied_labels) - dataset = datasets.Dataset.from_dict( - {"text": texts, "label": tokenzied_labels} - ) + dataset = datasets.Dataset.from_dict({"text": texts, "label": tokenzied_labels}) def _tokenizer_fn(example): return self.tokenizer(example["text"], **self.tokenizer_config) @@ -73,12 +69,8 @@ def train(self): self.data.valid_indices, self.data.test_indices, ) = split_data(self.texts, self.labels) - self.train_dataset = self._tokenize( - self.data.x_train, self.data.y_train - ) - self.valid_dataset = self._tokenize( - self.data.x_valid, self.data.y_valid - ) + self.train_dataset = self._tokenize(self.data.x_train, self.data.y_train) + self.valid_dataset = self._tokenize(self.data.x_valid, self.data.y_valid) self.test_dataset = self._tokenize(self.data.x_test, self.data.y_test) train_args = TrainingArguments(**self.train_args) data_collator = DataCollatorWithPadding(tokenizer=self.tokenizer) @@ -94,15 +86,11 @@ def train(self): def test(self): trainer = Trainer(model=self.model) predictions = trainer.predict(self.test_dataset) - self.data.y_pred = torch.argmax( - torch.tensor(predictions.predictions), axis=1 - ) + self.data.y_pred = torch.argmax(torch.tensor(predictions.predictions), axis=1) self.data.y_test = self.label_encoder.transform(self.data.y_test) def show_score(self): metric = evaluate.load("accuracy") - metric.compute( - predictions=self.data.y_pred, references=self.data.y_test - ) + metric.compute(predictions=self.data.y_pred, references=self.data.y_test) logging.info("Accuracy: %s", metric) diff --git a/src/everyai/classifier/huggingface_classifier.py b/src/everyai/classifier/huggingface_classifier.py index 4afecfa..0a1911e 100644 --- a/src/everyai/classifier/huggingface_classifier.py +++ b/src/everyai/classifier/huggingface_classifier.py @@ -32,9 +32,7 @@ def __init__( **classfiy_config, ) self.tokenizer = AutoTokenizer.from_pretrained(self.tokenizer_name) - self.model = AutoModelForSequenceClassification.from_pretrained( - self.model_name - ) + self.model = AutoModelForSequenceClassification.from_pretrained(self.model_name) self.label_encoder = None self.train_dataset, self.valid_dataset, self.test_dataset = ( None, @@ -46,9 +44,7 @@ def __init__( def _tokenize(self, texts: list[str], labels: list[str]): self.label_encoder, tokenzied_labels = label_encode(labels) tokenzied_labels = torch.tensor(tokenzied_labels) - dataset = datasets.Dataset.from_dict( - {"text": texts, "label": tokenzied_labels} - ) + dataset = datasets.Dataset.from_dict({"text": texts, "label": tokenzied_labels}) def _tokenizer_fn(example): return self.tokenizer(example["text"], **self.tokenizer_config) @@ -72,12 +68,8 @@ def train(self): self.data.valid_indices, self.data.test_indices, ) = split_data(self.texts, self.labels) - self.train_dataset = self._tokenize( - self.data.x_train, self.data.y_train - ) - self.valid_dataset = self._tokenize( - self.data.x_valid, self.data.y_valid - ) + self.train_dataset = self._tokenize(self.data.x_train, self.data.y_train) + self.valid_dataset = self._tokenize(self.data.x_valid, self.data.y_valid) self.test_dataset = self._tokenize(self.data.x_test, self.data.y_test) train_args = TrainingArguments(**self.train_args) data_collator = DataCollatorWithPadding(tokenizer=self.tokenizer) @@ -93,15 +85,11 @@ def train(self): def test(self): trainer = Trainer(model=self.model) predictions = trainer.predict(self.test_dataset) - self.data.y_pred = torch.argmax( - torch.tensor(predictions.predictions), axis=1 - ) + self.data.y_pred = torch.argmax(torch.tensor(predictions.predictions), axis=1) self.data.y_test = self.label_encoder.transform(self.data.y_test) def show_score(self): metric = evaluate.load("accuracy") - metric.compute( - predictions=self.data.y_pred, references=self.data.y_test - ) + metric.compute(predictions=self.data.y_pred, references=self.data.y_test) logging.info("Accuracy: %s", metric) diff --git a/src/everyai/classifier/multi_feature_model/fusionBert.py b/src/everyai/classifier/multi_feature_model/fusionBert.py index 993d596..d4ff7d2 100644 --- a/src/everyai/classifier/multi_feature_model/fusionBert.py +++ b/src/everyai/classifier/multi_feature_model/fusionBert.py @@ -4,9 +4,12 @@ import torch import torch.nn as nn import torch.nn.functional as F -from datasets import load_dataset,Dataset -from transformers import (BertForSequenceClassification, BertTokenizer, - PreTrainedTokenizer) +from datasets import Dataset, load_dataset +from transformers import ( + BertForSequenceClassification, + BertTokenizer, + PreTrainedTokenizer, +) from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer @@ -26,9 +29,7 @@ def __init__(self, semantic_tokenizer: PreTrainedTokenizer, **kwargs): self.all_tags = self.nlp.get_pipe("tagger").labels self.sentiment_analyzer = SentimentIntensityAnalyzer() - def analyze_word_level_sentiment( - self, text: str, max_length=512 - ) -> torch.tensor: + def analyze_word_level_sentiment(self, text: str, max_length=512) -> torch.tensor: # 使用SpaCy进行词汇级分析 doc = self.nlp(text) @@ -38,9 +39,9 @@ def analyze_word_level_sentiment( if token.is_stop or token.is_punct: sentiment = 0.0 else: - sentiment = self.sentiment_analyzer.polarity_scores( - token.text - )["compound"] + sentiment = self.sentiment_analyzer.polarity_scores(token.text)[ + "compound" + ] word_sentiment.append(sentiment) sentiment = torch.tensor(word_sentiment, dtype=torch.float) sentiment = truncate_and_pad_single_sequence(sentiment, max_length) @@ -116,9 +117,7 @@ def forward(self, features: list[torch.tensor]): outputs = [] for i in range(self.feature_num): if len(self.projections) <= i: - self.projections.append( - nn.Linear(features[i].size(-1), self.proj_dim) - ) + self.projections.append(nn.Linear(features[i].size(-1), self.proj_dim)) projected_features.append(self.projections[i](features[i])) for i in range(self.feature_num): @@ -133,9 +132,7 @@ def forward(self, features: list[torch.tensor]): class FeatureFusionBertClassfier(nn.Module): - def __init__( - self, feature_num=3, proj_dim=64, bert_input_dim=768, num_labels=2 - ): + def __init__(self, feature_num=3, proj_dim=64, bert_input_dim=768, num_labels=2): super(FeatureFusionBertClassfier, self).__init__() bert_classifier = BertForSequenceClassification.from_pretrained( "bert-base-uncased", num_labels=num_labels @@ -177,8 +174,8 @@ def forward(self, features: list[torch.tensor], input_ids: torch.tensor): semantic_tokenzier, sentiment_max_length=20 ) - def tokenzier_funtion(examples:Dataset): - featuress_input_ids =tokenizer( + def tokenzier_funtion(examples: Dataset): + featuress_input_ids = tokenizer( examples["text"], padding=True, truncation=True, diff --git a/src/everyai/classifier/sklearn_classifier.py b/src/everyai/classifier/sklearn_classifier.py index f9426a3..2e72b86 100644 --- a/src/everyai/classifier/sklearn_classifier.py +++ b/src/everyai/classifier/sklearn_classifier.py @@ -10,9 +10,16 @@ from sklearn.ensemble import RandomForestClassifier from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer from sklearn.linear_model import LogisticRegression -from sklearn.metrics import (accuracy_score, auc, confusion_matrix, f1_score, - precision_recall_curve, precision_score, - recall_score, roc_curve) +from sklearn.metrics import ( + accuracy_score, + auc, + confusion_matrix, + f1_score, + precision_recall_curve, + precision_score, + recall_score, + roc_curve, +) from sklearn.naive_bayes import GaussianNB from sklearn.neighbors import KNeighborsClassifier from sklearn.pipeline import make_pipeline @@ -141,9 +148,7 @@ def _init_sklearn_pipeline(pipeline_config: list[dict]): step_params = pipeline_config[step_name] steps.append((step_name, step_dict[step_name](**step_params))) else: - logging.warning( - "Step %s not recognized and will be skipped", step_name - ) + logging.warning("Step %s not recognized and will be skipped", step_name) return make_pipeline(*[step[1] for step in steps]) @@ -234,7 +239,11 @@ def predict(self, x): return self.model.predict(x) def show_score(self): - output_path = RESULT_PATH/ "classfiy_result"/ f"{self.model_name}_{self.tokenizer_name}_{self.data_name}" + output_path = ( + RESULT_PATH + / "classfiy_result" + / f"{self.model_name}_{self.tokenizer_name}_{self.data_name}" + ) self.score = evaluate_classification_model( self.data.y_test, self.data.y_pred, diff --git a/src/everyai/data_loader/data_load.py b/src/everyai/data_loader/data_load.py index d5c4c8c..bcadd71 100644 --- a/src/everyai/data_loader/data_load.py +++ b/src/everyai/data_loader/data_load.py @@ -1,5 +1,5 @@ -from collections.abc import Callable import logging +from collections.abc import Callable from pathlib import Path import pandas as pd @@ -35,10 +35,7 @@ def __init__( def load_data( self, max_count: int = None, return_type: str = "list" ) -> list[dict] | pd.DataFrame: - if ( - Path(self.file_name_or_path).exists() - or self.file_type == "huggingface" - ): + if Path(self.file_name_or_path).exists() or self.file_type == "huggingface": logging.info("Loading data from %s", self.file_name_or_path) match self.file_type: case "csv": @@ -46,9 +43,7 @@ def load_data( case "xlsx": loaded_data = pd.read_excel(self.file_name_or_path) case "jsonl": - loaded_data = pd.read_json( - self.file_name_or_path, lines=True - ) + loaded_data = pd.read_json(self.file_name_or_path, lines=True) case "json": loaded_data = pd.read_json(self.file_name_or_path) case "huggingface": @@ -72,9 +67,7 @@ def load_data( if max_count is not None and loaded_data is not None: loaded_data = loaded_data.head(max_count) else: - logging.info( - "Max count is None and all the records will be loaded" - ) + logging.info("Max count is None and all the records will be loaded") loaded_data.rename( columns={ self.question_column: "question", @@ -85,9 +78,7 @@ def load_data( if return_type == "pandas": result = loaded_data elif return_type == "list": - result = loaded_data[["question", "answer"]].to_dict( - orient="records" - ) + result = loaded_data[["question", "answer"]].to_dict(orient="records") else: logging.error("Invalid return type") return result diff --git a/src/everyai/data_loader/everyai_dataset.py b/src/everyai/data_loader/everyai_dataset.py index af28fe4..f8b2b8f 100644 --- a/src/everyai/data_loader/everyai_dataset.py +++ b/src/everyai/data_loader/everyai_dataset.py @@ -24,9 +24,7 @@ def __init__( if datas is not None: self.datas: pd.DataFrame = datas else: - self.datas: pd.DataFrame = pd.DataFrame( - columns=["question", "human"] - ) + self.datas: pd.DataFrame = pd.DataFrame(columns=["question", "human"]) if ai_list is not None: for ai_name in ai_list: self.datas[ai_name] = None @@ -71,17 +69,13 @@ def insert_ai_response(self, question, ai_name: str, ai_response: str): if question_exists: self._update_new_row(question, ai_name, ai_response) else: - self.datas.loc[self.datas["question"] == question, ai_name] = ( - ai_response - ) + self.datas.loc[self.datas["question"] == question, ai_name] = ai_response def insert_human_response(self, question, human_response: str): if self.datas[self.datas["question"] == question].empty: self._update_new_row(question, "human", human_response) else: - self.datas.loc[self.datas["question"] == question, "human"] = ( - human_response - ) + self.datas.loc[self.datas["question"] == question, "human"] = human_response def _update_new_row(self, question, arg1, arg2): logging.info("Inserting new question: %s", question) @@ -109,15 +103,15 @@ def _save2mongodb(self, database: pymongo.database.Database): if col != "question": update["$set"][col] = row[col] - bulk_operations.append({ - "updateOne": { - "filter": query, - "update": update, - "upsert": True - } - }) + bulk_operations.append( + {"updateOne": {"filter": query, "update": update, "upsert": True}} + ) result = collection.bulk_write(bulk_operations) - logging.info("update %d records and insert %d records ", result.matched_count, result.upserted_count) + logging.info( + "update %d records and insert %d records ", + result.matched_count, + result.upserted_count, + ) def _load_from_mongodb(self, database: pymongo.database.Database): logging.info("Loading dataset from mongodb: %s", database) @@ -129,9 +123,7 @@ def _load_from_mongodb(self, database: pymongo.database.Database): data = data.drop(columns=["timestamp"]) self.datas = data - def load( - self, path_or_database: str | Path = None, file_format: str = "csv" - ): + def load(self, path_or_database: str | Path = None, file_format: str = "csv"): if file_format == "mongodb": if path_or_database is None: path_or_database = self._initialize_mongo_connection() @@ -140,17 +132,13 @@ def load( self._load_from_mongodb(path_or_database) else: if path_or_database is None: - path_or_database = ( - DATA_PATH / f"{self.data_name}.{file_format}" - ) + path_or_database = DATA_PATH / f"{self.data_name}.{file_format}" logging.info("Load dataset from %s", path_or_database) if isinstance(path_or_database, str): path_or_database = Path(path_or_database) if path_or_database.suffix != f".{file_format}": logging.warning("Change file format to %s", file_format) - path_or_database = path_or_database.with_suffix( - f".{file_format}" - ) + path_or_database = path_or_database.with_suffix(f".{file_format}") match path_or_database.suffix: case ".csv": self.datas = pd.read_csv(path_or_database) @@ -165,9 +153,7 @@ def load( set(self.datas.columns) - {"question", "human", "timestamp"} ) - def save( - self, path_or_database: str | Path = None, file_format: str = "csv" - ): + def save(self, path_or_database: str | Path = None, file_format: str = "csv"): if file_format == "mongodb": if path_or_database is None: path_or_database = self._initialize_mongo_connection() @@ -180,9 +166,7 @@ def save( path_or_database = Path(path_or_database) if path_or_database.suffix != f".{file_format}": logging.warning("Change file format to %s", file_format) - path_or_database = path_or_database.with_suffix( - f".{file_format}" - ) + path_or_database = path_or_database.with_suffix(f".{file_format}") match path_or_database.suffix: case ".csv": self.datas.to_csv(path_or_database, index=False) diff --git a/src/everyai/data_loader/mongo_connection.py b/src/everyai/data_loader/mongo_connection.py index ebfdbe5..05ac6af 100644 --- a/src/everyai/data_loader/mongo_connection.py +++ b/src/everyai/data_loader/mongo_connection.py @@ -5,7 +5,8 @@ from pymongo import MongoClient -def get_mongo_connection(connection_string:str, database_name:str):# -> Database: +# -> Database: +def get_mongo_connection(connection_string: str, database_name: str): if connection_string is None or not connection_string: logging.info("Use the connection string in environment variable") connection_string = os.getenv("MONGO_CONNECTION_STRING") diff --git a/src/everyai/generator/generate.py b/src/everyai/generator/generate.py index 30cb655..b6bd5ce 100644 --- a/src/everyai/generator/generate.py +++ b/src/everyai/generator/generate.py @@ -31,9 +31,7 @@ def _openai_generate( ) -> str: client = OpenAI(api_key=api_key, base_url=base_url) messages = [{"role": "user", "content": user_input}] - result = client.chat.completions.create( - messages=messages, model=model_name - ) + result = client.chat.completions.create(messages=messages, model=model_name) return result.choices[0].message.content def _huggingface_generate( diff --git a/src/everyai/main.py b/src/everyai/main.py index 92a3911..82d414c 100644 --- a/src/everyai/main.py +++ b/src/everyai/main.py @@ -1,6 +1,7 @@ import logging import pandas as pd +import torch from tqdm import tqdm from everyai.classifier.huggingface_classifier import HuggingfaceClassifer @@ -22,7 +23,6 @@ MONGO_CONFIG_PATH, ) from everyai.utils.load_config import get_config -import torch def generate(): @@ -50,9 +50,7 @@ def generate(): ) for generate_config in generate_list_configs["generate_list"]: generator = Generator(config=generate_config) - for data in tqdm( - qa_datas, desc="Generating data", total=len(qa_datas) - ): + for data in tqdm(qa_datas, desc="Generating data", total=len(qa_datas)): ai_response: str = generator.generate(data["question"]) everyai_dataset.insert_ai_response( question=data["question"], diff --git a/src/everyai/test.py b/src/everyai/test.py index cdbf311..960b807 100644 --- a/src/everyai/test.py +++ b/src/everyai/test.py @@ -1,6 +1,6 @@ +import torch from datasets import Dataset from transformers import BertTokenizer -import torch # Sample dataset data = {"text": ["Hello, world!", "How are you?"]} @@ -10,14 +10,19 @@ tokenizer = BertTokenizer.from_pretrained("bert-base-uncased") # Function to tokenize the text and convert to torch tensors + + def tokenize_function(examples): # Tokenize the text - encoding = tokenizer(examples["text"], padding=True, truncation=True, return_tensors="pt") + encoding = tokenizer( + examples["text"], padding=True, truncation=True, return_tensors="pt" + ) # Convert to pytorch tensors (they will already be tensors if using return_tensors="pt") return {key: torch.tensor(value) for key, value in encoding.items()} + # Apply the function with batched=True to process a batch at a time tokenized_dataset = dataset.map(tokenize_function, batched=True) # Check the result -print(tokenized_dataset['input_ids']) +print(tokenized_dataset["input_ids"]) diff --git a/src/everyai/topic/my_bertopic.py b/src/everyai/topic/my_bertopic.py index c2e0727..f3d1c72 100644 --- a/src/everyai/topic/my_bertopic.py +++ b/src/everyai/topic/my_bertopic.py @@ -10,8 +10,9 @@ def create_topic( output_folder: Union[str, Path], embedding_model=None, topic_config: dict = None, -) : +): from bertopic import BERTopic + if topic_config is None: topic_config = {} topic_model = BERTopic(embedding_model=embedding_model, min_topic_size=5) diff --git a/src/everyai/utils/load_args.py b/src/everyai/utils/load_args.py index aad6fff..3ace55c 100644 --- a/src/everyai/utils/load_args.py +++ b/src/everyai/utils/load_args.py @@ -1,6 +1,5 @@ import logging - classfiy_allowed_keys = [ "model_name", "tokenizer_name", @@ -12,7 +11,7 @@ ] -def set_attrs_2class(self, classify_config, allowed_keys,necessary_keys): +def set_attrs_2class(self, classify_config, allowed_keys, necessary_keys): for key, value in classify_config.items(): if key in allowed_keys: setattr(self, key, value) @@ -22,4 +21,3 @@ def set_attrs_2class(self, classify_config, allowed_keys,necessary_keys): if key not in classify_config: setattr(self, key, None) logging.warning("Necessary key not provided: %s", key) -