From 9681b7ad930c0bf9f2ad19cc3e4ec9a81cd3f219 Mon Sep 17 00:00:00 2001 From: "deepsource-autofix[bot]" <62050782+deepsource-autofix[bot]@users.noreply.github.com> Date: Fri, 27 Dec 2024 09:54:46 +0000 Subject: [PATCH] style: format code with Autopep8, isort and Ruff Formatter This commit fixes the style issues introduced in ab54ba7 according to the output from Autopep8, isort and Ruff Formatter. Details: None --- src/everyai/classfier/classfy.py | 22 ++++++-------------- src/everyai/classfier/multi_feature/model.py | 4 +--- src/everyai/data_loader/data_load.py | 9 ++------ src/everyai/main.py | 20 +++++++----------- 4 files changed, 16 insertions(+), 39 deletions(-) diff --git a/src/everyai/classfier/classfy.py b/src/everyai/classfier/classfy.py index 03d3a94..cb22d05 100644 --- a/src/everyai/classfier/classfy.py +++ b/src/everyai/classfier/classfy.py @@ -175,8 +175,7 @@ def __init__( ) self.model_config = None self.model_path = ( - MODEL_PATH - / f"{self.model_name}_{self.tokenizer_name}_{self.data_name}.pkl" + MODEL_PATH / f"{self.model_name}_{self.tokenizer_name}_{self.data_name}.pkl" ) self.pipeline = pipeline if pipeline is not None else None @@ -186,9 +185,7 @@ def load_data(self, texts, labels, data_name): raise ValueError("Length of texts and labels should be same") self.texts = texts self.labels = labels - logging.info( - f"Loading data: {data_name} to classfier {self.model_name}" - ) + logging.info(f"Loading data: {data_name} to classfier {self.model_name}") self.data_name = data_name self.classfier_name = ( f"{self.model_name}_{self.tokenizer_name}_{self.data_name}" @@ -237,9 +234,7 @@ def _init_sklearn_pipeline(pipeline_config: list[dict]): if step_name in step_dict: steps.append((step_name, step_dict[step_name](**step_params))) else: - logging.warning( - f"Step {step_name} not recognized and will be skipped" - ) + logging.warning(f"Step {step_name} not recognized and will be skipped") return make_pipeline(*[step[1] for step in steps]) @@ -266,9 +261,7 @@ def __init__(self, **classfiy_config): else: logging.warning("Split size not provided or not valid") if self.texts is None or self.labels is None or self.data_name is None: - logging.warning( - "Data not provided, please use the load_data method" - ) + logging.warning("Data not provided, please use the load_data method") if "device" in classfiy_config and classfiy_config["device"] == "cuda": logging.warning( "Cuda is not supported in sklearn and setting device to cpu" @@ -298,9 +291,7 @@ def __init__(self, **classfiy_config): raise ValueError("Model not supported") if classfiy_config["tokenizer_name"] in tokenizer_dict: - self.tokenizer = tokenizer_dict[ - classfiy_config["tokenizer_name"] - ]() + self.tokenizer = tokenizer_dict[classfiy_config["tokenizer_name"]]() else: logging.error("Tokenizer not supported") raise ValueError("Tokenizer not supported") @@ -345,8 +336,7 @@ def _split_data(self, x, y): x_train, y_train, train_indices, - test_size=self.valid_size - / (self.train_size + self.valid_size), + test_size=self.valid_size / (self.train_size + self.valid_size), random_state=42, ) ) diff --git a/src/everyai/classfier/multi_feature/model.py b/src/everyai/classfier/multi_feature/model.py index 80320c8..2783d77 100644 --- a/src/everyai/classfier/multi_feature/model.py +++ b/src/everyai/classfier/multi_feature/model.py @@ -12,9 +12,7 @@ def __init__(self, d1, d2, d3, hidden_dim, output_dim, num_heads): def forward(self, f1, f2, f3): # 将所有特征拼接后输入注意力模块 - merged_features = torch.cat((f1, f2, f3), dim=-1).unsqueeze( - 0 - ) # 添加batch维度 + merged_features = torch.cat((f1, f2, f3), dim=-1).unsqueeze(0) # 添加batch维度 attn_output, _ = self.cross_attention( merged_features, merged_features, merged_features ) diff --git a/src/everyai/data_loader/data_load.py b/src/everyai/data_loader/data_load.py index 87f91db..01c9635 100644 --- a/src/everyai/data_loader/data_load.py +++ b/src/everyai/data_loader/data_load.py @@ -33,10 +33,7 @@ def __init__( self.filter = data_filter def load_data2list(self, max_count: int = None): - if ( - Path(self.file_name_or_path).exists() - or self.file_type == "huggingface" - ): + if Path(self.file_name_or_path).exists() or self.file_type == "huggingface": match self.file_type: case "csv": dataset = pd.read_csv(self.file_name_or_path) @@ -69,9 +66,7 @@ def load_data2list(self, max_count: int = None): if max_count is not None and dataset is not None: dataset = dataset.head(max_count) else: - logging.info( - "Max count is None and all the records will be loaded" - ) + logging.info("Max count is None and all the records will be loaded") dataset.rename( columns={ self.question_column: "question", diff --git a/src/everyai/main.py b/src/everyai/main.py index 5d6bf2a..0c74b20 100644 --- a/src/everyai/main.py +++ b/src/everyai/main.py @@ -39,16 +39,12 @@ def generate(): file_path=data_config["file_path"], data_type=data_config["data_type"], ) - qa_datas = data_loader.load_data2list( - max_count=data_config["max_count"] - ) + qa_datas = data_loader.load_data2list(max_count=data_config["max_count"]) everyai_dataset = EveryaiDataset( dataname=data_config["data_name"], ai_list=[generate_config["model_name"]], ) - for data in tqdm( - qa_datas, desc="Generating data", total=len(qa_datas) - ): + for data in tqdm(qa_datas, desc="Generating data", total=len(qa_datas)): ai_response: str = generator.generate(data["question"]) everyai_dataset.insert_ai_response( question=data["question"], @@ -84,7 +80,9 @@ def topic(): logging.info(f"Number of documents: {len(docs)}") new_docs = [] new_docs = [ - split_remove_stopwords_punctuation(doc, language=everyai_dataset.language) + split_remove_stopwords_punctuation( + doc, language=everyai_dataset.language + ) for doc in docs ] create_topic( @@ -105,9 +103,7 @@ def classfiy(): ) everyai_dataset.load(format="mongodb") logging.info(f"Loaded data: {everyai_dataset.data_name}") - texts, labels = everyai_dataset.get_records_with_1ai( - ["THUDM/glm-4-9b-chat-hf"] - ) + texts, labels = everyai_dataset.get_records_with_1ai(["THUDM/glm-4-9b-chat-hf"]) for classfiy_config in get_config(file_path=CLASSFIY_CONFIG_PATH)[ "classfier_list" ]: @@ -118,9 +114,7 @@ def classfiy(): ) case _: raise ValueError("Classfier type not supported") - text_classfier.load_data( - texts, labels, data_name=everyai_dataset.data_name - ) + text_classfier.load_data(texts, labels, data_name=everyai_dataset.data_name) text_classfier.train() text_classfier.test() text_classfier.save_model()