From d0a7724e17e5fdfab14717f438e6b3a7db63264a Mon Sep 17 00:00:00 2001 From: "deepsource-autofix[bot]" <62050782+deepsource-autofix[bot]@users.noreply.github.com> Date: Fri, 27 Dec 2024 12:25:53 +0000 Subject: [PATCH] style: format code with Autopep8, isort and Ruff Formatter This commit fixes the style issues introduced in 648f7ba according to the output from Autopep8, isort and Ruff Formatter. Details: None --- src/everyai/classfier/classfy.py | 22 ++++++---------------- src/everyai/data_loader/data_load.py | 5 +---- src/everyai/generator/generate.py | 8 ++------ src/everyai/main.py | 22 ++++++++-------------- 4 files changed, 17 insertions(+), 40 deletions(-) diff --git a/src/everyai/classfier/classfy.py b/src/everyai/classfier/classfy.py index d598103..e2beb3f 100644 --- a/src/everyai/classfier/classfy.py +++ b/src/everyai/classfier/classfy.py @@ -175,8 +175,7 @@ def __init__( ) self.model_config = None self.model_path = ( - MODEL_PATH - / f"{self.model_name}_{self.tokenizer_name}_{self.data_name}.pkl" + MODEL_PATH / f"{self.model_name}_{self.tokenizer_name}_{self.data_name}.pkl" ) self.pipeline = pipeline if pipeline is not None else None @@ -186,9 +185,7 @@ def load_data(self, texts, labels, data_name): raise ValueError("Length of texts and labels should be same") self.texts = texts self.labels = labels - logging.info( - "Loading data: %s to classfier %s", data_name, self.model_name - ) + logging.info("Loading data: %s to classfier %s", data_name, self.model_name) self.data_name = data_name self.classfier_name = ( f"{self.model_name}_{self.tokenizer_name}_{self.data_name}" @@ -237,9 +234,7 @@ def _init_sklearn_pipeline(pipeline_config: list[dict]): if step_name in step_dict: steps.append((step_name, step_dict[step_name](**step_params))) else: - logging.warning( - "Step %s not recognized and will be skipped", step_name - ) + logging.warning("Step %s not recognized and will be skipped", step_name) return make_pipeline(*[step[1] for step in steps]) @@ -266,9 +261,7 @@ def __init__(self, **classfiy_config): else: logging.warning("Split size not provided or not valid") if self.texts is None or self.labels is None or self.data_name is None: - logging.warning( - "Data not provided, please use the load_data method" - ) + logging.warning("Data not provided, please use the load_data method") if "device" in classfiy_config and classfiy_config["device"] == "cuda": logging.warning( "Cuda is not supported in sklearn and setting device to cpu" @@ -298,9 +291,7 @@ def __init__(self, **classfiy_config): raise ValueError("Model not supported") if classfiy_config["tokenizer_name"] in tokenizer_dict: - self.tokenizer = tokenizer_dict[ - classfiy_config["tokenizer_name"] - ]() + self.tokenizer = tokenizer_dict[classfiy_config["tokenizer_name"]]() else: logging.error("Tokenizer not supported") raise ValueError("Tokenizer not supported") @@ -345,8 +336,7 @@ def _split_data(self, x, y): x_train, y_train, train_indices, - test_size=self.valid_size - / (self.train_size + self.valid_size), + test_size=self.valid_size / (self.train_size + self.valid_size), random_state=42, ) ) diff --git a/src/everyai/data_loader/data_load.py b/src/everyai/data_loader/data_load.py index 069b244..5a4bc9d 100644 --- a/src/everyai/data_loader/data_load.py +++ b/src/everyai/data_loader/data_load.py @@ -33,10 +33,7 @@ def __init__( self.filter = data_filter def load_data2list(self, max_count: int = None): - if ( - Path(self.file_name_or_path).exists() - or self.file_type == "huggingface" - ): + if Path(self.file_name_or_path).exists() or self.file_type == "huggingface": match self.file_type: case "csv": dataset = pd.read_csv(self.file_name_or_path) diff --git a/src/everyai/generator/generate.py b/src/everyai/generator/generate.py index cfa8750..1894df4 100644 --- a/src/everyai/generator/generate.py +++ b/src/everyai/generator/generate.py @@ -31,9 +31,7 @@ def _openai_generate( ) -> str: client = OpenAI(api_key=api_key, base_url=base_url) messages = [{"role": "user", "content": user_input}] - result = client.chat.completions.create( - messages=messages, model=model_name - ) + result = client.chat.completions.create(messages=messages, model=model_name) return result.choices[0].message.content def _huggingface_generate( @@ -48,9 +46,7 @@ def _huggingface_generate( logging.info("Using glm-4-9b-chat model") logging.info("load model from %s", model_path_or_name) if self.tokenizer is None: - self.tokenizer = AutoTokenizer.from_pretrained( - model_path_or_name - ) + self.tokenizer = AutoTokenizer.from_pretrained(model_path_or_name) else: self.tokenizer = self.tokenizer logging.info("Use existing tokenizer") diff --git a/src/everyai/main.py b/src/everyai/main.py index d3a5df1..bec8ba2 100644 --- a/src/everyai/main.py +++ b/src/everyai/main.py @@ -39,16 +39,12 @@ def generate(): file_path=data_config["file_path"], data_type=data_config["data_type"], ) - qa_datas = data_loader.load_data2list( - max_count=data_config["max_count"] - ) + qa_datas = data_loader.load_data2list(max_count=data_config["max_count"]) everyai_dataset = EveryaiDataset( dataname=data_config["data_name"], ai_list=[generate_config["model_name"]], ) - for data in tqdm( - qa_datas, desc="Generating data", total=len(qa_datas) - ): + for data in tqdm(qa_datas, desc="Generating data", total=len(qa_datas)): ai_response: str = generator.generate(data["question"]) everyai_dataset.insert_ai_response( question=data["question"], @@ -83,7 +79,9 @@ def topic(): docs = everyai_dataset.datas[catogeory].tolist() logging.info("Number of documents: %d", len(docs)) new_docs = [ - split_remove_stopwords_punctuation(doc, language=everyai_dataset.language) + split_remove_stopwords_punctuation( + doc, language=everyai_dataset.language + ) for doc in docs ] create_topic( @@ -104,9 +102,7 @@ def classfiy(): ) everyai_dataset.load(format="mongodb") logging.info("Loaded data: %s", everyai_dataset.data_name) - texts, labels = everyai_dataset.get_records_with_1ai( - ["THUDM/glm-4-9b-chat-hf"] - ) + texts, labels = everyai_dataset.get_records_with_1ai(["THUDM/glm-4-9b-chat-hf"]) for classfiy_config in get_config(file_path=CLASSFIY_CONFIG_PATH)[ "classfier_list" ]: @@ -117,14 +113,12 @@ def classfiy(): ) case _: raise ValueError("Classfier type not supported") - text_classfier.load_data( - texts, labels, data_name=everyai_dataset.data_name - ) + text_classfier.load_data(texts, labels, data_name=everyai_dataset.data_name) text_classfier.train() text_classfier.test() text_classfier.save_model() text_classfier.show_score() - logging.info("Model saved for %s", classfiy_config['model_name']) + logging.info("Model saved for %s", classfiy_config["model_name"]) lime_explanation = LimeExplanation(classfier=text_classfier) lime_explanation.explain() shap_explanation = ShapExplanation(classfier=text_classfier)