Skip to content

style: format code with Autopep8, isort and Ruff Formatter #14

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
deepsource-autofix wants to merge 1 commit into
base: main
Choose a base branch
from
Open

style: format code with Autopep8, isort and Ruff Formatter #14

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 6 additions & 16 deletions src/everyai/classfier/classfy.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -175,8 +175,7 @@ def __init__(
)
self.model_config = None
self.model_path = (
MODEL_PATH
/ f"{self.model_name}_{self.tokenizer_name}_{self.data_name}.pkl"
MODEL_PATH / f"{self.model_name}_{self.tokenizer_name}_{self.data_name}.pkl"
)
self.pipeline = pipeline if pipeline is not None else None

Expand All @@ -186,9 +185,7 @@ def load_data(self, texts, labels, data_name):
raise ValueError("Length of texts and labels should be same")
self.texts = texts
self.labels = labels
logging.info(
"Loading data: %s to classfier %s", data_name, self.model_name
)
logging.info("Loading data: %s to classfier %s", data_name, self.model_name)
self.data_name = data_name
self.classfier_name = (
f"{self.model_name}_{self.tokenizer_name}_{self.data_name}"
Expand Down Expand Up @@ -237,9 +234,7 @@ def _init_sklearn_pipeline(pipeline_config: list[dict]):
if step_name in step_dict:
steps.append((step_name, step_dict[step_name](**step_params)))
else:
logging.warning(
"Step %s not recognized and will be skipped", step_name
)
logging.warning("Step %s not recognized and will be skipped", step_name)
return make_pipeline(*[step[1] for step in steps])


Expand All @@ -266,9 +261,7 @@ def __init__(self, **classfiy_config):
else:
logging.warning("Split size not provided or not valid")
if self.texts is None or self.labels is None or self.data_name is None:
logging.warning(
"Data not provided, please use the load_data method"
)
logging.warning("Data not provided, please use the load_data method")
if "device" in classfiy_config and classfiy_config["device"] == "cuda":
logging.warning(
"Cuda is not supported in sklearn and setting device to cpu"
Expand Down Expand Up @@ -298,9 +291,7 @@ def __init__(self, **classfiy_config):
raise ValueError("Model not supported")

if classfiy_config["tokenizer_name"] in tokenizer_dict:
self.tokenizer = tokenizer_dict[
classfiy_config["tokenizer_name"]
]()
self.tokenizer = tokenizer_dict[classfiy_config["tokenizer_name"]]()
else:
logging.error("Tokenizer not supported")
raise ValueError("Tokenizer not supported")
Expand Down Expand Up @@ -345,8 +336,7 @@ def _split_data(self, x, y):
x_train,
y_train,
train_indices,
test_size=self.valid_size
/ (self.train_size + self.valid_size),
test_size=self.valid_size / (self.train_size + self.valid_size),
random_state=42,
)
)
Expand Down
5 changes: 1 addition & 4 deletions src/everyai/data_loader/data_load.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,10 +33,7 @@ def __init__(
self.filter = data_filter

def load_data2list(self, max_count: int = None):
if (
Path(self.file_name_or_path).exists()
or self.file_type == "huggingface"
):
if Path(self.file_name_or_path).exists() or self.file_type == "huggingface":
match self.file_type:
case "csv":
dataset = pd.read_csv(self.file_name_or_path)
Expand Down
8 changes: 2 additions & 6 deletions src/everyai/generator/generate.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,7 @@ def _openai_generate(
) -> str:
client = OpenAI(api_key=api_key, base_url=base_url)
messages = [{"role": "user", "content": user_input}]
result = client.chat.completions.create(
messages=messages, model=model_name
)
result = client.chat.completions.create(messages=messages, model=model_name)
return result.choices[0].message.content

def _huggingface_generate(
Expand All @@ -48,9 +46,7 @@ def _huggingface_generate(
logging.info("Using glm-4-9b-chat model")
logging.info("load model from %s", model_path_or_name)
if self.tokenizer is None:
self.tokenizer = AutoTokenizer.from_pretrained(
model_path_or_name
)
self.tokenizer = AutoTokenizer.from_pretrained(model_path_or_name)
else:
self.tokenizer = self.tokenizer
logging.info("Use existing tokenizer")
Expand Down
22 changes: 8 additions & 14 deletions src/everyai/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,16 +39,12 @@ def generate():
file_path=data_config["file_path"],
data_type=data_config["data_type"],
)
qa_datas = data_loader.load_data2list(
max_count=data_config["max_count"]
)
qa_datas = data_loader.load_data2list(max_count=data_config["max_count"])
everyai_dataset = EveryaiDataset(
dataname=data_config["data_name"],
ai_list=[generate_config["model_name"]],
)
for data in tqdm(
qa_datas, desc="Generating data", total=len(qa_datas)
):
for data in tqdm(qa_datas, desc="Generating data", total=len(qa_datas)):
ai_response: str = generator.generate(data["question"])
everyai_dataset.insert_ai_response(
question=data["question"],
Expand Down Expand Up @@ -83,7 +79,9 @@ def topic():
docs = everyai_dataset.datas[catogeory].tolist()
logging.info("Number of documents: %d", len(docs))
new_docs = [
split_remove_stopwords_punctuation(doc, language=everyai_dataset.language)
split_remove_stopwords_punctuation(
doc, language=everyai_dataset.language
)
for doc in docs
]
create_topic(
Expand All @@ -104,9 +102,7 @@ def classfiy():
)
everyai_dataset.load(format="mongodb")
logging.info("Loaded data: %s", everyai_dataset.data_name)
texts, labels = everyai_dataset.get_records_with_1ai(
["THUDM/glm-4-9b-chat-hf"]
)
texts, labels = everyai_dataset.get_records_with_1ai(["THUDM/glm-4-9b-chat-hf"])
for classfiy_config in get_config(file_path=CLASSFIY_CONFIG_PATH)[
"classfier_list"
]:
Expand All @@ -117,14 +113,12 @@ def classfiy():
)
case _:
raise ValueError("Classfier type not supported")
text_classfier.load_data(
texts, labels, data_name=everyai_dataset.data_name
)
text_classfier.load_data(texts, labels, data_name=everyai_dataset.data_name)
text_classfier.train()
text_classfier.test()
text_classfier.save_model()
text_classfier.show_score()
logging.info("Model saved for %s", classfiy_config['model_name'])
logging.info("Model saved for %s", classfiy_config["model_name"])
lime_explanation = LimeExplanation(classfier=text_classfier)
lime_explanation.explain()
shap_explanation = ShapExplanation(classfier=text_classfier)
Expand Down