From 0a3dd0db516ff09ba3d97ab8a99a8624e754ad03 Mon Sep 17 00:00:00 2001 From: Mircea Lungu Date: Thu, 5 Nov 2026 16:43:17 +0100 Subject: [PATCH] Move function-level stdlib imports to the top of the file Also drops function-level imports that repeat a top-level one, and adds a test that keeps stdlib imports out of functions. article.py now uses one form of datetime. user.py keeps its three local datetime imports: its top-level `import datetime` means something else, and about 20 uses depend on that. Closes #935 Co-Authored-By: Claude Opus 5.5 --- zeeguu/api/app.py | 7 ++- zeeguu/api/endpoints/accounts.py | 6 +-- zeeguu/api/endpoints/activity_tracking.py | 8 ++- zeeguu/api/endpoints/article.py | 9 +--- zeeguu/api/endpoints/article_upload.py | 3 +- zeeguu/api/endpoints/audio_lessons.py | 4 +- zeeguu/api/endpoints/bookmarks_and_words.py | 5 -- zeeguu/api/endpoints/crawl_stats.py | 4 +- zeeguu/api/endpoints/generated_examples.py | 2 +- zeeguu/api/endpoints/own_texts.py | 3 +- zeeguu/api/endpoints/speech.py | 2 +- .../endpoints/teacher_dashboard/cohorts.py | 4 -- zeeguu/api/endpoints/translation.py | 7 +-- zeeguu/api/endpoints/user_article.py | 2 +- zeeguu/api/endpoints/user_articles.py | 4 +- zeeguu/api/endpoints/user_stats.py | 2 - zeeguu/api/utils/route_wrappers.py | 8 +-- zeeguu/core/article_pruning.py | 4 +- .../audio_lessons/daily_lesson_generator.py | 2 +- .../bookmark_operations/update_bookmark.py | 6 +-- .../bookmark_quality/negative_qualities.py | 7 ++- .../content_retriever/article_downloader.py | 15 ++---- .../parse_with_readability_server.py | 2 +- zeeguu/core/elastic/indexing.py | 3 +- zeeguu/core/emailer/zeeguu_mailer.py | 5 +- zeeguu/core/language/ml_cefr_classifier.py | 3 +- .../simplification_and_classification.py | 18 ++----- zeeguu/core/model/article.py | 28 +++-------- .../core/model/article_tokenization_cache.py | 2 +- .../model/audio_lesson_generation_progress.py | 3 +- zeeguu/core/model/bookmark.py | 5 +- zeeguu/core/model/bookmark_context.py | 1 - zeeguu/core/model/daily_audio_lesson.py | 3 +- zeeguu/core/model/exercise.py | 3 +- zeeguu/core/model/feed.py | 3 +- zeeguu/core/model/friendship.py | 1 - zeeguu/core/model/meaning.py | 1 - zeeguu/core/model/phrase.py | 6 +-- zeeguu/core/model/translation_correction.py | 4 +- zeeguu/core/model/url.py | 2 - zeeguu/core/model/user.py | 8 --- zeeguu/core/model/user_article.py | 8 +-- .../core/model/user_article_broken_report.py | 4 +- zeeguu/core/model/user_word.py | 3 -- .../elastic_semantic_search.py | 7 ++- zeeguu/core/test/test_article.py | 13 +++++ .../core/test/test_stdlib_imports_at_top.py | 50 +++++++++++++++++++ .../translation_services/deepl_translate.py | 3 +- .../core/translation_services/translator.py | 8 +-- zeeguu/operations/crawler/crawl_report.py | 3 +- zeeguu/operations/crawler/feed_retrieval.py | 3 +- 51 files changed, 138 insertions(+), 179 deletions(-) create mode 100644 zeeguu/core/test/test_stdlib_imports_at_top.py diff --git a/zeeguu/api/app.py b/zeeguu/api/app.py index 7ff40c592..a48a9515a 100644 --- a/zeeguu/api/app.py +++ b/zeeguu/api/app.py @@ -15,6 +15,9 @@ # apimux is quite noisy; supress it's output import logging +import subprocess +import sys +import traceback from apimux.log import logger # Suppress verbose Elasticsearch connection error logs @@ -148,7 +151,6 @@ def create_app(testing=False): # Add SQL query listener to catch source_text queries from sqlalchemy import event from sqlalchemy.engine import Engine - import sys @event.listens_for(Engine, "before_cursor_execute") def detect_source_text_query(conn, cursor, statement, parameters, context, executemany): @@ -159,7 +161,6 @@ def detect_source_text_query(conn, cursor, statement, parameters, context, execu print(f"\n!!! FULL TABLE SCAN DETECTED ON source_text !!!", file=sys.stderr) print(f"Statement: {statement[:500]}", file=sys.stderr) print(f"Parameters: {parameters}", file=sys.stderr) - import traceback print("Call stack:", file=sys.stderr) traceback.print_stack(file=sys.stderr) sys.stderr.flush() @@ -191,7 +192,6 @@ def shutdown_session_request(exception=None): # the session has already been cleaned up or doesn't need cleanup pass if exception: - import sys print(f"[DB-SESSION-TEARDOWN] Exception during request: {exception}", file=sys.stderr) sys.stderr.flush() @@ -232,7 +232,6 @@ def serve_speech(filename): # print(app.config) # Log the current git commit hash try: - import subprocess git_hash = subprocess.check_output( ["git", "rev-parse", "HEAD"], cwd=os.path.dirname(__file__), diff --git a/zeeguu/api/endpoints/accounts.py b/zeeguu/api/endpoints/accounts.py index 2700b4630..3b681784f 100644 --- a/zeeguu/api/endpoints/accounts.py +++ b/zeeguu/api/endpoints/accounts.py @@ -1,4 +1,5 @@ import hmac +import re import sqlalchemy import flask @@ -248,7 +249,6 @@ def request_email_verification(): This avoids the "limbo state" where the user is no longer anonymous but hasn't verified their email yet. """ - import re from zeeguu.core.emailer.email_confirmation import send_email_confirmation email = request.form.get("email", None) @@ -388,8 +388,6 @@ def _email_change_error(user, new_email, password): by address, not by user, so the code only proves the inbox, and the password is what proves the person. """ - import re - if user.is_anonymous(): return "Anonymous accounts add an email by upgrading the account" if not new_email or not password: @@ -504,8 +502,6 @@ def send_code(email): @api.route("/reset_password/", methods=["POST"]) @cross_domain def reset_password(email): - from zeeguu.logging import log - log(f"PASSWORD RESET ATTEMPT: email='{email}'") code = request.form.get("code", None) diff --git a/zeeguu/api/endpoints/activity_tracking.py b/zeeguu/api/endpoints/activity_tracking.py index 58df084de..6c2ff7ebb 100644 --- a/zeeguu/api/endpoints/activity_tracking.py +++ b/zeeguu/api/endpoints/activity_tracking.py @@ -1,3 +1,6 @@ +from datetime import datetime +import json + import flask from flask import request @@ -59,8 +62,6 @@ def days_since_last_use(): or an empty string in case there is no user activity event. """ - from datetime import datetime - last_active_time = UserActivityData.get_last_activity_timestamp(flask.g.user_id) if last_active_time: @@ -82,7 +83,6 @@ def _update_reading_completion_on_scroll(user, form_data): try: from zeeguu.core.model import Article, UserArticle from zeeguu.core.behavioral_modeling import find_last_reading_percentage - import json article_id = int(form_data.get("article_id")) article = Article.find_by_id(article_id) @@ -131,8 +131,6 @@ def _update_reading_completion_on_scroll(user, form_data): # Check if article is completed (>90%) and not already marked if completion_percentage > 0.9 and not user_article.completed_at: - from datetime import datetime - user_article.completed_at = datetime.now() from zeeguu.core import events diff --git a/zeeguu/api/endpoints/article.py b/zeeguu/api/endpoints/article.py index 9bf674cfa..984d920ac 100644 --- a/zeeguu/api/endpoints/article.py +++ b/zeeguu/api/endpoints/article.py @@ -28,6 +28,8 @@ import requests from io import BytesIO from urllib.parse import quote +from datetime import datetime +import traceback from markupsafe import escape from PIL import Image from zeeguu.config import ZEEGUU_DATA_FOLDER @@ -311,8 +313,6 @@ def find_or_create_article(): print_and_log_to_sentry(e) - import traceback - traceback.print_exc() flask.abort(500) @@ -498,7 +498,6 @@ def translate_and_adapt_article(): JSON with translated article data and fragments """ from zeeguu.core.llm_services.simplification_service import SimplificationService - from zeeguu.core.model import User from zeeguu.core.model.article import Article from zeeguu.core.content_retriever import download_and_parse @@ -637,7 +636,6 @@ def translate_and_adapt_article(): from zeeguu.api.endpoints.helpers.translated_article import ( save_translated_article, ) - from datetime import datetime # Create URL and language objects with unique translated URL translated_url = Url.find_or_create(db_session, translated_url_key) @@ -740,7 +738,6 @@ def simplify_article(article_id): flask.abort(404, "Article not found") # Get the current user's CEFR level - from zeeguu.core.model import User user = User.find_by_id(flask.g.user_id) try: @@ -812,8 +809,6 @@ def simplify_article(article_id): ) except Exception as e: - from zeeguu.logging import log - log(f"Simplification failed for article {article_id}: {str(e)}") return json_result({"status": "failed", "error": str(e)[:200]}) diff --git a/zeeguu/api/endpoints/article_upload.py b/zeeguu/api/endpoints/article_upload.py index cc7e101f7..bea4022c7 100644 --- a/zeeguu/api/endpoints/article_upload.py +++ b/zeeguu/api/endpoints/article_upload.py @@ -1,4 +1,6 @@ """Endpoints for ArticleUpload — lightweight per-user ingestion entity.""" +from datetime import datetime + import flask from flask import request @@ -245,7 +247,6 @@ def article_upload_translate_and_adapt(upload_id): from zeeguu.api.endpoints.helpers.translated_article import ( save_translated_article, ) - from datetime import datetime user = User.find_by_id(flask.g.user_id) upload = _find_upload_or_404(upload_id, user) diff --git a/zeeguu/api/endpoints/audio_lessons.py b/zeeguu/api/endpoints/audio_lessons.py index 5b871f877..74b827640 100644 --- a/zeeguu/api/endpoints/audio_lessons.py +++ b/zeeguu/api/endpoints/audio_lessons.py @@ -1,3 +1,5 @@ +from datetime import datetime, timedelta + import flask from markupsafe import escape @@ -134,7 +136,6 @@ def generate_daily_lesson(): is_general_topic = False if suggestion and lesson_type in ("topic", "situation"): # Reuse canonical form if this user has typed the same thing before - from zeeguu.core.model import DailyAudioLesson cached_canonical = DailyAudioLesson.find_canonical_for_raw_suggestion(user, suggestion) if cached_canonical: canonical_suggestion = cached_canonical @@ -528,7 +529,6 @@ def get_audio_lesson_generation_progress(): return json_result({"progress": None}) # Detect stuck generations (e.g., server restarted mid-generation) - from datetime import datetime, timedelta if progress.updated_at and datetime.utcnow() - progress.updated_at > timedelta(minutes=2): progress.mark_error("Generation appears to have stopped. Please try again.") db.session.commit() diff --git a/zeeguu/api/endpoints/bookmarks_and_words.py b/zeeguu/api/endpoints/bookmarks_and_words.py index 005cae7c0..b0fd95287 100644 --- a/zeeguu/api/endpoints/bookmarks_and_words.py +++ b/zeeguu/api/endpoints/bookmarks_and_words.py @@ -291,10 +291,6 @@ def delete_bookmark(bookmark_id): @cross_domain @requires_session def set_user_word_exercise_preference(bookmark_id): - from zeeguu.core.word_scheduling.basicSR.four_levels_per_word import ( - FourLevelsPerWord, - ) - bookmark = get_bookmark_if_owned(bookmark_id) user_word = bookmark.user_word @@ -673,7 +669,6 @@ def set_preferred_bookmark(user_word_id): } """ from zeeguu.core.model import UserWord, Bookmark - from flask import request user = User.find_by_id(flask.g.user_id) user_word = UserWord.query.get(user_word_id) diff --git a/zeeguu/api/endpoints/crawl_stats.py b/zeeguu/api/endpoints/crawl_stats.py index 1d40e2acd..e29c55cde 100644 --- a/zeeguu/api/endpoints/crawl_stats.py +++ b/zeeguu/api/endpoints/crawl_stats.py @@ -4,6 +4,7 @@ """ from datetime import datetime, timedelta +from collections import defaultdict from flask import request from markupsafe import escape @@ -158,7 +159,6 @@ def crawl_stats_today(): def crawl_stats_week(): """Get stats for the last 7 days.""" # Temporarily set days param - from flask import request request.args = {"days": "7"} return crawl_stats() @@ -245,7 +245,6 @@ def crawl_stats_dashboard(): .all() ) # Group by feed_id - from collections import defaultdict topics_by_feed = defaultdict(list) for feed_id, topic_title, count in feed_topic_stats: topics_by_feed[feed_id].append((topic_title, count)) @@ -392,7 +391,6 @@ def crawl_stats_dashboard():

🏷️ Simplified by Topic per Language (Today)

""" # Group topic stats by language - from collections import defaultdict topics_by_lang = defaultdict(list) for lang_code, lang_name, topic_id, topic_title, count in topic_stats_by_lang: topics_by_lang[lang_code].append((lang_name, topic_id, topic_title, count)) diff --git a/zeeguu/api/endpoints/generated_examples.py b/zeeguu/api/endpoints/generated_examples.py index 8468b0020..d6c2e126e 100644 --- a/zeeguu/api/endpoints/generated_examples.py +++ b/zeeguu/api/endpoints/generated_examples.py @@ -1,5 +1,6 @@ import json import traceback +import re import flask import sentry_sdk @@ -697,7 +698,6 @@ def preview_learning_card(): # Parse CEFR level from level_note (e.g., "appropriate for B1 - ..." → "B1") level_note = learning_card.get("level_note", "") - import re cefr_match = re.search(r'\b(A1|A2|B1|B2|C1|C2)\b', level_note) if cefr_match: meaning.word_cefr_level = cefr_match.group(1) diff --git a/zeeguu/api/endpoints/own_texts.py b/zeeguu/api/endpoints/own_texts.py index c09688979..b95620104 100644 --- a/zeeguu/api/endpoints/own_texts.py +++ b/zeeguu/api/endpoints/own_texts.py @@ -1,3 +1,5 @@ +import os + import flask import sqlalchemy from flask import request @@ -161,7 +163,6 @@ def estimate_article_cefr(): if ml_assessment is None: # Check if model exists for this language - import os data_folder = os.environ.get("ZEEGUU_DATA_FOLDER") model_path = None if data_folder: diff --git a/zeeguu/api/endpoints/speech.py b/zeeguu/api/endpoints/speech.py index f376666ba..ec2e4a72f 100644 --- a/zeeguu/api/endpoints/speech.py +++ b/zeeguu/api/endpoints/speech.py @@ -1,5 +1,6 @@ import os.path import re +import hashlib from flask import request @@ -148,7 +149,6 @@ def _file_name_for_phrase(phrase, language_id): def _file_name_for_full_article(full_article_text, language_id, article_id): # create md5 hash of the phrase and return it - import hashlib m = hashlib.md5() m.update(full_article_text.encode("utf-8")) diff --git a/zeeguu/api/endpoints/teacher_dashboard/cohorts.py b/zeeguu/api/endpoints/teacher_dashboard/cohorts.py index 09f3d2ec7..267cc3ad3 100644 --- a/zeeguu/api/endpoints/teacher_dashboard/cohorts.py +++ b/zeeguu/api/endpoints/teacher_dashboard/cohorts.py @@ -64,8 +64,6 @@ def remove_cohort(cohort_id): Also removes all students from the cohort. """ - from zeeguu.core.model import TeacherCohortMap, UserCohortMap - check_permission_for_cohort(cohort_id) try: @@ -122,8 +120,6 @@ def _link_teacher_cohort(user_id, cohort_id): """ Takes user_id and cohort_id and links them together in teacher_cohort_map table. """ - from zeeguu.core.model import TeacherCohortMap - user = User.find_by_id(user_id) cohort = Cohort.find(cohort_id) db.session.add(TeacherCohortMap(user, cohort)) diff --git a/zeeguu/api/endpoints/translation.py b/zeeguu/api/endpoints/translation.py index 67653f801..760c8d32a 100644 --- a/zeeguu/api/endpoints/translation.py +++ b/zeeguu/api/endpoints/translation.py @@ -1,6 +1,8 @@ import os from string import punctuation from urllib.parse import unquote_plus +import json +import time import flask from flask import request @@ -133,7 +135,6 @@ def translate_word(from_lang_code, to_lang_code): else: # TODO: must remove theurl, and title - they are not used in the calling method. from zeeguu.logging import log - import time correction = None if is_separated_mwe else _shared_correction( word_str, full_sentence_context or context, from_lang_code, to_lang_code @@ -324,7 +325,6 @@ def get_translations_stream(from_lang_code, to_lang_code): :return: SSE stream with translation events """ - import json from flask import Response from zeeguu.core.translation_services.translator import get_translations_streaming @@ -376,7 +376,6 @@ def translation_alternatives(from_lang_code, to_lang_code): from zeeguu.core.translation_services.translator import ( _vote_translation, ) - from python_translators.translation_query import TranslationQuery word_str = request.json.get("word", "").strip(punctuation_extended) context = request.json.get("context", "").strip() @@ -613,8 +612,6 @@ def update_bookmark_translation_only(bookmark_id): :param bookmark_id: ID of bookmark to update :return: Updated bookmark as JSON """ - from zeeguu.api.utils.json_result import json_result - # Find bookmark with proper error handling # Fixes ZEEGUU-WEB-B2: Bookmark.find() throws NoResultFound, not returns None try: diff --git a/zeeguu/api/endpoints/user_article.py b/zeeguu/api/endpoints/user_article.py index accd0008b..463b4decb 100644 --- a/zeeguu/api/endpoints/user_article.py +++ b/zeeguu/api/endpoints/user_article.py @@ -9,6 +9,7 @@ from datetime import datetime import json +import traceback import newspaper @@ -112,7 +113,6 @@ def generate(): yield f"event: complete\ndata: {json.dumps(article_info)}\n\n" except Exception as e: - import traceback traceback.print_exc() yield f"event: error\ndata: {json.dumps({'error': str(e)})}\n\n" diff --git a/zeeguu/api/endpoints/user_articles.py b/zeeguu/api/endpoints/user_articles.py index 086df72aa..33ac05524 100644 --- a/zeeguu/api/endpoints/user_articles.py +++ b/zeeguu/api/endpoints/user_articles.py @@ -1,3 +1,5 @@ +import traceback + import flask from zeeguu.core.content_recommender import ( @@ -320,8 +322,6 @@ def user_articles_foryou(): articles = content_recommendations(user.id, user.learned_language_id) print("Sending CB recommendations") except Exception as e: - import traceback - traceback.print_exc() print("Failed with: ", e) capture_exception(e) diff --git a/zeeguu/api/endpoints/user_stats.py b/zeeguu/api/endpoints/user_stats.py index 3c47b27fb..66d48eec7 100644 --- a/zeeguu/api/endpoints/user_stats.py +++ b/zeeguu/api/endpoints/user_stats.py @@ -232,8 +232,6 @@ def get_browsing_stats_for_user(user_id, start, end): def get_translations_for_user(user_id, start, end): """Get translations/bookmarks made by user in the given period.""" - from zeeguu.core.model.bookmark import Bookmark - bookmarks = ( Bookmark.query.join(UserWord, Bookmark.user_word_id == UserWord.id) .filter(UserWord.user_id == user_id) diff --git a/zeeguu/api/utils/route_wrappers.py b/zeeguu/api/utils/route_wrappers.py index 799d05757..d88a7cc5f 100644 --- a/zeeguu/api/utils/route_wrappers.py +++ b/zeeguu/api/utils/route_wrappers.py @@ -6,6 +6,10 @@ from zeeguu.core.model.session import Session from datetime import datetime, timedelta +import sys +import threading +import time as time_module +import traceback import zeeguu # Saves a database lookup per request. The cache is per worker: evicting a @@ -41,9 +45,6 @@ def requires_session(view): @functools.wraps(view) def wrapped_view(*args, **kwargs): - import sys - import threading - import time as time_module from zeeguu.api.utils.abort_handling import make_error request_start = time_module.time() @@ -120,7 +121,6 @@ def wrapped_view(*args, **kwargs): flask.abort(401) except Exception as e: - import traceback from sentry_sdk import capture_exception capture_exception(e) diff --git a/zeeguu/core/article_pruning.py b/zeeguu/core/article_pruning.py index 2cef618be..7faf5cb26 100644 --- a/zeeguu/core/article_pruning.py +++ b/zeeguu/core/article_pruning.py @@ -19,6 +19,8 @@ caption are kept. """ +import time + from sqlalchemy import text, event from sqlalchemy.exc import IntegrityError @@ -182,8 +184,6 @@ def delete_articles_in_batches(db_session, ids, batch_size=BATCH_SIZE, skip_binl connection's user; if the SET is denied we log it and continue with binlog ON rather than fail. Never use it where a replica reads this binlog. """ - import time - if skip_binlog: _disable_binlog_on_connect(db_session) diff --git a/zeeguu/core/audio_lessons/daily_lesson_generator.py b/zeeguu/core/audio_lessons/daily_lesson_generator.py index c58233e8c..4467e216e 100644 --- a/zeeguu/core/audio_lessons/daily_lesson_generator.py +++ b/zeeguu/core/audio_lessons/daily_lesson_generator.py @@ -5,6 +5,7 @@ import os import time from datetime import datetime, timezone, timedelta +import traceback from sqlalchemy.orm import sessionmaker @@ -701,7 +702,6 @@ def generate_daily_lesson( log( f"[generate_daily_lesson] Unexpected error in daily lesson generation: {str(e)}" ) - import traceback log(f"[generate_daily_lesson] Traceback: {traceback.format_exc()}") return {"error": f"Unexpected error: {str(e)}"} diff --git a/zeeguu/core/bookmark_operations/update_bookmark.py b/zeeguu/core/bookmark_operations/update_bookmark.py index 62c76007a..81bbf926e 100644 --- a/zeeguu/core/bookmark_operations/update_bookmark.py +++ b/zeeguu/core/bookmark_operations/update_bookmark.py @@ -8,6 +8,9 @@ See docs/BOOKMARK_UPDATE_LOGIC.md for detailed documentation. """ +from string import punctuation +from urllib.parse import unquote_plus + from zeeguu.logging import log from zeeguu.core.model import Meaning, Text, BookmarkContext, UserWord, Bookmark, Exercise @@ -48,9 +51,6 @@ def parse_update_params(request_json, bookmark): context_identifier, context_type, origin_lang_code, translation_lang_code """ - from urllib.parse import unquote_plus - from string import punctuation - punctuation_extended = "»«" + punctuation return { diff --git a/zeeguu/core/bookmark_quality/negative_qualities.py b/zeeguu/core/bookmark_quality/negative_qualities.py index 0e3f18e47..22943e655 100644 --- a/zeeguu/core/bookmark_quality/negative_qualities.py +++ b/zeeguu/core/bookmark_quality/negative_qualities.py @@ -1,3 +1,6 @@ +import re +import unicodedata + from zeeguu.core.model.meaning import MeaningFrequency, PhraseType from zeeguu.core.model.study_choice import UnfitReason @@ -127,8 +130,6 @@ def word_not_in_context(bookmark): These bookmarks cannot be properly highlighted in exercises. """ - import unicodedata - try: word = bookmark.user_word.meaning.origin.content context = bookmark.get_context() @@ -144,8 +145,6 @@ def word_not_in_context(bookmark): def _split_words_from_context(bookmark): - import re - result = [] bookmark_content_words = re.findall(r"(?u)\w+", bookmark.get_context()) for word in bookmark_content_words: diff --git a/zeeguu/core/content_retriever/article_downloader.py b/zeeguu/core/content_retriever/article_downloader.py index 1825e500f..3719d39df 100644 --- a/zeeguu/core/content_retriever/article_downloader.py +++ b/zeeguu/core/content_retriever/article_downloader.py @@ -12,6 +12,10 @@ from time import time from pymysql import DataError from datetime import datetime, timedelta +from collections import Counter +from io import BytesIO +import html +import traceback from simhash import Simhash from zeeguu.core.content_retriever.crawler_exceptions import * @@ -273,8 +277,6 @@ def _url_after_redirects(url): def _date_in_the_future(time): - from datetime import datetime - return time > datetime.now() @@ -306,8 +308,6 @@ def extract_article_image(np_article): if np_article.top_image != "": # from https://stackoverflow.com/questions/7391945/how-do-i-read-image-data-from-a-url-in-python from PIL import Image - import requests - from io import BytesIO try: # timeout is essential: this lead-image fetch is the one outbound @@ -375,7 +375,6 @@ def download_from_feed( if is_http_error or is_connection_error: log(f"[REQUEST FAILED] Feed unavailable: {feed.title} - {e}") else: - import traceback traceback.print_stack() capture_to_sentry(e) return "" @@ -568,8 +567,6 @@ def download_from_feed( continue except Exception as e: - import traceback - # Disarm before the slow work below (Sentry, rollback): a timeout # raised inside this handler would escape the loop and drop the # rest of the feed. @@ -634,8 +631,6 @@ def download_from_feed( def download_feed_item(session, feed, feed_item, url, crawl_report, simplification_provider=None): - import html - title = html.unescape(feed_item["title"]) log(f" → Processing: {title[:80]}") log(f" URL: {url}") @@ -934,8 +929,6 @@ def add_topics(new_article, feed, url_keywords, session): t.topic.title for t in new_article.topics ] - from collections import Counter - # Add based on KK neighbours: topic_to_assign = get_topic_classification_based_on_similar_content( new_article.content, filter_ids=[new_article.id], verbose=True diff --git a/zeeguu/core/content_retriever/parse_with_readability_server.py b/zeeguu/core/content_retriever/parse_with_readability_server.py index 50372361e..5492ed494 100644 --- a/zeeguu/core/content_retriever/parse_with_readability_server.py +++ b/zeeguu/core/content_retriever/parse_with_readability_server.py @@ -1,5 +1,6 @@ import json import re +import time import newspaper from langdetect import detect @@ -79,7 +80,6 @@ def download_and_parse(url, html_content=None, request_timeout=TIMEOUT_SECONDS): # Call readability server - always use POST with the HTML we already fetched log(f" Calling readability server...") - import time readability_start = time.time() result_dict = None # Initialize to None for exception handling diff --git a/zeeguu/core/elastic/indexing.py b/zeeguu/core/elastic/indexing.py index 77c764b61..a3f55f0f3 100644 --- a/zeeguu/core/elastic/indexing.py +++ b/zeeguu/core/elastic/indexing.py @@ -1,3 +1,5 @@ +import traceback + from zeeguu.core.model import UrlKeyword, Topic from zeeguu.core.model.article_url_keyword_map import ArticleUrlKeywordMap from zeeguu.core.model.article_topic_map import TopicOriginType, ArticleTopicMap @@ -268,7 +270,6 @@ def index_in_elasticsearch(new_article, session): from sentry_sdk import capture_exception capture_exception(e) - import traceback traceback.print_exc() diff --git a/zeeguu/core/emailer/zeeguu_mailer.py b/zeeguu/core/emailer/zeeguu_mailer.py index e337fef34..3a5f5db3a 100644 --- a/zeeguu/core/emailer/zeeguu_mailer.py +++ b/zeeguu/core/emailer/zeeguu_mailer.py @@ -1,4 +1,6 @@ from smtplib import SMTP +from datetime import datetime, timedelta +from email.mime.text import MIMEText import yagmail import zeeguu @@ -38,8 +40,6 @@ def send(self): capture_exception(e) def _content_of_email(self): - from email.mime.text import MIMEText - message = MIMEText(self.message_body) message["From"] = self.our_email message["To"] = self.to_email @@ -163,7 +163,6 @@ def _learner_line(user): def _recent_activity(user, minutes=15, max_lines=12): """What the user did right before sending feedback, oldest first, with consecutive repeats of an event collapsed into one line.""" - from datetime import datetime, timedelta from zeeguu.core.model.user_activitiy_data import UserActivityData try: diff --git a/zeeguu/core/language/ml_cefr_classifier.py b/zeeguu/core/language/ml_cefr_classifier.py index 6ab20e93b..47b4f6086 100644 --- a/zeeguu/core/language/ml_cefr_classifier.py +++ b/zeeguu/core/language/ml_cefr_classifier.py @@ -12,6 +12,7 @@ import os import re import pickle +import traceback import numpy as np from zeeguu.logging import log @@ -161,7 +162,6 @@ def load_model(language_code): return model except Exception as e: - import traceback log(f"Failed to load ML model for {language_code}: {e}") log(f"ML classifier traceback: {traceback.format_exc()}") _MODEL_CACHE[language_code] = None @@ -190,7 +190,6 @@ def predict_cefr_level(content, language_code, fk_difficulty, word_count): try: return model.predict(features)[0] except Exception as e: - import traceback log(f"ML prediction failed for {language_code}: {e}") log(f"ML classifier prediction traceback: {traceback.format_exc()}") return None diff --git a/zeeguu/core/llm_services/simplification_and_classification.py b/zeeguu/core/llm_services/simplification_and_classification.py index 7f4dc3c0d..ed4136cd9 100644 --- a/zeeguu/core/llm_services/simplification_and_classification.py +++ b/zeeguu/core/llm_services/simplification_and_classification.py @@ -13,6 +13,9 @@ import requests import time from collections import namedtuple +from datetime import datetime +import copy +import re from requests.exceptions import Timeout, RequestException from zeeguu.logging import log from zeeguu.core.language.language_check import describe_mismatches @@ -411,8 +414,6 @@ def _clean_text(text): def _strip_markdown_from_summary(text): """Remove markdown bold/italic formatting from summary text.""" - import re - text = re.sub(r"\*\*(.+?)\*\*", r"\1", text) text = re.sub(r"__(.+?)__", r"\1", text) text = re.sub(r"\*(.+?)\*", r"\1", text) @@ -1087,7 +1088,6 @@ def generate(correction): grammar_service = get_grammar_correction_service() # Keep a copy of uncorrected versions for logging - import copy uncorrected_versions = copy.deepcopy(versions) @@ -1511,8 +1511,6 @@ def create_user_specific_simplified_version(session, article, target_level): Returns: Simplified Article object or None if failed """ - from zeeguu.logging import log - log(f"Creating simplified version for article {article.id} at level {target_level}") try: @@ -1592,8 +1590,6 @@ def create_recipient_derivative_from_article( Returns the Article, or ``None`` (recipient opens the original and adapts) when no simplification is needed (level ≥ the article's) or generation fails. """ - from zeeguu.core.model.article import Article - if not article.language: return None source_language_code = article.language.code @@ -1616,11 +1612,9 @@ def create_recipient_derivative_from_article( # Cross language → translate + adapt into the recipient's language, as a # child of the original. Cached under the #translated-from URL key. from zeeguu.core.model import Language - from zeeguu.core.model.url import Url from zeeguu.core.model.source import Source from zeeguu.core.model.source_type import SourceType from zeeguu.core.llm_services.simplification_service import SimplificationService - from datetime import datetime import markdown2 target_lang = Language.find(target_language_code) @@ -1704,14 +1698,10 @@ def create_recipient_derivative(session, upload, target_language_code, target_le language+level reuses the first one's article. Returns the Article, or ``None`` if generation failed (too long / LLM error). """ - from zeeguu.core.model.article import Article from zeeguu.core.model import Language - from zeeguu.core.model.url import Url from zeeguu.core.model.source import Source from zeeguu.core.model.source_type import SourceType from zeeguu.core.llm_services.simplification_service import SimplificationService - from datetime import datetime - from zeeguu.logging import log if not upload.language: log(f"Upload {upload.id} has no language; cannot build recipient derivative") @@ -1803,8 +1793,6 @@ def create_simplified_version_from_upload(session, upload, target_level): is created; the resulting simplified Article stores source_upload_id as the back-reference. Returns the simplified Article or None. """ - from zeeguu.logging import log - log(f"Creating simplified article from upload {upload.id} at level {target_level}") cefr_order = ["A1", "A2", "B1", "B2", "C1", "C2"] diff --git a/zeeguu/core/model/article.py b/zeeguu/core/model/article.py index aab10ffbf..0855445e5 100644 --- a/zeeguu/core/model/article.py +++ b/zeeguu/core/model/article.py @@ -1,6 +1,9 @@ import re import unicodedata -from datetime import datetime +from datetime import date, datetime, timedelta +import difflib +import json +import uuid from sqlalchemy import ( Column, @@ -82,7 +85,6 @@ def strip_trailing_ellipsis_tokens_json(tokens_json): parsing fails or no trailing ellipsis token is found).""" if not tokens_json: return tokens_json - import json try: sentences = json.loads(tokens_json) @@ -378,8 +380,6 @@ def get_content(self): def update_content(self, session, content=None, commit=True): from zeeguu.core.content_retriever import download_and_parse - from zeeguu.core.model.source import Source - from zeeguu.core.model.source_type import SourceType from zeeguu.core.content_quality.quality_filter import ( sufficient_quality_plain_text, ) @@ -599,8 +599,6 @@ def get_tokenized_content(self): from zeeguu.core.tokenization import get_tokenizer, TOKENIZER_MODEL from zeeguu.core.tokenization.zeeguu_tokenizer import ZeeguuTokenizer from zeeguu.core.mwe import tokenize_for_reading - from zeeguu.core.model.db import db - import json content = self.get_content() @@ -748,7 +746,6 @@ def _cache_tokenized_content(self, session, tokenized_data): ArticleTokenizationCache, ) from zeeguu.core.mwe import tokenize_for_reading - import json tokenized_content = json.dumps( { @@ -795,7 +792,6 @@ def article_info_loads(cls): list, so a relationship added to article_info is warmed everywhere. """ from sqlalchemy.orm import selectinload - from zeeguu.core.model.article_topic_map import ArticleTopicMap from zeeguu.core.model.url import Url return [ @@ -1043,10 +1039,7 @@ def create_simplified_version( Article (parent_article) or directly from an ArticleUpload (source_upload). Exactly one of the two must be provided. """ - from zeeguu.core.model.source import Source - from zeeguu.core.model.source_type import SourceType from zeeguu.core.model.url import Url - import uuid if (parent_article is None) == (source_upload is None): raise ValueError("Provide exactly one of parent_article or source_upload") @@ -1548,8 +1541,6 @@ def find_or_create( - else download and parse article """ from zeeguu.core.model import Url, Language - from zeeguu.core.model.source import Source - from zeeguu.core.model.source_type import SourceType from zeeguu.core.content_retriever.article_downloader import ( extract_article_image, @@ -1710,7 +1701,6 @@ def find_by_content_and_source( # If no exact match, try fuzzy title matching for similar titles # This catches typos like "Gittes Von G" vs "Gitte Von G" - import difflib # Get all articles from same feed with similar content length candidates = cls.query.filter( @@ -1744,10 +1734,7 @@ def find_by_content_and_source( @classmethod def all_older_than(cls, days, limit=None): - import datetime - - today = datetime.date.today() - long_ago = today - datetime.timedelta(days) + long_ago = date.today() - timedelta(days) query = cls.query.filter(cls.published_time < long_ago).order_by(desc(cls.id)) if limit: query = query.limit(limit) @@ -1759,10 +1746,7 @@ def all_older_than(cls, days, limit=None): @classmethod def all_younger_than(cls, days): - import datetime - - today = datetime.date.today() - some_time_ago = today - datetime.timedelta(days) + some_time_ago = date.today() - timedelta(days) try: return cls.query.filter(cls.published_time > some_time_ago).all() diff --git a/zeeguu/core/model/article_tokenization_cache.py b/zeeguu/core/model/article_tokenization_cache.py index 84ec24289..f1f95b332 100644 --- a/zeeguu/core/model/article_tokenization_cache.py +++ b/zeeguu/core/model/article_tokenization_cache.py @@ -5,6 +5,7 @@ from sqlalchemy.orm import relationship from sqlalchemy.exc import IntegrityError, OperationalError from datetime import datetime, timedelta +from collections import defaultdict from zeeguu.core.model.db import db log = logging.getLogger(__name__) @@ -109,7 +110,6 @@ def ensure_populated_batch(cls, session, articles): Returns (populated, failed). Caller commits. """ - from collections import defaultdict from zeeguu.core.mwe import enrich_tokens_with_mwe from zeeguu.core.tokenization import get_tokenizer, TOKENIZER_MODEL diff --git a/zeeguu/core/model/audio_lesson_generation_progress.py b/zeeguu/core/model/audio_lesson_generation_progress.py index 7048078d0..a5a11bf8f 100644 --- a/zeeguu/core/model/audio_lesson_generation_progress.py +++ b/zeeguu/core/model/audio_lesson_generation_progress.py @@ -1,4 +1,4 @@ -from datetime import datetime +from datetime import datetime, timedelta from sqlalchemy import Column, Integer, String, DateTime, ForeignKey, Enum from sqlalchemy.orm import relationship @@ -161,7 +161,6 @@ def create_for_user(cls, user, total_segments=1, language=None): @classmethod def cleanup_old_records(cls, hours=1): - from datetime import timedelta cutoff = datetime.utcnow() - timedelta(hours=hours) cls.query.filter(cls.started_at < cutoff).delete() db.session.commit() diff --git a/zeeguu/core/model/bookmark.py b/zeeguu/core/model/bookmark.py index 3f67a4c88..d48d56d21 100644 --- a/zeeguu/core/model/bookmark.py +++ b/zeeguu/core/model/bookmark.py @@ -1,4 +1,5 @@ from datetime import datetime +import time import sqlalchemy from sqlalchemy.exc import NoResultFound @@ -152,7 +153,6 @@ def to_json( def get_source_title(self): from zeeguu.core.model.context_type import ContextType - from zeeguu.core.model.article import Article from zeeguu.core.model.video import Video if self.context.context_type.type == ContextType.ARTICLE_TITLE: @@ -421,7 +421,6 @@ def _self_heal_unanchorable_preferred(self): ): return - from zeeguu.core.model.db import db from zeeguu.logging import log try: @@ -498,7 +497,6 @@ def add_new_exercise_result( ) def get_context_identifier(self): - from zeeguu.core.model.context_identifier import ContextIdentifier from zeeguu.core.model.context_type import ContextType context_type = self.context.context_type.type @@ -601,7 +599,6 @@ def find_or_create( sentence is not a translation. """ from zeeguu.logging import log - import time # Checked up front, before the Meaning and UserWord side effects below if translation_source not in cls.TRANSLATION_SOURCES: diff --git a/zeeguu/core/model/bookmark_context.py b/zeeguu/core/model/bookmark_context.py index 6cf871ac3..84dd1bcf9 100644 --- a/zeeguu/core/model/bookmark_context.py +++ b/zeeguu/core/model/bookmark_context.py @@ -116,7 +116,6 @@ def find_all(cls, text, language): in multiple articles actually... """ from zeeguu.core.util import long_hash - from zeeguu.core.model.new_text import NewText hash_string = long_hash(text) return ( diff --git a/zeeguu/core/model/daily_audio_lesson.py b/zeeguu/core/model/daily_audio_lesson.py index c02b9fb26..29b8e19d3 100644 --- a/zeeguu/core/model/daily_audio_lesson.py +++ b/zeeguu/core/model/daily_audio_lesson.py @@ -1,5 +1,6 @@ import uuid from datetime import datetime +from collections import Counter from sqlalchemy import Column, DateTime, Integer, JSON, ForeignKey, TIMESTAMP from sqlalchemy.orm import relationship @@ -251,8 +252,6 @@ def cefr_level(self): across segments, which tracks the user's generation level (ties resolve to the first segment encountered). """ - from collections import Counter - levels = [] for segment in self.segments: if segment.segment_type == "dialogue_lesson" and segment.audio_lesson_dialogue: diff --git a/zeeguu/core/model/exercise.py b/zeeguu/core/model/exercise.py index 538930ef7..9183854d9 100644 --- a/zeeguu/core/model/exercise.py +++ b/zeeguu/core/model/exercise.py @@ -1,3 +1,4 @@ + import zeeguu.core from zeeguu.core.model.exercise_outcome import ExerciseOutcome @@ -71,8 +72,6 @@ def find(cls, user_id=None): return: list of exercises sorted by time in ascending order """ - from zeeguu.core.model.user_word import UserWord - query = cls.query if user_id is not None: query = query.join(UserWord).filter(UserWord.user_id == user_id) diff --git a/zeeguu/core/model/feed.py b/zeeguu/core/model/feed.py index fffdf70db..8f0a6ee07 100644 --- a/zeeguu/core/model/feed.py +++ b/zeeguu/core/model/feed.py @@ -1,6 +1,7 @@ # -*- coding: utf8 -*- from datetime import datetime, timedelta +import traceback import sqlalchemy.orm.exc from sqlalchemy.orm.exc import NoResultFound @@ -214,8 +215,6 @@ def find_by_id(cls, i): result = cls.query.filter(cls.id == i).one() return result except Exception as e: - import traceback - traceback.print_exc() from sentry_sdk import capture_exception diff --git a/zeeguu/core/model/friendship.py b/zeeguu/core/model/friendship.py index 697037b73..321dfc670 100644 --- a/zeeguu/core/model/friendship.py +++ b/zeeguu/core/model/friendship.py @@ -170,7 +170,6 @@ def find_friend_details(cls, user_id: int, friend_username: str): - or friend request between user_id and friend_username, if exists. Returns None if the target user is not found. """ - from zeeguu.core.model.user import User from zeeguu.core.model.friend_request import FriendRequest friend_user: User = User.find_by_username(friend_username) diff --git a/zeeguu/core/model/meaning.py b/zeeguu/core/model/meaning.py index 2cb21fa4f..957a91182 100644 --- a/zeeguu/core/model/meaning.py +++ b/zeeguu/core/model/meaning.py @@ -272,7 +272,6 @@ def find_or_create( potential_meanings = [] try: from sqlalchemy.orm import aliased, contains_eager - from zeeguu.core.model.db import db OriginPhrase = aliased(Phrase) TranslationPhrase = aliased(Phrase) diff --git a/zeeguu/core/model/phrase.py b/zeeguu/core/model/phrase.py index d147d7611..30cdbd121 100644 --- a/zeeguu/core/model/phrase.py +++ b/zeeguu/core/model/phrase.py @@ -1,3 +1,5 @@ +import time + import sqlalchemy.orm from sqlalchemy.orm.exc import NoResultFound from sqlalchemy import Computed @@ -68,8 +70,6 @@ def _calculate_rank_async(cls, phrase_id): from zeeguu.api.utils.background import run_in_background def calc(pid): - from zeeguu.core.model.db import db - phrase = db.session.query(cls).get(pid) if phrase is None or phrase.rank is not None: return @@ -230,8 +230,6 @@ def find_or_create(cls, session, _word: str, language: Language): print("successfully avoided race condition. nice! ") return w except sqlalchemy.orm.exc.NoResultFound: - import time - time.sleep(0.3) continue break diff --git a/zeeguu/core/model/translation_correction.py b/zeeguu/core/model/translation_correction.py index 3e2a9c52a..0f3a2df4c 100644 --- a/zeeguu/core/model/translation_correction.py +++ b/zeeguu/core/model/translation_correction.py @@ -20,6 +20,8 @@ verdict. """ +from datetime import datetime + from sqlalchemy import Column, DateTime, Enum, ForeignKey, Integer, String, Text, UniqueConstraint, and_, func, or_ from sqlalchemy.orm import aliased, relationship @@ -179,8 +181,6 @@ def judge(session, correction, validator): teachers and the validator disagree before trusting either more. No verdict leaves it as it was, to be asked again the next night. Commits. Returns the verdict, or None.""" - from datetime import datetime - meaning = correction.meaning result = validator.validate_translation( word=meaning.origin.content, diff --git a/zeeguu/core/model/url.py b/zeeguu/core/model/url.py index b357695f0..4e52c5840 100644 --- a/zeeguu/core/model/url.py +++ b/zeeguu/core/model/url.py @@ -153,8 +153,6 @@ def find(cls, url, title=""): @classmethod def extract_canonical_url(self, url: str): - from urllib.parse import urlparse - u = urlparse(url) return f"{u.scheme}://{u.netloc}{u.path}" diff --git a/zeeguu/core/model/user.py b/zeeguu/core/model/user.py index 69a3aec03..f208ef190 100644 --- a/zeeguu/core/model/user.py +++ b/zeeguu/core/model/user.py @@ -757,8 +757,6 @@ def update_password(self, password: str): :param password: str :return: """ - from zeeguu.logging import log - log( f"UPDATE_PASSWORD: user_id={self.id}, email='{self.email}' - Generating bcrypt password hash" ) @@ -781,8 +779,6 @@ def upgrade_to_full_account(self, email, name=None, password=None): :param password: Optional new password (keeps existing if not provided) :raises ValueError: If user is not anonymous or email is invalid/taken """ - from zeeguu.logging import log - if not self.is_anonymous(): raise ValueError("Only anonymous accounts can be upgraded") @@ -949,7 +945,6 @@ def learned_user_words(self, count=50, language_id=None): Unlike learned_bookmarks which can have duplicates for the same word from different contexts, this returns unique user words that have been learned. """ - from sqlalchemy.orm import joinedload from zeeguu.core.model import Phrase, Meaning, UserWord from zeeguu.core.model.bookmark import Bookmark @@ -1311,7 +1306,6 @@ def word_count(self): def _stored_cefr_level(self, language): """The stored level as A0..C2; ValueError when there is none to use.""" - from sqlalchemy.orm.exc import NoResultFound from zeeguu.core.model import UserLanguage try: @@ -1617,8 +1611,6 @@ def exists(cls, user): @classmethod def authorize(cls, email, password): - from zeeguu.logging import log - try: log(f"AUTHORIZE: Looking up user with email '{email}'") user = cls.find(email) diff --git a/zeeguu/core/model/user_article.py b/zeeguu/core/model/user_article.py index 84758eb29..dc0bcbb4c 100644 --- a/zeeguu/core/model/user_article.py +++ b/zeeguu/core/model/user_article.py @@ -1,4 +1,6 @@ from datetime import datetime +import copy +import json from sqlalchemy import ( Column, @@ -426,8 +428,6 @@ def filter_hidden_content(cls, user, content_list): Returns: List of content items with hidden articles removed """ - from zeeguu.core.model import Article - filtered = [] for item in content_list: # Only filter Article objects, preserve other types @@ -686,7 +686,6 @@ def _apply_mwe_overrides_to_summary_tokens(summary_tokens, overrides_by_hash): """ if not overrides_by_hash or not summary_tokens: return summary_tokens - import copy tokens = copy.deepcopy(summary_tokens) for paragraph in tokens: @@ -755,7 +754,6 @@ def _level_matched_title_payload(cls, article, level_row, page): @classmethod def _summary_tokens(cls, article, cache): """Cached summary tokens if we have them, else an immediate cheap pass.""" - import json from zeeguu.core.model.article import strip_trailing_ellipsis_tokens_json from zeeguu.core.model.article_tokenization_cache import ArticleTokenizationCache @@ -771,7 +769,6 @@ def _summary_tokens(cls, article, cache): @classmethod def _title_tokens(cls, article, cache): """Cached title tokens if we have them, else an immediate cheap pass.""" - import json from zeeguu.core.model.article_tokenization_cache import ArticleTokenizationCache if cache and cache.tokenized_title: @@ -896,7 +893,6 @@ def _saved_simplified_by_root(cls, user, articles): lines further down (simplified_pc_by_parent) was already built to avoid -- except that one runs after selection, so it could never help it. """ - from zeeguu.core.model.personal_copy import PersonalCopy from . import db root_ids = {a.parent_article_id or a.id for a in articles} diff --git a/zeeguu/core/model/user_article_broken_report.py b/zeeguu/core/model/user_article_broken_report.py index e1456fbb1..18aa21e8e 100644 --- a/zeeguu/core/model/user_article_broken_report.py +++ b/zeeguu/core/model/user_article_broken_report.py @@ -5,7 +5,7 @@ marked with the USER_REPORTED code. """ -from datetime import datetime +from datetime import datetime, timedelta from sqlalchemy import Column, Integer, DateTime, String, ForeignKey, UniqueConstraint from sqlalchemy.orm import relationship import sqlalchemy @@ -137,7 +137,5 @@ def all_for_user(cls, session, user: User): @classmethod def recent_reports(cls, session, days: int = 7): """Get reports from last N days.""" - from datetime import timedelta - cutoff = datetime.now() - timedelta(days=days) return cls.query.filter(cls.report_time >= cutoff).all() diff --git a/zeeguu/core/model/user_word.py b/zeeguu/core/model/user_word.py index 0acc3dff9..a5db051f6 100644 --- a/zeeguu/core/model/user_word.py +++ b/zeeguu/core/model/user_word.py @@ -140,8 +140,6 @@ def validate_data_integrity(self, strict=False): so when strict=False it MUST NOT modify the database permanently. Any repairs should happen during write operations. """ - from zeeguu.logging import log - bookmarks = self.bookmarks() # Check 1: Verify preferred_bookmark actually belongs to this UserWord @@ -295,7 +293,6 @@ def as_dictionary(self, schedule=None, pre_tokenized_context=None, with_context_ if self.is_user_added: scheduling_reason = "manually_added" elif next_practice_time: - from datetime import datetime # Use naive datetime to match next_practice_time from database now = datetime.now() diff --git a/zeeguu/core/semantic_search/elastic_semantic_search.py b/zeeguu/core/semantic_search/elastic_semantic_search.py index f075eb4c3..0d97e4604 100644 --- a/zeeguu/core/semantic_search/elastic_semantic_search.py +++ b/zeeguu/core/semantic_search/elastic_semantic_search.py @@ -1,3 +1,6 @@ +from collections import Counter +from pprint import pprint + from elasticsearch import Elasticsearch from elastic_transport import ConnectionError @@ -105,8 +108,6 @@ def get_topic_classification_based_on_similar_content( filter_ids: list[int] = None, verbose=False, ): - from collections import Counter - if filter_ids is None: filter_ids = [] @@ -118,8 +119,6 @@ def get_topic_classification_based_on_similar_content( topics_counter = Counter(neighbouring_topics) if verbose: - from pprint import pprint - pprint(topics_counter) top_topic, count = topics_counter.most_common(1)[0] diff --git a/zeeguu/core/test/test_article.py b/zeeguu/core/test/test_article.py index aa8eb6c66..a5e83a597 100644 --- a/zeeguu/core/test/test_article.py +++ b/zeeguu/core/test/test_article.py @@ -1,8 +1,10 @@ +from datetime import datetime, timedelta from unittest import TestCase from zeeguu.core.test.model_test_mixin import ModelTestMixIn import zeeguu.core +from zeeguu.core.model import Article from zeeguu.core.model.article_topic_map import TopicOriginType from zeeguu.core.test.rules.article_rule import ArticleRule from zeeguu.core.test.rules.topic_rule import TopicRule @@ -16,6 +18,17 @@ def setUp(self): self.article1 = ArticleRule().article self.article2 = ArticleRule().article + def test_all_older_and_younger_than_split_on_published_time(self): + self.article1.published_time = datetime.now() - timedelta(days=10) + self.article2.published_time = datetime.now() + session.commit() + + older = Article.all_older_than(5) + younger = Article.all_younger_than(5) + + assert self.article1 in older and self.article2 not in older + assert self.article2 in younger and self.article1 not in younger + def test_article_topics_and_basics(self): # articles are different assert self.article1.title != self.article2.title diff --git a/zeeguu/core/test/test_stdlib_imports_at_top.py b/zeeguu/core/test/test_stdlib_imports_at_top.py new file mode 100644 index 000000000..638f5f2e9 --- /dev/null +++ b/zeeguu/core/test/test_stdlib_imports_at_top.py @@ -0,0 +1,50 @@ +"""Standard-library imports belong at the top of the file, where the reader +sees what a module depends on. Unlike our own modules, they can't be part of +an import cycle, so nothing forces them into a function.""" +import ast +import sys +from pathlib import Path + +import zeeguu + +ROOT = Path(zeeguu.__file__).parent + + +def _names(node): + """The name each alias binds, with the statement that binds it.""" + if isinstance(node, ast.Import): + return {a.asname or a.name.split(".")[0]: a.name for a in node.names} + return {a.asname or a.name: f"{node.module}.{a.name}" for a in node.names} + + +def _is_stdlib(node): + if isinstance(node, ast.ImportFrom): + return not node.level and node.module.split(".")[0] in sys.stdlib_module_names + return node.names[0].name.split(".")[0] in sys.stdlib_module_names + + +def _function_level_stdlib_imports(tree): + for function in ast.walk(tree): + if isinstance(function, (ast.FunctionDef, ast.AsyncFunctionDef)): + for node in ast.walk(function): + if isinstance(node, (ast.Import, ast.ImportFrom)) and _is_stdlib(node): + yield node + + +def _offenders(path): + tree = ast.parse(path.read_text()) + at_top = {} + for node in tree.body: + if isinstance(node, (ast.Import, ast.ImportFrom)): + at_top.update(_names(node)) + + for node in _function_level_stdlib_imports(tree): + # `import datetime` next to a top-level `from datetime import datetime` + # can't move: one name, two meanings. + if all(at_top.get(name, source) == source for name, source in _names(node).items()): + yield f"{path.relative_to(ROOT.parent)}:{node.lineno}" + + +def test_stdlib_imports_are_at_the_top_of_the_file(): + sources = [p for p in ROOT.rglob("*.py") if "test" not in p.relative_to(ROOT).parts] + assert sorted(o for p in sources for o in _offenders(p)) == [] diff --git a/zeeguu/core/translation_services/deepl_translate.py b/zeeguu/core/translation_services/deepl_translate.py index 01d3c8e99..4decf4a60 100644 --- a/zeeguu/core/translation_services/deepl_translate.py +++ b/zeeguu/core/translation_services/deepl_translate.py @@ -10,6 +10,7 @@ import logging import os from typing import Optional +from html import escape, unescape logger = logging.getLogger(__name__) @@ -111,8 +112,6 @@ def deepl_translate_article(title: str, markdown: str, target_language: str) -> if client is None: return None - from html import escape, unescape - from zeeguu.core.content_cleaning.html_to_markdown import html_to_markdown, markdown_to_html try: diff --git a/zeeguu/core/translation_services/translator.py b/zeeguu/core/translation_services/translator.py index 057ed1383..5a24b494f 100644 --- a/zeeguu/core/translation_services/translator.py +++ b/zeeguu/core/translation_services/translator.py @@ -25,6 +25,7 @@ from python_translators.translators.wordnik_translator import WordnikTranslator import logging +from concurrent.futures import ThreadPoolExecutor, as_completed logging.getLogger("python_translators").setLevel(logging.CRITICAL) @@ -308,7 +309,6 @@ def translate_mwe_phrase(word, from_lang, to_lang): dict with 'translation', 'source', 'likelihood' keys, or None """ from python_translators.translation_query import TranslationQuery - from python_translators.factories.google_translator_factory import GoogleTranslatorFactory try: # Use contextless translation for the phrase @@ -384,8 +384,6 @@ def _vote_translation(data): Falls back to Microsoft contextual if all three contextual providers fail. """ - from concurrent.futures import ThreadPoolExecutor, as_completed - def azure_with_fallback(): # Azure alignment doesn't support every language pair (e.g. fr→nl # returns no alignment data). When it returns None, fall back to @@ -642,7 +640,6 @@ def get_translations_streaming(word, context, from_lang, to_lang, is_separated_m one LLM call on every menu open of a phrase. """ from python_translators.translation_query import TranslationQuery - from concurrent.futures import ThreadPoolExecutor, as_completed seen_translations = set() @@ -786,8 +783,6 @@ def get_all_translations(word, context, from_lang, to_lang, is_separated_mwe=Fal else: # For contiguous MWEs: use phrase translation plus other services and LLM if " " in word: - from concurrent.futures import ThreadPoolExecutor, as_completed - query = TranslationQuery(word, "", "", 1) data = { "source_language": from_lang, @@ -835,7 +830,6 @@ def get_all_translations(word, context, from_lang, to_lang, is_separated_mwe=Fal } # Run translation services in parallel - from concurrent.futures import ThreadPoolExecutor, as_completed def get_azure(): return azure_alignment_contextual_translate(data) diff --git a/zeeguu/operations/crawler/crawl_report.py b/zeeguu/operations/crawler/crawl_report.py index d6206e8a8..32482e243 100644 --- a/zeeguu/operations/crawler/crawl_report.py +++ b/zeeguu/operations/crawler/crawl_report.py @@ -4,6 +4,7 @@ import inspect import json import pathlib +from pprint import pprint STR_DATETIME_FORMAT = "%d_%m_%y_%H_%M_%S" CRAWL_REPORT_DATA = os.environ.get( @@ -231,8 +232,6 @@ def get_total_removed_sents_counts(self, langs_to_load: list[str] = None): feed_dict["article_report"]["sents_removed"] ) except Exception as e: - from pprint import pprint - pprint(feed_dict) print(e, type(e)) input("Continue?") diff --git a/zeeguu/operations/crawler/feed_retrieval.py b/zeeguu/operations/crawler/feed_retrieval.py index 91101d797..ffdcca912 100755 --- a/zeeguu/operations/crawler/feed_retrieval.py +++ b/zeeguu/operations/crawler/feed_retrieval.py @@ -20,6 +20,7 @@ from sqlalchemy.exc import PendingRollbackError from time import time +import datetime import zeeguu.core @@ -90,8 +91,6 @@ def retrieve_articles_for_language(language_code, send_email=False): log("sending summary email") - import datetime - mailer = ZeeguuMailer( f"{language.name} Crawl Summary " + datetime.datetime.now().strftime("%H:%M"),