diff --git a/.gitignore b/.gitignore index 7110ae72..bf70d4a8 100644 --- a/.gitignore +++ b/.gitignore @@ -58,4 +58,4 @@ z/jsons/pages.json throttle.ctrl scan.py .env -/src/db/tools1 +/src/db/tools diff --git a/src/md_core/mdpy/fix_duplicate.py b/src1/md_core/mdpy/fix_duplicate.py similarity index 100% rename from src/md_core/mdpy/fix_duplicate.py rename to src1/md_core/mdpy/fix_duplicate.py diff --git a/src/md_core/mdpy/fixred.py b/src1/md_core/mdpy/fixred.py similarity index 100% rename from src/md_core/mdpy/fixred.py rename to src1/md_core/mdpy/fixred.py diff --git a/src/md_core/mdpy/imp.py b/src1/md_core/mdpy/imp.py similarity index 100% rename from src/md_core/mdpy/imp.py rename to src1/md_core/mdpy/imp.py diff --git a/src/md_core/mdpy/mdjobs.sh b/src1/md_core/mdpy/mdjobs.sh similarity index 100% rename from src/md_core/mdpy/mdjobs.sh rename to src1/md_core/mdpy/mdjobs.sh diff --git a/src/md_core/mdpy/orred.py b/src1/md_core/mdpy/orred.py similarity index 100% rename from src/md_core/mdpy/orred.py rename to src1/md_core/mdpy/orred.py diff --git a/src/md_core/mdpy/red.py b/src1/md_core/mdpy/red.py similarity index 100% rename from src/md_core/mdpy/red.py rename to src1/md_core/mdpy/red.py diff --git a/src/md_core/mdpy/replace1.py b/src1/md_core/mdpy/replace1.py similarity index 100% rename from src/md_core/mdpy/replace1.py rename to src1/md_core/mdpy/replace1.py diff --git a/toolforge/deploy_repo.sh b/toolforge/deploy_repo.sh new file mode 100644 index 00000000..6a3a5eef --- /dev/null +++ b/toolforge/deploy_repo.sh @@ -0,0 +1,131 @@ +#!/bin/bash +# toolforge-jobs run installar --image python3.11 --command "~/shs/update_ArWikiCats.sh" --wait +# sed -i 's/\r$//' ~/shs/*.sh && chmod +x ~/shs/*.sh +# sed -i 's/\r$//' ~/jobs/*.sh && chmod +x ~/jobs/*.sh + +# use bash strict mode +set -euo pipefail + +REPO_NAME="$1" # e.g. cats_maker +TARGET_DIR="$2" # e.g. bots/cats_maker + +if [ -z "${1:-}" ] || [ -z "${2:-}" ]; then + echo "Usage: $0 [branch]" >&2 + exit 1 +fi + +COPY_TO_TARGET="${COPY_TO_TARGET:-}" + +BRANCH="${3:-main}" +SUB_DIR_COPY="${SUB_DIR_COPY:-}" +USER_NAME="${USER_NAME:-MrIbrahem}" +BASE_DIR="${BASE_DIR:-${HOME}}" + +CLONE_DIR="${BASE_DIR}/${REPO_NAME}_tmp" + +# Optional clean of TARGET_DIR +CLEAN_INSTALL="${CLEAN_INSTALL:-0}" + +# Define the centralized archive directory in the home folder +OLD_REPOS_BASE="${HOME}/old_repos" + +# Optional clean of jsons files before copy to avoid issues with old jsons files +REMOVE_SRC_JSONS_BEFORE_COPY="${REMOVE_SRC_JSONS_BEFORE_COPY:-0}" + +# Ensure the Python3 binary exists before compiling +PYTHON_BIN="${PYTHON_BIN:-$HOME/local/bin/python3}" + +COMPILE_PYTHON_FILES="${COMPILE_PYTHON_FILES:-0}" + +REPO_URL="https://github.com/${USER_NAME}/${REPO_NAME}.git" + +# Navigate to the project directory +cd "$BASE_DIR" || exit + +echo ">>> Deploying ${REPO_NAME} (branch: ${BRANCH})" + +# Remove any existing clone directory +rm -rf "$CLONE_DIR" + +echo ">>> clone --branch ${BRANCH}" +echo "${REPO_URL}" + +git clone --branch "$BRANCH" "$REPO_URL" "$CLONE_DIR" + +rm -rf "$CLONE_DIR/.git" + +if [ "$CLEAN_INSTALL" = "1" ] && [ -d "$TARGET_DIR" ]; then + echo ">>> Clean install enabled" + + # Ensure the directory is writable before moving/deleting + chmod -R u+w "$TARGET_DIR" 2>/dev/null || true + + # Ensure the archive directory exists + mkdir -p "$OLD_REPOS_BASE" + + # Extract the directory name (e.g., cats_maker) to use in the archive name + DIR_NAME=$(basename "$TARGET_DIR") + + # Set the destination path with a timestamp (e.g., ~/old_repos/src_backup_1715000) + DESTINATION="${OLD_REPOS_BASE}/${REPO_NAME}_${DIR_NAME}_backup_$(date +%s%N)_$$" + + echo ">>> Archiving old version to: $DESTINATION" + mv "$TARGET_DIR" "$DESTINATION" +fi + +mkdir -p "$TARGET_DIR" + +# Copy +SRC_DIR="$CLONE_DIR" + +if [ -n "$SUB_DIR_COPY" ]; then + SRC_DIR="$CLONE_DIR/$SUB_DIR_COPY" + echo ">>> Copying sub-directory: $SRC_DIR" + + if [ ! -d "$SRC_DIR" ]; then + echo ">>> ERROR: $SRC_DIR does not exist in repo" + echo ">>> exit." + exit 1 + fi +fi + +if [ "$REMOVE_SRC_JSONS_BEFORE_COPY" = "1" ]; then + echo ">>> Removing old JSON files from $SRC_DIR" + find "$SRC_DIR" -name "*.json" -delete +fi + +cp -rf "$SRC_DIR/"* "$TARGET_DIR/" -v + +if [ -n "$COPY_TO_TARGET" ]; then + echo ">>> Copying additional file: $COPY_TO_TARGET" + cp -f "$CLONE_DIR/$COPY_TO_TARGET" "$TARGET_DIR/" -v +fi + +if [ "$COMPILE_PYTHON_FILES" = "1" ]; then + echo ">>> Compiling Python files to .pyc" + + # Compile all Python files to .pyc explicitly to avoid race conditions + # Ensure the Python3 binary exists before compiling + if command -v "$PYTHON_BIN" >/dev/null 2>&1; then + export PYTHONDONTWRITEBYTECODE=1 + + # Compile all Python files in the TARGET_DIR + "$PYTHON_BIN" -m compileall -q -f "$TARGET_DIR" + unset PYTHONDONTWRITEBYTECODE + else + echo ">>> Warning: Python binary not found at $PYTHON_BIN, skipping bytecode compilation" + fi + + # Optional: Set permissions + # chmod -R 770 "$TARGET_DIR" + + find "$TARGET_DIR" -type f ! -name "*.pyc" -exec chmod 770 {} \; +fi + +# Optional: Install dependencies +#"$PYTHON_BIN" -m pip install -r "$TARGET_DIR/requirements.in" + +# Remove the "$CLONE_DIR" directory. +rm -rf "$CLONE_DIR" + +echo ">>> ${REPO_NAME} deployed successfully" diff --git a/toolforge/mdwiki-jobs.yaml b/toolforge/mdwiki-jobs.yaml new file mode 100644 index 00000000..d4c238c9 --- /dev/null +++ b/toolforge/mdwiki-jobs.yaml @@ -0,0 +1,115 @@ +# sed -i 's/\r$//' ~/shs/*.sh && chmod +x ~/shs/*.sh +# sed -i 's/\r$//' ~/jobs/*.sh && chmod +x ~/jobs/*.sh + +- name: medwiki + command: "$HOME/local/bin/python3 c9/pwb.py copy_to_en/revid" + schedule: "1 * * * 1" + image: python3.9 + emails: none + +# rm $HOME/cookies/* -v +- name: rmv + command: "$HOME/jobs/rmc.sh" + schedule: "0 1 * * *" + image: bullseye + emails: none + +# sql backup +# tfj run dbbackup --image mariadb --command "$HOME/jobs/db_backup_gzip.sh" +- name: backup + command: "$HOME/jobs/db_backup_gzip.sh" + schedule: "@daily" + image: mariadb + emails: none + +# $HOME/local/bin/python3 c9/pwb.py td_core/mdcount/countref newpages +# $HOME/local/bin/python3 c9/pwb.py td_core/mdcount/countref sql +# $HOME/local/bin/python3 c9/pwb.py td_core/mdcount/words newpages +- name: countref + command: "$HOME/jobs/count_refs_words.sh" + schedule: "30 3 * * *" + image: tf-python39 + emails: none + +# $HOME/local/bin/python3 c9/pwb.py md_core/fix_cs1/bot +# $HOME/local/bin/python3 c9/pwb.py md_core/fix_cs1/fix_cs_params/bot +- name: fixrefs + command: "$HOME/jobs/fix_refs.sh" + schedule: "@weekly" + image: tf-python39 + emails: none + +# Find qids and fix qids +# $HOME/local/bin/python3 c9/pwb.py td_core/td_other_qids/make_list all add +# $HOME/local/bin/python3 c9/pwb.py td_core/mdpages/find_qids redirects add +# $HOME/local/bin/python3 c9/pwb.py td_core/db_work/get_red +# $HOME/local/bin/python3 c9/pwb.py td_core/td_other_qids/fix_qids fix all +# $HOME/local/bin/python3 c9/pwb.py md_core/p11143_bot/bot all add addq +# $HOME/local/bin/python3 c9/pwb.py md_core/unlinked_wb/bot + +- name: qids1 + command: "$HOME/pybot/td_core/mdpages/findqids.sh" + schedule: "40 */5 * * *" + image: tf-python39 + emails: none + +- name: days7 + command: "$HOME/local/bin/python3 c9/pwb.py td_core/db_work/days_7" + schedule: "1 1 * * 1" + image: python3.9 + emails: none + +# make views +- name: views1 + command: "$HOME/local/bin/python3 c9/pwb.py td_core/mdpyget/sqlviews_new" + schedule: "15 6 * * *" + image: python3.9 + emails: none + +# important +- name: getas + command: "$HOME/local/bin/python3 c9/pwb.py td_core/mdpyget/getas" + schedule: "12 2 * * *" + image: python3.9 + emails: none + +- name: enpv + command: "$HOME/local/bin/python3 c9/pwb.py td_core/mdpyget/enwiki_views" + schedule: "1 1 * * 3" + image: python3.9 + emails: none + +# find old pages +- name: updates + command: "$HOME/local/bin/python3 c9/pwb.py md_core/updates/io && $HOME/local/bin/python3 c9/pwb.py md_core/updates/listo" + schedule: "55 21 * * *" + image: python3.9 + emails: none + +- name: upstats + command: "$HOME/local/bin/python3 c9/pwb.py md_core/updates/Medicine_articles" + schedule: "1 1 1 * *" + image: python3.9 + emails: none + + +# $HOME/local/bin/python3 c9/pwb.py td_core/mdpages/cashwd +# $HOME/local/bin/python3 c9/pwb.py td_core/copy_data/by_qid/sitelinks +# $HOME/local/bin/python3 c9/pwb.py md_core_helps/apis/cat_cach newlist + +# $HOME/local/bin/python3 c9/pwb.py td_core/copy_data/by_title/all_articles +# $HOME/local/bin/python3 c9/pwb.py td_core/wd_works/recheck + +# $HOME/local/bin/python3 c9/pwb.py td_core/db_work/check_titles + +- name: cach-jobs + command: "$HOME/jobs/cach_jobs.sh" + schedule: "10 6 * * *" + image: tf-python39 + emails: none + +- name: fixusers + command: "$HOME/local/bin/python3 c9/pwb.py td_core/fix_user_pages/bot" + schedule: "@daily" + image: python3.9 + emails: none diff --git a/mdwiki-jobs.yaml.list b/toolforge/mdwiki-jobs.yaml.list similarity index 100% rename from mdwiki-jobs.yaml.list rename to toolforge/mdwiki-jobs.yaml.list diff --git a/toolforge/update_pybot.sh b/toolforge/update_pybot.sh new file mode 100644 index 00000000..8990bf0f --- /dev/null +++ b/toolforge/update_pybot.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# toolforge-jobs run updatepybot --image python3.11 --command "~/shs/update_pybot.sh" --wait + +export USER_NAME="Mdwiki-TD" +export SUB_DIR_COPY="src" +export CLEAN_INSTALL=0 + +# Optional clean of jsons files before copy to avoid issues with old jsons files +export REMOVE_SRC_JSONS_BEFORE_COPY=0 + +# Ensure the Python3 binary exists before compiling +export PYTHON_BIN="$HOME/local/bin/python3" +export COMPILE_PYTHON_FILES=1 + +# additional file to copy to TARGET_DIR +export COPY_TO_TARGET="" + +REPO_NAME="mdwiki-python-files" +TARGET_DIR="pybot" +BRANCH="${1:-main}" + +# Run deploy +$HOME/shs/deploy_repo.sh "$REPO_NAME" "$TARGET_DIR" "$BRANCH"