From edb24dde45a9346c4dc2473c14601a5725317a95 Mon Sep 17 00:00:00 2001 From: UsatovPavel Date: Tue, 29 Sep 2026 00:02:04 +0300 Subject: [PATCH] fix(ml): finish window snapshot --- .github/workflows/deploy.yml | 6 ++++-- deploy/backup-equipment-window.sh | 24 ++++++++++++------------ tests/test_equipment_window_backup.py | 23 +++++++++++++++++++---- 3 files changed, 35 insertions(+), 18 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index a4be9f6..fb26fb1 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -78,8 +78,10 @@ jobs: # inference:current. При ошибке новый образ не запускается. - name: Preserve equipment window run: | - ssh vps "sudo -n env SHA=${{ github.sha }} REMOTE_DIR=\$HOME/$REMOTE_DIR bash -s" \ - < deploy/backup-equipment-window.sh + result=$(ssh vps "sudo -n env SHA=${{ github.sha }} REMOTE_DIR=\$HOME/$REMOTE_DIR bash -s" \ + < deploy/backup-equipment-window.sh) + printf '%s\n' "$result" + grep -Fq 'equipment window saved:' <<<"$result" # Прежний current — точка отката. Повторный прогон того же sha его # не трогает, иначе previous совпал бы с новым образом. diff --git a/deploy/backup-equipment-window.sh b/deploy/backup-equipment-window.sh index f99cc1e..68d2ba7 100644 --- a/deploy/backup-equipment-window.sh +++ b/deploy/backup-equipment-window.sh @@ -10,12 +10,12 @@ backup_mount=${BACKUP_MOUNT:-/mnt/backups} mountpoint -q "$backup_mount" || { echo "$backup_mount is not mounted" >&2; exit 1; } cd "${REMOTE_DIR:-$HOME/backend}" -services=$(docker compose config --services) +services=$(docker compose config --services &2 exit 1 fi -container=$(docker compose ps -q inference) +container=$(docker compose ps -q inference &2; exit 1; } [[ $(docker inspect -f '{{.State.Running}}' "$container") == true ]] || { echo 'equipment inference is not running' >&2 @@ -53,7 +53,7 @@ cleanup() { local result=$? trap - EXIT if (( stopped )); then - if ! docker compose start inference || ! verify_old_running; then + if ! docker compose start inference &2 result=1 fi @@ -88,28 +88,28 @@ verify_old_running() { # Stop before copying the cursor so it matches a completed batch. stopped=1 -docker compose stop inference +docker compose stop inference "$tmp/runtime.tar" -docker cp "$container:/app/data/best_model.cbm" "$tmp/equipment_model.cbm" -docker cp "$container:/app/data/feature_encoding.json" "$tmp/feature_encoding.json" -docker cp "$container:/app/config.yml" "$tmp/config.yml" +docker cp "$container:/app/runtime/." - > "$tmp/runtime.tar" /dev/null -docker compose start inference +docker compose start inference &2 exit 1 } -db=$(docker compose exec -T postgres printenv POSTGRES_DB | tr -d '\r') -dbuser=$(docker compose exec -T postgres printenv POSTGRES_USER | tr -d '\r') +db=$(docker compose exec -T postgres printenv POSTGRES_DB &2; exit 1; } copy_query() { local query=$1 destination=$2 docker compose exec -T postgres \ psql -X -q -v ON_ERROR_STOP=1 -U "$dbuser" -d "$db" \ - -c "COPY ($query) TO STDOUT WITH (FORMAT csv, HEADER true)" \ + -c "COPY ($query) TO STDOUT WITH (FORMAT csv, HEADER true)" "$tmp/$destination" gzip -t "$tmp/$destination" } diff --git a/tests/test_equipment_window_backup.py b/tests/test_equipment_window_backup.py index 54f3dc5..8bd2e39 100644 --- a/tests/test_equipment_window_backup.py +++ b/tests/test_equipment_window_backup.py @@ -37,7 +37,9 @@ def _mock_tools(tmp_path: Path) -> Path: echo 'column "journal_is_alarm" does not exist'; fi ;; 'image inspect -f {{.Id}} inference:current') echo sha256:old-image ;; 'compose stop inference') : ;; - 'compose start inference') [ "${MOCK_FAIL_START:-0}" != 1 ] ;; + 'compose start inference') + if [ "${MOCK_CONSUME_STDIN:-0}" = 1 ]; then cat >/dev/null; fi + [ "${MOCK_FAIL_START:-0}" != 1 ] ;; 'cp old-container:/app/runtime/. -') if [ "${MOCK_FAIL_CP:-0}" = 1 ]; then exit 2; fi tar -C "$MOCK_RUNTIME" -cf - . ;; @@ -60,7 +62,7 @@ def _mock_tools(tmp_path: Path) -> Path: def _run(tmp_path: Path, fail_copy: bool = False, fail_start: bool = False, no_inference: bool = False, schema_error: bool = False, - healthy_error: bool = False + healthy_error: bool = False, streamed: bool = False ) -> tuple[subprocess.CompletedProcess[str], Path]: bin_dir = _mock_tools(tmp_path) runtime = tmp_path / "runtime" @@ -86,9 +88,13 @@ def _run(tmp_path: Path, fail_copy: bool = False, fail_start: bool = False, "MOCK_NO_INFERENCE": "1" if no_inference else "0", "MOCK_SCHEMA_ERROR": "1" if schema_error else "0", "MOCK_HEALTHY_ERROR": "1" if healthy_error else "0", + "MOCK_CONSUME_STDIN": "1" if streamed else "0", } - result = subprocess.run(["bash", str(SCRIPT)], env=env, text=True, - capture_output=True, check=False) + result = subprocess.run( + ["bash", "-s"] if streamed else ["bash", str(SCRIPT)], + env=env, input=SCRIPT.read_text() if streamed else None, + text=True, capture_output=True, check=False, + ) return result, backup @@ -114,6 +120,15 @@ def test_snapshot_contains_cursor_window_and_model(tmp_path: Path) -> None: assert calls.index("compose stop inference") < calls.index("compose start inference") +def test_streamed_deploy_script_completes_snapshot(tmp_path: Path) -> None: + result, backup = _run(tmp_path, streamed=True) + assert result.returncode == 0, result.stderr + assert "equipment window saved:" in result.stdout + snapshots = list((backup / "inference-windows").glob("equipment-*")) + assert len(snapshots) == 1 + assert (snapshots[0] / "SHA256SUMS").is_file() + + def test_failed_cursor_copy_restarts_old_service(tmp_path: Path) -> None: result, backup = _run(tmp_path, fail_copy=True) assert result.returncode != 0