Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions .github/workflows/deploy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -78,8 +78,10 @@ jobs:
# inference:current. При ошибке новый образ не запускается.
- name: Preserve equipment window
run: |
ssh vps "sudo -n env SHA=${{ github.sha }} REMOTE_DIR=\$HOME/$REMOTE_DIR bash -s" \
< deploy/backup-equipment-window.sh
result=$(ssh vps "sudo -n env SHA=${{ github.sha }} REMOTE_DIR=\$HOME/$REMOTE_DIR bash -s" \
< deploy/backup-equipment-window.sh)
printf '%s\n' "$result"
grep -Fq 'equipment window saved:' <<<"$result"

# Прежний current — точка отката. Повторный прогон того же sha его
# не трогает, иначе previous совпал бы с новым образом.
Expand Down
24 changes: 12 additions & 12 deletions deploy/backup-equipment-window.sh
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,12 @@ backup_mount=${BACKUP_MOUNT:-/mnt/backups}
mountpoint -q "$backup_mount" || { echo "$backup_mount is not mounted" >&2; exit 1; }
cd "${REMOTE_DIR:-$HOME/backend}"

services=$(docker compose config --services)
services=$(docker compose config --services </dev/null)
if ! grep -qx inference <<<"$services"; then
echo 'equipment inference is not configured; window cannot be preserved' >&2
exit 1
fi
container=$(docker compose ps -q inference)
container=$(docker compose ps -q inference </dev/null)
[[ -n $container ]] || { echo 'equipment inference container is missing' >&2; exit 1; }
[[ $(docker inspect -f '{{.State.Running}}' "$container") == true ]] || {
echo 'equipment inference is not running' >&2
Expand Down Expand Up @@ -53,7 +53,7 @@ cleanup() {
local result=$?
trap - EXIT
if (( stopped )); then
if ! docker compose start inference || ! verify_old_running; then
if ! docker compose start inference </dev/null || ! verify_old_running; then
echo 'CRITICAL: old equipment inference did not recover after backup failure' >&2
result=1
fi
Expand Down Expand Up @@ -88,28 +88,28 @@ verify_old_running() {

# Stop before copying the cursor so it matches a completed batch.
stopped=1
docker compose stop inference
docker compose stop inference </dev/null
snapshot_at=$(date -u +%Y-%m-%dT%H:%M:%S.%NZ)
docker cp "$container:/app/runtime/." - > "$tmp/runtime.tar"
docker cp "$container:/app/data/best_model.cbm" "$tmp/equipment_model.cbm"
docker cp "$container:/app/data/feature_encoding.json" "$tmp/feature_encoding.json"
docker cp "$container:/app/config.yml" "$tmp/config.yml"
docker cp "$container:/app/runtime/." - > "$tmp/runtime.tar" </dev/null
docker cp "$container:/app/data/best_model.cbm" "$tmp/equipment_model.cbm" </dev/null
docker cp "$container:/app/data/feature_encoding.json" "$tmp/feature_encoding.json" </dev/null
docker cp "$container:/app/config.yml" "$tmp/config.yml" </dev/null
tar -tf "$tmp/runtime.tar" > /dev/null
docker compose start inference
docker compose start inference </dev/null
stopped=0
[[ $(docker inspect -f '{{.State.Running}}' "$container") == true ]] || {
echo 'equipment inference failed to restart after snapshot' >&2
exit 1
}

db=$(docker compose exec -T postgres printenv POSTGRES_DB | tr -d '\r')
dbuser=$(docker compose exec -T postgres printenv POSTGRES_USER | tr -d '\r')
db=$(docker compose exec -T postgres printenv POSTGRES_DB </dev/null | tr -d '\r')
dbuser=$(docker compose exec -T postgres printenv POSTGRES_USER </dev/null | tr -d '\r')
[[ -n $db && -n $dbuser ]] || { echo 'database name/user missing' >&2; exit 1; }
copy_query() {
local query=$1 destination=$2
docker compose exec -T postgres \
psql -X -q -v ON_ERROR_STOP=1 -U "$dbuser" -d "$db" \
-c "COPY ($query) TO STDOUT WITH (FORMAT csv, HEADER true)" \
-c "COPY ($query) TO STDOUT WITH (FORMAT csv, HEADER true)" </dev/null \
| gzip -1 > "$tmp/$destination"
gzip -t "$tmp/$destination"
}
Expand Down
23 changes: 19 additions & 4 deletions tests/test_equipment_window_backup.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,9 @@ def _mock_tools(tmp_path: Path) -> Path:
echo 'column "journal_is_alarm" does not exist'; fi ;;
'image inspect -f {{.Id}} inference:current') echo sha256:old-image ;;
'compose stop inference') : ;;
'compose start inference') [ "${MOCK_FAIL_START:-0}" != 1 ] ;;
'compose start inference')
if [ "${MOCK_CONSUME_STDIN:-0}" = 1 ]; then cat >/dev/null; fi
[ "${MOCK_FAIL_START:-0}" != 1 ] ;;
'cp old-container:/app/runtime/. -')
if [ "${MOCK_FAIL_CP:-0}" = 1 ]; then exit 2; fi
tar -C "$MOCK_RUNTIME" -cf - . ;;
Expand All @@ -60,7 +62,7 @@ def _mock_tools(tmp_path: Path) -> Path:

def _run(tmp_path: Path, fail_copy: bool = False, fail_start: bool = False,
no_inference: bool = False, schema_error: bool = False,
healthy_error: bool = False
healthy_error: bool = False, streamed: bool = False
) -> tuple[subprocess.CompletedProcess[str], Path]:
bin_dir = _mock_tools(tmp_path)
runtime = tmp_path / "runtime"
Expand All @@ -86,9 +88,13 @@ def _run(tmp_path: Path, fail_copy: bool = False, fail_start: bool = False,
"MOCK_NO_INFERENCE": "1" if no_inference else "0",
"MOCK_SCHEMA_ERROR": "1" if schema_error else "0",
"MOCK_HEALTHY_ERROR": "1" if healthy_error else "0",
"MOCK_CONSUME_STDIN": "1" if streamed else "0",
}
result = subprocess.run(["bash", str(SCRIPT)], env=env, text=True,
capture_output=True, check=False)
result = subprocess.run(
["bash", "-s"] if streamed else ["bash", str(SCRIPT)],
env=env, input=SCRIPT.read_text() if streamed else None,
text=True, capture_output=True, check=False,
)
return result, backup


Expand All @@ -114,6 +120,15 @@ def test_snapshot_contains_cursor_window_and_model(tmp_path: Path) -> None:
assert calls.index("compose stop inference") < calls.index("compose start inference")


def test_streamed_deploy_script_completes_snapshot(tmp_path: Path) -> None:
result, backup = _run(tmp_path, streamed=True)
assert result.returncode == 0, result.stderr
assert "equipment window saved:" in result.stdout
snapshots = list((backup / "inference-windows").glob("equipment-*"))
assert len(snapshots) == 1
assert (snapshots[0] / "SHA256SUMS").is_file()


def test_failed_cursor_copy_restarts_old_service(tmp_path: Path) -> None:
result, backup = _run(tmp_path, fail_copy=True)
assert result.returncode != 0
Expand Down
Loading