Monitor cleanup operations

group_vars/maintenance.yml

@@ -193,6 +193,13 @@ telegraf_plugins_extra:
       - timeout = "10s"
       - data_format = "influx"
       - interval = "15s"
+  galaxy_cleanup_storage:
+    plugin: "exec"
+    config:
+      - commands = ["/usr/bin/cat {{ galaxy_cleanup_status_file }}"]
+      - timeout = "10s"
+      - data_format = "influx"
+      - interval = "15s"
   galaxy_uploaded:
     plugin: "exec"
     config:

roles/usegalaxy_eu.galaxy_cleanup/defaults/main.yml

@@ -16,3 +16,6 @@ galaxy_cleanup_scratchstorage: {}
 # Default: runs daily at 1:00 AM
 galaxy_cleanup_minute: 0
 galaxy_cleanup_hour: 1
+
+# Influx line-protocol status emitted after each cleanup run
+galaxy_cleanup_status_file: {{ galaxy_log_dir }}/storage_cleanup_status.influx

roles/usegalaxy_eu.galaxy_cleanup/tasks/main.yml

@@ -16,6 +16,16 @@
     dest: "/usr/local/bin/cleanup_storage.sh"
     mode: "0555"
 
+- name: Initialize cleanup status metric
+  ansible.builtin.file:
+    path: "{{ galaxy_cleanup_status_file }}"
+    state: touch
+    owner: "{{ galaxy_user.name }}"
+    group: root
+    mode: "0644"
+    access_time: preserve
+    modification_time: preserve
+
 - name: Schedule Galaxy cleanup objectstore script
   ansible.builtin.cron:
     name: "Galaxy cleanup scratch"

roles/usegalaxy_eu.galaxy_cleanup/templates/cleanup_storage.sh.j2

@@ -1,8 +1,22 @@
-#!/bin/sh
+#!/bin/bash
 ##
 ## This file is maintained by Ansible - CHANGES WILL BE OVERWRITTEN
 ##
-set -e
+set -uo pipefail
+umask 022
+
+status=0
+run_timestamp="$(date --rfc-3339=seconds)"
+status_file="{{ galaxy_cleanup_status_file }}"
+status_file_tmp="${status_file}.tmp.$$"
+
+trap 'rm -f "$status_file_tmp"' EXIT
+
+record_failure() {
+	if [ "$status" -eq 0 ]; then
+		status="$1"
+	fi
+}
 
 {#
 ## Under normal curcumstances the (non-scratch storage) the pgcleanup script will never purge data, that has not been deleted before.
@@ -17,18 +31,31 @@ set -e
 {% for objectstore_id, config in galaxy_cleanup_scratchstorage.items() %}
 # Stage 1: Purge old HDAs at day 60 for our Scratch storage - short term storage
 {{ galaxy_venv_dir }}/bin/python {{ galaxy_server_dir }}/scripts/cleanup_datasets/pgcleanup.py -c {{ galaxy_config_file }} -U -b 5000 -o {{ (config.days | int) }} -l {{ galaxy_log_dir }} -w 128MB --object-store-id {{ objectstore_id }} purge_old_hdas 2>&1 | tee -a {{ galaxy_log_dir }}/cleanup-$(date --rfc-3339=seconds)-purgeOldHdas.log
+if [ "$command_status" -ne 0 ]; then
+	record_failure "$command_status"
+fi
 {% endfor %}
 
 # Stage 4: Galaxy Default Cleanup
 for action in {delete_userless_histories,delete_exported_histories,purge_deleted_users,purge_deleted_histories,purge_deleted_hdas,purge_historyless_hdas,purge_hdas_of_purged_histories,delete_datasets,purge_datasets}; do
-		{{ galaxy_venv_dir }}/bin/python \
-			"{{ galaxy_server_dir }}/scripts/cleanup_datasets/pgcleanup.py" \
+        {{ galaxy_venv_dir }}/bin/python \
+ 			"{{ galaxy_server_dir }}/scripts/cleanup_datasets/pgcleanup.py" \
 			-U \
 			-o "60" \
 			-b "5000" \
 			-l "{{ galaxy_log_dir }}" \
 			-s $action \
 			-w 128MB \
 			 >> "{{ galaxy_log_dir }}/cleanup-$(date --rfc-3339=seconds)-${action}.log" \
-			2>> "{{ galaxy_log_dir }}/cleanup-$(date --rfc-3339=seconds)-${action}.err";
-	done
+			2>> "{{ galaxy_log_dir }}/cleanup-$(date --rfc-3339=seconds)-${action}.err"
+	command_status=$?
+	if [ "$command_status" -ne 0 ]; then
+		record_failure "$command_status"
+	fi
+done
+
+printf 'galaxy_cleanup_storage exit_code=%di %s\n' "$status" "$(date +%s%N)" > "$status_file_tmp"
+mv "$status_file_tmp" "$status_file"
+trap - EXIT
+
+exit "$status"