Hardening and Refactoring

.dockerignore

@@ -0,0 +1,6 @@
+.git
+.github
+.DS_Store
+__pycache__
+*.pyc
+test_sync.py

.github/dependabot.yml

@@ -0,0 +1,14 @@
+version: 2
+updates:
+  - package-ecosystem: pip
+    directory: /
+    schedule:
+      interval: monthly
+  - package-ecosystem: docker
+    directory: /
+    schedule:
+      interval: monthly
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: monthly

.github/workflows/test.yml

@@ -0,0 +1,61 @@
+name: Test
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+
+permissions:
+  contents: read
+
+jobs:
+  unit-tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.12"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+      - run: python -m pip install --upgrade pip
+      - run: python -m pip install -r requirements.txt
+      - run: python -m unittest -v
+      - run: python -m py_compile sync.py test_sync.py
+
+  container-build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: docker/setup-buildx-action@v3
+      - uses: docker/build-push-action@v6
+        with:
+          context: .
+          load: true
+          push: false
+          tags: mythic-sync:test
+      - name: Verify Redis AOF persistence across container recreation
+        run: |
+          docker volume create mythic_sync_test_redis
+          docker run --detach --name mythic-sync-redis-before \
+            --volume mythic_sync_test_redis:/data \
+            --entrypoint redis-server mythic-sync:test \
+            --appendonly yes --appendfsync always --dir /data
+          until docker exec mythic-sync-redis-before redis-cli ping; do sleep 1; done
+          docker exec mythic-sync-redis-before redis-cli set mythic_sync:persistence_test queued
+          docker stop mythic-sync-redis-before
+          docker rm mythic-sync-redis-before
+          docker run --detach --name mythic-sync-redis-after \
+            --volume mythic_sync_test_redis:/data \
+            --entrypoint redis-server mythic-sync:test \
+            --appendonly yes --appendfsync always --dir /data
+          until docker exec mythic-sync-redis-after redis-cli ping; do sleep 1; done
+          test "$(docker exec mythic-sync-redis-after redis-cli get mythic_sync:persistence_test)" = queued
+      - name: Clean up Redis persistence test
+        if: always()
+        run: |
+          docker rm --force mythic-sync-redis-before mythic-sync-redis-after 2>/dev/null || true
+          docker volume rm mythic_sync_test_redis 2>/dev/null || true

.gitignore

@@ -1 +1,4 @@
-.idea/*
+.idea/*
+.DS_Store
+__pycache__/
+*.py[cod]

CHANGELOG.md

@@ -5,6 +5,51 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+
+## [3.1.0] - 18 July 2026
+
+### Added
+
+* Added an AOF-backed Redis retry queue so Ghostwriter tag failures do not block log entry ingestion and pending jobs survive `mythic_sync` restarts and service recreation, provided the Redis data volume is preserved.
+* Added Redis health checks, pending-job reporting, and automatic migration of legacy mappings and queued tag jobs.
+* Added tests for query retries, stale entry reconciliation, deleted entry recreation, tag retries, and source IP formatting.
+* Added a container-level CI check that verifies queued Redis data survives a Redis container restart.
+* Added startup warnings for embedded Redis volume requirements and actionable dead-letter inspection details.
+* Added optional `REDIS_URL` support for authenticated and TLS-protected external Redis services without logging URL credentials.
+* Added GitHub Actions coverage for Python 3.10, Python 3.12, and production container builds, plus Dependabot configuration.
+
+### Changed
+
+* Changed Ghostwriter GraphQL retries to use exponential backoff with jitter, capped at five minutes.
+* Improved GraphQL error messages with the operation name and variables, including actionable context for ambiguous `ModelDoesNotExist` responses.
+* Changed source IP formatting from a JSON array to a sorted, comma-separated string.
+* Made Redis hostname, port, and database configurable and scoped Redis mappings by Mythic host, Mythic port, and Ghostwriter oplog.
+* Made `MYTHIC_PORT` optional and defaulted it to Mythic's standard HTTPS port, `7443`, preserving existing install behavior.
+* Enabled Redis append-only persistence for embedded Mythic deployments and for the standalone Compose Redis service with a named volume. Raw Mythic subscription events remain live-streamed and are not persisted locally.
+* Changed Redis AOF flushing to `appendfsync always` and added graceful embedded Redis shutdown handling.
+* Updated supported dependency pins and moved the production container to Python 3.11 on Debian Bookworm.
+* Made Mythic timestamp and IP parsing tolerant of timezone variants, IPv6, plain strings, CIDR notation, and malformed individual addresses.
+* Filtered loopback, unspecified, multicast, and IPv6 link-local source addresses while preserving potentially useful private, CGNAT, ULA, and global addresses.
+* Limited GraphQL error context to identifiers while redacting commands, comments, and other potentially sensitive values.
+* Made the Ghostwriter initialization entry idempotent and corrected Mythic API-key authentication so user credentials are not also required.
+* Made subscription and tag worker shutdown explicit so worker failures cancel and await the remaining tasks before the Ghostwriter client closes.
+* Documented dead-letter inspection, the single-replica constraint, and a manual Mythic/Ghostwriter acceptance checklist.
+
+### Fixed
+
+* Fixed stale Redis entry mappings by looking up the Ghostwriter entry by `entryIdentifier` and repairing the mapping or recreating a deleted entry.
+* Fixed `ModelDoesNotExist` update failures retrying the same stale ID forever instead of reconciling by `entryIdentifier`.
+* Fixed pending tag jobs targeting deleted entries retrying forever; jobs now move to a replacement entry or enter a durable Redis dead-letter hash with an operator notification.
+* Prevented repeated Mythic error notifications and notification delivery failures from interfering with GraphQL retries.
+* Fixed Redis startup checks reporting success without issuing a command.
+* Prevented Redis startup failures from attempting Mythic notifications before Mythic authentication is established.
+* Ensured task cancellation exits Redis, service, authentication, GraphQL, and tag retry loops instead of being handled as a retryable failure.
+* Prevented Mythic instances sharing an IP but using different ports from colliding in Redis state or initialization entry identifiers; existing IP-only Redis state migrates on startup.
+* Stopped the standalone Compose file from overriding Redis host, port, and database values supplied through `settings.env`.
+* Fixed conversion, creation, update, and Redis failures being swallowed after logging, which could allow processing to continue after an entry failed.
+* Fixed timezone-aware token expiration parsing for timestamps ending in `Z`.
+
 ## [3.0.8] - 25 July 2025
 
 ### Changed

Dockerfile

@@ -1,11 +1,22 @@
-FROM redis:7-bullseye
+FROM redis:7-bookworm
 
-RUN apt update && apt install python3 python3-pip -y  \
-    --no-install-recommends
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends python3 python3-venv \
+    && rm -rf /var/lib/apt/lists/*
 
-COPY requirements.txt .
-RUN python3 -m pip install -r requirements.txt
+WORKDIR /app
 
-COPY sync.py .
+COPY requirements.txt ./
+RUN python3 -m venv /opt/venv \
+    && /opt/venv/bin/pip install --no-cache-dir -r requirements.txt
 
-CMD ["bash", "-c", "redis-server &  python3 -u sync.py"]
+COPY sync.py docker-entrypoint.sh ./
+RUN chmod +x /app/docker-entrypoint.sh
+
+VOLUME ["/data"]
+
+STOPSIGNAL SIGTERM
+
+USER redis
+
+ENTRYPOINT ["/app/docker-entrypoint.sh"]

README.md

@@ -36,6 +36,7 @@ For the easiest experience with `mythic_sync`, install it via the `mythic-cli` t
 On your Mythic server, run: `sudo ./mythic-cli mythic_sync install github https://github.com/GhostManager/mythic_sync`
 
 Follow the prompts to configure `mythic_sync` with your Mythic and Ghostwriter server configuration.
+If `MYTHIC_PORT` is not provided, `mythic_sync` uses Mythic's default HTTPS port, `7443`.
 
 You can get your Ghostwriter Oplog ID by visiting your log in your web browser and looking at the top of the page or the URL. A URL with `/oplog/12/entries` means your Oplog ID is `12`.
 
@@ -61,11 +62,39 @@ After cloning repository, open the `settings.env` file and fill in the variables
 
 ```text
 MYTHIC_IP=10.10.1.100
+MYTHIC_PORT=7443
 MYTHIC_USERNAME=mythic_admin
 MYTHIC_PASSWORD=SuperSecretPassword
-GHOSTWRITER_API_KEY=eyJ0eXAiO...
+GHOSTWRITER_API_KEY=gwst_...
 GHOSTWRITER_URL=https://ghostwriter.mydomain.com
 GHOSTWRITER_OPLOG_ID=12
+REDIS_HOSTNAME=redis
+REDIS_PORT=6379
+REDIS_DB=1
+```
+
+The standalone Compose deployment reads these Redis settings directly from `settings.env`; edit
+that file to select a different Redis host, port, or database without modifying
+`docker-compose.yml`.
+
+Set `MYTHIC_API_KEY` to authenticate with a Mythic API key instead of `MYTHIC_USERNAME` and
+`MYTHIC_PASSWORD`. `MYTHIC_PORT` is optional and defaults to `7443`.
+
+The standalone Compose deployment stores entry mappings and pending tag updates in the named
+`mythic_sync_redis` volume. Redis append-only persistence with `appendfsync always` is enabled so recreating the
+`mythic_sync` application container does not discard pending work. Mythic installations default
+to an embedded append-only Redis instance for compatibility. To preserve that data across service
+recreation, configure Mythic's generated Compose service to mount a stable named volume at `/data`;
+the Dockerfile's volume declaration alone does not guarantee that a replacement container will
+reattach the same storage. Alternatively, `REDIS_HOSTNAME`, `REDIS_PORT`, and `REDIS_DB` can select
+an external persistent Redis instance.
+
+For an authenticated or TLS-protected external Redis service, set `REDIS_URL` instead of the three
+individual Redis settings. The URL takes precedence and may include the database number, username,
+and password. Its credentials are never written to the service log:
+
+```text
+REDIS_URL=rediss://sync-user:password@redis.example.com:6380/1
 ```
 
 Once the environment variables are set up, you can launch the service by using `docker-compose`:
@@ -78,7 +107,7 @@ docker-compose up
 
 Open your Ghostwriter log and look for an initial entry. You should see something like the following:
 
-    > Initial entry from mythic_sync at: <server_ip>. If you're seeing this then oplog syncing is working for this C2 server!
+    > Initial entry from mythic_sync at: <server_ip>:<server_port>. If you're seeing this then oplog syncing is working for this C2 server!
 
 If so, you're all set! Otherwise, check the logs from the docker container for error messages. Fetch the logs with:
 
@@ -88,9 +117,85 @@ If so, you're all set! Otherwise, check the logs from the docker container for e
 
 Ensure the host where `mythic_sync` is running has network access to the Ghostwriter and Mythic servers.
 
-`mythic_sync` uses an internal Redis database to sync what events have already been sent to Ghostwriter, avoiding duplicates.
+`mythic_sync` uses Redis to track events already sent to Ghostwriter and to retain pending tag
+updates. Redis mappings are scoped by Mythic host, Mythic port, and Ghostwriter oplog. Legacy
+mappings and pending tag jobs are migrated automatically when first accessed after an upgrade.
+Migration from the earlier IP-only namespace assumes the old state belongs to the first upgraded
+instance; state that was already mixed by multiple Mythic ports cannot be separated automatically.
+
+Source IP normalization removes duplicate, loopback, unspecified, multicast, and IPv6 link-local
+addresses to keep multi-adapter hosts readable. Private IPv4, CGNAT, IPv6 ULA, and globally routable
+addresses are preserved because interface metadata is not available to identify container bridges
+reliably.
+If a queued tag job's entry no longer resolves by `entryIdentifier`, the worker removes it from
+active retries but preserves its payload in the deployment's namespaced Redis dead-letter hash and
+notifies Mythic. A later task update can create a fresh tag job for the current entry.
+
+The dead-letter hash is named:
+
+```text
+mythic_sync:<oplog-id>:<mythic-host>:<mythic-port>:pending_tag_updates:dead_letter
+```
+
+The startup warning prints the exact key. For the standalone Compose deployment, inspect it with:
+
+```bash
+docker compose exec redis redis-cli -n 1 HGETALL \
+  "mythic_sync:<oplog-id>:<mythic-host>:<mythic-port>:pending_tag_updates:dead_letter"
+```
+
+After investigating an individual job, remove only that field with `HDEL <hash-key> <entry-id>`.
+There is intentionally no automatic dead-letter replay in this release; updating the corresponding
+Mythic task creates a fresh tag job if Ghostwriter can resolve or recreate its oplog entry.
+
+The embedded Redis process flushes and shuts down when the application container receives
+`SIGTERM`. Both embedded and standalone configurations use synchronous AOF writes. Do not remove
+or renew the Redis data volume during upgrades if queued work must survive.
+
+At startup, the service logs the selected Redis endpoint and number of pending tag updates. A
+successful Redis client construction is not sufficient: the service waits for `PING` to succeed
+before subscribing to Mythic events.
+
+Run only one `mythic_sync` instance for a given Mythic host, port, and Ghostwriter oplog namespace.
+This release does not coordinate tag workers across multiple active replicas.
+
+### Manual Acceptance Checklist
+
+Perform destructive checks only in a disposable test oplog.
+
+1. Start `mythic_sync` and confirm the initialization entry appears once in Ghostwriter. Restart
+   the service and confirm a duplicate initialization entry is not created.
+2. Run a Mythic task and confirm Ghostwriter receives an oplog entry whose `entryIdentifier`
+   matches the Mythic `agent_task_id`. Update the task and confirm the existing entry changes rather
+   than creating a duplicate.
+3. Delete that Ghostwriter entry, then update the Mythic task. Confirm the logs identify the stale
+   Ghostwriter ID and `entryIdentifier`, and that the entry is found again or recreated.
+4. Add, change, and remove tags on the Mythic task. Confirm Ghostwriter's corresponding `mythic:`
+   tags converge to the current Mythic tags while any Ghostwriter tags without the `mythic:` prefix
+   remain unchanged. Stale-target and dead-letter behavior is covered by the automated test suite.
+5. With a stable Redis volume attached, note the startup pending/dead-letter counts, restart or
+   recreate only the `mythic_sync` container, and confirm the counts and Redis data remain. Do not
+   remove or renew the volume during this check.
+6. Temporarily use a service token without the required test-oplog permission and confirm the error
+   identifies the GraphQL operation, oplog or entry identifier, and likely permission problem without
+   logging command or comment contents.
+
+### Testing
+
+Run the unit suite in the production dependency image:
+
+```bash
+docker build -t mythic-sync-test .
+docker run --rm --entrypoint /opt/venv/bin/python \
+  -e PYTHONDONTWRITEBYTECODE=1 \
+  -v "$PWD:/workspace" -w /workspace \
+  mythic-sync-test -m unittest -v
+```
 
-If the `mythic_sync` service goes down, it is safe to stand it back up and avoid duplicates as long as nothing has forcefully stopped Mythic's Redis container.
+The suite covers retries, `ModelDoesNotExist` reconciliation, stale and deleted entries, Redis
+migrations, tag queue processing,
+authentication selection, timestamp and IP normalization, initialization idempotency, and
+diagnostic redaction.
 
 ## References
 

docker-compose.yml

@@ -1,10 +1,24 @@
-version: "3"
 services:
     mythic_sync:
         build: .
         depends_on:
-            - redis
+            redis:
+                condition: service_healthy
         env_file:
             - settings.env
+        restart: unless-stopped
     redis:
-        image: redis:5-alpine
+        image: redis:7-alpine
+        command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always"]
+        healthcheck:
+            test: ["CMD", "redis-cli", "ping"]
+            interval: 5s
+            timeout: 3s
+            retries: 10
+        restart: unless-stopped
+        stop_grace_period: 30s
+        volumes:
+            - mythic_sync_redis:/data
+
+volumes:
+    mythic_sync_redis:

docker-entrypoint.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+set -eu
+
+redis_hostname="${REDIS_HOSTNAME:-127.0.0.1}"
+redis_url="${REDIS_URL:-}"
+
+if [ -z "$redis_url" ] && { [ "$redis_hostname" = "127.0.0.1" ] || [ "$redis_hostname" = "localhost" ]; }; then
+    redis-server --appendonly yes --appendfsync always --dir /data --daemonize yes
+
+    /opt/venv/bin/python -u /app/sync.py &
+    app_pid=$!
+
+    shutdown() {
+        trap - TERM INT
+        kill -TERM "$app_pid" 2>/dev/null || true
+        redis-cli -h 127.0.0.1 shutdown save 2>/dev/null || true
+    }
+
+    trap shutdown TERM INT
+    set +e
+    wait "$app_pid"
+    app_status=$?
+    set -e
+    redis-cli -h 127.0.0.1 shutdown save 2>/dev/null || true
+    exit "$app_status"
+fi
+
+exec /opt/venv/bin/python -u /app/sync.py

requirements.txt

@@ -1,4 +1,4 @@
-aiohttp==3.8.4
-redis==4.5.4
-mythic==0.2.6
-gql==3.4.0
+aiohttp==3.14.1
+redis==8.0.1
+mythic==0.2.10
+gql==3.5.3

settings.env

@@ -7,3 +7,5 @@ GHOSTWRITER_URL=https://ghostwriter.mydomain.com
 GHOSTWRITER_OPLOG_ID=123
 REDIS_HOSTNAME=redis
 REDIS_PORT=6379
+REDIS_DB=1
+# REDIS_URL=rediss://username:password@redis.example.com:6380/1