From 9111149cd8a33133a3e7de084aa2a2ca1c7fa008 Mon Sep 17 00:00:00 2001 From: Mathieu Leplatre Date: Mon, 13 Jul 2026 15:12:00 +0200 Subject: [PATCH] cronjobs: RMST-459: publish a manifest.json in the bid/cid folder --- .../build_compression_dictionaries.py | 22 +++++++++++++++++++ .../test_build_compression_dictionaries.py | 8 +++++++ 2 files changed, 30 insertions(+) diff --git a/cronjobs/src/commands/build_compression_dictionaries.py b/cronjobs/src/commands/build_compression_dictionaries.py index dea77fa6..871ef767 100644 --- a/cronjobs/src/commands/build_compression_dictionaries.py +++ b/cronjobs/src/commands/build_compression_dictionaries.py @@ -4,6 +4,7 @@ It then uploads these `.dcz` files to Google Cloud Storage. """ +import json import os import tempfile import typing @@ -218,3 +219,24 @@ def build_compression_dictionaries(): content_type="application/zstd", client=gcs_client, ) + + # Publish a `manifest.json` for each impacted collection. + # It shows the list of all pairs, giving the list of possible + # sources for each target. + impacted_cids = {(p.bid, p.cid) for p in missing_dictionaries} + all_pairs_of_impacted_cids = [ + p for p in all_pairs if (p.bid, p.cid) in impacted_cids + ] + pairs_by_cid_by_target = {} + for d in all_pairs_of_impacted_cids: + # Strip `bid/cid` out of `bid/cid/yyyymmdd--rid--filename.ext` + pairs_by_cid_by_target.setdefault((d.bid, d.cid), {}).setdefault( + d.new.split("/")[2], [] + ).append(d.old.split("/")[2]) + for (bid, cid), pairs_by_target in pairs_by_cid_by_target.items(): + manifest_path = f"{DESTINATION_FOLDER}/{bid}/{cid}/manifest.json" + storage.Blob(bucket=dicts_bucket, name=manifest_path).upload_from_string( + json.dumps(pairs_by_target), + content_type="application/json", + client=gcs_client, + ) diff --git a/cronjobs/tests/commands/test_build_compression_dictionaries.py b/cronjobs/tests/commands/test_build_compression_dictionaries.py index 1d52061e..bf1a1963 100644 --- a/cronjobs/tests/commands/test_build_compression_dictionaries.py +++ b/cronjobs/tests/commands/test_build_compression_dictionaries.py @@ -246,6 +246,13 @@ def test_build_compression_dictionaries( client=mock.ANY, ) + # Assert that a manifest.json file was published. + mock_blob.return_value.upload_from_string.assert_called_once_with( + """{"20260601--rid1--file.txt": ["20260101--rid1--file.txt"]}""", + content_type="application/json", + client=mock.ANY, + ) + def test_build_compression_dictionaries_up_to_date( mock_kinto_client, mock_fetch_all_changesets, mock_storage_bucket, mock_blob @@ -271,3 +278,4 @@ def test_build_compression_dictionaries_up_to_date( mock_blob.return_value.download_to_file.assert_not_called() mock_blob.return_value.upload_from_file.assert_not_called() + mock_blob.return_value.upload_from_string.assert_not_called()