diff --git a/cronjobs/src/commands/build_compression_dictionaries.py b/cronjobs/src/commands/build_compression_dictionaries.py index 58ae44ae..dd2e0238 100644 --- a/cronjobs/src/commands/build_compression_dictionaries.py +++ b/cronjobs/src/commands/build_compression_dictionaries.py @@ -5,6 +5,7 @@ """ import hashlib +import json import os import tempfile import typing @@ -232,3 +233,24 @@ def build_compression_dictionaries(): content_type=dictpair.mimetype, client=gcs_client, ) + + # Publish a `manifest.json` for each impacted collection. + # It shows the list of all pairs, giving the list of possible + # sources for each target. + impacted_cids = {(p.bid, p.cid) for p in missing_dictionaries} + all_pairs_of_impacted_cids = [ + p for p in all_pairs if (p.bid, p.cid) in impacted_cids + ] + pairs_by_cid_by_target = {} + for d in all_pairs_of_impacted_cids: + # Strip `bid/cid` out of `bid/cid/yyyymmdd--rid--filename.ext` + pairs_by_cid_by_target.setdefault((d.bid, d.cid), {}).setdefault( + d.new.split("/")[2], [] + ).append(d.old.split("/")[2]) + for (bid, cid), pairs_by_target in pairs_by_cid_by_target.items(): + manifest_path = f"{DESTINATION_FOLDER}/{bid}/{cid}/manifest.json" + storage.Blob(bucket=dicts_bucket, name=manifest_path).upload_from_string( + json.dumps(pairs_by_target), + content_type="application/json", + client=gcs_client, + ) diff --git a/cronjobs/tests/commands/test_build_compression_dictionaries.py b/cronjobs/tests/commands/test_build_compression_dictionaries.py index 1ad5e6f6..b515cde9 100644 --- a/cronjobs/tests/commands/test_build_compression_dictionaries.py +++ b/cronjobs/tests/commands/test_build_compression_dictionaries.py @@ -260,6 +260,13 @@ def test_build_compression_dictionaries( client=mock.ANY, ) + # Assert that a manifest.json file was published. + mock_blob.return_value.upload_from_string.assert_called_once_with( + """{"20260601--rid1--file.txt": ["20260101--rid1--file.txt"]}""", + content_type="application/json", + client=mock.ANY, + ) + def test_build_compression_dictionaries_up_to_date( mock_kinto_client, mock_fetch_all_changesets, mock_storage_bucket, mock_blob @@ -285,3 +292,4 @@ def test_build_compression_dictionaries_up_to_date( mock_blob.return_value.download_to_file.assert_not_called() mock_blob.return_value.upload_from_file.assert_not_called() + mock_blob.return_value.upload_from_string.assert_not_called()