Skip to content

⚡ Optimize snapshot loading with shared cache #176

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
badMade wants to merge 2 commits into
base: main
Choose a base branch
from
Open

⚡ Optimize snapshot loading with shared cache #176

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
127a32f
Optimize snapshot loading with shared cache
badMade May 11, 2026
ee2c053
Fix syntax error in auto-merge CI workflow
badMade May 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 26 additions & 7 deletions .github/workflows/auto-merge.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,18 +1,15 @@
name: auto-merge


on:
pull_request:
types: [opened, synchronize, reopened, labeled]


permissions:
pull-requests: write
contents: write
checks: read
statuses: read


jobs:
auto-merge:
# Skip fork PRs
Expand All @@ -24,7 +21,6 @@ jobs:
- name: Wait for CI checks to initialize
run: sleep 300


- name: Wait for checks to complete
uses: actions/github-script@v7
id: wait-checks
Expand All @@ -34,11 +30,34 @@ jobs:
const owner = context.repo.owner;
const repo = context.repo.repo;


// Wait up to 5 minutes for checks to complete
for (let i = 0; i < 10; i++) {
await new Promise(r => setTimeout(r, 30000));


const { data: checkRuns } = await github.rest.checks.listForRef({
owner, repo,
owner,
repo,
ref: pr.head.sha
});

// Filter out the auto-merge check itself
const relevantChecks = checkRuns.check_runs.filter(check => check.name !== 'auto-merge');
const allCompleted = relevantChecks.every(check => check.status === 'completed');

if (allCompleted) {
const allSuccess = relevantChecks.every(check =>
check.conclusion === 'success' || check.conclusion === 'skipped'
);
Comment on lines +43 to +50
if (allSuccess) {
console.log("All checks passed!");
return true;
} else {
console.log("Some checks failed.");
core.setFailed("Checks failed.");
return false;
}
}
console.log("Waiting for checks to complete...");
}
core.setFailed("Timed out waiting for checks.");
return false;
8 changes: 2 additions & 6 deletions src/commands.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,17 +1,13 @@
from __future__ import annotations

import json
from collections.abc import Mapping
from dataclasses import dataclass
from functools import lru_cache
from pathlib import Path
from types import MappingProxyType
Comment on lines 5 to 7

from .models import PortingBacklog, PortingModule

SNAPSHOT_PATH = (
Path(__file__).resolve().parent / "reference_data" / "commands_snapshot.json"
)
from .snapshot import load_modules_snapshot


@dataclass(frozen=True)
Expand All @@ -25,7 +21,7 @@ class CommandExecution:

@lru_cache(maxsize=1)
def load_command_snapshot() -> tuple[PortingModule, ...]:
raw_entries = json.loads(SNAPSHOT_PATH.read_text())
raw_entries = load_modules_snapshot()["commands"]
return tuple(
PortingModule(
name=entry["name"],
Expand Down
11 changes: 5 additions & 6 deletions src/parity_audit.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,7 @@
ARCHIVE_ROOT = Path(__file__).resolve().parent.parent / 'archive' / 'claude_code_ts_snapshot' / 'src'
CURRENT_ROOT = Path(__file__).resolve().parent
REFERENCE_SURFACE_PATH = CURRENT_ROOT / 'reference_data' / 'archive_surface_snapshot.json'
COMMAND_SNAPSHOT_PATH = CURRENT_ROOT / 'reference_data' / 'commands_snapshot.json'
TOOL_SNAPSHOT_PATH = CURRENT_ROOT / 'reference_data' / 'tools_snapshot.json'
MODULES_SNAPSHOT_PATH = CURRENT_ROOT / 'reference_data' / 'modules_snapshot.json'

ARCHIVE_ROOT_FILES = {
'QueryEngine.ts': 'QueryEngine.py',
Expand Down Expand Up @@ -114,8 +113,8 @@ def _reference_surface() -> dict[str, object]:
return json.loads(REFERENCE_SURFACE_PATH.read_text())


def _snapshot_count(path: Path) -> int:
return len(json.loads(path.read_text()))
def _snapshot_count(path: Path, key: str) -> int:
return len(json.loads(path.read_text())[key])


def run_parity_audit() -> ParityAuditResult:
Expand All @@ -131,8 +130,8 @@ def run_parity_audit() -> ParityAuditResult:
root_file_coverage=(len(root_hits), len(ARCHIVE_ROOT_FILES)),
directory_coverage=(len(dir_hits), len(ARCHIVE_DIR_MAPPINGS)),
total_file_ratio=(current_python_files, int(reference['total_ts_like_files'])),
command_entry_ratio=(_snapshot_count(COMMAND_SNAPSHOT_PATH), int(reference['command_entry_count'])),
tool_entry_ratio=(_snapshot_count(TOOL_SNAPSHOT_PATH), int(reference['tool_entry_count'])),
command_entry_ratio=(_snapshot_count(MODULES_SNAPSHOT_PATH, 'commands'), int(reference['command_entry_count'])),
tool_entry_ratio=(_snapshot_count(MODULES_SNAPSHOT_PATH, 'tools'), int(reference['tool_entry_count'])),
Comment on lines +133 to +134
Copy link
Copy Markdown

@gemini-code-assist gemini-code-assist Bot May 11, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

These two lines cause the modules_snapshot.json file to be read and parsed twice, which is inefficient. To align with the optimization goal of this PR, you should load the snapshot once at the beginning of run_parity_audit and then get the counts from the parsed dictionary. This would also allow you to remove the _snapshot_count helper function.

For example:

def run_parity_audit() -> ParityAuditResult:
    # ... other setup ...
    reference = _reference_surface()
    modules_snapshot = json.loads(MODULES_SNAPSHOT_PATH.read_text())

    return ParityAuditResult(
        # ... other fields ...
        command_entry_ratio=(len(modules_snapshot.get('commands', [])), int(reference['command_entry_count'])),
        tool_entry_ratio=(len(modules_snapshot.get('tools', [])), int(reference['tool_entry_count'])),
        # ... other fields ...
    )

Copy link
Copy Markdown
Owner Author

@badMade badMade May 12, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@gemini-code-assist, what are the steps and line numbers to make this fix?

Copy link
Copy Markdown

@gemini-code-assist gemini-code-assist Bot May 12, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

To implement the optimization, you should modify src/parity_audit.py to load the modules_snapshot.json file once at the start of the run_parity_audit function, and then pass that data to the _snapshot_count helper (or simply use the dictionary directly).

Here are the specific changes:

  1. Update run_parity_audit in src/parity_audit.py (around line 120):
import json
# ...

def run_parity_audit() -> ParityAuditResult:
    # ... existing setup ...
    reference = _reference_surface()
    modules_snapshot = json.loads(MODULES_SNAPSHOT_PATH.read_text())

    return ParityAuditResult(
        # ... other fields ...
        command_entry_ratio=(len(modules_snapshot.get('commands', [])), int(reference['command_entry_count'])),
        tool_entry_ratio=(len(modules_snapshot.get('tools', [])), int(reference['tool_entry_count'])),
        # ... other fields ...
    )
  1. Remove the _snapshot_count helper function from src/parity_audit.py as it will no longer be needed.
Suggested change
command_entry_ratio=(_snapshot_count(MODULES_SNAPSHOT_PATH, 'commands'), int(reference['command_entry_count'])),
tool_entry_ratio=(_snapshot_count(MODULES_SNAPSHOT_PATH, 'tools'), int(reference['tool_entry_count'])),
def run_parity_audit() -> ParityAuditResult:
# ... other setup ...
reference = _reference_surface()
modules_snapshot = json.loads(MODULES_SNAPSHOT_PATH.read_text())
return ParityAuditResult(
root_file_coverage=(len(root_hits), len(ARCHIVE_ROOT_FILES)),
directory_coverage=(len(dir_hits), len(ARCHIVE_DIR_MAPPINGS)),
total_file_ratio=(current_python_files, int(reference['total_ts_like_files'])),
command_entry_ratio=(len(modules_snapshot.get('commands', [])), int(reference['command_entry_count'])),
tool_entry_ratio=(len(modules_snapshot.get('tools', [])), int(reference['tool_entry_count'])),
missing_root_targets=missing_roots,
missing_directory_targets=missing_dirs,
)

Comment on lines 132 to +134
missing_root_targets=missing_roots,
missing_directory_targets=missing_dirs,
)
Loading
Loading