BenchmarkGenerator.yaml

name: Macrobenchmark Generator

on:
    workflow_dispatch:
        inputs:
            benchmarks-count:
                description: 'Number of user scenarios you would like to generate'
                required: true
                default: 1
                type: number
            custom-instructions:
                description: 'Extra user provided instructions passed to scenario generator prompt'
                required: false
                type: string
            scenario-steps:
                description: 'Maximum steps the AI can take before forcing a MAX_STEPS_REACHED scenario termination'
                required: false
                default: 20
                type: number

jobs:
    generate-benchmarks:
        runs-on: blacksmith-8vcpu-ubuntu-2404
        permissions:
            contents: write
            pull-requests: write

        steps:
            - name: Free Disk Space
              run: |
                sudo rm -rf /usr/share/dotnet
                sudo rm -rf /opt/ghc
                sudo rm -rf "/usr/local/share/boost"

            - name: Enable KVM
              run: |
                echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
                sudo udevadm control --reload-rules
                sudo udevadm trigger --name-match=kvm

            - name: Checkout
              uses: actions/checkout@v4
              with:
                lfs: true

            - name: Setup Python
              uses: actions/setup-python@v6
              with:
                python-version: '3.13'

            - name: Install latest version of Ollama
              run: |
                curl -fsSL https://ollama.com/install.sh | sh

            - name: Start Ollama server
              run: |
                ollama serve &
                timeout 30 bash -c 'until curl -s http://localhost:11434 > /dev/null; do sleep 1; done'

            - name: GELab-Zero-4B Model Install & Setup
              run: |
                pip install huggingface_hub

                hf download --no-force-download stepfun-ai/GELab-Zero-4B-preview --local-dir gelab-zero-4b-preview

                cd gelab-zero-4b-preview
                ollama create gelab-zero-4b-preview -f Modelfile

            - name: Ensure the model was installed correctly
              run: |
                curl -X POST http://localhost:11434/v1/chat/completions \
                -H "Content-Type: application/json" \
                -d '{
                      "model": "gelab-zero-4b-preview",
                      "messages": [{"role": "user", "content": "Hello, GELab-Zero!"}]
                    }'

            - name: Clone gelab-zero repo fork & install requirements.txt
              run: |
                git clone https://github.com/Frozen-Bytes/gelab-zero
                cd gelab-zero

                # Install dependencies
                pip install -r requirements.txt
                # pip install --upgrade openai

            - name: Install apktool
              run: |
                # Wrapper script
                wget -q https://raw.githubusercontent.com/iBotPeaches/Apktool/master/scripts/linux/apktool -O /tmp/apktool
                # Latest jar from official download page
                LATEST_URL=$(curl -s https://api.github.com/repos/iBotPeaches/Apktool/releases/latest | grep browser_download_url | grep '\.jar' | cut -d '"' -f 4)
                echo "Downloading latest apktool version: $LATEST"
                wget -q "$LATEST_URL" -O /tmp/apktool.jar
                sudo mv /tmp/apktool /usr/local/bin/apktool
                sudo mv /tmp/apktool.jar /usr/local/bin/apktool.jar
                sudo chmod +x /usr/local/bin/apktool
                sudo chmod +x /usr/local/bin/apktool.jar
                apktool --version

            - name: Run AI scenario generation
              uses: reactivecircus/android-emulator-runner@v2
              env:
                  GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
                  PYTHONUNBUFFERED: "1"
              with:
                api-level: 34
                target: google_apis
                arch: x86_64
                profile: pixel_6
                force-avd-creation: false
                emulator-options: -no-window -no-audio -no-boot-anim -gpu swiftshader_indirect -no-snapshot
                disable-animations: true
                script: |
                  cd gelab-zero && python apk_test_agent.py "../apks/org.wikipedia_r_50595.apk" --num_goals ${{ github.event.inputs.benchmarks-count }} --steps ${{ github.event.inputs.scenario-steps }} --custom_instructions "${{ github.event.inputs.custom-instructions }}"

            - name: Upload gelab-zero log as artifact
              if: always()
              uses: actions/upload-artifact@v4
              with:
                name: gelab-zero-running-log-${{ github.run_id }}
                path: gelab-zero/running_log/

            - name: Install ai-action-to-mp4 and run it
              shell: bash
              run: |
                pip install git+https://github.com/Frozen-Bytes/ai-action-to-mp4.git@main
                ai-action-to-mp4 --jsons gelab-zero/running_log/action_logs/ --imgs gelab-zero/running_log/server_log/os-copilot-local-eval-logs/images/ --out ./benchmarks/src/main/kotlin/com/google/samples/apps/nowinandroid/Generator

            - name: Generate github app token
              id: app-token
              uses: actions/create-github-app-token@v3
              with:
                client-id: ${{ vars.GH_APP_CLIENT_ID }}
                private-key: ${{ secrets.GH_APP_PRIVATE_KEY }}

            - name: Create pull request branch
              id: create-branch
              run: |
                SHORT_SHA=$(head -c 30 /dev/urandom | sha1sum | cut -c 1-7)
                BRANCH_NAME="macrobenchmark/$SHORT_SHA"
                git checkout -b "$BRANCH_NAME"
                echo "branch-name=$BRANCH_NAME" >> $GITHUB_OUTPUT

            - name: Convert JSON actions to macrobenchmark files
              run: |
                git clone https://github.com/Frozen-Bytes/actions-to-macrobenchmark.git
                python -u actions-to-macrobenchmark/bench_generator.py --target-package-name "org.wikipedia"

            - name: Commit generated benchmarks
              run: |
                git config user.name "${{ vars.GH_APP_NAME }}[bot]"
                git config user.email "${{ vars.GH_APP_NAME }}[bot]@users.noreply.github.com"
                git add benchmarks/
                git commit -m "test(perf): add generated macrobenchmarks

                Generated ${{ inputs.benchmarks-count }} macrobenchmark(s)
                Triggered by: @${{ github.actor }}
                Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" || echo "No changes to commit"

            - name: Push benchmark branch
              run: |
                git remote set-url origin https://x-access-token:${{ steps.app-token.outputs.token }}@github.com/${{ github.repository }}
                git push origin "${{ steps.create-branch.outputs.branch-name }}"

            - name: Compute benchmark total
              id: compute-total
              run: echo "total=$(( 1 + 2 * ${{ inputs.benchmarks-count }} ))" >> $GITHUB_OUTPUT

            - name: Collect generated video links
              id: collect-videos
              shell: bash
              run: |
                VIDEO_DIR="./benchmarks/src/main/kotlin/com/google/samples/apps/nowinandroid/Generator"
                VIDEO_LINKS=""
                while IFS= read -r -d '' file; do
                  REL_PATH="https://raw.githubusercontent.com/${{ github.repository }}/${{ steps.create-branch.outputs.branch-name }}/${file#./}"
                  NAME=$(basename "$file")
                  VIDEO_LINKS="${VIDEO_LINKS}"$'\n'"* [${NAME}](${REL_PATH})"
                done < <(find "$VIDEO_DIR" -name "*.mp4" -print0 | sort -z)

                {
                  echo "video-links<<EOF"
                  echo "$VIDEO_LINKS"
                  echo "EOF"
                } >> "$GITHUB_OUTPUT"

            - name: Open Pull Request
              env:
                GH_TOKEN: ${{ steps.app-token.outputs.token }}
              run: |
                gh pr create \
                --title "Generated ${{ steps.compute-total.outputs.total }} macrobenchmarks" \
                --body "## 🤖 Macrobenchmark Generation
                This PR was automatically generated by the Macrobenchmark Generator workflow.
                **Triggered by:** @${{ github.actor }}
                **Benchmarks generated:** ${{ steps.compute-total.outputs.total }}
                - 1 startup benchmark
                - ${{ inputs.benchmarks-count }} frametime benchmark(s) (one per user scenario)
                - ${{ inputs.benchmarks-count }} memory benchmark(s) (one per user scenario)

                **🎥 Generated User Scenarios:**
                ${{ steps.collect-videos.outputs.video-links }}" \
                --head "${{ steps.create-branch.outputs.branch-name }}"