From fb3bf5b4ada289fd2f24037f45206ad8aa8bccf3 Mon Sep 17 00:00:00 2001 From: ayush Date: Sat, 4 Jul 2026 02:21:46 +0530 Subject: [PATCH 1/7] feat(bonus): implement github actions ci/cd pipeline for automated PR validation --- .github/workflows/jiopc-agent-ci.yml | 58 ++++++++++++++++++++++++++++ README.md | 12 ++++++ 2 files changed, 70 insertions(+) create mode 100644 .github/workflows/jiopc-agent-ci.yml diff --git a/.github/workflows/jiopc-agent-ci.yml b/.github/workflows/jiopc-agent-ci.yml new file mode 100644 index 0000000..cef2df1 --- /dev/null +++ b/.github/workflows/jiopc-agent-ci.yml @@ -0,0 +1,58 @@ +name: JioPC Automated Testing CI + +on: + pull_request: + branches: [ "main" ] + +jobs: + validate-os-patch: + runs-on: ubuntu-24.04 + + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Install System Dependencies + run: | + sudo apt-get update + sudo apt-get install -y dpkg xvfb jq + + - name: Build Debian Package + run: | + dpkg-deb --build packaging/ jiopc-testing-agent.deb + + - name: Install Agent + run: | + sudo apt --reinstall install -y ./jiopc-testing-agent.deb + + - name: Run Testing Agent & Generate Analysis + id: run-agent + env: + LLM_BASE_URL: ${{ secrets.LLM_BASE_URL }} + LLM_MODEL: ${{ secrets.LLM_MODEL }} + LLM_API_KEY: ${{ secrets.LLM_API_KEY }} + run: | + # Run the agent in a headless X server to ensure native UI apps launch safely + # We pipe the output to a file so we can extract the LLM report later + xvfb-run jiopc-agent --analyse > analysis_output.txt || true + + # Extract the LLM report block safely (everything after "LLM ANALYSIS RESULT") + # The agent prints exactly "LLM ANALYSIS RESULT" before printing the report string + sed -n '/LLM ANALYSIS RESULT/,$p' analysis_output.txt > llm_report.txt + + # Pass the report content to GitHub Actions Environment Variables safely + echo "REPORT<> $GITHUB_ENV + cat llm_report.txt >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + + - name: Post LLM Analysis as PR Comment + uses: peter-evans/create-or-update-comment@v4 + with: + issue-number: ${{ github.event.pull_request.number }} + body: | + ## 🤖 JioPC Automated Testing Report + *The testing agent executed the full validation suite against this OS Patch.* + + ```text + ${{ env.REPORT }} + ``` diff --git a/README.md b/README.md index 41b2f16..781c465 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,18 @@ The logs are written in **JSON Lines (JSONL)** format, making them highly machin > **IMPORTANT:** `BLOCKED` is **NOT** a failure. It means a Jio-protected URL triggered expected bot detection in the headless browser. The LLM is explicitly trained to understand this distinction. +## CI/CD Integration (Bonus Feature) + +The repository includes a production-ready **GitHub Actions Pipeline** (`.github/workflows/jiopc-agent-ci.yml`). + +Whenever a Pull Request is opened against the `main` branch, the pipeline will automatically: +1. Spin up an Ubuntu 24.04 runner. +2. Build and install the `.deb` package dynamically. +3. Run the full validation suite (Parts A, B, and C) inside a headless `xvfb` frame. +4. Extract the LLM Analysis and **post it as a comment directly on the Pull Request** so engineers can immediately see if the patch is safe to promote. + +*(To enable this on your fork, simply add `LLM_API_KEY`, `LLM_MODEL`, and `LLM_BASE_URL` to your GitHub Repository Secrets).* + ## Benchmarking To prove that the agent complies with the strict hackathon resource constraints (< 150MB RAM, < 20% CPU), run the included benchmark script: From af58c4512ade6b609ab539614a3012052d124ca7 Mon Sep 17 00:00:00 2001 From: ayush Date: Sat, 4 Jul 2026 03:14:24 +0530 Subject: [PATCH 2/7] ci: grant write permissions for github actions bot to post PR comments --- .github/workflows/jiopc-agent-ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/jiopc-agent-ci.yml b/.github/workflows/jiopc-agent-ci.yml index cef2df1..64624db 100644 --- a/.github/workflows/jiopc-agent-ci.yml +++ b/.github/workflows/jiopc-agent-ci.yml @@ -4,6 +4,10 @@ on: pull_request: branches: [ "main" ] +permissions: + pull-requests: write + issues: write + jobs: validate-os-patch: runs-on: ubuntu-24.04 From f75b7122558e452a158cc92ce921527d64db88db Mon Sep 17 00:00:00 2001 From: ayush Date: Sat, 4 Jul 2026 03:30:51 +0530 Subject: [PATCH 3/7] ci: rebuild pipeline strictly according to hackathon specifications --- .github/workflows/README.md | 64 +++++++++++ .github/workflows/ci.yml | 163 +++++++++++++++++++++++++++ .github/workflows/jiopc-agent-ci.yml | 2 +- 3 files changed, 228 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/README.md create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 0000000..cdda190 --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,64 @@ +# JioPC Testing Agent CI/CD Pipeline + +## What This Pipeline Does + +The GitHub Actions pipeline automatically triggers on every Pull Request to the repository. It spins up a clean, ephemeral Ubuntu 24.04 environment and installs the JioPC testing agent dependencies. The pipeline then runs the web validation suite (Part A), securely calls the OpenAI API to analyze the JSON Lines log file via the agent's internal LLM module, and automatically extracts and posts the test counts and the LLM recommendation directly as a comment on the Pull Request. This gives engineers immediate AI-powered feedback on whether the OS patch is safe to promote. + +## Trigger Conditions + +The CI/CD pipeline is designed to execute automatically on: +- **Pull Requests:** Whenever a PR is opened, reopened, or when new commits are synchronized to an open PR targeting `main`. +- **Manual Trigger:** The workflow supports `workflow_dispatch`, allowing developers to manually run the pipeline from the GitHub Actions UI for testing or demonstration purposes. + +## Pipeline Steps + +| Step | Name | What it does | +|------|------|--------------| +| 1 | Checkout | Clones the repository code into the runner | +| 2 | Python setup | Installs Python 3.11 | +| 3 | System deps | Installs underlying system requirements (`xvfb`, `xdg-utils`, `wmctrl`) | +| 4 | Python deps | Installs all required pip packages including `playwright` and `openai` | +| 5 | Playwright | Downloads headless Chromium and its OS-level dependencies (`--with-deps`) | +| 6 | Log dir | Creates `~/.local/share/jiopc/agent/` to safely capture test logs | +| 7 | Run agent | Executes the testing agent (Part A) and captures the standard output | +| 8 | Find log | Automatically locates the latest JSONL generated log file by timestamp | +| 9 | LLM analysis | Runs `analyse.py` against the log file with the OpenAI API | +| 10 | Extract summary | Uses Python to safely parse the `summary: true` block for pass/fail counts | +| 11 | PR comment | Uses `actions/github-script` to post the extracted metrics and LLM report to the PR | +| 12 | Check result | Forcefully fails the pipeline if any `FAIL` results are detected | + +## Required GitHub Secrets + +To allow the pipeline to securely communicate with the LLM without exposing credentials in the public codebase, the following repository secret must be configured: + +| Secret name | What it is | Where to set it | +|-------------|------------|-----------------| +| `OPENAI_API_KEY` | OpenAI API key for LLM analysis | GitHub repo → Settings → Secrets and variables → Actions → New repository secret | + +## How to Read the PR Comment + +When the pipeline posts the automated PR comment, it provides a comprehensive overview: +- ✅ **Green (PASS):** Indicates that all executed tests passed successfully (or were safely blocked). +- ❌ **Red (FAIL):** Indicates that at least one functional failure was detected during the run. +- **BLOCKED Results:** These are explicitly expected for Jio URLs (like JioSaavn, JioCloud) that sit behind Cloudflare/Bot protection when accessed via headless Chromium. They **do not** indicate failures. +- **LLM Analysis:** This section contains the deep-dive diagnostic report and states the clear **PROMOTE** or **HOLD** recommendation from the AI. + +## What `GITHUB_TOKEN` is + +The `GITHUB_TOKEN` is a special authentication token automatically provided by GitHub Actions at runtime. No manual setup is needed. It is strictly used in Step 11 (`actions/github-script`) to authenticate the bot to post the PR comment on your behalf. Because the workflow file explicitly sets `permissions: pull-requests: write`, the token operates securely and with least-privilege. + +## Running Manually + +If you need to trigger a test run without opening a Pull Request: +1. Go to your GitHub repository and click the **Actions** tab. +2. Select **JioPC Testing Agent** from the left-hand sidebar. +3. Click the **Run workflow** dropdown on the right side. +4. Select your target branch and click the green **Run workflow** button. + +--- + +> **Note on Scope:** +> The CI pipeline runs Part A (web testing) only. Parts B and C require the JioPC Gold Image environment with pre-installed native applications and desktop folder structure. To run the full agent including all three components, use a JioPC VM or an Ubuntu 24.04 + LxQt environment with the Gold Image installed. +> +> Full run command (on JioPC VM): +> `jiopc-agent --config /usr/lib/jiopc-agent/config/jiopc-agent.yaml --analyse` diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..4b68ff2 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,163 @@ +name: JioPC Testing Agent + +on: + pull_request: + types: [opened, reopened, synchronize] + workflow_dispatch: + +jobs: + test-and-analyse: + runs-on: ubuntu-24.04 + permissions: + pull-requests: write + contents: read + env: + LLM_BASE_URL: https://api.openai.com/v1 + LLM_MODEL: gpt-3.5-turbo + LLM_API_KEY: ${{ secrets.OPENAI_API_KEY }} + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install system dependencies + run: | + sudo apt-get update -y + sudo apt-get install -y xvfb xdg-utils wmctrl + + - name: Install Python dependencies + run: | + pip install --upgrade pip + pip install pyyaml psutil pyxdg playwright openai + + - name: Install Playwright Chromium + run: python3 -m playwright install chromium --with-deps + + - name: Create log directory + run: mkdir -p ~/.local/share/jiopc/agent + + - name: Run agent + run: | + python3 src/jiopc_agent.py \ + --config config/jiopc-agent.yaml \ + --part A \ + 2>&1 | tee /tmp/agent_output.txt + continue-on-error: true + + - name: Find log file + run: | + LOG_FILE=$(ls -t ~/.local/share/jiopc/agent/test_run_*.log 2>/dev/null | head -1) + if [ -z "$LOG_FILE" ]; then + echo "ERROR: No log file found" + exit 1 + fi + echo "LOG_FILE=$LOG_FILE" >> $GITHUB_ENV + echo "Found log file: $LOG_FILE" + cat "$LOG_FILE" + + - name: Run LLM Analysis + run: | + python3 src/analyse.py --log "$LOG_FILE" > /tmp/analysis_output.txt 2>&1 + echo "--- LLM Analysis Output ---" + cat /tmp/analysis_output.txt + + # Save analysis output to env for PR comment + { + echo 'ANALYSIS_OUTPUT<> $GITHUB_ENV + env: + LLM_BASE_URL: ${{ env.LLM_BASE_URL }} + LLM_MODEL: ${{ env.LLM_MODEL }} + LLM_API_KEY: ${{ secrets.OPENAI_API_KEY }} + continue-on-error: true + + - name: Extract test summary + run: | + # Read the summary line from the JSON Lines log + SUMMARY=$(python3 -c " + import json, sys + with open('$LOG_FILE') as f: + for line in f: + try: + obj = json.loads(line.strip()) + if obj.get('summary'): + print(json.dumps(obj, indent=2)) + sys.exit(0) + except: + pass + print('{}') + ") + + TOTAL=$(echo "$SUMMARY" | python3 -c "import json,sys; d=json.load(sys.stdin); print(d.get('total',0))") + PASS=$(echo "$SUMMARY" | python3 -c "import json,sys; d=json.load(sys.stdin); print(d.get('pass',0))") + FAIL=$(echo "$SUMMARY" | python3 -c "import json,sys; d=json.load(sys.stdin); print(d.get('fail',0))") + BLOCKED=$(echo "$SUMMARY" | python3 -c "import json,sys; d=json.load(sys.stdin); print(d.get('blocked',0))") + + echo "TOTAL=$TOTAL" >> $GITHUB_ENV + echo "PASS=$PASS" >> $GITHUB_ENV + echo "FAIL=$FAIL" >> $GITHUB_ENV + echo "BLOCKED=$BLOCKED" >> $GITHUB_ENV + + echo "Total: $TOTAL | Pass: $PASS | Fail: $FAIL | Blocked: $BLOCKED" + + - name: Post PR comment + uses: actions/github-script@v7 + if: github.event_name == 'pull_request' + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const total = process.env.TOTAL || '0'; + const pass = process.env.PASS || '0'; + const fail = process.env.FAIL || '0'; + const blocked = process.env.BLOCKED || '0'; + const analysis = process.env.ANALYSIS_OUTPUT || 'Analysis not available'; + + const statusEmoji = parseInt(fail) === 0 ? '✅' : '❌'; + const statusText = parseInt(fail) === 0 ? 'PASS' : 'FAIL'; + + const body = [ + '## ' + statusEmoji + ' JioPC Testing Agent — ' + statusText, + '', + '| Metric | Count |', + '|--------|-------|', + '| Total Tests | ' + total + ' |', + '| ✅ Passed | ' + pass + ' |', + '| ❌ Failed | ' + fail + ' |', + '| 🔒 Blocked (bot protection) | ' + blocked + ' |', + '', + '> **Note:** BLOCKED results are expected for Jio URLs behind', + '> Cloudflare protection and do not indicate failures.', + '', + '---', + '', + '## 🤖 LLM Analysis', + '', + '```', + analysis, + '```', + '', + '---', + '_Triggered by commit ' + context.sha.substring(0, 7) + '_' + ].join('\n'); + + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: body + }); + + - name: Check result + run: | + if [ "$FAIL" -gt "0" ]; then + echo "Pipeline failed: $FAIL test(s) failed" + exit 1 + fi + echo "All tests passed or blocked. Pipeline successful." diff --git a/.github/workflows/jiopc-agent-ci.yml b/.github/workflows/jiopc-agent-ci.yml index 64624db..fdae652 100644 --- a/.github/workflows/jiopc-agent-ci.yml +++ b/.github/workflows/jiopc-agent-ci.yml @@ -54,7 +54,7 @@ jobs: with: issue-number: ${{ github.event.pull_request.number }} body: | - ## 🤖 JioPC Automated Testing Report + ## JioPC Automated Testing Report *The testing agent executed the full validation suite against this OS Patch.* ```text From 39a52f869bfe1c29844afadd97161667deb95437 Mon Sep 17 00:00:00 2001 From: ayush Date: Sat, 4 Jul 2026 03:30:58 +0530 Subject: [PATCH 4/7] ci: remove old workflow file --- .github/workflows/jiopc-agent-ci.yml | 62 ---------------------------- 1 file changed, 62 deletions(-) delete mode 100644 .github/workflows/jiopc-agent-ci.yml diff --git a/.github/workflows/jiopc-agent-ci.yml b/.github/workflows/jiopc-agent-ci.yml deleted file mode 100644 index fdae652..0000000 --- a/.github/workflows/jiopc-agent-ci.yml +++ /dev/null @@ -1,62 +0,0 @@ -name: JioPC Automated Testing CI - -on: - pull_request: - branches: [ "main" ] - -permissions: - pull-requests: write - issues: write - -jobs: - validate-os-patch: - runs-on: ubuntu-24.04 - - steps: - - name: Checkout Code - uses: actions/checkout@v4 - - - name: Install System Dependencies - run: | - sudo apt-get update - sudo apt-get install -y dpkg xvfb jq - - - name: Build Debian Package - run: | - dpkg-deb --build packaging/ jiopc-testing-agent.deb - - - name: Install Agent - run: | - sudo apt --reinstall install -y ./jiopc-testing-agent.deb - - - name: Run Testing Agent & Generate Analysis - id: run-agent - env: - LLM_BASE_URL: ${{ secrets.LLM_BASE_URL }} - LLM_MODEL: ${{ secrets.LLM_MODEL }} - LLM_API_KEY: ${{ secrets.LLM_API_KEY }} - run: | - # Run the agent in a headless X server to ensure native UI apps launch safely - # We pipe the output to a file so we can extract the LLM report later - xvfb-run jiopc-agent --analyse > analysis_output.txt || true - - # Extract the LLM report block safely (everything after "LLM ANALYSIS RESULT") - # The agent prints exactly "LLM ANALYSIS RESULT" before printing the report string - sed -n '/LLM ANALYSIS RESULT/,$p' analysis_output.txt > llm_report.txt - - # Pass the report content to GitHub Actions Environment Variables safely - echo "REPORT<> $GITHUB_ENV - cat llm_report.txt >> $GITHUB_ENV - echo "EOF" >> $GITHUB_ENV - - - name: Post LLM Analysis as PR Comment - uses: peter-evans/create-or-update-comment@v4 - with: - issue-number: ${{ github.event.pull_request.number }} - body: | - ## JioPC Automated Testing Report - *The testing agent executed the full validation suite against this OS Patch.* - - ```text - ${{ env.REPORT }} - ``` From 2f46bd9e0d64cd4451b247681a58443537d8365a Mon Sep 17 00:00:00 2001 From: ayush Date: Sat, 4 Jul 2026 03:32:16 +0530 Subject: [PATCH 5/7] ci: migrate LLM configuration to BharatCode and explicitly map SMTP secret --- .github/workflows/README.md | 3 ++- .github/workflows/ci.yml | 9 +++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index cdda190..2d5f16f 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -33,7 +33,8 @@ To allow the pipeline to securely communicate with the LLM without exposing cred | Secret name | What it is | Where to set it | |-------------|------------|-----------------| -| `OPENAI_API_KEY` | OpenAI API key for LLM analysis | GitHub repo → Settings → Secrets and variables → Actions → New repository secret | +| `LLM_API_KEY` | BharatCode API key for LLM analysis | GitHub repo → Settings → Secrets and variables → Actions → New repository secret | +| `SMTP_PASS` | Gmail App Password for email summary (Optional) | GitHub repo → Settings → Secrets and variables → Actions → New repository secret | ## How to Read the PR Comment diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4b68ff2..90d59db 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,9 +12,10 @@ jobs: pull-requests: write contents: read env: - LLM_BASE_URL: https://api.openai.com/v1 - LLM_MODEL: gpt-3.5-turbo - LLM_API_KEY: ${{ secrets.OPENAI_API_KEY }} + LLM_BASE_URL: https://bharatcode.ai/api/model/v1 + LLM_MODEL: bharatcode:qwen36-35b-q6-256k-vision + LLM_API_KEY: ${{ secrets.LLM_API_KEY }} + SMTP_PASS: ${{ secrets.SMTP_PASS }} steps: - name: Checkout code @@ -75,7 +76,7 @@ jobs: env: LLM_BASE_URL: ${{ env.LLM_BASE_URL }} LLM_MODEL: ${{ env.LLM_MODEL }} - LLM_API_KEY: ${{ secrets.OPENAI_API_KEY }} + LLM_API_KEY: ${{ secrets.LLM_API_KEY }} continue-on-error: true - name: Extract test summary From 74f3b9aa6319d3882cc50012e76b709e3b373d70 Mon Sep 17 00:00:00 2001 From: ayush Date: Sat, 4 Jul 2026 03:34:27 +0530 Subject: [PATCH 6/7] ci: fix log file search pattern to correctly match jsonl output --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 90d59db..dcb5a86 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,7 +52,7 @@ jobs: - name: Find log file run: | - LOG_FILE=$(ls -t ~/.local/share/jiopc/agent/test_run_*.log 2>/dev/null | head -1) + LOG_FILE=$(ls -t ~/.local/share/jiopc/agent/*.jsonl 2>/dev/null | head -1) if [ -z "$LOG_FILE" ]; then echo "ERROR: No log file found" exit 1 From cb9b2368dfdbd99913ec477c93f2772e320d04d3 Mon Sep 17 00:00:00 2001 From: ayush Date: Sat, 4 Jul 2026 03:37:49 +0530 Subject: [PATCH 7/7] ci: fix bare except swallowing SystemExit during summary extraction --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dcb5a86..1b0733f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -91,7 +91,7 @@ jobs: if obj.get('summary'): print(json.dumps(obj, indent=2)) sys.exit(0) - except: + except Exception: pass print('{}') ")