Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion .dockerignore
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,13 @@
**/*.swo

node_modules
**/node_modules
.nx
**/.nx
.angular
**/.angular
dist
**/dist
.docker-fast-context
tmp
out-tsc
Expand All @@ -26,7 +30,7 @@ database.db
!.env.example

# Rust CLI build artifacts (not needed in Node image)
evaluator/target
rust/target
**/target

# Local tooling / docs not needed in image
Expand Down
3 changes: 3 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,15 +23,18 @@ jobs:
with:
node-version: "26"
cache: npm
cache-dependency-path: www/package-lock.json

- name: Install
working-directory: www
run: npm ci
env:
PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: "true"
PUPPETEER_SKIP_DOWNLOAD: "true"
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: "1"

- name: Build
working-directory: www
run: npm run build
env:
PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: "true"
Expand Down
16 changes: 8 additions & 8 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -40,18 +40,18 @@ jobs:
exit 1
fi
VERSION="${TAG#v}"
PKG_VERSION="$(node -p "require('./package.json').version")"
CARGO_VERSION="$(awk '/^version = /{gsub(/"/, "", $3); print $3; exit}' evaluator/Cargo.toml)"
PKG_VERSION="$(node -p "require('./www/package.json').version")"
CARGO_VERSION="$(awk '/^version = /{gsub(/"/, "", $3); print $3; exit}' rust/Cargo.toml)"
if [ "$VERSION" != "$PKG_VERSION" ]; then
echo "Release tag version ($VERSION) does not match package.json ($PKG_VERSION)"
echo "Release tag version ($VERSION) does not match www/package.json ($PKG_VERSION)"
exit 1
fi
if [ "$VERSION" != "$CARGO_VERSION" ]; then
echo "Release tag version ($VERSION) does not match evaluator/Cargo.toml ($CARGO_VERSION)"
echo "Release tag version ($VERSION) does not match rust/Cargo.toml ($CARGO_VERSION)"
exit 1
fi
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
echo "OK: tag $TAG matches package.json + Cargo.toml $VERSION"
echo "OK: tag $TAG matches www/package.json + rust/Cargo.toml $VERSION"

build-and-push:
name: Build and Push Images
Expand All @@ -75,15 +75,15 @@ jobs:
uses: dtolnay/rust-toolchain@stable

- name: Build release binaries (CLI + MCP)
working-directory: evaluator
working-directory: rust
run: cargo build --release --features apps --bin evaluator --bin evaluator-mcp

- name: Upload binaries to GitHub Release
uses: softprops/action-gh-release@v3
with:
files: |
evaluator/target/release/evaluator
evaluator/target/release/evaluator-mcp
rust/target/release/evaluator
rust/target/release/evaluator-mcp

- name: Log in to Docker Hub
uses: docker/login-action@v4
Expand Down
19 changes: 10 additions & 9 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -15,18 +15,19 @@ HUB_IMAGE ?= interchouette/evaluator
GHCR_PERSONAL_IMAGE ?= ghcr.io/groussac/evaluator
GHCR_ORG_IMAGE ?= ghcr.io/interchouette-itc/evaluator
TAG ?= latest
APP_VERSION ?= $(shell node -p "require('./package.json').version")
APP_VERSION ?= $(shell node -p "require('./www/package.json').version")
DOCKERFILE ?= docker/Dockerfile
DOCKER_BUILDKIT ?= 1
CI ?= 0
COMPOSE_PROD ?= docker/docker-compose.yml
WWW ?= www

# Host Rust CLI / MCP — same shape as tvscreener-rs
unexport CARGO_TARGET_DIR
CARGO_BIN ?= cargo
CARGO = env -u CARGO_TARGET_DIR $(CARGO_BIN)
CARGO_FLAGS ?= --features apps
CLI_MANIFEST ?= evaluator/Cargo.toml
CLI_MANIFEST ?= rust/Cargo.toml
CLI_BIN ?= evaluator

.DEFAULT_GOAL := help
Expand All @@ -38,14 +39,14 @@ help:
@echo " make build cargo build $(CARGO_FLAGS) → evaluator + evaluator-mcp"
@echo " make build-release cargo build --release $(CARGO_FLAGS)"
@echo " make check / check-lib cargo check (apps) / lean lib"
@echo " make install cargo install --path evaluator --features apps"
@echo " make install cargo install --path rust --features apps"
@echo " make run ARGS='…' cargo run --bin evaluator --features apps -- …"
@echo " e.g. make run ARGS='evaluate --url http://127.0.0.1:8765/demo1shop.html --fn window.eval'"
@echo " make run-batch-capped same as run, wrapped in systemd --scope MemoryMax=$(MEMORY_MAX)"
@echo " e.g. make run-batch-capped ARGS='batch -p archive/test.csv -f window.eval'"
@echo " make run-mcp cargo run --bin evaluator-mcp (stdio)"
@echo " make run-mcp-http cargo run --bin evaluator-mcp -- --http"
@echo " (needs: npm run build + Chromium / PUPPETEER_EXECUTABLE_PATH)"
@echo " (needs: cd www && npm run build + Chromium / PUPPETEER_EXECUTABLE_PATH)"
@echo ""
@echo "Docker:"
@echo " make docker-pull-dev Pull Hub :dev (preferred local test)"
Expand Down Expand Up @@ -76,13 +77,13 @@ check-lib:
$(CARGO) check --manifest-path $(CLI_MANIFEST) --lib

install:
$(CARGO) install --path evaluator --force --features apps
$(CARGO) install --path rust --force --features apps

clean:
$(CARGO) clean --manifest-path $(CLI_MANIFEST)

run:
cd evaluator && $(CARGO) run $(CARGO_FLAGS) --bin $(CLI_BIN) -- $(if $(strip $(ARGS)),$(ARGS),--help)
cd rust && $(CARGO) run $(CARGO_FLAGS) --bin $(CLI_BIN) -- $(if $(strip $(ARGS)),$(ARGS),--help)

# Cap batch RAM so the OOM killer hits the job scope, not Cursor/desktop.
MEMORY_MAX ?= 4G
Expand All @@ -92,13 +93,13 @@ run-batch-capped:
$(MAKE) run ARGS='$(ARGS)'

run-mcp:
cd evaluator && $(CARGO) run $(CARGO_FLAGS) --bin evaluator-mcp -- $(ARGS)
cd rust && $(CARGO) run $(CARGO_FLAGS) --bin evaluator-mcp -- $(ARGS)

run-mcp-http:
cd evaluator && $(CARGO) run $(CARGO_FLAGS) --bin evaluator-mcp -- --http $(ARGS)
cd rust && $(CARGO) run $(CARGO_FLAGS) --bin evaluator-mcp -- --http $(ARGS)

version-show:
@echo "package.json version: $(APP_VERSION)"
@echo "www/package.json version: $(APP_VERSION)"
@echo "suggested tags: $(HUB_IMAGE):$(APP_VERSION) $(HUB_IMAGE):latest $(HUB_IMAGE):dev"

docker-build:
Expand Down
6 changes: 3 additions & 3 deletions docker/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ ARG RUST_IMAGE=rust:1.88-bookworm

FROM ${RUST_IMAGE} AS rust-builder
WORKDIR /src
COPY evaluator/ ./
COPY rust/ ./
RUN cargo build --release --features apps --bin evaluator --bin evaluator-mcp

FROM ${NODE_IMAGE} AS builder
Expand All @@ -24,11 +24,11 @@ ENV PUPPETEER_SKIP_DOWNLOAD=true \
USE_PUPPETEER=0 \
NX_NO_CLOUD=true

COPY package.json package-lock.json ./
COPY www/package.json www/package-lock.json ./
RUN --mount=type=cache,target=/root/.npm \
npm ci

COPY . .
COPY www/ .
RUN --mount=type=cache,target=/root/.npm \
npm run build \
&& npm prune --omit=dev
Expand Down
2 changes: 1 addition & 1 deletion docker/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ services:
- FUNCTIONS_PATH=/app/db/functions.json
volumes:
- evaluator-data:/app/db
- ../evaluator:/data:ro
- ../rust:/data:ro

volumes:
evaluator-data:
Expand Down
1 change: 1 addition & 0 deletions docs/CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

### Changed

- Move Nx workspace under `www/`; Rust CLI/MCP tree to `rust/`; remove unused `tools/`
- Docs live under [`docs/`](.) (`README.md`, this changelog); Docker Hub Overview maintained privately in `.cursor/scripts/DOCKERHUB.md`
- Runtime SQLite default path is `db/database.db` (Docker: `/app/db/database.db`)

Expand Down
48 changes: 24 additions & 24 deletions docs/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@ compromised / demo page
| Doc | Description |
| --- | --- |
| [`CHANGELOG.md`](CHANGELOG.md) | Release notes |
| [`evaluator/MAGECART.md`](../evaluator/MAGECART.md) | Magecart / Grelos product notes |
| [`evaluator/README.md`](../evaluator/README.md) | Rust CLI details |
| [`evaluator/MAGECART.md`](../rust/MAGECART.md) | Magecart / Grelos product notes |
| [`evaluator/README.md`](../rust/README.md) | Rust CLI details |
| [`docker/`](../docker/) | Dockerfile / Compose (Hub overview lives in private `.cursor/scripts/DOCKERHUB.md`) |


Expand All @@ -45,14 +45,14 @@ Typically helps with deobfuscating patterns like [this Stack Overflow case](http

Static scanners (Willem’s rules, VT + YARA) see files on disk. Skimmers often stay opaque until the browser runs them. **Evaluator is the complementary runtime tool:** after the packer calls `eval`, you see `grelos_v`, `checkout`, gate URLs, and friends in the hook output.

Full product notes (warnings, canary vs demos, batch): **[`evaluator/MAGECART.md`](../evaluator/MAGECART.md)** — start there for Magecart / Grelos context.
Full product notes (warnings, canary vs demos, batch): **[`evaluator/MAGECART.md`](../rust/MAGECART.md)** — start there for Magecart / Grelos context.

### References

- Jérôme Segura (Malwarebytes), *Hunting web skimmers with VirusTotal and YARA*, VB2021 — [PDF](https://vblocalhost.com/uploads/VB2021-Segura.pdf)
- Product notes (Magecart / Grelos / Blogspot demo fixtures): [`evaluator/MAGECART.md`](../evaluator/MAGECART.md)
- Product notes (Magecart / Grelos / Blogspot demo fixtures): [`evaluator/MAGECART.md`](../rust/MAGECART.md)
- Astra — signs of hacked OpenCart / Magento / PrestaShop stores (malicious JS): [getastra.com article](https://www.getastra.com/e/malware/infections/the-presence-of-these-malicious-javascript-are-the-sign-of-hacked-opencart-magento-or-prestashop-store)
- Willem de Groot — magento-malware-scanner frontend rules: [`rules/frontend.txt`](https://github.com/gwillem/magento-malware-scanner/blob/master/rules/frontend.txt) (local snapshot: [`evaluator/rules/frontend.txt`](../evaluator/rules/frontend.txt))
- Willem de Groot — magento-malware-scanner frontend rules: [`rules/frontend.txt`](https://github.com/gwillem/magento-malware-scanner/blob/master/rules/frontend.txt) (local snapshot: [`evaluator/rules/frontend.txt`](../rust/rules/frontend.txt))

<details open>
<summary><strong>Screenshots</strong> — web UI in action</summary>
Expand Down Expand Up @@ -86,16 +86,16 @@ https://evaluator.interchouette.net/evaluate/?url=https://www.w3schools.com/jsre

## Demos — illustrate the PDF story on localhost

Research fixtures under [`evaluator/archive/fixtures/`](../evaluator/archive/fixtures/). **Serve only on localhost.** Do **not** expose them through Nest, Express, the Docker public image, or any internet-facing route. Details: [`DEOBFUSCATED.md`](../evaluator/archive/fixtures/DEOBFUSCATED.md), [`MAGECART.md`](../evaluator/MAGECART.md).
Research fixtures under [`evaluator/archive/fixtures/`](../rust/archive/fixtures/). **Serve only on localhost.** Do **not** expose them through Nest, Express, the Docker public image, or any internet-facing route. Details: [`DEOBFUSCATED.md`](../rust/archive/fixtures/DEOBFUSCATED.md), [`MAGECART.md`](../rust/MAGECART.md).

| # | Obfuscated | Deobfuscated | What you learn |
| --- | --- | --- | --- |
| **1** | [`demo1shop.html`](../evaluator/archive/fixtures/demo1shop.html) | [`demo1shop.deobfuscated.js`](../evaluator/archive/fixtures/demo1shop.deobfuscated.js) | Neutral shop JS; historical `_0xd419` hex packer → `eval` → `checkout` / `cart` |
| **2** | [`demo2grelos.html`](../evaluator/archive/fixtures/demo2grelos.html) | [`demo2grelos.deobfuscated.js`](../evaluator/archive/fixtures/demo2grelos.deobfuscated.js) | Grelos-shaped marker; light teaching packer |
| **3** | [`demo3grelos.html`](../evaluator/archive/fixtures/demo3grelos.html) | [`demo3grelos.deobfuscated.js`](../evaluator/archive/fixtures/demo3grelos.deobfuscated.js) | Same marker; Magento-era `_0x` hex-table → `eval` (demo‑1 packing family) |
| **1** | [`demo1shop.html`](../rust/archive/fixtures/demo1shop.html) | [`demo1shop.deobfuscated.js`](../rust/archive/fixtures/demo1shop.deobfuscated.js) | Neutral shop JS; historical `_0xd419` hex packer → `eval` → `checkout` / `cart` |
| **2** | [`demo2grelos.html`](../rust/archive/fixtures/demo2grelos.html) | [`demo2grelos.deobfuscated.js`](../rust/archive/fixtures/demo2grelos.deobfuscated.js) | Grelos-shaped marker; light teaching packer |
| **3** | [`demo3grelos.html`](../rust/archive/fixtures/demo3grelos.html) | [`demo3grelos.deobfuscated.js`](../rust/archive/fixtures/demo3grelos.deobfuscated.js) | Same marker; Magento-era `_0x` hex-table → `eval` (demo‑1 packing family) |

```bash
cd evaluator/archive/fixtures && python3 -m http.server 8765
cd rust/archive/fixtures && python3 -m http.server 8765
# or: ./smoke-demos.sh
```

Expand All @@ -108,9 +108,9 @@ evaluator evaluate --url http://127.0.0.1:8765/demo3grelos.html --fn window.eval
Filter live/batch hits with keywords, regex, or Willem’s rules:

```bash
evaluator batch -p evaluator/archive/All-Live-Magento-Sites.csv \
evaluator batch -p rust/archive/All-Live-Magento-Sites.csv \
-f window.eval -n 1 \
--rules evaluator/rules/frontend.txt
--rules rust/rules/frontend.txt
```

**Authorized research only.** If you confirm a live compromise, report it responsibly to the merchant / hoster.
Expand Down Expand Up @@ -204,7 +204,7 @@ Visit http://localhost:4000/
make docker-build-dev # or make docker-build
```

Optional local packaging after `npm run build`: `make docker-build`.
Optional local packaging after `cd www && npm run build`: `make docker-build`.

CLI / MCP (built into the all-in-one image; CLI does **not** need web up):

Expand All @@ -222,13 +222,13 @@ make docker-run-mcp # stdio (evaluator-mcp)

| Workflow | Trigger | What |
| --- | --- | --- |
| `ci.yml` | PR / push to `dev` | `npm ci` + `npm run build` |
| `ci.yml` | PR / push to `dev` | `www/`: `npm ci` + `npm run build` |
| `docker-build-push-dev.yml` | manual | monolith `:dev` + `:latest` → Hub + GHCR; then Render via `RENDER_DEPLOY_HOOK` (Hub Overview: `python3 .cursor/scripts/sync-hub-description.py`) |
| `release.yml` | GitHub Release `vX.Y.Z` | attach host `evaluator` + `evaluator-mcp` binaries; push monolith `:X.Y.Z` + `:latest` → Hub + GHCR; Render redeploy |

Secret `RENDER_DEPLOY_HOOK` = full Render Deploy Hook URL (repo secret, not an app env). Without it, Hub still updates; Render stays on the old digests until a manual redeploy.

To publish a release: bump **both** `package.json` and `evaluator/Cargo.toml` to the same `X.Y.Z`, tag `vX.Y.Z`, create the GitHub Release. Workflow validates the tag against both versions.
To publish a release: bump **both** `www/package.json` and `rust/Cargo.toml` to the same `X.Y.Z`, tag `vX.Y.Z`, create the GitHub Release. Workflow validates the tag against both versions.

</details>

Expand All @@ -243,35 +243,35 @@ To publish a release: bump **both** `package.json` and `evaluator/Cargo.toml` to
- Evaluate engine: **Playwright by default** on distro Chromium (`PUPPETEER_EXECUTABLE_PATH`); set `USE_PUPPETEER=1` for Puppeteer

```shell
npm install
cd www && npm install
```

## Development server

```shell
npm start
cd www && npm start
```

Dev UI: http://localhost:4200/ (API proxied; Nest backend on 3333).

## Build

```shell
npm run build
cd www && npm run build
```

Artifacts land in `dist/`. Serve production Node gateway + Nest:
Artifacts land in `www/dist/`. Serve production Node gateway + Nest:

```shell
npm run serve
cd www && npm run serve
```

Then open http://localhost:4000/

## Test

```shell
npm test
cd www && npm test
```

</details>
Expand All @@ -282,20 +282,20 @@ npm test
Same shape as sibling ITC crates (`make build` / `make run` / `make install`):

```shell
make build # → evaluator/target/debug/evaluator (+ evaluator-mcp)
make build # → rust/target/debug/evaluator (+ evaluator-mcp)
make install # → ~/.cargo/bin (features apps)
make run ARGS='evaluate --url http://127.0.0.1:8765/demo1shop.html --fn window.eval'
```

Needs `npm run build` (Node entry at `dist/evaluator/server/server.js`) + Chromium.
Needs `cd www && npm run build` (Node entry at `www/dist/evaluator/server/server.js`) + Chromium.

| Mode | Command |
| --- | --- |
| Interactive | `make run` or `evaluator` |
| One-shot | `evaluator evaluate --url … [--fn …]` |
| CSV batch | `evaluator batch -p archive/test.csv -f window.eval -n 1` |

See [evaluator/README.md](../evaluator/README.md) for details.
See [evaluator/README.md](../rust/README.md) for details.

</details>

Expand Down
File renamed without changes.
File renamed without changes.
8 changes: 4 additions & 4 deletions evaluator/MAGECART.md → rust/MAGECART.md
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ evaluator evaluate \
| 3 | `demo3grelos.html` | `demo3grelos.deobfuscated.js` | Grelos marker; Magento-era `_0x` hex-table → `eval` |

```bash
cd evaluator/archive/fixtures && python3 -m http.server 8765
cd rust/archive/fixtures && python3 -m http.server 8765
# or: ./smoke-demos.sh
```

Expand All @@ -78,13 +78,13 @@ See [`archive/fixtures/README.md`](./archive/fixtures/README.md) and [`archive/f
### Batch

```bash
evaluator batch -p evaluator/archive/test.csv -f window.eval -n 1
evaluator batch -p rust/archive/test.csv -f window.eval -n 1
```

```bash
evaluator batch -p evaluator/archive/All-Live-Magento-Sites.csv \
evaluator batch -p rust/archive/All-Live-Magento-Sites.csv \
-f window.eval -n 2 \
--rules evaluator/rules/frontend.txt
--rules rust/rules/frontend.txt
```

## How this fits together
Expand Down
2 changes: 1 addition & 1 deletion evaluator/README.md → rust/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

Host binary under this directory. Build with `make build` / `make install` from the repo root (`--features apps`), or `cargo build --features apps` here.

**One engine:** CLI/MCP always spawn the shared Node entry (`dist/evaluator/server/server.js`) with `evaluate` or `batch`. No gateway HTTP, no `--legacy`. Needs `npm run build` at the repo root + Chromium (`PUPPETEER_EXECUTABLE_PATH`).
**One engine:** CLI/MCP always spawn the shared Node entry (`dist/evaluator/server/server.js`) with `evaluate` or `batch`. No gateway HTTP, no `--legacy`. Needs `npm run build` in `www/` + Chromium (`PUPPETEER_EXECUTABLE_PATH`).

Override the Node entry with `EVALUATOR_NODE_ENTRY`.

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ Raw Blogspot dump (do not scan): [`demo1shop-blogspot-raw.html`](./demo1shop-blo
## Run

```bash
cd evaluator/archive/fixtures
cd rust/archive/fixtures
python3 -m http.server 8765
```

Expand Down
Loading