superdoc · caio-pizzol · May 19, 2026 · May 19, 2026 · May 19, 2026 · May 19, 2026
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -13,6 +13,10 @@ jobs:
     steps:
       - uses: actions/checkout@v6
 
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '22'
+
       - uses: oven-sh/setup-bun@v2
 
       - run: bun install
@@ -31,3 +35,11 @@ jobs:
 
       - name: Build
         run: bun run build
+
+      - name: Type check (web)
+        run: bun run --cwd apps/web typecheck
+
+      - name: Build (web)
+        env:
+          DATABASE_URL: ${{ secrets.DATABASE_URL }}
+        run: bun run --cwd apps/web build
diff --git a/.github/workflows/deploy-site.yml b/.github/workflows/deploy-site.yml
@@ -15,12 +15,19 @@ jobs:
     steps:
       - uses: actions/checkout@v6
 
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '22'
+
       - uses: oven-sh/setup-bun@v2
 
-      - name: Generate pages
+      - name: Install dependencies
+        run: bun install --frozen-lockfile
+
+      - name: Build site
         env:
           DATABASE_URL: ${{ secrets.DATABASE_URL }}
-        run: bun run apps/web/scripts/generate-pages.ts
+        run: bun run --cwd apps/web build
 
       - name: Upload site to R2
         env:
@@ -31,30 +38,26 @@ jobs:
           R2_ENDPOINT: https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com
           R2_BUCKET: s3://${{ secrets.R2_BUCKET_NAME }}
         run: |
-          # Static files
-          aws s3 cp apps/web/og-image.png "$R2_BUCKET/og-image.png" --endpoint-url "$R2_ENDPOINT"
-          aws s3 cp apps/web/index.html "$R2_BUCKET/index.html" --endpoint-url "$R2_ENDPOINT"
-          aws s3 cp apps/web/sitemap.xml "$R2_BUCKET/sitemap.xml" --endpoint-url "$R2_ENDPOINT"
-          aws s3 cp apps/web/robots.txt "$R2_BUCKET/robots.txt" --endpoint-url "$R2_ENDPOINT"
-          aws s3 cp apps/web/llms.txt "$R2_BUCKET/llms.txt" --endpoint-url "$R2_ENDPOINT"
-          aws s3 cp apps/web/public/ "$R2_BUCKET/public/" --recursive --endpoint-url "$R2_ENDPOINT"
-
-          # Generated type pages (clean URLs: /types/legal, not /types/legal.html)
-          for f in apps/web/types/*.html; do
-            name=$(basename "$f" .html)
-            if [ "$name" = "index" ]; then
-              aws s3 cp "$f" "$R2_BUCKET/types" --content-type "text/html; charset=utf-8" --endpoint-url "$R2_ENDPOINT"
-            else
-              aws s3 cp "$f" "$R2_BUCKET/types/$name" --content-type "text/html; charset=utf-8" --endpoint-url "$R2_ENDPOINT"
-            fi
-          done
-
-          # Generated topic pages
-          for f in apps/web/topics/*.html; do
-            name=$(basename "$f" .html)
-            if [ "$name" = "index" ]; then
-              aws s3 cp "$f" "$R2_BUCKET/topics" --content-type "text/html; charset=utf-8" --endpoint-url "$R2_ENDPOINT"
+          set -euo pipefail
+
+          # Non-HTML assets: sync with their natural extensions and content types.
+          # No --delete: the bucket also holds /documents/ and /extracted/ which
+          # are scraper-managed and must not be touched by this workflow.
+          aws s3 sync apps/web/dist/ "$R2_BUCKET/" \
+            --exclude "*.html" \
+            --endpoint-url "$R2_ENDPOINT"
+
+          # HTML files: upload to extensionless keys to match canonical URLs.
+          # /classification.html -> r2://classification, /types/legal.html -> r2://types/legal.
+          # index.html is the one exception; it stays as a static homepage at /.
+          find apps/web/dist -name "*.html" -print0 | while IFS= read -r -d '' f; do
+            rel="${f#apps/web/dist/}"
+            if [ "$rel" = "index.html" ]; then
+              key="index.html"
             else
-              aws s3 cp "$f" "$R2_BUCKET/topics/$name" --content-type "text/html; charset=utf-8" --endpoint-url "$R2_ENDPOINT"
+              key="${rel%.html}"
             fi
+            aws s3 cp "$f" "$R2_BUCKET/$key" \
+              --content-type "text/html; charset=utf-8" \
+              --endpoint-url "$R2_ENDPOINT"
           done
diff --git a/.gitignore b/.gitignore
@@ -19,8 +19,9 @@ dev/
 
 # Generated web data (local only)
 apps/web/data/
-apps/web/types/
-apps/web/topics/
+
+# Astro
+apps/web/.astro/
 
 # Tests
 coverage/

diff --git a/apps/web/astro.config.mjs b/apps/web/astro.config.mjs
@@ -0,0 +1,10 @@
+import { defineConfig } from 'astro/config';
+
+export default defineConfig({
+  site: 'https://docxcorp.us',
+  output: 'static',
+  trailingSlash: 'never',
+  build: {
+    format: 'file',
+  },
+});
diff --git a/apps/web/content/classification.md b/apps/web/content/classification.md
@@ -0,0 +1,121 @@
+---
+title: "How docx-corpus classifies Word documents: taxonomy, model, and procedure"
+description: "Two-dimensional taxonomy of 10 document types and 9 topics. Trained from a 3,500-document Claude-labeled sample using a fine-tuned XLM-RoBERTa classifier. Full methodology."
+canonicalPath: /classification
+status: draft
+lastVerified: 2026-05-19
+primaryQuery: "document classification taxonomy"
+secondaryQueries:
+  - "docx classification"
+  - "xlm-roberta document classifier"
+  - "document type taxonomy"
+---
+
+# Classification
+
+docx-corpus uses a two-dimensional taxonomy. Every classified document has one document_type label (its form/structure) and one document_topic label (its subject domain). The labels are independent; a `legal` document can have any topic, and a `government` topic can have any document_type.
+
+Taxonomy version: `docx-corpus-v2` (2.0.0). Source: [scripts/classification/taxonomy.json](https://github.com/superdoc-dev/docx-corpus/blob/main/scripts/classification/taxonomy.json).
+
+## Document types (10)
+
+Form or structure of the document.
+
+| ID | Label | Examples |
+|---|---|---|
+| `legal` | Legal Documents | contracts, NDAs, terms, regulations, statutes |
+| `forms` | Forms & Applications | applications, registrations, surveys, ballots |
+| `reports` | Reports & Analysis | annual reports, research papers, case studies, white papers |
+| `policies` | Policies & Procedures | privacy policies, employee handbooks, SOPs |
+| `educational` | Educational Materials | syllabi, lesson plans, theses, dissertations |
+| `correspondence` | Correspondence | letters, memos, press releases, newsletters |
+| `technical` | Technical Documentation | manuals, API docs, specifications, datasheets |
+| `administrative` | Administrative Documents | meeting minutes, agendas, organizational docs |
+| `creative` | Creative & Marketing | brochures, proposals, marketing plans, presentation scripts, pitch decks |
+| `reference` | Reference & Catalogs | product catalogs, directories, glossaries, FAQs, indexes |
+
+## Topics (9)
+
+Subject domain of the document.
+
+`government`, `education`, `healthcare`, `finance`, `legal_judicial`, `technology`, `environment`, `nonprofit`, `general`.
+
+The `general` topic is the catch-all when no domain-specific label fits. Full descriptions and examples live in `taxonomy.json`.
+
+## Pipeline
+
+The classification pipeline follows the [FineWeb-Edu](https://huggingface.co/spaces/HuggingFaceFW/blogpost-fineweb-v1) pattern: an LLM labels a small sample, that sample trains a lightweight classifier, and the classifier runs over the full corpus.
+
+```
+sample.py    -> stratified sample of 3,500 documents
+label.py     -> Claude Haiku 4.5 labels each one (type + topic)
+train.py     -> fine-tune xlm-roberta-base on the labels
+classify.py  -> apply trained classifiers to every extracted document
+```
+
+### Sampling
+
+`scripts/classification/sample.py` draws a stratified sample from the database:
+
+- Default total: 3,500 documents
+- Languages: en, ru, cs, pl, es (proportional to corpus distribution)
+- Stratification: word-count terciles (small / medium / large) plus source-domain diversity
+- Seed: 42 (reproducible)
+
+The goal is coverage of length and source, not random uniform sampling. A 10K-word policy doc and a 50-word memo both need to be representable.
+
+### LLM labeling
+
+`scripts/classification/label.py` calls Claude Haiku 4.5 for each sampled document. The prompt includes the full taxonomy with descriptions and examples. The labeler returns `(document_type, document_topic, confidence_per_dim)`.
+
+Async, rate-limited, resumable. Output is a JSONL file used as training input.
+
+### Classifier training
+
+`scripts/classification/train.py` fine-tunes `xlm-roberta-base` with these defaults:
+
+| Hyperparameter | Value |
+|---|---|
+| Base model | `xlm-roberta-base` (multilingual) |
+| Epochs | 5 |
+| Learning rate | 2e-5 |
+| Batch size | 16 |
+| Max token length | 512 |
+| Validation split | 15% |
+| Loss | Cross-entropy with class weights for class imbalance |
+
+Two independent classifiers are trained: one for `document_type` (10 classes), one for `document_topic` (9 classes). Both share the same base model and training data.
+
+Class weights are computed from the label distribution so that small classes (e.g. `creative`, `reference`) don't get drowned out by `legal` and `forms`.
+
+Training supports `--modal` for cloud GPUs (T4 default, configurable). Models are persisted to `./models/{dimension}/best/`.
+
+### Inference at scale
+
+`scripts/classification/classify.py` loads both trained classifiers and runs them over every extracted document.
+
+- Local mode: single device (CUDA / MPS / CPU)
+- Cloud mode: `--modal --workers 20` fans out across parallel containers (~160 docs/sec aggregate)
+- Resumable: already-classified documents are skipped on restart
+- Writes back to the `documents` table: `document_type`, `document_topic`, `classification_confidence`, `classification_model`
+
+The `classification_model` column records `{base_model}-{taxonomy_version}` (e.g. `xlm-roberta-base-2.0.0`) so future taxonomy revisions or model swaps can be distinguished.
+
+The two dimensions are independent so a government privacy policy can be `(policies, government)` and a finance handbook can be `(policies, finance)`. 90 combinations are possible; the API exposes filters for any subset.
+
+## Known limitations
+
+- **Labeler ceiling.** Claude Haiku 4.5 is the upper bound on accuracy. No independent human-annotated evaluation set has been published with this release, so absolute classifier accuracy is unmeasured against a human baseline.
+- **English-trained, multilingual-applied.** The training sample is drawn from 5 languages (en, ru, cs, pl, es). The classifier is run over 76 languages because `xlm-roberta-base` is multilingual, but accuracy on languages outside the training set has not been measured here.
+- **Topic skew.** `government` and `education` dominate the topic distribution (~58% combined). This reflects the public web (.gov, .edu domains over-publish .docx), not a sampling choice.
+- **Hard cases.** Documents that span types (e.g. a "report" that's mostly tables and forms) get a single label. The confidence score helps you filter these.
+
+See [/quality](/quality) for confidence distributions and the rest of the dataset's limitations.
+
+## Reproducing this
+
+Code in [scripts/classification/](https://github.com/superdoc-dev/docx-corpus/tree/main/scripts/classification). Run order: `sample.py` -> `label.py` -> `train.py` -> `classify.py`. Pipeline notes in [CLAUDE.md](https://github.com/superdoc-dev/docx-corpus/blob/main/scripts/classification/CLAUDE.md).
+
+---
+
+Built by [SuperDoc](https://superdoc.dev/?utm_source=docxcorp.us&utm_medium=referral&utm_campaign=classification) - DOCX editing and tooling.
diff --git a/apps/web/content/dataset.md b/apps/web/content/dataset.md
@@ -0,0 +1,99 @@
+---
+title: "docx-corpus dataset: 736K classified Word documents from the public web"
+description: "Schema, coverage, and access methods for docx-corpus. 736K classified .docx files from Common Crawl, plus 365K additional uploaded files not yet in the classified browser/API set. Open dataset for document AI research."
+canonicalPath: /dataset
+status: draft
+lastVerified: 2026-05-19
+primaryQuery: "docx dataset"
+secondaryQueries:
+  - "document corpus"
+  - "word document dataset"
+  - "open dataset for document AI"
+---
+
+# docx-corpus dataset
+
+docx-corpus is an open dataset of real Word documents collected from Common Crawl. Each document is validated, deduplicated by content hash, has text extracted, and is labeled by document type and topic with a fine-tuned XLM-RoBERTa classifier.
+
+## What's in it
+
+Live numbers (verified 2026-05-19). Hero totals come from [api.docxcorp.us/stats](https://api.docxcorp.us/stats); the pipeline-gap breakdown, duplicate/failed counts, and word-count statistics come from direct database queries.
+
+| Bucket | Count |
+|---|---:|
+| Total documents uploaded | 1,101,537 |
+| Classified (type + topic + language) | 736,242 |
+| Extracted, awaiting classification | 267,539 |
+| Uploaded, awaiting extraction | 93,440 |
+| Extracted with empty text (skipped) | 4,316 |
+| Duplicate records (cross-crawl) | 241,993 |
+| Failed records (WARC error, invalid .docx) | 117,862 |
+
+The classified subset is what the browser, the API, and the Hugging Face dataset expose by default. The extracted and awaiting-extraction uploaded buckets are pipeline backlog and move into the classified set as extraction and classification catch up. Failed, duplicate, and empty-text rows stay in the database for accounting and do not reach the classified set under the current pipeline.
+
+## Per-document schema
+
+Each classified row exposes:
+
+| Field | Type | Description |
+|---|---|---|
+| `id` | string | SHA-256 of the raw .docx bytes; also the storage key (`documents/{id}.docx`) |
+| `filename` | string | Original filename as observed in the source URL |
+| `document_type` | enum (10) | Form/structure label. See [classification](/classification). |
+| `document_topic` | enum (9) | Subject domain label. See [classification](/classification). |
+| `language` | ISO 639-1 | Detected by lingua. 76 distinct values present. |
+| `word_count` | integer | Words in extracted text. Median 566, mean 2,795. |
+| `classification_confidence` | float | `min(type_conf, topic_conf)`. See [quality](/quality) for distribution. |
+| `url` | string | `https://docxcorp.us/documents/{id}.docx` |
+
+The extracted text for any classified document is available at `https://docxcorp.us/extracted/{id}.txt`. Raw .docx and extracted text both return `X-Robots-Tag: noindex` so they don't compete with the dataset pages in search.
+
+## Coverage
+
+| Dimension | Count |
+|---|---:|
+| Document types | 10 |
+| Topics | 9 |
+| Distinct languages detected | 76 |
+| Common Crawl snapshots | 3+ (varies by snapshot, see scraper logs) |
+
+Top 5 languages in the classified subset (out of 76):
+
+| Lang | Documents | Share |
+|---|---:|---:|
+| en | 245,018 | 33.3% |
+| ru | 57,879 | 7.9% |
+| cs | 50,709 | 6.9% |
+| pl | 38,546 | 5.2% |
+| es | 36,442 | 4.9% |
+
+The dataset is multilingual but English-dominant. Smaller-language documents are present but underrepresented relative to web traffic. Top languages in the uploaded-but-not-yet-classified set differ slightly; see [/quality](/quality) for the per-bucket breakdown.
+
+## How to access
+
+| Method | Best for | See |
+|---|---|---|
+| Hugging Face dataset | Bulk download, Parquet, `datasets` library | [/download](/download) |
+| REST API at api.docxcorp.us | Programmatic queries, faceted filters | [/download](/download) |
+| Manifest endpoint | Bulk .docx URL list for `wget` / `curl` | [/download](/download) |
+| Per-doc URL | One file at a time | `https://docxcorp.us/documents/{id}.docx` |
+
+## License
+
+Dataset metadata is licensed under [ODC-BY 1.0](https://opendatacommons.org/licenses/by/1-0/). The pipeline source code is MIT-licensed. Individual document contents retain their original copyright; the dataset is a metadata index plus a content-addressed mirror.
+
+If you need a document removed from the corpus, email [help@docxcorp.us](mailto:help@docxcorp.us) with the URL or hash.
+
+## Citing
+
+> docx-corpus (2026). Open corpus of classified Word documents from the public web. [https://docxcorp.us](https://docxcorp.us). Built by SuperDoc.
+
+## See also
+
+- [/classification](/classification) - taxonomy, labeling, model, training procedure
+- [/quality](/quality) - validation rules, dedup, confidence distribution, known limitations
+- [/download](/download) - Hugging Face, API endpoints, manifest, code examples
+
+---
+
+Built by [SuperDoc](https://superdoc.dev/?utm_source=docxcorp.us&utm_medium=referral&utm_campaign=dataset) - DOCX editing and tooling.