diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5977928..b644de1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,6 +13,10 @@ jobs: steps: - uses: actions/checkout@v6 + - uses: actions/setup-node@v4 + with: + node-version: '22' + - uses: oven-sh/setup-bun@v2 - run: bun install @@ -31,3 +35,11 @@ jobs: - name: Build run: bun run build + + - name: Type check (web) + run: bun run --cwd apps/web typecheck + + - name: Build (web) + env: + DATABASE_URL: ${{ secrets.DATABASE_URL }} + run: bun run --cwd apps/web build diff --git a/.github/workflows/deploy-site.yml b/.github/workflows/deploy-site.yml index 009252e..891147b 100644 --- a/.github/workflows/deploy-site.yml +++ b/.github/workflows/deploy-site.yml @@ -15,12 +15,19 @@ jobs: steps: - uses: actions/checkout@v6 + - uses: actions/setup-node@v4 + with: + node-version: '22' + - uses: oven-sh/setup-bun@v2 - - name: Generate pages + - name: Install dependencies + run: bun install --frozen-lockfile + + - name: Build site env: DATABASE_URL: ${{ secrets.DATABASE_URL }} - run: bun run apps/web/scripts/generate-pages.ts + run: bun run --cwd apps/web build - name: Upload site to R2 env: @@ -31,30 +38,26 @@ jobs: R2_ENDPOINT: https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com R2_BUCKET: s3://${{ secrets.R2_BUCKET_NAME }} run: | - # Static files - aws s3 cp apps/web/og-image.png "$R2_BUCKET/og-image.png" --endpoint-url "$R2_ENDPOINT" - aws s3 cp apps/web/index.html "$R2_BUCKET/index.html" --endpoint-url "$R2_ENDPOINT" - aws s3 cp apps/web/sitemap.xml "$R2_BUCKET/sitemap.xml" --endpoint-url "$R2_ENDPOINT" - aws s3 cp apps/web/robots.txt "$R2_BUCKET/robots.txt" --endpoint-url "$R2_ENDPOINT" - aws s3 cp apps/web/llms.txt "$R2_BUCKET/llms.txt" --endpoint-url "$R2_ENDPOINT" - aws s3 cp apps/web/public/ "$R2_BUCKET/public/" --recursive --endpoint-url "$R2_ENDPOINT" - - # Generated type pages (clean URLs: /types/legal, not /types/legal.html) - for f in apps/web/types/*.html; do - name=$(basename "$f" .html) - if [ "$name" = "index" ]; then - aws s3 cp "$f" "$R2_BUCKET/types" --content-type "text/html; charset=utf-8" --endpoint-url "$R2_ENDPOINT" - else - aws s3 cp "$f" "$R2_BUCKET/types/$name" --content-type "text/html; charset=utf-8" --endpoint-url "$R2_ENDPOINT" - fi - done - - # Generated topic pages - for f in apps/web/topics/*.html; do - name=$(basename "$f" .html) - if [ "$name" = "index" ]; then - aws s3 cp "$f" "$R2_BUCKET/topics" --content-type "text/html; charset=utf-8" --endpoint-url "$R2_ENDPOINT" + set -euo pipefail + + # Non-HTML assets: sync with their natural extensions and content types. + # No --delete: the bucket also holds /documents/ and /extracted/ which + # are scraper-managed and must not be touched by this workflow. + aws s3 sync apps/web/dist/ "$R2_BUCKET/" \ + --exclude "*.html" \ + --endpoint-url "$R2_ENDPOINT" + + # HTML files: upload to extensionless keys to match canonical URLs. + # /classification.html -> r2://classification, /types/legal.html -> r2://types/legal. + # index.html is the one exception; it stays as a static homepage at /. + find apps/web/dist -name "*.html" -print0 | while IFS= read -r -d '' f; do + rel="${f#apps/web/dist/}" + if [ "$rel" = "index.html" ]; then + key="index.html" else - aws s3 cp "$f" "$R2_BUCKET/topics/$name" --content-type "text/html; charset=utf-8" --endpoint-url "$R2_ENDPOINT" + key="${rel%.html}" fi + aws s3 cp "$f" "$R2_BUCKET/$key" \ + --content-type "text/html; charset=utf-8" \ + --endpoint-url "$R2_ENDPOINT" done diff --git a/.gitignore b/.gitignore index c396185..4c23510 100644 --- a/.gitignore +++ b/.gitignore @@ -19,8 +19,9 @@ dev/ # Generated web data (local only) apps/web/data/ -apps/web/types/ -apps/web/topics/ + +# Astro +apps/web/.astro/ # Tests coverage/ diff --git a/apps/web/astro.config.mjs b/apps/web/astro.config.mjs new file mode 100644 index 0000000..a3cc3eb --- /dev/null +++ b/apps/web/astro.config.mjs @@ -0,0 +1,10 @@ +import { defineConfig } from 'astro/config'; + +export default defineConfig({ + site: 'https://docxcorp.us', + output: 'static', + trailingSlash: 'never', + build: { + format: 'file', + }, +}); diff --git a/apps/web/content/classification.md b/apps/web/content/classification.md new file mode 100644 index 0000000..5005ce0 --- /dev/null +++ b/apps/web/content/classification.md @@ -0,0 +1,121 @@ +--- +title: "How docx-corpus classifies Word documents: taxonomy, model, and procedure" +description: "Two-dimensional taxonomy of 10 document types and 9 topics. Trained from a 3,500-document Claude-labeled sample using a fine-tuned XLM-RoBERTa classifier. Full methodology." +canonicalPath: /classification +status: draft +lastVerified: 2026-05-19 +primaryQuery: "document classification taxonomy" +secondaryQueries: + - "docx classification" + - "xlm-roberta document classifier" + - "document type taxonomy" +--- + +# Classification + +docx-corpus uses a two-dimensional taxonomy. Every classified document has one document_type label (its form/structure) and one document_topic label (its subject domain). The labels are independent; a `legal` document can have any topic, and a `government` topic can have any document_type. + +Taxonomy version: `docx-corpus-v2` (2.0.0). Source: [scripts/classification/taxonomy.json](https://github.com/superdoc-dev/docx-corpus/blob/main/scripts/classification/taxonomy.json). + +## Document types (10) + +Form or structure of the document. + +| ID | Label | Examples | +|---|---|---| +| `legal` | Legal Documents | contracts, NDAs, terms, regulations, statutes | +| `forms` | Forms & Applications | applications, registrations, surveys, ballots | +| `reports` | Reports & Analysis | annual reports, research papers, case studies, white papers | +| `policies` | Policies & Procedures | privacy policies, employee handbooks, SOPs | +| `educational` | Educational Materials | syllabi, lesson plans, theses, dissertations | +| `correspondence` | Correspondence | letters, memos, press releases, newsletters | +| `technical` | Technical Documentation | manuals, API docs, specifications, datasheets | +| `administrative` | Administrative Documents | meeting minutes, agendas, organizational docs | +| `creative` | Creative & Marketing | brochures, proposals, marketing plans, presentation scripts, pitch decks | +| `reference` | Reference & Catalogs | product catalogs, directories, glossaries, FAQs, indexes | + +## Topics (9) + +Subject domain of the document. + +`government`, `education`, `healthcare`, `finance`, `legal_judicial`, `technology`, `environment`, `nonprofit`, `general`. + +The `general` topic is the catch-all when no domain-specific label fits. Full descriptions and examples live in `taxonomy.json`. + +## Pipeline + +The classification pipeline follows the [FineWeb-Edu](https://huggingface.co/spaces/HuggingFaceFW/blogpost-fineweb-v1) pattern: an LLM labels a small sample, that sample trains a lightweight classifier, and the classifier runs over the full corpus. + +``` +sample.py -> stratified sample of 3,500 documents +label.py -> Claude Haiku 4.5 labels each one (type + topic) +train.py -> fine-tune xlm-roberta-base on the labels +classify.py -> apply trained classifiers to every extracted document +``` + +### Sampling + +`scripts/classification/sample.py` draws a stratified sample from the database: + +- Default total: 3,500 documents +- Languages: en, ru, cs, pl, es (proportional to corpus distribution) +- Stratification: word-count terciles (small / medium / large) plus source-domain diversity +- Seed: 42 (reproducible) + +The goal is coverage of length and source, not random uniform sampling. A 10K-word policy doc and a 50-word memo both need to be representable. + +### LLM labeling + +`scripts/classification/label.py` calls Claude Haiku 4.5 for each sampled document. The prompt includes the full taxonomy with descriptions and examples. The labeler returns `(document_type, document_topic, confidence_per_dim)`. + +Async, rate-limited, resumable. Output is a JSONL file used as training input. + +### Classifier training + +`scripts/classification/train.py` fine-tunes `xlm-roberta-base` with these defaults: + +| Hyperparameter | Value | +|---|---| +| Base model | `xlm-roberta-base` (multilingual) | +| Epochs | 5 | +| Learning rate | 2e-5 | +| Batch size | 16 | +| Max token length | 512 | +| Validation split | 15% | +| Loss | Cross-entropy with class weights for class imbalance | + +Two independent classifiers are trained: one for `document_type` (10 classes), one for `document_topic` (9 classes). Both share the same base model and training data. + +Class weights are computed from the label distribution so that small classes (e.g. `creative`, `reference`) don't get drowned out by `legal` and `forms`. + +Training supports `--modal` for cloud GPUs (T4 default, configurable). Models are persisted to `./models/{dimension}/best/`. + +### Inference at scale + +`scripts/classification/classify.py` loads both trained classifiers and runs them over every extracted document. + +- Local mode: single device (CUDA / MPS / CPU) +- Cloud mode: `--modal --workers 20` fans out across parallel containers (~160 docs/sec aggregate) +- Resumable: already-classified documents are skipped on restart +- Writes back to the `documents` table: `document_type`, `document_topic`, `classification_confidence`, `classification_model` + +The `classification_model` column records `{base_model}-{taxonomy_version}` (e.g. `xlm-roberta-base-2.0.0`) so future taxonomy revisions or model swaps can be distinguished. + +The two dimensions are independent so a government privacy policy can be `(policies, government)` and a finance handbook can be `(policies, finance)`. 90 combinations are possible; the API exposes filters for any subset. + +## Known limitations + +- **Labeler ceiling.** Claude Haiku 4.5 is the upper bound on accuracy. No independent human-annotated evaluation set has been published with this release, so absolute classifier accuracy is unmeasured against a human baseline. +- **English-trained, multilingual-applied.** The training sample is drawn from 5 languages (en, ru, cs, pl, es). The classifier is run over 76 languages because `xlm-roberta-base` is multilingual, but accuracy on languages outside the training set has not been measured here. +- **Topic skew.** `government` and `education` dominate the topic distribution (~58% combined). This reflects the public web (.gov, .edu domains over-publish .docx), not a sampling choice. +- **Hard cases.** Documents that span types (e.g. a "report" that's mostly tables and forms) get a single label. The confidence score helps you filter these. + +See [/quality](/quality) for confidence distributions and the rest of the dataset's limitations. + +## Reproducing this + +Code in [scripts/classification/](https://github.com/superdoc-dev/docx-corpus/tree/main/scripts/classification). Run order: `sample.py` -> `label.py` -> `train.py` -> `classify.py`. Pipeline notes in [CLAUDE.md](https://github.com/superdoc-dev/docx-corpus/blob/main/scripts/classification/CLAUDE.md). + +--- + +Built by [SuperDoc](https://superdoc.dev/?utm_source=docxcorp.us&utm_medium=referral&utm_campaign=classification) - DOCX editing and tooling. diff --git a/apps/web/content/dataset.md b/apps/web/content/dataset.md new file mode 100644 index 0000000..ea04d41 --- /dev/null +++ b/apps/web/content/dataset.md @@ -0,0 +1,99 @@ +--- +title: "docx-corpus dataset: 736K classified Word documents from the public web" +description: "Schema, coverage, and access methods for docx-corpus. 736K classified .docx files from Common Crawl, plus 365K additional uploaded files not yet in the classified browser/API set. Open dataset for document AI research." +canonicalPath: /dataset +status: draft +lastVerified: 2026-05-19 +primaryQuery: "docx dataset" +secondaryQueries: + - "document corpus" + - "word document dataset" + - "open dataset for document AI" +--- + +# docx-corpus dataset + +docx-corpus is an open dataset of real Word documents collected from Common Crawl. Each document is validated, deduplicated by content hash, has text extracted, and is labeled by document type and topic with a fine-tuned XLM-RoBERTa classifier. + +## What's in it + +Live numbers (verified 2026-05-19). Hero totals come from [api.docxcorp.us/stats](https://api.docxcorp.us/stats); the pipeline-gap breakdown, duplicate/failed counts, and word-count statistics come from direct database queries. + +| Bucket | Count | +|---|---:| +| Total documents uploaded | 1,101,537 | +| Classified (type + topic + language) | 736,242 | +| Extracted, awaiting classification | 267,539 | +| Uploaded, awaiting extraction | 93,440 | +| Extracted with empty text (skipped) | 4,316 | +| Duplicate records (cross-crawl) | 241,993 | +| Failed records (WARC error, invalid .docx) | 117,862 | + +The classified subset is what the browser, the API, and the Hugging Face dataset expose by default. The extracted and awaiting-extraction uploaded buckets are pipeline backlog and move into the classified set as extraction and classification catch up. Failed, duplicate, and empty-text rows stay in the database for accounting and do not reach the classified set under the current pipeline. + +## Per-document schema + +Each classified row exposes: + +| Field | Type | Description | +|---|---|---| +| `id` | string | SHA-256 of the raw .docx bytes; also the storage key (`documents/{id}.docx`) | +| `filename` | string | Original filename as observed in the source URL | +| `document_type` | enum (10) | Form/structure label. See [classification](/classification). | +| `document_topic` | enum (9) | Subject domain label. See [classification](/classification). | +| `language` | ISO 639-1 | Detected by lingua. 76 distinct values present. | +| `word_count` | integer | Words in extracted text. Median 566, mean 2,795. | +| `classification_confidence` | float | `min(type_conf, topic_conf)`. See [quality](/quality) for distribution. | +| `url` | string | `https://docxcorp.us/documents/{id}.docx` | + +The extracted text for any classified document is available at `https://docxcorp.us/extracted/{id}.txt`. Raw .docx and extracted text both return `X-Robots-Tag: noindex` so they don't compete with the dataset pages in search. + +## Coverage + +| Dimension | Count | +|---|---:| +| Document types | 10 | +| Topics | 9 | +| Distinct languages detected | 76 | +| Common Crawl snapshots | 3+ (varies by snapshot, see scraper logs) | + +Top 5 languages in the classified subset (out of 76): + +| Lang | Documents | Share | +|---|---:|---:| +| en | 245,018 | 33.3% | +| ru | 57,879 | 7.9% | +| cs | 50,709 | 6.9% | +| pl | 38,546 | 5.2% | +| es | 36,442 | 4.9% | + +The dataset is multilingual but English-dominant. Smaller-language documents are present but underrepresented relative to web traffic. Top languages in the uploaded-but-not-yet-classified set differ slightly; see [/quality](/quality) for the per-bucket breakdown. + +## How to access + +| Method | Best for | See | +|---|---|---| +| Hugging Face dataset | Bulk download, Parquet, `datasets` library | [/download](/download) | +| REST API at api.docxcorp.us | Programmatic queries, faceted filters | [/download](/download) | +| Manifest endpoint | Bulk .docx URL list for `wget` / `curl` | [/download](/download) | +| Per-doc URL | One file at a time | `https://docxcorp.us/documents/{id}.docx` | + +## License + +Dataset metadata is licensed under [ODC-BY 1.0](https://opendatacommons.org/licenses/by/1-0/). The pipeline source code is MIT-licensed. Individual document contents retain their original copyright; the dataset is a metadata index plus a content-addressed mirror. + +If you need a document removed from the corpus, email [help@docxcorp.us](mailto:help@docxcorp.us) with the URL or hash. + +## Citing + +> docx-corpus (2026). Open corpus of classified Word documents from the public web. [https://docxcorp.us](https://docxcorp.us). Built by SuperDoc. + +## See also + +- [/classification](/classification) - taxonomy, labeling, model, training procedure +- [/quality](/quality) - validation rules, dedup, confidence distribution, known limitations +- [/download](/download) - Hugging Face, API endpoints, manifest, code examples + +--- + +Built by [SuperDoc](https://superdoc.dev/?utm_source=docxcorp.us&utm_medium=referral&utm_campaign=dataset) - DOCX editing and tooling. diff --git a/apps/web/content/download.md b/apps/web/content/download.md new file mode 100644 index 0000000..24aae5a --- /dev/null +++ b/apps/web/content/download.md @@ -0,0 +1,181 @@ +--- +title: "Download docx-corpus: Hugging Face, REST API, manifest, and code examples" +description: "Three ways to access docx-corpus: Hugging Face Parquet dataset, REST API with faceted filters, and bulk URL manifests. Working examples for Python, curl, and wget." +canonicalPath: /download +status: draft +lastVerified: 2026-05-19 +primaryQuery: "download docx dataset" +secondaryQueries: + - "huggingface docx dataset" + - "docx corpus api" + - "bulk download word documents" +--- + +# Download + +Three access paths. Pick by use case. + +| Use case | Path | +|---|---| +| Bulk metadata + extracted text for ML training | [Hugging Face](#hugging-face) | +| Programmatic filtering, faceted browse, exact counts | [REST API](#rest-api) | +| Bulk .docx files by filter (any subset, full corpus) | [Manifest + wget/curl](#manifest) | +| Single document | `https://docxcorp.us/documents/{id}.docx` | + +All three return the same underlying classified subset (736,242 documents). The full uploaded set, including unclassified rows, is only in the database, not in any public endpoint. + +## Hugging Face + +```python +from datasets import load_dataset + +ds = load_dataset("superdoc-dev/docx-corpus", split="train") +print(ds[0]) +``` + +Returned fields per row: `id`, `filename`, `type`, `topic`, `language`, `word_count`, `confidence`, `url`. The `url` points at the raw .docx; extracted text is at `https://docxcorp.us/extracted/{id}.txt`. + +License: [ODC-BY 1.0](https://opendatacommons.org/licenses/by/1-0/) (metadata). Individual document copyright remains with the original author. + +## REST API + +Base URL: `https://api.docxcorp.us` + +### `GET /stats` + +Corpus-wide counters: total documents, language list, type/topic breakdowns with counts and percentages. No parameters. 5-minute cache. + +```bash +curl https://api.docxcorp.us/stats | jq '.hero' +``` + +### `GET /documents` + +Paginated, filterable. Returns rows plus faceted counts (so the UI can show option counts that update with active filters). + +| Param | Default | Notes | +|---|---|---| +| `type` | (any) | One of the 10 document types | +| `topic` | (any) | One of the 9 topics | +| `lang` | (any) | ISO 639-1 code | +| `min_confidence` | `0` | 0.0 - 1.0, inclusive lower bound | +| `page` | `1` | 1-indexed | +| `limit` | `25` | Max 100 | + +```bash +# First page of high-confidence legal documents in English +curl "https://api.docxcorp.us/documents?type=legal&lang=en&min_confidence=0.8" +``` + +Response shape: + +```json +{ + "documents": [ + { + "id": "...", + "filename": "...", + "type": "legal", + "topic": "government", + "language": "en", + "word_count": 1234, + "confidence": 0.95 + } + ], + "total": 12345, + "page": 1, + "pages": 124, + "facets": { + "types": [{"id": "legal", "label": "Legal", "count": 12345}], + "topics": [{"id": "government", "label": "Government", "count": 4567}], + "languages": [{"code": "en", "name": "English", "count": 9012, "percentage": 33.3}] + } +} +``` + +`facets.languages` is capped at the top 20 entries with `percentage` computed against the filtered set; the other two facets return all values matching the active (excluding-own-dimension) filters. + +Rows are ordered by `classification_confidence DESC NULLS LAST` so the most confidently labeled documents appear first. + +### `GET /manifest` + +Returns a newline-delimited text file of .docx URLs matching the given filters. Same filter params as `/documents` (no pagination; capped at 2,000,000 rows). Use this when you want to download files in bulk. + +```bash +# Manifest of English legal documents, classified at 0.8+ confidence +curl "https://api.docxcorp.us/manifest?type=legal&lang=en&min_confidence=0.8" -o manifest.txt +wc -l manifest.txt +``` + +Filename is auto-named based on the filters: `docx-corpus-legal-en-manifest.txt`. + +## Manifest + wget/curl + +Once you have a manifest, bulk-download with standard tools: + +```bash +# wget, one file at a time, into ./corpus/ +wget -i manifest.txt -P ./corpus/ + +# wget, parallel (4 concurrent) +xargs -n 1 -P 4 -a manifest.txt wget -q -P ./corpus/ + +# curl, write each to a hash-named file +xargs -n 1 curl -OJL < manifest.txt +``` + +Files are content-addressed: `{sha256}.docx`. Two URLs pointing at the same bytes deduplicate to one file on disk because they share an ID. + +For the full classified set, fetch the empty manifest: + +```bash +curl "https://api.docxcorp.us/manifest" -o all-manifest.txt +# ~736K URLs, plain text, ~73 MB +``` + +## Programmatic patterns + +### Page through everything in Python + +```python +import requests + +url = "https://api.docxcorp.us/documents" +page = 1 +while True: + r = requests.get(url, params={"limit": 100, "page": page}) + r.raise_for_status() + data = r.json() + if not data["documents"]: + break + for doc in data["documents"]: + ... # do something with each row + page += 1 +``` + +### Stream extracted text without downloading .docx + +```python +import requests +for doc_id in doc_ids: + text = requests.get(f"https://docxcorp.us/extracted/{doc_id}.txt").text + ... +``` + +The extracted text endpoint is cached at the edge and returns `X-Robots-Tag: noindex`, so it doesn't pollute search results. + +The API accepts one value per dimension. For multi-value filters, query each value separately or concatenate per-value manifests. + +## Rate limits + +The API is unmetered and behind Cloudflare. Free, no signup, no API key. For full-corpus pulls, use the Hugging Face dataset rather than paging through `/documents`. If your pipeline will fetch heavily on a schedule, email [help@docxcorp.us](mailto:help@docxcorp.us) first. + +## See also + +- [/dataset](/dataset) - schema, coverage, license +- [/classification](/classification) - what the labels mean +- [/quality](/quality) - confidence distribution, limitations + +--- + +Built by [SuperDoc](https://superdoc.dev/?utm_source=docxcorp.us&utm_medium=referral&utm_campaign=download) - DOCX editing and tooling. diff --git a/apps/web/content/quality.md b/apps/web/content/quality.md new file mode 100644 index 0000000..993c5a8 --- /dev/null +++ b/apps/web/content/quality.md @@ -0,0 +1,111 @@ +--- +title: "Quality, validation, and known limitations of docx-corpus" +description: "How docx-corpus validates .docx files, deduplicates by content hash, distributes classification confidence, and where the dataset is weakest. First-class limitations." +canonicalPath: /quality +status: draft +lastVerified: 2026-05-19 +primaryQuery: "docx dataset quality" +secondaryQueries: + - "document classification confidence" + - "common crawl docx coverage" + - "docx corpus methodology" +--- + +# Quality + +This page is for researchers deciding whether docx-corpus is fit for their task. It documents validation rules, deduplication, the confidence distribution, and the limitations we know about. If you find one we don't list, email [help@docxcorp.us](mailto:help@docxcorp.us). + +## Validation + +Every file that enters the corpus passes these checks (in [`packages/scraper/validation.ts`](https://github.com/superdoc-dev/docx-corpus/blob/main/packages/scraper/validation.ts)): + +1. Minimum size: at least 100 bytes. +2. ZIP magic bytes: starts with `PK\x03\x04`. +3. Contains `[Content_Types].xml`. +4. Contains `word/document.xml` (or `word/document` for older variants). + +Files that fail any check are stored as `failed` records (URL hash, no content) so re-runs don't re-fetch them. Failed count is 117,862 (current). + +## Deduplication + +Storage is content-addressed. The document ID is the SHA-256 of the raw .docx bytes; the storage key is `documents/{id}.docx`. Two URLs that point to byte-identical files collapse to one storage record. + +When a duplicate is detected, the second URL is preserved as a `duplicate` record (`dup-{sha256(url+crawlId)}`) that points back to the canonical content hash. This keeps a 1:1 mapping between CDX URLs and database rows per crawl, without storing the bytes twice. + +Current dedup outcome: 241,993 duplicate records pointing at 1,101,537 unique-content uploads. + +This is exact-content dedup only. **Near-duplicates** (same template with different filled-in fields, or the same document re-saved by Word with minor metadata changes) are NOT detected. Researchers building dedup-sensitive benchmarks should re-dedup at the text level themselves. + +## Classification confidence + +The `classification_confidence` field is `min(type_confidence, topic_confidence)` from the two XLM-RoBERTa classifiers. Distribution across the 736,242 classified documents (verified 2026-05-19): + +| Range | Count | Share | +|---|---:|---:| +| 0.9 - 1.0 | 366,389 | 49.8% | +| 0.8 - 0.9 | 108,863 | 14.8% | +| 0.7 - 0.8 | 74,462 | 10.1% | +| 0.6 - 0.7 | 67,280 | 9.1% | +| 0.5 - 0.6 | 64,795 | 8.8% | +| 0.4 - 0.5 | 39,678 | 5.4% | +| 0.3 - 0.4 | 13,127 | 1.8% | +| 0.2 - 0.3 | 1,596 | 0.2% | +| 0.1 - 0.2 | 52 | <0.01% | + +Half the corpus is at 0.9+ confidence. About 14% sits below 0.6. For tasks where label noise matters, filter on `min_confidence=0.7` (76% of the corpus) or `0.8` (65%). + +The classifier was not calibrated against a held-out human-annotated test set. The confidence score is the softmax probability from the model, not a probability of correctness. Treat it as a relative ranking, not a calibrated probability. + +## Coverage + +### Sources + +Documents come from Common Crawl WARC archives. Coverage inherits Common Crawl's coverage of the public web: heavy on government, education, and large public-facing sites; thin on intranets, paywalled content, and short-lived web pages. + +### Languages + +76 distinct languages were detected by lingua. The top 5 (en, ru, cs, pl, es) make up 55% of the corpus. The long tail (down to fewer than 100 documents) covers regional and minority languages, but reliability of the lang code drops on very short documents. + +`unknown` is its own bucket (103,869 documents, ~9.4% of total) where lingua couldn't make a confident call, typically very short or symbol-heavy documents. + +### Document length + +| Statistic | Value | +|---|---:| +| Min word count | 1 | +| Median word count | 566 | +| Mean word count | 2,795 | +| Max word count | 15,811,488 | + +The mean is pulled by a long tail of very large documents (legal compendia, technical manuals). The median is the better summary of "what a typical document looks like." + +Documents with `word_count = 0` (4,316 records) extracted as zero words even though the .docx itself was valid. The records have not been sampled to classify the causes. + +## Pipeline gaps + +The 365K-document gap between "uploaded" (1.1M) and "classified" (736K) breaks down as: + +| Bucket | Count | What it is | +|---|---:|---| +| Extracted, awaiting classification | 267,539 | Text exists, classifier hasn't run on these rows yet | +| Uploaded, awaiting extraction | 93,440 | File on R2, extraction pipeline hasn't processed them | +| Extracted with empty text | 4,316 | Extraction returned zero words; root causes not yet sampled | + +This gap is operational, not a quality decision. As extraction and classification batches run, the classified set will grow. + +## Known limitations + +1. **Source bias.** Common Crawl reflects what's publicly linked, not what's privately authored. Government and education domains over-publish .docx, so they're over-represented. Intranets and content behind login are absent. +2. **Topic skew.** `government` (33%) and `education` (25%) dominate. Filter and sample if you need balance. +3. **English-heavy.** English is 33% of the classified subset. Smaller languages are present but small in share. +4. **No human evaluation set.** Classification accuracy is bounded by the LLM labeler (Claude Haiku 4.5). No human-annotated test set has been published. +5. **Near-duplicate templates not collapsed.** Exact byte-identical duplicates are removed; near-duplicates (same template, different fields) remain. +6. **Language detection on short docs.** Lingua is reliable on full text and less so on short, mixed-language, or symbol-heavy content (hence the `unknown` bucket). +7. **Word counts cover extracted text only.** Text inside images or scanned content is not counted. +8. **License covers the metadata, not the originals.** Individual document copyright stays with the author. The corpus is a metadata index plus content-addressed mirror, distributed under ODC-BY. + +See [/classification](/classification) for model caveats and [/dataset](/dataset) for schema and access. + +--- + +Built by [SuperDoc](https://superdoc.dev/?utm_source=docxcorp.us&utm_medium=referral&utm_campaign=quality) - DOCX editing and tooling. diff --git a/apps/web/llms.txt b/apps/web/llms.txt deleted file mode 100644 index 7033fad..0000000 --- a/apps/web/llms.txt +++ /dev/null @@ -1,46 +0,0 @@ -# docx-corpus - -> Every Word document on the public web. Classified and open. - -The largest open corpus of classified Word documents. 736K+ real .docx files from the public web, classified into 10 document types and 9 topics across 46+ languages. Built for document processing research, NLP benchmarking, and training models that work with real-world Word documents. - -The entire document AI ecosystem runs on scanned images and PDFs. The format people actually write in had no dataset — until docx-corpus. - -Documents are classified using fine-tuned XLM-RoBERTa text classifiers with an average confidence of 82%. - -Built by [SuperDoc](https://www.superdoc.dev) — DOCX editing and tooling. - -## Document Types - -- [Legal](https://docxcorp.us/types/legal): Contracts, agreements, legal notices, court filings -- [Forms](https://docxcorp.us/types/forms): Application forms, surveys, questionnaires, fillable templates -- [Educational](https://docxcorp.us/types/educational): Course materials, syllabi, assignments, lecture notes -- [Administrative](https://docxcorp.us/types/administrative): Meeting minutes, agendas, organizational documents -- [Policies](https://docxcorp.us/types/policies): Policy documents, procedures, guidelines, handbooks -- [Correspondence](https://docxcorp.us/types/correspondence): Letters, memos, formal communications -- [Reports](https://docxcorp.us/types/reports): Annual reports, research reports, financial reports -- [Reference](https://docxcorp.us/types/reference): Reference materials, glossaries, directories, catalogs -- [Technical](https://docxcorp.us/types/technical): Technical documentation, specifications, user manuals -- [Creative](https://docxcorp.us/types/creative): Creative writing, marketing materials, newsletters - -## Topics - -- [Government](https://docxcorp.us/topics/government): Public administration and civic organizations -- [Education](https://docxcorp.us/topics/education): Schools, universities, research institutions -- [Healthcare](https://docxcorp.us/topics/healthcare): Hospitals, clinics, health organizations -- [General](https://docxcorp.us/topics/general): Cross-sector documents -- [Legal / Judicial](https://docxcorp.us/topics/legal_judicial): Law firms, courts, regulatory bodies -- [Finance](https://docxcorp.us/topics/finance): Banks, investment firms, insurance -- [Environment](https://docxcorp.us/topics/environment): Environmental agencies, sustainability -- [Nonprofit](https://docxcorp.us/topics/nonprofit): NGOs, charities, foundations -- [Technology](https://docxcorp.us/topics/technology): Tech companies, software, IT - -## Links - -- Homepage: https://docxcorp.us -- Browse all types: https://docxcorp.us/types -- Browse all topics: https://docxcorp.us/topics -- GitHub: https://github.com/superdoc-dev/docx-corpus -- HuggingFace: https://huggingface.co/datasets/superdoc-dev/docx-corpus -- API: https://api.docxcorp.us -- Takedown requests: help@docxcorp.us diff --git a/apps/web/package.json b/apps/web/package.json new file mode 100644 index 0000000..3751611 --- /dev/null +++ b/apps/web/package.json @@ -0,0 +1,20 @@ +{ + "name": "@docx-corpus/web", + "private": true, + "version": "0.0.0", + "type": "module", + "scripts": { + "dev": "astro dev", + "build": "astro build", + "preview": "astro preview", + "typecheck": "astro check" + }, + "dependencies": { + "@neondatabase/serverless": "^1.0.0", + "astro": "^6.1.0" + }, + "devDependencies": { + "@astrojs/check": "^0.9.4", + "typescript": "^5.9.0" + } +} diff --git a/apps/web/index.html b/apps/web/public/index.html similarity index 99% rename from apps/web/index.html rename to apps/web/public/index.html index 6b12295..6fde6ac 100644 --- a/apps/web/index.html +++ b/apps/web/public/index.html @@ -66,10 +66,10 @@ ] } - - - - + + + + = { - legal: { label: "Legal", desc: "Contracts, agreements, legal notices, court filings, and compliance documentation." }, - forms: { label: "Forms", desc: "Application forms, registration forms, surveys, questionnaires, and fillable templates." }, - reports: { label: "Reports", desc: "Annual reports, research reports, progress reports, financial reports, and analysis documents." }, - policies: { label: "Policies", desc: "Policy documents, procedures, guidelines, handbooks, and organizational rules." }, - educational: { label: "Educational", desc: "Course materials, syllabi, assignments, lecture notes, and academic resources." }, - correspondence: { label: "Correspondence", desc: "Letters, memos, emails, notices, and formal communications." }, - technical: { label: "Technical", desc: "Technical documentation, specifications, API docs, user manuals, and engineering documents." }, - administrative: { label: "Administrative", desc: "Meeting minutes, agendas, organizational documents, and administrative records." }, - creative: { label: "Creative", desc: "Creative writing, marketing materials, brochures, newsletters, and promotional content." }, - reference: { label: "Reference", desc: "Reference materials, glossaries, directories, catalogs, and lookup documents." }, - general: { label: "General", desc: "General-purpose Word documents that span multiple categories." }, -}; - -const TOPIC_META: Record = { - government: { label: "Government", desc: "Documents from government agencies, public administration, and civic organizations." }, - education: { label: "Education", desc: "Documents from schools, universities, research institutions, and educational programs." }, - healthcare: { label: "Healthcare", desc: "Documents from hospitals, clinics, pharmaceutical companies, and health organizations." }, - finance: { label: "Finance", desc: "Documents from banks, investment firms, insurance companies, and financial institutions." }, - legal_judicial: { label: "Legal / Judicial", desc: "Documents from law firms, courts, regulatory bodies, and judicial institutions." }, - technology: { label: "Technology", desc: "Documents from tech companies, software firms, IT departments, and digital services." }, - environment: { label: "Environment", desc: "Documents related to environmental agencies, sustainability, and conservation efforts." }, - nonprofit: { label: "Nonprofit", desc: "Documents from NGOs, charities, foundations, and community organizations." }, - general: { label: "General", desc: "Documents from various sectors that span multiple topic categories." }, -}; - -const LANG_NAMES: Record = { - en: "English", ru: "Russian", cs: "Czech", pl: "Polish", es: "Spanish", - zh: "Chinese", lt: "Lithuanian", sk: "Slovak", de: "German", id: "Indonesian", - fr: "French", pt: "Portuguese", ar: "Arabic", ja: "Japanese", ko: "Korean", - it: "Italian", sv: "Swedish", nl: "Dutch", bg: "Bulgarian", tr: "Turkish", - vi: "Vietnamese", th: "Thai", uk: "Ukrainian", ro: "Romanian", hu: "Hungarian", - hr: "Croatian", fi: "Finnish", da: "Danish", nb: "Norwegian", el: "Greek", - he: "Hebrew", hi: "Hindi", unknown: "Unknown", -}; - -// --------------------------------------------------------------------------- -// Types -// --------------------------------------------------------------------------- - -interface FacetRow { id: string; count: number } -interface LangRow { code: string; count: number } -interface DocRow { - id: string; - filename: string | null; - document_type: string; - document_topic: string; - language: string | null; - word_count: number | null; - classification_confidence: number | null; -} - -interface PageData { - kind: "type" | "topic"; - id: string; - label: string; - description: string; - total: number; - langCount: number; - avgConfidence: number; - languages: LangRow[]; - crossDimension: FacetRow[]; // topics for a type page, types for a topic page - crossMeta: Record; - crossKind: "topic" | "type"; - siblings: FacetRow[]; - documents: DocRow[]; -} - -// --------------------------------------------------------------------------- -// Queries -// --------------------------------------------------------------------------- - -async function queryTypeStats(typeId: string) { - const [[{ total, lang_count, avg_conf }], languages, topics, documents] = await Promise.all([ - sql<{ total: number; lang_count: number; avg_conf: number }[]>` - SELECT COUNT(*)::int AS total, - COUNT(DISTINCT language) FILTER (WHERE language IS NOT NULL)::int AS lang_count, - ROUND(AVG(classification_confidence)::numeric, 1)::float AS avg_conf - FROM documents - WHERE document_type = ${typeId} AND status = 'uploaded' - `, - sql` - SELECT COALESCE(NULLIF(language, 'unknown'), 'unknown') AS code, COUNT(*)::int AS count - FROM documents - WHERE document_type = ${typeId} AND status = 'uploaded' - GROUP BY 1 ORDER BY count DESC LIMIT 8 - `, - sql` - SELECT document_topic AS id, COUNT(*)::int AS count - FROM documents - WHERE document_type = ${typeId} AND document_topic IS NOT NULL - GROUP BY 1 ORDER BY count DESC - `, - sql` - SELECT id, original_filename AS filename, document_type, document_topic, - language, word_count, classification_confidence - FROM documents - WHERE document_type = ${typeId} AND classification_confidence IS NOT NULL - ORDER BY classification_confidence DESC - LIMIT 10 - `, - ]); - return { total, langCount: lang_count, avgConfidence: avg_conf, languages, topics, documents }; -} - -async function queryTopicStats(topicId: string) { - const [[{ total, lang_count, avg_conf }], languages, types, documents] = await Promise.all([ - sql<{ total: number; lang_count: number; avg_conf: number }[]>` - SELECT COUNT(*)::int AS total, - COUNT(DISTINCT language) FILTER (WHERE language IS NOT NULL)::int AS lang_count, - ROUND(AVG(classification_confidence)::numeric, 1)::float AS avg_conf - FROM documents - WHERE document_topic = ${topicId} AND status = 'uploaded' - `, - sql` - SELECT COALESCE(NULLIF(language, 'unknown'), 'unknown') AS code, COUNT(*)::int AS count - FROM documents - WHERE document_topic = ${topicId} AND status = 'uploaded' - GROUP BY 1 ORDER BY count DESC LIMIT 8 - `, - sql` - SELECT document_type AS id, COUNT(*)::int AS count - FROM documents - WHERE document_topic = ${topicId} AND document_type IS NOT NULL - GROUP BY 1 ORDER BY count DESC - `, - sql` - SELECT id, original_filename AS filename, document_type, document_topic, - language, word_count, classification_confidence - FROM documents - WHERE document_topic = ${topicId} AND classification_confidence IS NOT NULL - ORDER BY classification_confidence DESC - LIMIT 10 - `, - ]); - return { total, langCount: lang_count, avgConfidence: avg_conf, languages, types, documents }; -} - -async function queryAllFacets() { - const [types, topics] = await Promise.all([ - sql` - SELECT document_type AS id, COUNT(*)::int AS count - FROM documents WHERE document_type IS NOT NULL - GROUP BY 1 ORDER BY count DESC - `, - sql` - SELECT document_topic AS id, COUNT(*)::int AS count - FROM documents WHERE document_topic IS NOT NULL - GROUP BY 1 ORDER BY count DESC - `, - ]); - return { types, topics }; -} - -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - -function fmt(n: number): string { - return n.toLocaleString("en-US"); -} - -function esc(s: string): string { - return s.replace(/&/g, "&").replace(//g, ">").replace(/"/g, """); -} - -function barWidth(count: number, max: number): number { - return Math.max(4, Math.round((count / max) * 100)); -} - -function barClass(i: number): string { - if (i === 0) return "bar-fill primary"; - if (i < 3) return "bar-fill muted"; - return "bar-fill light"; -} - -// --------------------------------------------------------------------------- -// Template: shared pieces -// --------------------------------------------------------------------------- - -function sharedStyles(): string { - return ` - * { margin: 0; padding: 0; box-sizing: border-box; } - body { font-family: 'Inter', system-ui, sans-serif; background: #fff; color: #2D2D2D; min-height: 100vh; } - a { color: inherit; } - header { display: flex; align-items: center; justify-content: space-between; padding: 20px 48px; max-width: 1200px; margin: 0 auto; } - .brand { font-size: 0.95rem; font-weight: 700; letter-spacing: -0.02em; } - .brand a { text-decoration: none; color: #2D2D2D; } - .brand .accent { color: #C9493D; } - .links { display: flex; gap: 24px; } - .links a { text-decoration: none; color: #6B7280; font-size: 0.82rem; font-weight: 500; transition: color 0.15s; } - .links a:hover { color: #2D2D2D; } - .links a.active { color: #C9493D; } - .container { max-width: 1200px; margin: 0 auto; padding: 0 48px; } - .breadcrumb { font-size: 0.78rem; color: #6B7280; padding: 16px 0 0; } - .breadcrumb a { color: #6B7280; text-decoration: none; } - .breadcrumb a:hover { color: #C9493D; } - .breadcrumb .sep { margin: 0 6px; } - .hero { padding: 40px 0 48px; border-bottom: 1px solid #f0f0f0; margin-bottom: 40px; } - .hero-kicker { font-size: 0.72rem; font-weight: 600; color: #C9493D; text-transform: uppercase; letter-spacing: 0.1em; margin-bottom: 12px; } - .hero h1 { font-size: 2.4rem; font-weight: 800; letter-spacing: -0.04em; line-height: 1.1; margin-bottom: 16px; max-width: 700px; } - .hero-description { font-size: 1rem; color: #6B7280; line-height: 1.7; max-width: 640px; margin-bottom: 12px; } - .hero-by { font-size: 0.85rem; color: #6B7280; margin-bottom: 32px; } - .hero-by a { color: #C9493D; text-decoration: none; font-weight: 600; } - .hero-by a:hover { text-decoration: underline; } - .numbers { display: flex; gap: 48px; } - .num-val { font-family: 'JetBrains Mono', monospace; font-size: 1.8rem; font-weight: 700; letter-spacing: -0.02em; } - .num-label { font-size: 0.72rem; color: #6B7280; letter-spacing: 0.02em; } - .section-label { font-size: 0.72rem; font-weight: 600; color: #6B7280; text-transform: uppercase; letter-spacing: 0.08em; margin-bottom: 20px; } - .stats-section { padding: 0 0 40px; border-bottom: 1px solid #f0f0f0; margin-bottom: 40px; } - .stats-grid { display: grid; grid-template-columns: 1fr 1fr; gap: 32px; } - .stat-card { border: 1px solid #f0f0f0; border-radius: 12px; padding: 24px; } - .stat-card h3 { font-size: 0.82rem; font-weight: 600; color: #6B7280; text-transform: uppercase; letter-spacing: 0.05em; margin-bottom: 16px; } - .bar-chart { display: flex; flex-direction: column; gap: 10px; } - .bar-row { display: flex; align-items: center; gap: 12px; } - .bar-label { font-size: 0.78rem; font-weight: 500; width: 90px; flex-shrink: 0; text-align: right; } - .bar-track { flex: 1; height: 24px; background: #f9fafb; border-radius: 4px; overflow: hidden; } - .bar-fill { height: 100%; border-radius: 4px; display: flex; align-items: center; padding-left: 8px; font-size: 0.68rem; font-weight: 600; color: white; font-family: 'JetBrains Mono', monospace; min-width: 20px; } - .bar-fill.primary { background: #C9493D; } - .bar-fill.muted { background: #D4706A; } - .bar-fill.light { background: #E8A19C; color: #4B2B27; } - .related-section { padding: 0 0 40px; border-bottom: 1px solid #f0f0f0; margin-bottom: 40px; } - .related-grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(180px, 1fr)); gap: 12px; } - .related-card { padding: 16px 20px; border: 1px solid #f0f0f0; border-radius: 8px; text-decoration: none; transition: border-color 0.15s; display: block; } - .related-card:hover { border-color: #F97B6D; } - .related-card .name { font-size: 0.85rem; font-weight: 600; margin-bottom: 4px; } - .related-card .count { font-family: 'JetBrains Mono', monospace; font-size: 0.72rem; color: #6B7280; } - .explore-section { padding-bottom: 64px; } - .explore-header { display: flex; align-items: center; justify-content: space-between; margin-bottom: 20px; } - .explore-count { font-size: 0.72rem; color: #6B7280; font-family: 'JetBrains Mono', monospace; } - .btn-outline { display: inline-flex; align-items: center; gap: 6px; padding: 8px 16px; border: 1px solid #f0f0f0; border-radius: 6px; font-size: 0.78rem; font-weight: 600; text-decoration: none; background: white; transition: border-color 0.15s; } - .btn-outline:hover { border-color: #F97B6D; } - table { width: 100%; border-collapse: collapse; font-size: 0.82rem; } - thead th { text-align: left; padding: 10px 12px; font-weight: 600; font-size: 0.72rem; text-transform: uppercase; letter-spacing: 0.05em; color: #6B7280; border-bottom: 1px solid #f0f0f0; } - tbody td { padding: 12px; border-bottom: 1px solid #fafafa; vertical-align: middle; } - tbody tr:hover { background: #fafafa; } - .url-cell { font-family: 'JetBrains Mono', monospace; font-size: 0.72rem; color: #6B7280; max-width: 360px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } - .badge { display: inline-block; padding: 2px 8px; border-radius: 4px; font-size: 0.68rem; font-weight: 600; } - .badge-type { background: #FEF0EE; color: #C9493D; } - .badge-topic { background: #EEF2FF; color: #6366F1; } - .badge-lang { background: #F0FDF4; color: #16A34A; } - .badge-conf { background: #fef8f7; color: #C9493D; font-family: 'JetBrains Mono', monospace; } - .view-all-row { text-align: center; padding: 20px 0; } - .view-all-row a { color: #C9493D; text-decoration: none; font-weight: 600; font-size: 0.85rem; } - .view-all-row a:hover { text-decoration: underline; } - .cta-section { padding: 40px 0; border-bottom: 1px solid #f0f0f0; } - .cta-box { background: #fef8f7; border: 1px solid #fdd8d4; border-radius: 12px; padding: 32px; display: flex; align-items: center; justify-content: space-between; } - .cta-text h3 { font-size: 1.1rem; font-weight: 700; margin-bottom: 6px; } - .cta-text p { font-size: 0.85rem; color: #6B7280; } - .btn-primary { display: inline-flex; align-items: center; gap: 6px; padding: 10px 20px; background: #F97B6D; color: white; border: none; border-radius: 6px; font-size: 0.82rem; font-weight: 600; text-decoration: none; transition: background 0.15s; } - .btn-primary:hover { background: #e8685a; } - .site-footer { border-top: 1px solid #f0f0f0; padding: 24px 48px; margin-top: 0; display: flex; align-items: center; justify-content: space-between; font-size: 0.78rem; color: #6B7280; max-width: 1200px; margin: 0 auto; } - .site-footer a { color: #6B7280; text-decoration: none; } - .site-footer a:hover { text-decoration: underline; } - @media (max-width: 768px) { - header, .container { padding-left: 24px; padding-right: 24px; } - .hero h1 { font-size: 1.8rem; } - .numbers { gap: 24px; flex-wrap: wrap; } - .stats-grid { grid-template-columns: 1fr; } - .cta-box { flex-direction: column; gap: 16px; text-align: center; } - }`; -} - -function sharedHead(title: string, description: string, canonicalPath: string, kind: string, id: string): string { - const url = `${SITE}${canonicalPath}`; - return ` - - - - - - ${esc(title)} - - - - - - - - - - - - - - - - - - - `; -} - -function sharedHeader(activeKind: "type" | "topic"): string { - return ` -
-
docx-corpus
- -
`; -} - -function sharedFooter(): string { - return ` -
- Built by SuperDoc — DOCX editing and tooling. - Takedown requests: help@docxcorp.us -
`; -} - -// --------------------------------------------------------------------------- -// Template: bar chart component -// --------------------------------------------------------------------------- - -function renderBarChart(title: string, items: { label: string; count: number }[]): string { - if (items.length === 0) return ""; - const max = items[0].count; - const rows = items.map((item, i) => ` -
-
${esc(item.label)}
-
-
${fmt(item.count)}
-
-
`).join(""); - - return ` -
-

${esc(title)}

-
${rows} -
-
`; -} - -// --------------------------------------------------------------------------- -// Template: documents table component -// --------------------------------------------------------------------------- - -function renderDocTable(docs: DocRow[], total: number, kind: string, id: string, label: string): string { - const filterParam = kind === "type" ? `type=${id}` : `topic=${id}`; - const rows = docs.map((d) => { - const displayName = d.filename || `${d.id.slice(0, 12)}...docx`; - const conf = d.classification_confidence != null - ? `${(d.classification_confidence * 100).toFixed(1)}%` - : "—"; - const topicLabel = TOPIC_META[d.document_topic]?.label || d.document_topic || "—"; - const typeLabel = TYPE_META[d.document_type]?.label || d.document_type || "—"; - const langName = d.language ? (LANG_NAMES[d.language] || d.language) : "—"; - - return ` - - ${esc(displayName)} - ${kind === "type" - ? `${esc(topicLabel)}` - : `${esc(typeLabel)}`} - ${esc(d.language || "?")} ${esc(langName)} - ${conf} - ${d.word_count != null ? fmt(d.word_count) : "—"} - `; - }).join(""); - - const crossHeader = kind === "type" ? "Topic" : "Type"; - - return ` -
-
Sample Documents
-
- Showing ${docs.length} of ${fmt(total)} ${esc(label.toLowerCase())} documents - View all in Explorer → -
- - - - - - - - - - - ${rows} - -
Filename${crossHeader}LanguageConfidenceWords
-
- View all ${fmt(total)} documents → -
-
`; -} - -// --------------------------------------------------------------------------- -// Template: siblings grid component -// --------------------------------------------------------------------------- - -function renderSiblings( - title: string, - items: FacetRow[], - currentId: string, - kind: "type" | "topic", - meta: Record, -): string { - const others = items.filter((item) => item.id !== currentId); - if (others.length === 0) return ""; - - const cards = others.map((item) => { - const label = meta[item.id]?.label || item.id; - const urlKind = kind === "type" ? "types" : "topics"; - return ` - -
${esc(label)}
-
${fmt(item.count)}
- `; - }).join(""); - - return ` -
-
${esc(title)}
-
${cards} -
-
`; -} - -// --------------------------------------------------------------------------- -// Template: full page -// --------------------------------------------------------------------------- - -function renderPage(data: PageData): string { - const { - kind, id, label, description, total, langCount, avgConfidence, - languages, crossDimension, crossMeta, crossKind, siblings, documents, - } = data; - - const kindLabel = kind === "type" ? "Document Type" : "Document Topic"; - const kindPlural = kind === "type" ? "Types" : "Topics"; - const urlPrefix = kind === "type" ? "types" : "topics"; - const crossLabel = crossKind === "topic" ? "Top Topics" : "Top Types"; - const siblingTitle = `Other ${kindPlural}`; - const filterParam = kind === "type" ? `type=${id}` : `topic=${id}`; - - const title = `${label} Word Documents — docx-corpus`; - const metaDesc = `${fmt(total)}+ real ${label.toLowerCase()} .docx files across ${langCount} languages. Open dataset for document processing research.`; - - const langItems = languages.map((l) => ({ - label: LANG_NAMES[l.code] || l.code, - count: l.count, - })); - - const crossItems = crossDimension.map((c) => ({ - label: crossMeta[c.id]?.label || c.id, - count: c.count, - })); - - return ` - -${sharedHead(title, metaDesc, `/${urlPrefix}/${id}`, kind, id)} - - - - ${sharedHeader(kind)} -
-
- Home/ - ${kindPlural}/ - ${esc(label)} -
-
-
${esc(kindLabel)}
-

${esc(label)} Documents

-

- ${fmt(total)}+ real ${label.toLowerCase()} Word documents from the public web. - ${esc(description)} -

-

Built by SuperDoc — DOCX editing and tooling.

-
-
${fmt(total)}
documents
-
${langCount}
languages
-
${avgConfidence}%
avg confidence
-
-
-
-
Distribution
-
- ${renderBarChart("Top Languages", langItems)} - ${renderBarChart(crossLabel, crossItems)} -
-
- ${renderSiblings(siblingTitle, siblings, id, kind, kind === "type" ? TYPE_META : TOPIC_META)} - ${renderDocTable(documents, total, kind, id, label)} -
-
-
-

Download this subset

-

Get a manifest of all ${fmt(total)} ${label.toLowerCase()} documents for wget or curl.

-
- Generate Manifest ↓ -
-
-
- ${sharedFooter()} - -`; -} - -// --------------------------------------------------------------------------- -// Template: index page (lists all types or all topics) -// --------------------------------------------------------------------------- - -function renderIndexPage( - kind: "type" | "topic", - items: FacetRow[], - meta: Record, -): string { - const kindPlural = kind === "type" ? "Types" : "Topics"; - const urlPrefix = kind === "type" ? "types" : "topics"; - const title = `Document ${kindPlural} — docx-corpus`; - const description = `Browse all ${items.length} document ${kindPlural.toLowerCase()} in the docx-corpus dataset. ${fmt(items.reduce((s, i) => s + i.count, 0))}+ classified Word documents.`; - - const cards = items.map((item) => { - const m = meta[item.id] || { label: item.id, desc: "" }; - return ` - -
${esc(m.label)}
-
${fmt(item.count)} documents
-
${esc(m.desc)}
- `; - }).join(""); - - return ` - -${sharedHead(title, description, `/${urlPrefix}`, kind, "")} - - - - - ${sharedHeader(kind)} -
-
- Home/ - ${kindPlural} -
-
-
Browse by ${kind === "type" ? "document type" : "topic"}
-

Document ${kindPlural}

-

- ${items.length} ${kindPlural.toLowerCase()} across 46+ languages. - Click any ${kind} to explore its documents, language distribution, and download options. -

-

Built by SuperDoc — DOCX editing and tooling.

-
-
${cards} -
-
- ${sharedFooter()} - -`; -} - -// --------------------------------------------------------------------------- -// Sitemap -// --------------------------------------------------------------------------- - -function renderSitemap(types: FacetRow[], topics: FacetRow[]): string { - const urls = [ - { loc: "/", priority: "1.0" }, - { loc: "/types", priority: "0.8" }, - { loc: "/topics", priority: "0.8" }, - ...types.map((t) => ({ loc: `/types/${t.id}`, priority: "0.7" })), - ...topics.map((t) => ({ loc: `/topics/${t.id}`, priority: "0.7" })), - ]; - - const entries = urls.map((u) => ` - ${SITE}${u.loc} - weekly - ${u.priority} - `).join("\n"); - - return ` - -${entries} - -`; -} - -// --------------------------------------------------------------------------- -// Main -// --------------------------------------------------------------------------- - -async function main() { - console.log("Fetching data..."); - const { types, topics } = await queryAllFacets(); - - // Generate type pages - console.log(`Generating ${types.length} type pages...`); - for (const type of types) { - const stats = await queryTypeStats(type.id); - const meta = TYPE_META[type.id] || { label: type.id, desc: "" }; - const html = renderPage({ - kind: "type", - id: type.id, - label: meta.label, - description: meta.desc, - total: stats.total, - langCount: stats.langCount, - avgConfidence: stats.avgConfidence, - languages: stats.languages, - crossDimension: stats.topics, - crossMeta: TOPIC_META, - crossKind: "topic", - siblings: types, - documents: stats.documents, - }); - await Bun.write(`${OUT_DIR}types/${type.id}.html`, html); - console.log(` types/${type.id}.html (${fmt(stats.total)} docs)`); - } - - // Generate topic pages - console.log(`Generating ${topics.length} topic pages...`); - for (const topic of topics) { - const stats = await queryTopicStats(topic.id); - const meta = TOPIC_META[topic.id] || { label: topic.id, desc: "" }; - const html = renderPage({ - kind: "topic", - id: topic.id, - label: meta.label, - description: meta.desc, - total: stats.total, - langCount: stats.langCount, - avgConfidence: stats.avgConfidence, - languages: stats.languages, - crossDimension: stats.types, - crossMeta: TYPE_META, - crossKind: "type", - siblings: topics, - documents: stats.documents, - }); - await Bun.write(`${OUT_DIR}topics/${topic.id}.html`, html); - console.log(` topics/${topic.id}.html (${fmt(stats.total)} docs)`); - } - - // Generate index pages - console.log("Generating index pages..."); - await Bun.write(`${OUT_DIR}types/index.html`, renderIndexPage("type", types, TYPE_META)); - await Bun.write(`${OUT_DIR}topics/index.html`, renderIndexPage("topic", topics, TOPIC_META)); - console.log(" types/index.html"); - console.log(" topics/index.html"); - - // Generate sitemap - const sitemap = renderSitemap(types, topics); - await Bun.write(`${OUT_DIR}sitemap.xml`, sitemap); - console.log(" sitemap.xml"); - - await sql.close(); - console.log("Done!"); -} - -main().catch((err) => { - console.error("Failed:", err); - process.exit(1); -}); diff --git a/apps/web/scripts/generate-stats.ts b/apps/web/scripts/generate-stats.ts deleted file mode 100644 index 6222f0c..0000000 --- a/apps/web/scripts/generate-stats.ts +++ /dev/null @@ -1,242 +0,0 @@ -#!/usr/bin/env bun -/** - * Generate static stats.json for the docxcorp.us landing page. - * - * Queries Neon Postgres for aggregate counts, type/topic distributions, - * language breakdown, and a sample of documents. Output is written to - * apps/web/data/stats.json and deployed alongside the HTML. - * - * Usage: - * bun run apps/web/scripts/generate-stats.ts - * - * Requires DATABASE_URL in environment (loaded automatically by Bun from .env). - */ - -import { SQL } from "bun"; - -const DATABASE_URL = process.env.DATABASE_URL; -if (!DATABASE_URL) { - console.error("DATABASE_URL is required"); - process.exit(1); -} - -const sql = new SQL({ url: DATABASE_URL }); - -// ---------- queries ---------- - -async function heroStats() { - const [row] = await sql<{ - total: number; - languages: number; - avg_confidence: number; - classified: number; - }[]>` - SELECT - COUNT(*)::int AS total, - COUNT(DISTINCT language) FILTER (WHERE language IS NOT NULL)::int AS languages, - ROUND(AVG(classification_confidence)::numeric, 2)::float AS avg_confidence, - COUNT(*) FILTER (WHERE document_type IS NOT NULL)::int AS classified - FROM documents - WHERE status = 'uploaded' - `; - return row; -} - -async function typeDistribution() { - return sql<{ id: string; count: number }[]>` - SELECT document_type AS id, COUNT(*)::int AS count - FROM documents - WHERE document_type IS NOT NULL - GROUP BY document_type - ORDER BY count DESC - `; -} - -async function topicDistribution() { - return sql<{ id: string; count: number }[]>` - SELECT document_topic AS id, COUNT(*)::int AS count - FROM documents - WHERE document_topic IS NOT NULL - GROUP BY document_topic - ORDER BY count DESC - `; -} - -async function languageDistribution() { - return sql<{ code: string; count: number }[]>` - SELECT - COALESCE(NULLIF(language, 'unknown'), 'unknown') AS code, - SUM(cnt)::int AS count - FROM ( - SELECT COALESCE(language, 'unknown') AS language, COUNT(*) AS cnt - FROM documents - WHERE status = 'uploaded' - GROUP BY language - ) sub - GROUP BY COALESCE(NULLIF(language, 'unknown'), 'unknown') - ORDER BY count DESC - LIMIT 20 - `; -} - -async function sampleDocuments() { - return sql<{ - id: string; - filename: string | null; - document_type: string; - document_topic: string; - language: string; - word_count: number | null; - classification_confidence: number; - }[]>` - SELECT - id, - original_filename AS filename, - document_type, - document_topic, - language, - word_count, - classification_confidence - FROM documents - WHERE document_type IS NOT NULL - AND classification_confidence IS NOT NULL - ORDER BY RANDOM() - LIMIT 25 - `; -} - -// ---------- type labels ---------- - -const TYPE_LABELS: Record = { - legal: "Legal", - forms: "Forms", - reports: "Reports", - policies: "Policies", - educational: "Educational", - correspondence: "Correspondence", - technical: "Technical", - administrative: "Administrative", - creative: "Creative", - reference: "Reference", - general: "General", -}; - -const TOPIC_LABELS: Record = { - government: "Government", - education: "Education", - healthcare: "Healthcare", - finance: "Finance", - legal_judicial: "Legal / Judicial", - technology: "Technology", - environment: "Environment", - nonprofit: "Nonprofit", - general: "General", -}; - -const LANG_NAMES: Record = { - en: "English", - ru: "Russian", - cs: "Czech", - pl: "Polish", - es: "Spanish", - zh: "Chinese", - lt: "Lithuanian", - sk: "Slovak", - de: "German", - id: "Indonesian", - fr: "French", - pt: "Portuguese", - ar: "Arabic", - ja: "Japanese", - ko: "Korean", - it: "Italian", - sv: "Swedish", - nl: "Dutch", - bg: "Bulgarian", - tr: "Turkish", - vi: "Vietnamese", - th: "Thai", - uk: "Ukrainian", - ro: "Romanian", - hu: "Hungarian", - hr: "Croatian", - fi: "Finnish", - da: "Danish", - nb: "Norwegian", - el: "Greek", - he: "Hebrew", - hi: "Hindi", - unknown: "Unknown", -}; - -// ---------- main ---------- - -async function main() { - console.log("Querying database..."); - - const [hero, types, topics, languages, samples] = await Promise.all([ - heroStats(), - typeDistribution(), - topicDistribution(), - languageDistribution(), - sampleDocuments(), - ]); - - const totalClassified = types.reduce((sum, t) => sum + t.count, 0); - - const stats = { - generated_at: new Date().toISOString(), - hero: { - total_documents: hero.total, - languages: hero.languages, - types: types.length, - topics: topics.length, - avg_confidence: hero.avg_confidence, - }, - types: types.map((t) => ({ - id: t.id, - label: TYPE_LABELS[t.id] || t.id, - count: t.count, - percentage: Math.round((1000 * t.count) / totalClassified) / 10, - })), - topics: topics.map((t) => ({ - id: t.id, - label: TOPIC_LABELS[t.id] || t.id, - count: t.count, - percentage: Math.round((1000 * t.count) / totalClassified) / 10, - })), - languages: languages.map((l) => ({ - code: l.code, - name: LANG_NAMES[l.code] || l.code, - count: l.count, - percentage: Math.round((1000 * l.count) / hero.total) / 10, - })), - sample_documents: samples.map((d) => ({ - id: d.id, - filename: d.filename, - type: d.document_type, - topic: d.document_topic, - language: d.language, - word_count: d.word_count, - confidence: d.classification_confidence, - })), - }; - - const outPath = new URL("../data/stats.json", import.meta.url).pathname; - await Bun.write(outPath, JSON.stringify(stats, null, 2)); - - console.log(`Written to ${outPath}`); - console.log(` Documents: ${stats.hero.total_documents.toLocaleString()}`); - console.log(` Languages: ${stats.hero.languages}`); - console.log(` Types: ${stats.types.length} (${totalClassified.toLocaleString()} classified)`); - console.log(` Topics: ${stats.topics.length}`); - console.log(` Avg confidence: ${stats.hero.avg_confidence}`); - console.log(` Sample docs: ${stats.sample_documents.length}`); - - await sql.close(); -} - -main().catch((err) => { - console.error("Failed:", err); - process.exit(1); -}); diff --git a/apps/web/scripts/preview.ts b/apps/web/scripts/preview.ts deleted file mode 100644 index 42d14f3..0000000 --- a/apps/web/scripts/preview.ts +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bun -/** - * Local preview server for the landing page. - * Serves static files and proxies /api/* to the Worker (if running). - * - * Usage: bun run apps/web/scripts/preview.ts - */ - -const WEB_DIR = new URL("../", import.meta.url).pathname; - -Bun.serve({ - port: 3000, - async fetch(req) { - const url = new URL(req.url); - - // Serve static files - let path = url.pathname; - if (path === "/") path = "/index.html"; - - const file = Bun.file(WEB_DIR + path); - if (await file.exists()) { - return new Response(file); - } - - // Clean URLs: /types/legal → /types/legal.html (generated pages) - const htmlFile = Bun.file(WEB_DIR + path + ".html"); - if (await htmlFile.exists()) { - return new Response(htmlFile, { headers: { "Content-Type": "text/html; charset=utf-8" } }); - } - - // Directory index: /types → /types/index.html - const indexFile = Bun.file(WEB_DIR + path + "/index.html"); - if (await indexFile.exists()) { - return new Response(indexFile, { headers: { "Content-Type": "text/html; charset=utf-8" } }); - } - - return new Response("Not found", { status: 404 }); - }, -}); - -console.log("Preview: http://localhost:3000"); diff --git a/apps/web/sitemap.xml b/apps/web/sitemap.xml deleted file mode 100644 index 8be2d90..0000000 --- a/apps/web/sitemap.xml +++ /dev/null @@ -1,113 +0,0 @@ - - - - https://docxcorp.us/ - weekly - 1.0 - - - https://docxcorp.us/types - weekly - 0.8 - - - https://docxcorp.us/topics - weekly - 0.8 - - - https://docxcorp.us/types/legal - weekly - 0.7 - - - https://docxcorp.us/types/forms - weekly - 0.7 - - - https://docxcorp.us/types/educational - weekly - 0.7 - - - https://docxcorp.us/types/administrative - weekly - 0.7 - - - https://docxcorp.us/types/policies - weekly - 0.7 - - - https://docxcorp.us/types/correspondence - weekly - 0.7 - - - https://docxcorp.us/types/reports - weekly - 0.7 - - - https://docxcorp.us/types/reference - weekly - 0.7 - - - https://docxcorp.us/types/technical - weekly - 0.7 - - - https://docxcorp.us/types/creative - weekly - 0.7 - - - https://docxcorp.us/topics/government - weekly - 0.7 - - - https://docxcorp.us/topics/education - weekly - 0.7 - - - https://docxcorp.us/topics/healthcare - weekly - 0.7 - - - https://docxcorp.us/topics/general - weekly - 0.7 - - - https://docxcorp.us/topics/legal_judicial - weekly - 0.7 - - - https://docxcorp.us/topics/finance - weekly - 0.7 - - - https://docxcorp.us/topics/environment - weekly - 0.7 - - - https://docxcorp.us/topics/nonprofit - weekly - 0.7 - - - https://docxcorp.us/topics/technology - weekly - 0.7 - - diff --git a/apps/web/src/env.d.ts b/apps/web/src/env.d.ts new file mode 100644 index 0000000..170746a --- /dev/null +++ b/apps/web/src/env.d.ts @@ -0,0 +1,9 @@ +/// + +interface ImportMetaEnv { + readonly DATABASE_URL: string; +} + +interface ImportMeta { + readonly env: ImportMetaEnv; +} diff --git a/apps/web/src/layouts/Layout.astro b/apps/web/src/layouts/Layout.astro new file mode 100644 index 0000000..48042a2 --- /dev/null +++ b/apps/web/src/layouts/Layout.astro @@ -0,0 +1,156 @@ +--- +import { canonicalUrl, stringifyLd, breadcrumbList } from '../lib/seo'; + +export interface Props { + title: string; + description: string; + canonicalPath: string; + ogImage?: string; + robots?: string; + structuredData?: object; + breadcrumbs?: { name: string; path: string }[]; +} + +const { + title, + description, + canonicalPath, + ogImage = '/og-image.png', + robots, + structuredData, + breadcrumbs, +} = Astro.props; + +const canonical = canonicalUrl(canonicalPath); +const ogImageUrl = canonicalUrl(ogImage); +const breadcrumbLd = breadcrumbs?.length ? breadcrumbList(breadcrumbs) : null; +--- + + + + + + + + + + {title} + + {robots && } + + + + + + + + + + + + + + + + + {structuredData && ( +