From 4c491ebe6646323d3827ec86960eb65b9d86ace1 Mon Sep 17 00:00:00 2001 From: Caio Pizzol Date: Tue, 28 Jul 2026 13:38:34 -0300 Subject: [PATCH] chore(repo): update GitHub organization references --- README.md | 10 +++++----- apps/site/content/classification.md | 4 ++-- apps/site/content/quality.md | 2 +- apps/site/src/pages/classification.astro | 2 +- apps/site/src/pages/index.astro | 2 +- apps/site/src/pages/llms.txt.ts | 2 +- packages/scraper/commoncrawl/warc.ts | 2 +- scripts/export-hf.py | 2 +- 8 files changed, 13 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 7cdea6d..85313d7 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ logo -[![CLI](https://img.shields.io/github/v/release/superdoc-dev/docx-corpus?filter=cli-v*&label=cli)](https://github.com/superdoc-dev/docx-corpus/releases) -[![CDX Filter](https://img.shields.io/github/v/release/superdoc-dev/docx-corpus?filter=cdx-filter-v*&label=cdx-filter)](https://github.com/superdoc-dev/docx-corpus/releases) -[![codecov](https://codecov.io/gh/superdoc-dev/docx-corpus/graph/badge.svg)](https://codecov.io/gh/superdoc-dev/docx-corpus) +[![CLI](https://img.shields.io/github/v/release/superdoc/docx-corpus?filter=cli-v*&label=cli)](https://github.com/superdoc/docx-corpus/releases) +[![CDX Filter](https://img.shields.io/github/v/release/superdoc/docx-corpus?filter=cdx-filter-v*&label=cdx-filter)](https://github.com/superdoc/docx-corpus/releases) +[![codecov](https://codecov.io/gh/superdoc/docx-corpus/graph/badge.svg)](https://codecov.io/gh/superdoc/docx-corpus) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) The largest open corpus of classified Word documents. 736K+ real `.docx` files from the public web, classified into 10 document types and 9 topics across 46+ languages. @@ -12,7 +12,7 @@ The entire document AI research ecosystem runs on scanned images and PDFs. The f **[docxcorp.us](https://docxcorp.us)** · **[HuggingFace](https://huggingface.co/datasets/superdoc-dev/docx-corpus)** · **[API](https://api.docxcorp.us/stats)** > [!IMPORTANT] -> **WARC trailing bytes fixed for new scrapes** — Documents scraped before [`477d1b9`](https://github.com/superdoc-dev/docx-corpus/commit/477d1b9) may contain a 4-byte WARC record terminator (`\r\n\r\n`) after ZIP EOCD. Future scrapes are fixed; use `python3 scripts/fix_trailing_warc_bytes.py` to inspect or repair older local copies. This was confirmed to trigger Word for Mac's unreadable-content warning for the sample in [#8](https://github.com/superdoc-dev/docx-corpus/issues/8). +> **WARC trailing bytes fixed for new scrapes** — Documents scraped before [`477d1b9`](https://github.com/superdoc/docx-corpus/commit/477d1b9) may contain a 4-byte WARC record terminator (`\r\n\r\n`) after ZIP EOCD. Future scrapes are fixed; use `python3 scripts/fix_trailing_warc_bytes.py` to inspect or repair older local copies. This was confirmed to trigger Word for Mac's unreadable-content warning for the sample in [#8](https://github.com/superdoc/docx-corpus/issues/8). ## How It Works @@ -33,7 +33,7 @@ Common Crawl (3B+ URLs/month) ## Quick Start ```bash -git clone https://github.com/superdoc-dev/docx-corpus.git +git clone https://github.com/superdoc/docx-corpus.git cd docx-corpus bun install ``` diff --git a/apps/site/content/classification.md b/apps/site/content/classification.md index 5005ce0..1a57d17 100644 --- a/apps/site/content/classification.md +++ b/apps/site/content/classification.md @@ -15,7 +15,7 @@ secondaryQueries: docx-corpus uses a two-dimensional taxonomy. Every classified document has one document_type label (its form/structure) and one document_topic label (its subject domain). The labels are independent; a `legal` document can have any topic, and a `government` topic can have any document_type. -Taxonomy version: `docx-corpus-v2` (2.0.0). Source: [scripts/classification/taxonomy.json](https://github.com/superdoc-dev/docx-corpus/blob/main/scripts/classification/taxonomy.json). +Taxonomy version: `docx-corpus-v2` (2.0.0). Source: [scripts/classification/taxonomy.json](https://github.com/superdoc/docx-corpus/blob/main/scripts/classification/taxonomy.json). ## Document types (10) @@ -114,7 +114,7 @@ See [/quality](/quality) for confidence distributions and the rest of the datase ## Reproducing this -Code in [scripts/classification/](https://github.com/superdoc-dev/docx-corpus/tree/main/scripts/classification). Run order: `sample.py` -> `label.py` -> `train.py` -> `classify.py`. Pipeline notes in [CLAUDE.md](https://github.com/superdoc-dev/docx-corpus/blob/main/scripts/classification/CLAUDE.md). +Code in [scripts/classification/](https://github.com/superdoc/docx-corpus/tree/main/scripts/classification). Run order: `sample.py` -> `label.py` -> `train.py` -> `classify.py`. Pipeline notes in [CLAUDE.md](https://github.com/superdoc/docx-corpus/blob/main/scripts/classification/CLAUDE.md). --- diff --git a/apps/site/content/quality.md b/apps/site/content/quality.md index 993c5a8..ff9f6f4 100644 --- a/apps/site/content/quality.md +++ b/apps/site/content/quality.md @@ -17,7 +17,7 @@ This page is for researchers deciding whether docx-corpus is fit for their task. ## Validation -Every file that enters the corpus passes these checks (in [`packages/scraper/validation.ts`](https://github.com/superdoc-dev/docx-corpus/blob/main/packages/scraper/validation.ts)): +Every file that enters the corpus passes these checks (in [`packages/scraper/validation.ts`](https://github.com/superdoc/docx-corpus/blob/main/packages/scraper/validation.ts)): 1. Minimum size: at least 100 bytes. 2. ZIP magic bytes: starts with `PK\x03\x04`. diff --git a/apps/site/src/pages/classification.astro b/apps/site/src/pages/classification.astro index 7281db2..1488800 100644 --- a/apps/site/src/pages/classification.astro +++ b/apps/site/src/pages/classification.astro @@ -47,7 +47,7 @@ const description = 'Two-dimensional taxonomy of 10 document types and 9 topics.
How labels are produced

We follow the FineWeb-Edu pattern: sample a small, stratified set of documents across the five most-common languages, label each with an LLM, then train a lightweight multilingual text classifier on those labels and apply it at scale.

-

The classifier is a fine-tuned XLM-RoBERTa, trained on roughly 3,500 LLM-labeled examples. Two independent classifiers are trained, one per dimension. Code is open: see the classification scripts on GitHub.

+

The classifier is a fine-tuned XLM-RoBERTa, trained on roughly 3,500 LLM-labeled examples. Two independent classifiers are trained, one per dimension. Code is open: see the classification scripts on GitHub.

diff --git a/apps/site/src/pages/index.astro b/apps/site/src/pages/index.astro index fed200c..fdb9f14 100644 --- a/apps/site/src/pages/index.astro +++ b/apps/site/src/pages/index.astro @@ -863,7 +863,7 @@ const structuredData = {
About the dataset

docx-corpus is the largest open corpus of classified Word documents on the public web. It contains 736,000+ real .docx files collected from Common Crawl, validated, deduplicated, and labeled by document type and topic. It is the first large-scale research dataset for DOCX, the world's most-used document creation format, which has historically been absent from document AI work built on scanned images and PDFs.

-

Each document has a content hash, file size, detected language, and a type and topic label from a fine-tuned XLM-RoBERTa text classifier. The corpus spans 46+ languages and uses a 10-type, 9-topic taxonomy. Available as a HuggingFace dataset, a REST API at api.docxcorp.us, and downloadable manifest files. Built by SuperDoc. Source on GitHub. Dataset is ODC-BY; source code is MIT.

+

Each document has a content hash, file size, detected language, and a type and topic label from a fine-tuned XLM-RoBERTa text classifier. The corpus spans 46+ languages and uses a 10-type, 9-topic taxonomy. Available as a HuggingFace dataset, a REST API at api.docxcorp.us, and downloadable manifest files. Built by SuperDoc. Source on GitHub. Dataset is ODC-BY; source code is MIT.

diff --git a/apps/site/src/pages/llms.txt.ts b/apps/site/src/pages/llms.txt.ts index f979ded..0f5e318 100644 --- a/apps/site/src/pages/llms.txt.ts +++ b/apps/site/src/pages/llms.txt.ts @@ -40,7 +40,7 @@ ${topics.map(t => `- [${t.label}](${absoluteUrl(routes.topic(t.id))}): ${t.count ## Links - Homepage: ${absoluteUrl(routes.home())} -- GitHub: https://github.com/superdoc-dev/docx-corpus +- GitHub: https://github.com/superdoc/docx-corpus - HuggingFace: https://huggingface.co/datasets/superdoc-dev/docx-corpus - API: https://api.docxcorp.us - Takedown requests: help@docxcorp.us diff --git a/packages/scraper/commoncrawl/warc.ts b/packages/scraper/commoncrawl/warc.ts index 5f9d46d..2dd2432 100644 --- a/packages/scraper/commoncrawl/warc.ts +++ b/packages/scraper/commoncrawl/warc.ts @@ -3,7 +3,7 @@ import type { RateLimiter } from "../rate-limiter"; import type { CdxRecord } from "./cdx-r2"; const WARC_BASE_URL = "https://data.commoncrawl.org"; -const USER_AGENT = "docx-corpus (https://github.com/superdoc-dev/docx-corpus)"; +const USER_AGENT = "docx-corpus (https://github.com/superdoc/docx-corpus)"; export interface WarcResult { content: Uint8Array; diff --git a/scripts/export-hf.py b/scripts/export-hf.py index 7124e34..4533442 100644 --- a/scripts/export-hf.py +++ b/scripts/export-hf.py @@ -127,7 +127,7 @@ ## Links - **Website**: [docxcorp.us](https://docxcorp.us) -- **GitHub**: [superdoc-dev/docx-corpus](https://github.com/superdoc-dev/docx-corpus) +- **GitHub**: [superdoc/docx-corpus](https://github.com/superdoc/docx-corpus) - **Built by**: 🦋 [SuperDoc](https://superdoc.dev) """