Add unit-circle embedding benchmark project

unit-circle-benchmark/.replit

@@ -0,0 +1,17 @@
+run = "python -m src.main"
+entrypoint = "src/main.py"
+
+[nix]
+channel = "stable-24_05"
+
+[deployment]
+run = ["sh", "-c", "python -m src.main"]
+deploymentTarget = "cloudrun"
+
+[env]
+# Switch to bge-m3 for Phase 2:  MODEL_PRESET=bge-m3
+MODEL_PRESET = "nomic"
+
+[[ports]]
+localPort = 8080
+externalPort = 80

unit-circle-benchmark/README.md

@@ -0,0 +1,112 @@
+# Unit Circle Embedding Benchmark
+
+This repository tests whether a unit-circle / number-base coordinate remapping can improve the usefulness of existing embedding spaces.
+
+## Principle
+
+We do not train a new encoder first.
+We begin with a proven embedding model, then test whether a derived coordinate system improves retrieval, clustering, and neighborhood structure.
+
+## Initial substrates
+
+- `nomic-ai/nomic-embed-text-v1.5` — Phase 1 (Matryoshka-style dim reduction, long context)
+- `BAAI/bge-m3` — Phase 2 (dense + sparse + multi-vector, multilingual)
+- `text-embedding-3-large` — optional Phase 3 hosted ceiling-check
+
+## Core question
+
+Does the transform preserve or improve semantic structure better than:
+1. native vectors
+2. matched-dimension PCA
+3. negative-control transforms (random rotation, broken phase)
+
+---
+
+## Ablation ladder
+
+| Step | Name            | Description                                              |
+|------|-----------------|----------------------------------------------------------|
+| A0   | `native`        | Plain L2-normalised vectors                              |
+| A1   | `pca_256/64`    | PCA to same target dim — is gain just dimensional cleanup? |
+| A2   | `rot_768`       | Random orthogonal rotation — is gain just reorientation? |
+| A3   | `uc_256/64`     | Unit-circle direct angle map — main hypothesis           |
+| A3b  | `uc_256_rank`   | Unit-circle rank angle map                               |
+| A4   | *(in notebook)* | UC → inverse PCA back to Euclidean                       |
+| A5   | `uc_256_broken` | Shuffled phase — negative control / theory destroyer     |
+
+---
+
+## Unit-circle transform variants
+
+**T1 — direct angle map** (`mode="direct"`)
+```
+theta_i = pi * x_i
+output_i = [cos(theta_i), sin(theta_i)]
+```
+
+**T2 — rank angle map** (`mode="rank"`)
+```
+theta_i = 2*pi * rank(x_i) / (D-1)
+output_i = [|x_i|*cos(theta_i), |x_i|*sin(theta_i)]
+```
+
+---
+
+## Metrics
+
+| Category    | Metrics                                       |
+|-------------|-----------------------------------------------|
+| Retrieval   | Recall@1/5/10, MRR, nDCG@10                  |
+| Clustering  | Silhouette, NMI, ARI                          |
+| Geometry    | Neighbor overlap@k, Spearman rank correlation, trustworthiness/continuity penalties |
+
+---
+
+## Quick start
+
+```bash
+pip install -r requirements.txt
+python -m src.main                     # Phase 1: nomic
+MODEL_PRESET=bge-m3 python -m src.main # Phase 2: BGE-M3
+```
+
+Results are written to `outputs/metrics/`.
+
+---
+
+## Project layout
+
+```
+unit-circle-benchmark/
+├── src/
+│   ├── main.py        — runner (ablation loop)
+│   ├── config.py      — model presets, transform configs
+│   ├── data.py        — JSONL loaders + validators
+│   ├── embed.py       — SentenceTransformer wrapper
+│   ├── transforms.py  — all A0–A5 transforms
+│   ├── benchmark.py   — retrieval, clustering, geometry suites
+│   ├── metrics.py     — table formatting, delta, success checks
+│   └── utils.py       — JSON I/O, dir helpers
+├── data/
+│   ├── corpus.jsonl   — {id, text, label}
+│   ├── queries.jsonl  — {id, text}
+│   └── qrels.jsonl    — {query_id, doc_id, relevance}
+├── outputs/
+│   ├── metrics/       — JSON result files
+│   ├── runs/          — per-run artefacts
+│   └── plots/         — visualisations from notebook
+└── notebooks/
+    └── sanity_checks.ipynb
+```
+
+---
+
+## Success criteria (spec)
+
+Call it promising only if:
+- gain appears on **at least two corpora or two models**
+- gain **survives against PCA control**
+- gain is **not confined to a single metric**
+- **negative control breaks** as expected (`uc_256_broken` < `native`)
+
+A single bump on one dataset is noise until proven otherwise.

unit-circle-benchmark/data/corpus.jsonl

@@ -0,0 +1,20 @@
+{"id":"d1","text":"The mitochondria are the powerhouse of the cell, responsible for generating most of the cell's supply of ATP through oxidative phosphorylation.","label":"biology"}
+{"id":"d2","text":"Photosynthesis is the process by which green plants and algae convert sunlight, water, and carbon dioxide into glucose and oxygen.","label":"biology"}
+{"id":"d3","text":"DNA replication occurs during the S phase of the cell cycle and involves unwinding the double helix and synthesising a complementary strand.","label":"biology"}
+{"id":"d4","text":"The Krebs cycle, also known as the citric acid cycle, is a series of chemical reactions used to generate energy through the oxidation of acetyl-CoA.","label":"biology"}
+{"id":"d5","text":"Neurons transmit signals through a combination of electrical impulses and chemical neurotransmitters released across synaptic gaps.","label":"biology"}
+{"id":"d6","text":"Quantum entanglement is a phenomenon where two particles remain connected such that the state of one instantly influences the other regardless of distance.","label":"physics"}
+{"id":"d7","text":"The Higgs boson, discovered at CERN in 2012, is a fundamental particle that gives other particles mass through interaction with the Higgs field.","label":"physics"}
+{"id":"d8","text":"General relativity describes gravity as the curvature of spacetime caused by mass and energy, generalising Newton's law of universal gravitation.","label":"physics"}
+{"id":"d9","text":"Thermodynamics describes the relationships between heat, work, temperature, and energy, underpinned by the four laws of thermodynamics.","label":"physics"}
+{"id":"d10","text":"Wave-particle duality states that every quantum entity exhibits both wave and particle properties depending on how it is observed or measured.","label":"physics"}
+{"id":"d11","text":"Gradient descent is an optimisation algorithm that iteratively adjusts model parameters in the direction that minimises a loss function.","label":"ml"}
+{"id":"d12","text":"Transformer architectures use self-attention mechanisms to model relationships between all tokens in a sequence simultaneously.","label":"ml"}
+{"id":"d13","text":"Overfitting occurs when a model learns the training data too well, including noise, leading to poor generalisation on unseen examples.","label":"ml"}
+{"id":"d14","text":"Convolutional neural networks apply learnable filters across input data, making them effective for image recognition and spatial feature extraction.","label":"ml"}
+{"id":"d15","text":"Reinforcement learning trains agents to maximise cumulative reward by taking actions in an environment and learning from feedback signals.","label":"ml"}
+{"id":"d16","text":"The Roman Empire at its height controlled territories stretching from Britain in the northwest to Mesopotamia in the east.","label":"history"}
+{"id":"d17","text":"The Industrial Revolution, beginning in Britain in the late 18th century, transformed manufacturing through mechanisation and the use of steam power.","label":"history"}
+{"id":"d18","text":"The French Revolution of 1789 led to the abolition of the monarchy, the rise of Napoleon, and the spread of republican ideals across Europe.","label":"history"}
+{"id":"d19","text":"The Silk Road was an ancient network of trade routes connecting China and East Asia with Central Asia, the Middle East, and Europe.","label":"history"}
+{"id":"d20","text":"The Renaissance was a cultural and intellectual movement in Europe from the 14th to 17th centuries, reviving interest in classical Greek and Roman thought.","label":"history"}

unit-circle-benchmark/data/qrels.jsonl

@@ -0,0 +1,12 @@
+{"query_id":"q1","doc_id":"d1","relevance":1}
+{"query_id":"q1","doc_id":"d4","relevance":1}
+{"query_id":"q2","doc_id":"d7","relevance":1}
+{"query_id":"q3","doc_id":"d13","relevance":1}
+{"query_id":"q3","doc_id":"d11","relevance":1}
+{"query_id":"q4","doc_id":"d18","relevance":1}
+{"query_id":"q5","doc_id":"d6","relevance":1}
+{"query_id":"q6","doc_id":"d3","relevance":1}
+{"query_id":"q7","doc_id":"d11","relevance":1}
+{"query_id":"q8","doc_id":"d19","relevance":1}
+{"query_id":"q9","doc_id":"d14","relevance":1}
+{"query_id":"q10","doc_id":"d17","relevance":1}

unit-circle-benchmark/data/queries.jsonl

@@ -0,0 +1,10 @@
+{"id":"q1","text":"How do cells produce energy?"}
+{"id":"q2","text":"What is the role of the Higgs boson?"}
+{"id":"q3","text":"How do neural networks avoid overfitting?"}
+{"id":"q4","text":"What caused the French Revolution?"}
+{"id":"q5","text":"How does quantum entanglement work?"}
+{"id":"q6","text":"What is the process of DNA copying?"}
+{"id":"q7","text":"How does gradient descent optimise a model?"}
+{"id":"q8","text":"What trade routes connected ancient East and West?"}
+{"id":"q9","text":"How do convolutional networks process images?"}
+{"id":"q10","text":"What were the effects of industrialisation in Britain?"}

unit-circle-benchmark/notebooks/sanity_checks.ipynb

@@ -0,0 +1,185 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": ["# Unit-circle embedding benchmark — sanity checks\n", "\n", "Run this notebook after `python -m src.main` to visualise results and inspect the transform geometry."]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import sys, os\n",
+    "sys.path.insert(0, os.path.abspath(\"..\"))\n",
+    "\n",
+    "import numpy as np\n",
+    "import matplotlib.pyplot as plt\n",
+    "from src.utils import load_json\n",
+    "from src.metrics import format_results_table, delta_vs_baseline"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": ["## 1. Load results"]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "runs = load_json(\"../outputs/metrics/nomic_results.json\")\n",
+    "print(format_results_table(runs))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": ["## 2. Deltas vs native baseline"]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "deltas = delta_vs_baseline(runs, baseline=\"native\")\n",
+    "for name, d in deltas.items():\n",
+    "    print(f\"\\n--- {name} ---\")\n",
+    "    for k, v in sorted(d.items()):\n",
+    "        sign = \"+\" if v >= 0 else \"\"\n",
+    "        print(f\"  {k:<40} {sign}{v:.4f}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": ["## 3. MRR bar chart: all transforms"]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "names = list(runs.keys())\n",
+    "mrrs  = [runs[n][\"retrieval\"][\"mrr\"] for n in names]\n",
+    "\n",
+    "fig, ax = plt.subplots(figsize=(10, 4))\n",
+    "bars = ax.bar(names, mrrs, color=[\"steelblue\" if \"uc\" in n else \"grey\" for n in names])\n",
+    "ax.axhline(runs[\"native\"][\"retrieval\"][\"mrr\"], color=\"red\", linestyle=\"--\", label=\"native\")\n",
+    "ax.set_ylabel(\"MRR\")\n",
+    "ax.set_title(\"MRR by transform (blue = unit-circle)\")\n",
+    "ax.legend()\n",
+    "plt.xticks(rotation=30, ha=\"right\")\n",
+    "plt.tight_layout()\n",
+    "os.makedirs(\"../outputs/plots\", exist_ok=True)\n",
+    "plt.savefig(\"../outputs/plots/mrr_bar.png\", dpi=150)\n",
+    "plt.show()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": ["## 4. Neighbor overlap: native vs uc_256 vs pca_256"]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "keys_of_interest = [\"native\", \"pca_256\", \"uc_256\", \"pca_64\", \"uc_64\"]\n",
+    "overlap_key = [k for k in runs[\"native\"][\"geometry\"] if \"overlap\" in k][0]\n",
+    "\n",
+    "labels = [k for k in keys_of_interest if k in runs]\n",
+    "overlaps = [runs[k][\"geometry\"][overlap_key] for k in labels]\n",
+    "\n",
+    "fig, ax = plt.subplots(figsize=(7, 4))\n",
+    "ax.bar(labels, overlaps, color=[\"steelblue\" if \"uc\" in l else \"grey\" for l in labels])\n",
+    "ax.set_ylabel(overlap_key)\n",
+    "ax.set_title(\"Neighbor overlap vs native space\")\n",
+    "plt.tight_layout()\n",
+    "plt.savefig(\"../outputs/plots/neighbor_overlap.png\", dpi=150)\n",
+    "plt.show()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": ["## 5. Direct geometry check: T1 transform on a toy vector"]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from src.transforms import _uc_map, l2_normalize\n",
+    "\n",
+    "# Construct a simple 4-dim test vector\n",
+    "x = np.array([[1.0, -1.0, 0.5, -0.5]], dtype=np.float32)\n",
+    "x_norm = l2_normalize(x)\n",
+    "mapped = _uc_map(x_norm, out_dim=4, mode=\"direct\")\n",
+    "\n",
+    "print(\"input (normalised):\", x_norm)\n",
+    "print(\"UC-mapped output  :\", mapped)\n",
+    "print(\"\\ncos half         :\", np.cos(np.pi * x_norm))\n",
+    "print(\"sin half         :\", np.sin(np.pi * x_norm))\n",
+    "\n",
+    "# Visualise on unit circle\n",
+    "thetas = np.pi * x_norm[0]\n",
+    "fig, ax = plt.subplots(figsize=(5, 5))\n",
+    "circle = plt.Circle((0, 0), 1, fill=False, color=\"lightgrey\")\n",
+    "ax.add_patch(circle)\n",
+    "for i, theta in enumerate(thetas):\n",
+    "    ax.plot([0, np.cos(theta)], [0, np.sin(theta)], label=f\"x_{i}={x_norm[0,i]:.2f}\")\n",
+    "    ax.scatter(np.cos(theta), np.sin(theta), s=80, zorder=5)\n",
+    "ax.set_xlim(-1.3, 1.3)\n",
+    "ax.set_ylim(-1.3, 1.3)\n",
+    "ax.set_aspect(\"equal\")\n",
+    "ax.legend(fontsize=8)\n",
+    "ax.set_title(\"T1 direct angle map\")\n",
+    "plt.tight_layout()\n",
+    "plt.savefig(\"../outputs/plots/uc_geometry.png\", dpi=150)\n",
+    "plt.show()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": ["## 6. Success criteria summary"]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from src.metrics import check_success_criteria\n",
+    "\n",
+    "checks = check_success_criteria(runs, uc_key=\"uc_256\", pca_key=\"pca_256\")\n",
+    "print(\"Success criteria:\")\n",
+    "for criterion, passed in checks.items():\n",
+    "    mark = \"PASS\" if passed else \"FAIL\"\n",
+    "    print(f\"  [{mark}] {criterion}\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "name": "python",
+   "version": "3.10.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

unit-circle-benchmark/pyproject.toml

@@ -0,0 +1,28 @@
+[build-system]
+requires = ["setuptools>=68"]
+build-backend = "setuptools.backends.legacy:build"
+
+[project]
+name = "unit-circle-benchmark"
+version = "0.1.0"
+description = "Benchmark: does a unit-circle coordinate remapping improve embedding usefulness?"
+requires-python = ">=3.10"
+dependencies = [
+    "numpy",
+    "pandas",
+    "scikit-learn",
+    "scipy",
+    "sentence-transformers",
+    "transformers",
+    "torch",
+    "tqdm",
+    "orjson",
+    "matplotlib",
+]
+
+[project.optional-dependencies]
+openai = ["openai"]
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["src*"]