From 1f6dca58e79a3a2eb2ad120cc4d31e958da1a13a Mon Sep 17 00:00:00 2001 From: "Akeem T. L. King" Date: Fri, 17 Jul 2026 10:47:03 -0400 Subject: [PATCH 01/10] Add persistent dataset search --- .env.example | 10 +- .gitignore | 4 + .gitmodules | 3 + CHANGELOG-2026-07-17.md | 23 ++ CHANGELOG.md | 16 +- Dockerfile | 1 + README.md | 92 ++++-- dataset_service/README.md | 35 +++ dataset_service/__init__.py | 1 + dataset_service/models.py | 52 ++++ dataset_service/repository.py | 501 +++++++++++++++++++++++++++++++ dataset_service/requirements.txt | 2 + dataset_service/server.py | 167 +++++++++++ docker-compose.yml | 20 +- nginx.conf | 42 ++- profile_search | 2 +- tests/test_dataset_service.py | 145 +++++++++ tests/test_health_endpoints.py | 31 +- whoisit | 1 + 19 files changed, 1111 insertions(+), 37 deletions(-) create mode 100644 .gitignore create mode 100644 CHANGELOG-2026-07-17.md create mode 100644 dataset_service/README.md create mode 100644 dataset_service/__init__.py create mode 100644 dataset_service/models.py create mode 100644 dataset_service/repository.py create mode 100644 dataset_service/requirements.txt create mode 100644 dataset_service/server.py create mode 100644 tests/test_dataset_service.py create mode 160000 whoisit diff --git a/.env.example b/.env.example index f31c356..a1fa4f5 100644 --- a/.env.example +++ b/.env.example @@ -1,8 +1,14 @@ # Copy this file to .env and fill in the required values before running docker compose. -# profile_search +# profile_search: X API v2 bearer token used by the Tweepy client. +# Create it in an X Developer Portal project/app. Keep it secret. TWEEPY_BEARER_TOKEN=your_tweepy_bearer_token -# phone_search +# phone_search: Twilio API credentials from the Twilio Console. +# The Account SID normally starts with AC. Keep the auth token secret. TWILIO_ACCOUNT_SID=your_twilio_account_sid TWILIO_AUTH_TOKEN=your_twilio_auth_token + +# dataset_service: Compose already persists this path in the dataset_data volume. +# Override only when you intentionally want a different in-container location. +# DATASET_DB_PATH=/data/datasets.db diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f02ff5d --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.env +__pycache__/ +*.py[cod] +finder.log diff --git a/.gitmodules b/.gitmodules index 12dca78..faca96f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,3 +7,6 @@ [submodule "phone_search"] path = phone_search url = git@github.com:osint-services/phone_search +[submodule "whoisit"] + path = whoisit + url = git@github.com:osint-services/whoisit.git diff --git a/CHANGELOG-2026-07-17.md b/CHANGELOG-2026-07-17.md new file mode 100644 index 0000000..fbc1c16 --- /dev/null +++ b/CHANGELOG-2026-07-17.md @@ -0,0 +1,23 @@ +# Changelog for 2026-07-17 + +## Added + +- Persistent SQLite-backed profile and phone datasets with mapped imports and local search. +- CSV, JSON, JSONL, and NDJSON parsing with automatic field suggestions and previews. +- Per-row validation, rejected-row reporting, provenance, observation time, confidence, and raw source retention. +- Search history and combined live/local results in the Electron application. +- Backend and renderer parser regression tests. + +## Changed + +- Reorganized the desktop application into Username, Phone, Datasets, and History workspaces. +- Expanded profile and caller details and made source provenance visible throughout the UI. +- Kept API failures scoped to their own source and search workflow. +- Added dataset routing, persistence, and readiness checks to the deployed architecture. + +## Documentation + +- Rewrote the root architecture and setup guide. +- Expanded environment-variable guidance. +- Rewrote the desktop UI guide and added a dataset service guide. +- Updated the OSINT Services organization profile README. diff --git a/CHANGELOG.md b/CHANGELOG.md index d0056f0..ae07bee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,12 +5,18 @@ All notable changes to this project will be documented in this file. ## [Unreleased] ### Added -- Added lightweight `/healthz` endpoints to the FastAPI services for smoke checks and container health probes. -- Added an initial regression test suite covering the new health endpoints. +- Added a persistent SQLite dataset service for mapped profile and phone imports. +- Added CSV, JSON, JSONL, and NDJSON parsing, previews, automatic field suggestions, per-row validation, provenance, and raw-record retention. +- Added combined live and imported-source searching, local search history, raw JSON inspection, and dataset management to the Electron UI. +- Expanded regression coverage for dataset import, fuzzy profile search, normalized phone search, deletion, health endpoints, and client-side parsers. ### Changed -- Parallelized profile checker requests with bounded concurrency to reduce scan latency across site checks. -- Removed the obsolete Compose `version` field from the deployment configuration to avoid startup warnings. +- Reworked the Electron interface into cohesive Username, Phone, Datasets, and History workspaces with master/detail results. +- Isolated live-provider failures so other sources can still return results and errors remain in the correct search workspace. +- Expanded profile and phone detail views with provider metadata, provenance, freshness, confidence, metrics, and raw source records. +- Added the dataset service to Docker Compose, nginx routing, persistent storage, and desktop readiness probes. +- Increased the desktop window size and stopped opening DevTools in normal runs. ### Documentation -- Updated the platform README to describe the new health checks and operational improvements. +- Updated the platform, environment, desktop UI, dataset service, and organization profile documentation for the current architecture. +- Added [2026-07-17 release notes](CHANGELOG-2026-07-17.md). diff --git a/Dockerfile b/Dockerfile index 82f3f8f..897b2cc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,6 +8,7 @@ WORKDIR /app COPY ./profile_checker /app/profile_checker COPY ./profile_search /app/profile_search COPY ./phone_search /app/phone_search +COPY ./dataset_service /app/dataset_service # Install the selected service dependencies ARG APP_DIR=profile_checker diff --git a/README.md b/README.md index 3ca93be..d1bbc9c 100644 --- a/README.md +++ b/README.md @@ -1,54 +1,94 @@ -# platform +# OSINT Services Platform -Stands up multiple microservices so they can be contacted using a single URI with `nginx` and `docker compose`. -Each service is stored as a git submodule; see each service directory for service-specific details. +The platform combines live public-data APIs with user-supplied datasets behind one nginx endpoint and one Electron investigation workspace. + +## Architecture + +```text +whoisit desktop app + | + v +nginx :80 + |-- /scan/* -> profile_checker (public username availability) + |-- /focus* -> profile_search (X profile metadata via Tweepy) + |-- /phone_search* -> phone_search (caller-name data via Twilio) + `-- /datasets* -> dataset_service (SQLite imports and local search) +``` + +Docker Compose runs the four FastAPI services, nginx, and a persistent `dataset_data` volume. The Electron UI queries live and local sources independently and combines successful results, so one unavailable provider does not prevent other sources from being searched. ## Requirements -- [`docker`](https://www.docker.com/) -- Docker Compose support (`docker compose`) +- Docker with `docker compose` +- Node.js 18 or newer to build or run the Electron UI +- An X API bearer token for profile inspection +- Twilio Account SID and Auth Token for live phone lookup ## Setup -1. Initialize submodules if needed: - ```bash git submodule update --init --recursive +cp .env.example .env ``` -2. Build and start all services: +Add your provider credentials to `.env`, then start the API stack: ```bash ./scripts/start.sh ``` -3. Stop the stack: +Run the desktop UI from a second terminal: ```bash -./scripts/stop.sh +cd whoisit +npm install +npm start ``` -4. Rebuild the services: +The UI checks service readiness on launch and can attempt to start the Compose stack when Docker is available. Packaged builds are created with `npm run package`. -```bash -./scripts/build.sh -``` +## Environment values -5. Stream logs: +| Variable | Required | Used by | Meaning | +| --- | --- | --- | --- | +| `TWEEPY_BEARER_TOKEN` | For live profile inspection | `profile_search` | X API v2 application bearer token. Despite the variable name, the credential is issued by X; Tweepy is the Python client. | +| `TWILIO_ACCOUNT_SID` | For live phone lookup | `phone_search` | Twilio account identifier, normally beginning with `AC`. | +| `TWILIO_AUTH_TOKEN` | For live phone lookup | `phone_search` | Secret used with the Account SID to authenticate Twilio API calls. | +| `DATASET_DB_PATH` | No | `dataset_service` | SQLite file location. Compose sets this to `/data/datasets.db` inside its persistent volume. | + +Do not commit `.env`. Imported datasets may contain sensitive or licensed information; only import data you are authorized to retain and use. + +## API routes + +| Route | Purpose | +| --- | --- | +| `GET /scan/{username}` | Check supported sites for a public username | +| `GET /focus?url=https://x.com/{username}` | Retrieve expanded X profile metadata | +| `GET /phone_search?phone_number=+18135551212` | Retrieve live caller-name metadata | +| `GET /datasets` | List imported datasets | +| `GET /datasets/schema/{profile\|phone}` | Get canonical import fields and a sample | +| `POST /datasets/import` | Import mapped profile or phone rows | +| `GET /datasets/search/profiles?query=...&fuzzy=true` | Search imported profiles | +| `GET /datasets/search/phones?phone_number=...` | Search imported phone records | +| `DELETE /datasets/{dataset_id}` | Delete a dataset and its records | + +The import API accepts JSON rows after the UI parses CSV, JSON, JSONL, or NDJSON. A single import is limited to 10,000 records and the UI limits files to 10 MB. Canonical records preserve their source row, dataset provenance, observation time, and optional confidence score. Phone search uses normalized E.164 values; profile search supports exact or fuzzy matching. + +## Operations and tests ```bash -./scripts/logs.sh +./scripts/build.sh # rebuild images +./scripts/logs.sh # stream Compose logs +./scripts/stop.sh # stop the stack +.venv/bin/pytest -q # backend tests +cd whoisit && npm test # importer parser tests ``` -## Notes - -- The proxy listens on port `80` and routes `/scan/...` to `profile_checker`, `/focus` to `profile_search`, and `/phone_search` to `phone_search`. -- `profile_search` requires `TWEEPY_BEARER_TOKEN` to be available in the environment when running the Compose stack. -- `phone_search` requires `TWILIO_ACCOUNT_SID` and `TWILIO_AUTH_TOKEN` to be set. -- Copy `.env.example` to `.env` and fill in the required values before starting Compose. -- Each service now exposes a lightweight `/healthz` endpoint for smoke checks and container health probes. +Liveness endpoints are available at `/scan/healthz`, `/phone_search/healthz`, and `/datasets/healthz`. Profile inspection also exposes `/focus/readyz`, which verifies that its X credential is configured. -## Changelog +## Documentation -- See [CHANGELOG.md](CHANGELOG.md) for the current release notes. -- See [CHANGELOG-2026-07-16.md](CHANGELOG-2026-07-16.md) for the latest dated entry. +- [Desktop UI guide](whoisit/README.md) +- [Dataset service guide](dataset_service/README.md) +- [Changelog](CHANGELOG.md) +- [2026-07-17 release notes](CHANGELOG-2026-07-17.md) diff --git a/dataset_service/README.md b/dataset_service/README.md new file mode 100644 index 0000000..adb9c27 --- /dev/null +++ b/dataset_service/README.md @@ -0,0 +1,35 @@ +# Dataset service + +The dataset service provides normalized, persistent local sources for the platform's profile and phone searches. It uses SQLite and has no external provider dependency. + +## Supported input + +The Electron UI parses `.csv`, `.json`, `.jsonl`, and `.ndjson` files, then maps source columns to the canonical fields returned by: + +```http +GET /datasets/schema/profile +GET /datasets/schema/phone +``` + +Profile records require either a username or profile URL. Phone records require an E.164 number such as `+18135551212`. Invalid booleans, negative metrics, confidence outside `0..1`, and malformed identifiers are rejected per row without discarding valid rows. + +Each accepted record includes: + +- its dataset and source; +- an observation timestamp; +- optional confidence; +- normalized searchable identifiers; and +- the complete original source row for audit. + +## Storage and limits + +Compose stores `/data/datasets.db` in the named `dataset_data` volume. Deleting a dataset cascades to its records. Each import accepts 1–10,000 JSON object rows; the desktop UI additionally enforces a 10 MB file limit. + +The API is intentionally local and currently has no authentication layer. Do not expose the nginx proxy to an untrusted network without adding access control. + +## Development + +```bash +uvicorn dataset_service.server:app --reload --port 8000 +.venv/bin/pytest -q tests/test_dataset_service.py +``` diff --git a/dataset_service/__init__.py b/dataset_service/__init__.py new file mode 100644 index 0000000..d8a5d3b --- /dev/null +++ b/dataset_service/__init__.py @@ -0,0 +1 @@ +"""Local dataset import and search service.""" diff --git a/dataset_service/models.py b/dataset_service/models.py new file mode 100644 index 0000000..d07f09c --- /dev/null +++ b/dataset_service/models.py @@ -0,0 +1,52 @@ +from enum import Enum +from typing import Any + +from pydantic import BaseModel, Field + + +class RecordType(str, Enum): + profile = "profile" + phone = "phone" + + +class DatasetImportRequest(BaseModel): + name: str = Field(min_length=1, max_length=120) + record_type: RecordType + filename: str | None = Field(default=None, max_length=255) + mapping: dict[str, str] = Field(default_factory=dict) + rows: list[dict[str, Any]] = Field(min_items=1, max_items=10_000) + + +class RejectedRow(BaseModel): + row_number: int + reason: str + + +class DatasetImportResult(BaseModel): + dataset_id: str + imported: int + rejected: int + rejected_rows: list[RejectedRow] + + +class DatasetSummary(BaseModel): + id: str + name: str + record_type: RecordType + filename: str | None + imported_at: str + row_count: int + rejected_count: int + + +class FieldDefinition(BaseModel): + key: str + label: str + required: bool = False + description: str + + +class DatasetSchema(BaseModel): + record_type: RecordType + fields: list[FieldDefinition] + sample: dict[str, Any] diff --git a/dataset_service/repository.py b/dataset_service/repository.py new file mode 100644 index 0000000..eedd01a --- /dev/null +++ b/dataset_service/repository.py @@ -0,0 +1,501 @@ +import json +import re +import sqlite3 +from datetime import datetime, timezone +from pathlib import Path +from typing import Any +from urllib.parse import urlparse +from uuid import uuid4 + +from .models import DatasetImportRequest, DatasetImportResult, RejectedRow + + +PROFILE_FIELDS = ( + "username", + "platform", + "profile_url", + "display_name", + "bio", + "location", + "website", + "verified", + "protected", + "created_at", + "profile_image_url", + "profile_banner_url", + "followers_count", + "following_count", + "post_count", + "listed_count", + "like_count", + "media_count", + "observed_at", + "confidence", + "source", +) + +PHONE_FIELDS = ( + "phone_number", + "valid", + "national_format", + "country_code", + "caller_name", + "caller_type", + "carrier_name", + "line_type", + "location", + "observed_at", + "confidence", + "source", +) + + +def utc_now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def clean_text(value: Any) -> str | None: + if value is None: + return None + text = str(value).strip() + return text or None + + +def parse_bool(value: Any) -> bool | None: + if value is None or value == "": + return None + if isinstance(value, bool): + return value + normalized = str(value).strip().lower() + if normalized in {"true", "1", "yes", "y"}: + return True + if normalized in {"false", "0", "no", "n"}: + return False + raise ValueError(f"'{value}' is not a boolean") + + +def parse_int(value: Any) -> int | None: + if value is None or value == "": + return None + number = int(str(value).replace(",", "").strip()) + if number < 0: + raise ValueError("metric values cannot be negative") + return number + + +def parse_confidence(value: Any) -> float | None: + if value is None or value == "": + return None + confidence = float(value) + if confidence < 0 or confidence > 1: + raise ValueError("confidence must be between 0 and 1") + return confidence + + +def normalize_username(value: Any) -> str | None: + username = clean_text(value) + return username.lstrip("@").lower() if username else None + + +def username_from_url(value: Any) -> str | None: + url = clean_text(value) + if not url: + return None + path = urlparse(url).path.strip("/") + return normalize_username(path.split("/")[0]) if path else None + + +def normalize_phone(value: Any) -> str: + phone = clean_text(value) + if not phone: + raise ValueError("phone_number is required") + has_plus = phone.startswith("+") + digits = re.sub(r"\D", "", phone) + if not has_plus or not 8 <= len(digits) <= 15: + raise ValueError("phone_number must use E.164 format, for example +18135551212") + return f"+{digits}" + + +class DatasetRepository: + def __init__(self, database_path: str | Path): + self.database_path = str(database_path) + Path(self.database_path).parent.mkdir(parents=True, exist_ok=True) + self.initialize() + + def connect(self) -> sqlite3.Connection: + connection = sqlite3.connect(self.database_path) + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + return connection + + def initialize(self) -> None: + with self.connect() as connection: + connection.executescript( + """ + CREATE TABLE IF NOT EXISTS datasets ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + record_type TEXT NOT NULL CHECK(record_type IN ('profile', 'phone')), + filename TEXT, + imported_at TEXT NOT NULL, + row_count INTEGER NOT NULL, + rejected_count INTEGER NOT NULL + ); + + CREATE TABLE IF NOT EXISTS profile_records ( + id TEXT PRIMARY KEY, + dataset_id TEXT NOT NULL REFERENCES datasets(id) ON DELETE CASCADE, + platform TEXT, + username TEXT, + username_normalized TEXT, + profile_url TEXT, + display_name TEXT, + bio TEXT, + location TEXT, + website TEXT, + verified INTEGER, + protected INTEGER, + created_at TEXT, + profile_image_url TEXT, + profile_banner_url TEXT, + followers_count INTEGER, + following_count INTEGER, + post_count INTEGER, + listed_count INTEGER, + like_count INTEGER, + media_count INTEGER, + observed_at TEXT, + confidence REAL, + source TEXT, + raw_json TEXT NOT NULL + ); + + CREATE TABLE IF NOT EXISTS phone_records ( + id TEXT PRIMARY KEY, + dataset_id TEXT NOT NULL REFERENCES datasets(id) ON DELETE CASCADE, + phone_number TEXT NOT NULL, + valid INTEGER, + national_format TEXT, + country_code TEXT, + caller_name TEXT, + caller_type TEXT, + carrier_name TEXT, + line_type TEXT, + location TEXT, + observed_at TEXT, + confidence REAL, + source TEXT, + raw_json TEXT NOT NULL + ); + + CREATE INDEX IF NOT EXISTS idx_profile_username + ON profile_records(username_normalized); + CREATE INDEX IF NOT EXISTS idx_profile_display_name + ON profile_records(display_name); + CREATE INDEX IF NOT EXISTS idx_phone_number + ON phone_records(phone_number); + """ + ) + + @staticmethod + def mapped_row( + row: dict[str, Any], mapping: dict[str, str], canonical_fields: tuple[str, ...] + ) -> dict[str, Any]: + return { + field: row.get(mapping.get(field, field)) + for field in canonical_fields + } + + def import_dataset(self, request: DatasetImportRequest) -> DatasetImportResult: + dataset_id = str(uuid4()) + accepted: list[dict[str, Any]] = [] + rejected: list[RejectedRow] = [] + + for row_number, raw_row in enumerate(request.rows, start=1): + try: + if request.record_type.value == "profile": + accepted.append( + self.prepare_profile(raw_row, request.mapping, request.name) + ) + else: + accepted.append( + self.prepare_phone(raw_row, request.mapping, request.name) + ) + except (TypeError, ValueError) as exc: + rejected.append(RejectedRow(row_number=row_number, reason=str(exc))) + + imported_at = utc_now() + with self.connect() as connection: + connection.execute( + """ + INSERT INTO datasets + (id, name, record_type, filename, imported_at, row_count, rejected_count) + VALUES (?, ?, ?, ?, ?, ?, ?) + """, + ( + dataset_id, + request.name.strip(), + request.record_type.value, + request.filename, + imported_at, + len(accepted), + len(rejected), + ), + ) + + if request.record_type.value == "profile": + connection.executemany( + """ + INSERT INTO profile_records ( + id, dataset_id, platform, username, username_normalized, + profile_url, display_name, bio, location, website, verified, + protected, created_at, profile_image_url, profile_banner_url, + followers_count, following_count, post_count, listed_count, + like_count, media_count, observed_at, confidence, source, raw_json + ) VALUES ( + :id, :dataset_id, :platform, :username, :username_normalized, + :profile_url, :display_name, :bio, :location, :website, :verified, + :protected, :created_at, :profile_image_url, :profile_banner_url, + :followers_count, :following_count, :post_count, :listed_count, + :like_count, :media_count, :observed_at, :confidence, :source, :raw_json + ) + """, + [dict(record, dataset_id=dataset_id) for record in accepted], + ) + else: + connection.executemany( + """ + INSERT INTO phone_records ( + id, dataset_id, phone_number, valid, national_format, + country_code, caller_name, caller_type, carrier_name, + line_type, location, observed_at, confidence, source, raw_json + ) VALUES ( + :id, :dataset_id, :phone_number, :valid, :national_format, + :country_code, :caller_name, :caller_type, :carrier_name, + :line_type, :location, :observed_at, :confidence, :source, :raw_json + ) + """, + [dict(record, dataset_id=dataset_id) for record in accepted], + ) + + return DatasetImportResult( + dataset_id=dataset_id, + imported=len(accepted), + rejected=len(rejected), + rejected_rows=rejected[:100], + ) + + def prepare_profile( + self, raw_row: dict[str, Any], mapping: dict[str, str], dataset_name: str + ) -> dict[str, Any]: + values = self.mapped_row(raw_row, mapping, PROFILE_FIELDS) + username = clean_text(values["username"]) + username_normalized = normalize_username(username) or username_from_url( + values["profile_url"] + ) + if not username_normalized: + raise ValueError("username or profile_url is required") + username = username or username_normalized + + return { + "id": str(uuid4()), + "platform": clean_text(values["platform"]) or "Imported", + "username": username.lstrip("@"), + "username_normalized": username_normalized, + "profile_url": clean_text(values["profile_url"]), + "display_name": clean_text(values["display_name"]), + "bio": clean_text(values["bio"]), + "location": clean_text(values["location"]), + "website": clean_text(values["website"]), + "verified": parse_bool(values["verified"]), + "protected": parse_bool(values["protected"]), + "created_at": clean_text(values["created_at"]), + "profile_image_url": clean_text(values["profile_image_url"]), + "profile_banner_url": clean_text(values["profile_banner_url"]), + "followers_count": parse_int(values["followers_count"]), + "following_count": parse_int(values["following_count"]), + "post_count": parse_int(values["post_count"]), + "listed_count": parse_int(values["listed_count"]), + "like_count": parse_int(values["like_count"]), + "media_count": parse_int(values["media_count"]), + "observed_at": clean_text(values["observed_at"]) or utc_now(), + "confidence": parse_confidence(values["confidence"]), + "source": clean_text(values["source"]) or dataset_name, + "raw_json": json.dumps(raw_row, default=str), + } + + def prepare_phone( + self, raw_row: dict[str, Any], mapping: dict[str, str], dataset_name: str + ) -> dict[str, Any]: + values = self.mapped_row(raw_row, mapping, PHONE_FIELDS) + return { + "id": str(uuid4()), + "phone_number": normalize_phone(values["phone_number"]), + "valid": parse_bool(values["valid"]), + "national_format": clean_text(values["national_format"]), + "country_code": clean_text(values["country_code"]), + "caller_name": clean_text(values["caller_name"]), + "caller_type": clean_text(values["caller_type"]), + "carrier_name": clean_text(values["carrier_name"]), + "line_type": clean_text(values["line_type"]), + "location": clean_text(values["location"]), + "observed_at": clean_text(values["observed_at"]) or utc_now(), + "confidence": parse_confidence(values["confidence"]), + "source": clean_text(values["source"]) or dataset_name, + "raw_json": json.dumps(raw_row, default=str), + } + + def list_datasets(self) -> list[dict[str, Any]]: + with self.connect() as connection: + rows = connection.execute( + "SELECT * FROM datasets ORDER BY imported_at DESC" + ).fetchall() + return [dict(row) for row in rows] + + def delete_dataset(self, dataset_id: str) -> bool: + with self.connect() as connection: + cursor = connection.execute( + "DELETE FROM datasets WHERE id = ?", (dataset_id,) + ) + return cursor.rowcount > 0 + + def search_profiles(self, query: str, fuzzy: bool = False) -> list[dict[str, Any]]: + normalized = normalize_username(query) + if not normalized: + return [] + with self.connect() as connection: + if fuzzy: + pattern = f"%{normalized}%" + rows = connection.execute( + """ + SELECT profile_records.*, datasets.name AS dataset_name + FROM profile_records + JOIN datasets ON datasets.id = profile_records.dataset_id + WHERE username_normalized LIKE ? + OR LOWER(COALESCE(display_name, '')) LIKE ? + OR LOWER(COALESCE(profile_url, '')) LIKE ? + ORDER BY confidence DESC, observed_at DESC + LIMIT 100 + """, + (pattern, pattern, pattern), + ).fetchall() + else: + rows = connection.execute( + """ + SELECT profile_records.*, datasets.name AS dataset_name + FROM profile_records + JOIN datasets ON datasets.id = profile_records.dataset_id + WHERE username_normalized = ? + ORDER BY confidence DESC, observed_at DESC + LIMIT 100 + """, + (normalized,), + ).fetchall() + return [self.profile_response(row) for row in rows] + + def search_phones(self, phone_number: str) -> list[dict[str, Any]]: + normalized = normalize_phone(phone_number) + with self.connect() as connection: + rows = connection.execute( + """ + SELECT phone_records.*, datasets.name AS dataset_name + FROM phone_records + JOIN datasets ON datasets.id = phone_records.dataset_id + WHERE phone_number = ? + ORDER BY confidence DESC, observed_at DESC + LIMIT 100 + """, + (normalized,), + ).fetchall() + return [self.phone_response(row) for row in rows] + + def dataset_records( + self, dataset_id: str, limit: int, offset: int + ) -> tuple[str, list[dict[str, Any]]] | None: + with self.connect() as connection: + dataset = connection.execute( + "SELECT record_type FROM datasets WHERE id = ?", (dataset_id,) + ).fetchone() + if not dataset: + return None + table = ( + "profile_records" + if dataset["record_type"] == "profile" + else "phone_records" + ) + rows = connection.execute( + f"SELECT * FROM {table} WHERE dataset_id = ? LIMIT ? OFFSET ?", + (dataset_id, limit, offset), + ).fetchall() + return dataset["record_type"], [ + self.profile_response(row) + if dataset["record_type"] == "profile" + else self.phone_response(row) + for row in rows + ] + + @staticmethod + def profile_response(row: sqlite3.Row) -> dict[str, Any]: + metrics = { + key: row[key] + for key in ( + "followers_count", + "following_count", + "post_count", + "listed_count", + "like_count", + "media_count", + ) + if row[key] is not None + } + return { + "id": row["id"], + "record_type": "profile", + "source_type": "dataset", + "dataset_id": row["dataset_id"], + "dataset_name": row["dataset_name"] if "dataset_name" in row.keys() else None, + "source": row["source"], + "observed_at": row["observed_at"], + "confidence": row["confidence"], + "platform": row["platform"], + "username": row["username"], + "profile_uri": row["profile_url"], + "name": row["display_name"], + "bio": row["bio"], + "location": row["location"], + "website": row["website"], + "verified": ( + bool(row["verified"]) if row["verified"] is not None else None + ), + "protected": ( + bool(row["protected"]) if row["protected"] is not None else None + ), + "created_at": row["created_at"], + "profile_image_url": row["profile_image_url"], + "profile_banner_url": row["profile_banner_url"], + "metrics": metrics, + "raw": json.loads(row["raw_json"]), + } + + @staticmethod + def phone_response(row: sqlite3.Row) -> dict[str, Any]: + return { + "id": row["id"], + "record_type": "phone", + "source_type": "dataset", + "dataset_id": row["dataset_id"], + "dataset_name": row["dataset_name"] if "dataset_name" in row.keys() else None, + "source": row["source"], + "observed_at": row["observed_at"], + "confidence": row["confidence"], + "phone_number": row["phone_number"], + "valid": bool(row["valid"]) if row["valid"] is not None else None, + "national_format": row["national_format"], + "country_code": row["country_code"], + "caller_name": row["caller_name"], + "caller_type": row["caller_type"], + "carrier_name": row["carrier_name"], + "line_type": row["line_type"], + "location": row["location"], + "raw": json.loads(row["raw_json"]), + } diff --git a/dataset_service/requirements.txt b/dataset_service/requirements.txt new file mode 100644 index 0000000..cb9becc --- /dev/null +++ b/dataset_service/requirements.txt @@ -0,0 +1,2 @@ +fastapi[standard]==0.115.0 +pydantic==2.9.2 diff --git a/dataset_service/server.py b/dataset_service/server.py new file mode 100644 index 0000000..2d27b33 --- /dev/null +++ b/dataset_service/server.py @@ -0,0 +1,167 @@ +import os +import tempfile +from pathlib import Path + +from fastapi import FastAPI, HTTPException, Query, Response, status +from fastapi.middleware.cors import CORSMiddleware + +from .models import ( + DatasetImportRequest, + DatasetImportResult, + DatasetSchema, + DatasetSummary, + FieldDefinition, + RecordType, +) +from .repository import DatasetRepository + + +PROFILE_SCHEMA = DatasetSchema( + record_type=RecordType.profile, + fields=[ + FieldDefinition(key="username", label="Username", required=True, description="Account handle, with or without @. A profile URL can be used instead."), + FieldDefinition(key="platform", label="Platform", description="Network or site name, for example X or GitHub."), + FieldDefinition(key="profile_url", label="Profile URL", description="Canonical public profile URL."), + FieldDefinition(key="display_name", label="Display name", description="Human-readable account name."), + FieldDefinition(key="bio", label="Bio", description="Public profile description."), + FieldDefinition(key="location", label="Location", description="Public or source-provided location."), + FieldDefinition(key="website", label="Website", description="Website associated with the account."), + FieldDefinition(key="verified", label="Verified", description="Boolean verification status."), + FieldDefinition(key="protected", label="Protected", description="Boolean protected/private status."), + FieldDefinition(key="created_at", label="Created at", description="Account creation timestamp."), + FieldDefinition(key="profile_image_url", label="Profile image URL", description="Avatar URL."), + FieldDefinition(key="profile_banner_url", label="Profile banner URL", description="Header/banner image URL."), + FieldDefinition(key="followers_count", label="Followers", description="Follower count."), + FieldDefinition(key="following_count", label="Following", description="Following count."), + FieldDefinition(key="post_count", label="Posts", description="Post or status count."), + FieldDefinition(key="listed_count", label="Listed", description="List membership count."), + FieldDefinition(key="like_count", label="Likes", description="Like or favorite count."), + FieldDefinition(key="media_count", label="Media", description="Media post count."), + FieldDefinition(key="observed_at", label="Observed at", description="Timestamp when the source observed this record."), + FieldDefinition(key="confidence", label="Confidence", description="Optional confidence score from 0 to 1."), + FieldDefinition(key="source", label="Source", description="Original source name or URL."), + ], + sample={ + "username": "example", + "platform": "X", + "profile_url": "https://x.com/example", + "display_name": "Example Person", + "verified": False, + "followers_count": 1250, + "observed_at": "2026-07-17T12:00:00Z", + "confidence": 0.9, + "source": "Public research export", + }, +) + +PHONE_SCHEMA = DatasetSchema( + record_type=RecordType.phone, + fields=[ + FieldDefinition(key="phone_number", label="Phone number", required=True, description="Number in E.164 format, for example +18135551212."), + FieldDefinition(key="valid", label="Valid", description="Boolean validity status."), + FieldDefinition(key="national_format", label="National format", description="Locally formatted phone number."), + FieldDefinition(key="country_code", label="Country code", description="ISO country code."), + FieldDefinition(key="caller_name", label="Caller name", description="Person or business caller name."), + FieldDefinition(key="caller_type", label="Caller type", description="Caller-name classification."), + FieldDefinition(key="carrier_name", label="Carrier", description="Network carrier name."), + FieldDefinition(key="line_type", label="Line type", description="Mobile, landline, VoIP, or other classification."), + FieldDefinition(key="location", label="Location", description="Source-provided phone location."), + FieldDefinition(key="observed_at", label="Observed at", description="Timestamp when the source observed this record."), + FieldDefinition(key="confidence", label="Confidence", description="Optional confidence score from 0 to 1."), + FieldDefinition(key="source", label="Source", description="Original source name or URL."), + ], + sample={ + "phone_number": "+18135551212", + "valid": True, + "national_format": "(813) 555-1212", + "country_code": "US", + "caller_name": "Example Business", + "line_type": "landline", + "observed_at": "2026-07-17T12:00:00Z", + "confidence": 0.85, + "source": "Licensed directory", + }, +) + + +def create_app(database_path: str | Path | None = None) -> FastAPI: + fallback = Path(tempfile.gettempdir()) / "whoisit-datasets.db" + path = database_path or os.getenv("DATASET_DB_PATH", str(fallback)) + repository = DatasetRepository(path) + app = FastAPI(title="OSINT Dataset Service") + app.state.repository = repository + + app.add_middleware( + CORSMiddleware, + allow_origins=["http://localhost:3000"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], + ) + + @app.get("/healthz") + def healthz() -> dict[str, str]: + return {"status": "ok"} + + @app.get("/datasets/schema/{record_type}", response_model=DatasetSchema) + def dataset_schema(record_type: RecordType) -> DatasetSchema: + return PROFILE_SCHEMA if record_type is RecordType.profile else PHONE_SCHEMA + + @app.post( + "/datasets/import", + response_model=DatasetImportResult, + status_code=status.HTTP_201_CREATED, + ) + def import_dataset(request: DatasetImportRequest) -> DatasetImportResult: + return repository.import_dataset(request) + + @app.get("/datasets", response_model=list[DatasetSummary]) + def list_datasets() -> list[dict]: + return repository.list_datasets() + + @app.delete("/datasets/{dataset_id}", status_code=status.HTTP_204_NO_CONTENT) + def delete_dataset(dataset_id: str) -> Response: + if not repository.delete_dataset(dataset_id): + raise HTTPException(status_code=404, detail="Dataset not found") + return Response(status_code=status.HTTP_204_NO_CONTENT) + + @app.get("/datasets/{dataset_id}/records") + def dataset_records( + dataset_id: str, + limit: int = Query(default=100, ge=1, le=500), + offset: int = Query(default=0, ge=0), + ) -> dict: + result = repository.dataset_records(dataset_id, limit, offset) + if result is None: + raise HTTPException(status_code=404, detail="Dataset not found") + record_type, records = result + return { + "dataset_id": dataset_id, + "record_type": record_type, + "records": records, + "limit": limit, + "offset": offset, + } + + @app.get("/datasets/search/profiles") + def search_profiles( + query: str = Query(min_length=1), + fuzzy: bool = False, + ) -> dict: + return { + "query": query, + "records": repository.search_profiles(query, fuzzy=fuzzy), + } + + @app.get("/datasets/search/phones") + def search_phones(phone_number: str = Query(min_length=1)) -> dict: + try: + records = repository.search_phones(phone_number) + except ValueError as exc: + raise HTTPException(status_code=422, detail=str(exc)) from exc + return {"query": phone_number, "records": records} + + return app + + +app = create_app() diff --git a/docker-compose.yml b/docker-compose.yml index 1be542a..e9cae25 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -32,7 +32,21 @@ services: environment: - TWILIO_ACCOUNT_SID=${TWILIO_ACCOUNT_SID} - TWILIO_AUTH_TOKEN=${TWILIO_AUTH_TOKEN} - command: uvicorn server:app --host 0.0.0.0 --port 8000 + command: uvicorn phone_search.server:app --host 0.0.0.0 --port 8000 + networks: + - platform_net + + dataset_service: + build: + context: . + dockerfile: Dockerfile + args: + APP_DIR: dataset_service + environment: + - DATASET_DB_PATH=/data/datasets.db + command: uvicorn dataset_service.server:app --host 0.0.0.0 --port 8000 + volumes: + - dataset_data:/data networks: - platform_net @@ -42,6 +56,7 @@ services: - profile_checker - profile_search - phone_search + - dataset_service ports: - "80:80" volumes: @@ -52,3 +67,6 @@ services: networks: platform_net: driver: bridge + +volumes: + dataset_data: diff --git a/nginx.conf b/nginx.conf index 1810a2f..a1dd81b 100644 --- a/nginx.conf +++ b/nginx.conf @@ -6,6 +6,14 @@ http { server { listen 80; + location = /scan/healthz { + proxy_pass http://profile_checker:8000/healthz; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + location /scan/ { proxy_pass http://profile_checker:8000; proxy_set_header Host $host; @@ -14,6 +22,14 @@ http { proxy_set_header X-Forwarded-Proto $scheme; } + location = /focus/readyz { + proxy_pass http://profile_search:8000/readyz; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + location /focus { proxy_pass http://profile_search:8000; proxy_set_header Host $host; @@ -22,6 +38,14 @@ http { proxy_set_header X-Forwarded-Proto $scheme; } + location = /phone_search/healthz { + proxy_pass http://phone_search:8000/healthz; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + location /phone_search { proxy_pass http://phone_search:8000/lookup/caller-name; proxy_set_header Host $host; @@ -30,5 +54,21 @@ http { proxy_set_header X-Forwarded-Proto $scheme; } + location = /datasets/healthz { + proxy_pass http://dataset_service:8000/healthz; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location /datasets { + proxy_pass http://dataset_service:8000; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + } -} \ No newline at end of file +} diff --git a/profile_search b/profile_search index 5710370..d134f8e 160000 --- a/profile_search +++ b/profile_search @@ -1 +1 @@ -Subproject commit 5710370a6f83e99ec558754b36ddfad5b3e63650 +Subproject commit d134f8e8a30bdb643b4f2050d16306cce6fa04fd diff --git a/tests/test_dataset_service.py b/tests/test_dataset_service.py new file mode 100644 index 0000000..1a5ab92 --- /dev/null +++ b/tests/test_dataset_service.py @@ -0,0 +1,145 @@ +import sys +from pathlib import Path + +from fastapi.testclient import TestClient + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from dataset_service.server import create_app + + +def make_client(tmp_path): + return TestClient(create_app(tmp_path / "datasets.db")) + + +def test_health_and_schema_are_available(tmp_path): + client = make_client(tmp_path) + + assert client.get("/healthz").json() == {"status": "ok"} + schema = client.get("/datasets/schema/profile") + assert schema.status_code == 200 + assert schema.json()["record_type"] == "profile" + assert any(field["key"] == "username" for field in schema.json()["fields"]) + + +def test_profile_import_preserves_provenance_and_supports_fuzzy_search(tmp_path): + client = make_client(tmp_path) + response = client.post( + "/datasets/import", + json={ + "name": "Conference profiles", + "record_type": "profile", + "filename": "profiles.csv", + "mapping": { + "username": "handle", + "platform": "network", + "display_name": "full_name", + "bio": "description", + "followers_count": "followers", + "verified": "is_verified", + "confidence": "score", + }, + "rows": [ + { + "handle": "@AdaLovelace", + "network": "X", + "full_name": "Ada Lovelace", + "description": "Computing pioneer", + "followers": "1200", + "is_verified": "yes", + "score": "0.92", + "unmapped_note": "preserve me", + }, + { + "handle": "", + "network": "X", + "full_name": "No identifier", + }, + ], + }, + ) + + assert response.status_code == 201 + result = response.json() + assert result["imported"] == 1 + assert result["rejected"] == 1 + assert result["rejected_rows"][0]["row_number"] == 2 + + exact = client.get( + "/datasets/search/profiles", params={"query": "adalovelace"} + ).json()["records"] + assert len(exact) == 1 + record = exact[0] + assert record["username"] == "AdaLovelace" + assert record["verified"] is True + assert record["metrics"]["followers_count"] == 1200 + assert record["dataset_name"] == "Conference profiles" + assert record["confidence"] == 0.92 + assert record["raw"]["unmapped_note"] == "preserve me" + + fuzzy = client.get( + "/datasets/search/profiles", + params={"query": "lovelac", "fuzzy": "true"}, + ).json()["records"] + assert len(fuzzy) == 1 + + datasets = client.get("/datasets").json() + assert datasets[0]["row_count"] == 1 + assert datasets[0]["rejected_count"] == 1 + + +def test_phone_import_search_records_and_delete(tmp_path): + client = make_client(tmp_path) + imported = client.post( + "/datasets/import", + json={ + "name": "Subscriber export", + "record_type": "phone", + "filename": "subscribers.jsonl", + "mapping": { + "phone_number": "number", + "caller_name": "name", + "country_code": "country", + "carrier_name": "carrier", + "line_type": "kind", + }, + "rows": [ + { + "number": "+1 (813) 555-1212", + "name": "Example Subscriber", + "country": "US", + "carrier": "Example Wireless", + "kind": "mobile", + }, + {"number": "not-a-number", "name": "Invalid"}, + ], + }, + ) + + assert imported.status_code == 201 + result = imported.json() + assert result["imported"] == 1 + dataset_id = result["dataset_id"] + + matches = client.get( + "/datasets/search/phones", params={"phone_number": "+18135551212"} + ).json()["records"] + assert len(matches) == 1 + assert matches[0]["phone_number"] == "+18135551212" + assert matches[0]["caller_name"] == "Example Subscriber" + assert matches[0]["carrier_name"] == "Example Wireless" + + records = client.get(f"/datasets/{dataset_id}/records") + assert records.status_code == 200 + assert len(records.json()["records"]) == 1 + + assert client.delete(f"/datasets/{dataset_id}").status_code == 204 + assert client.get("/datasets").json() == [] + assert ( + client.get( + "/datasets/search/phones", params={"phone_number": "+18135551212"} + ).json()["records"] + == [] + ) diff --git a/tests/test_health_endpoints.py b/tests/test_health_endpoints.py index d0ee796..5b4e234 100644 --- a/tests/test_health_endpoints.py +++ b/tests/test_health_endpoints.py @@ -11,7 +11,7 @@ @pytest.fixture -def apps(monkeypatch): +def apps(monkeypatch, tmp_path): monkeypatch.setenv("TWILIO_ACCOUNT_SID", "AC123") monkeypatch.setenv("TWILIO_AUTH_TOKEN", "secret") monkeypatch.setenv("TWEEPY_BEARER_TOKEN", "token") @@ -19,11 +19,13 @@ def apps(monkeypatch): phone_server = importlib.import_module("phone_search.server") profile_search_server = importlib.import_module("profile_search.server") profile_checker_main = importlib.import_module("profile_checker.main") + from dataset_service.server import create_app as create_dataset_app return { "phone": phone_server.app, "profile_search": profile_search_server.app, "profile_checker": profile_checker_main.app, + "dataset": create_dataset_app(tmp_path / "health-datasets.db"), } @@ -33,3 +35,30 @@ def test_health_endpoint_is_available(apps): response = client.get("/healthz") assert response.status_code == 200, f"{app_name} did not expose /healthz" assert response.json() == {"status": "ok"} + + +def test_profile_search_readiness_requires_bearer_token(apps, monkeypatch): + with TestClient(apps["profile_search"]) as client: + monkeypatch.delenv("TWEEPY_BEARER_TOKEN", raising=False) + monkeypatch.delenv("BEARER_TOKEN", raising=False) + + response = client.get("/readyz") + + assert response.status_code == 503 + assert "TWEEPY_BEARER_TOKEN is missing" in response.json()["detail"] + + +def test_profile_search_returns_configuration_error(apps, monkeypatch): + with TestClient(apps["profile_search"]) as client: + monkeypatch.delenv("TWEEPY_BEARER_TOKEN", raising=False) + monkeypatch.delenv("BEARER_TOKEN", raising=False) + + response = client.get( + "/focus", + params={"url": "https://x.com/example"}, + headers={"Origin": "http://localhost:3000"}, + ) + + assert response.status_code == 503 + assert "Missing TWEEPY_BEARER_TOKEN" in response.json()["detail"] + assert response.headers["access-control-allow-origin"] == "http://localhost:3000" diff --git a/whoisit b/whoisit new file mode 160000 index 0000000..e847a17 --- /dev/null +++ b/whoisit @@ -0,0 +1 @@ +Subproject commit e847a1728fc805c20e34bc4370d7cf0e5f4ea5c6 From e19eb9fb580b7025d526447cbe72cbd3e44db29f Mon Sep 17 00:00:00 2001 From: "Akeem T. L. King" Date: Fri, 17 Jul 2026 10:54:37 -0400 Subject: [PATCH 02/10] Document provider integration settings --- CHANGELOG-2026-07-17.md | 2 ++ CHANGELOG.md | 2 ++ README.md | 2 +- whoisit | 2 +- 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG-2026-07-17.md b/CHANGELOG-2026-07-17.md index fbc1c16..9c01844 100644 --- a/CHANGELOG-2026-07-17.md +++ b/CHANGELOG-2026-07-17.md @@ -6,6 +6,7 @@ - CSV, JSON, JSONL, and NDJSON parsing with automatic field suggestions and previews. - Per-row validation, rejected-row reporting, provenance, observation time, confidence, and raw source retention. - Search history and combined live/local results in the Electron application. +- Integration status and credential management for X/Tweepy and Twilio Lookup. - Backend and renderer parser regression tests. ## Changed @@ -14,6 +15,7 @@ - Expanded profile and caller details and made source provenance visible throughout the UI. - Kept API failures scoped to their own source and search workflow. - Added dataset routing, persistence, and readiness checks to the deployed architecture. +- Restricted saved provider credentials to the operating-system user and prevented existing secrets from being returned to the UI. ## Documentation diff --git a/CHANGELOG.md b/CHANGELOG.md index ae07bee..b9b5edc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ All notable changes to this project will be documented in this file. - Added a persistent SQLite dataset service for mapped profile and phone imports. - Added CSV, JSON, JSONL, and NDJSON parsing, previews, automatic field suggestions, per-row validation, provenance, and raw-record retention. - Added combined live and imported-source searching, local search history, raw JSON inspection, and dataset management to the Electron UI. +- Added an Integrations workspace for X/Tweepy, Twilio Lookup, and local dataset connection status with protected credential replacement. - Expanded regression coverage for dataset import, fuzzy profile search, normalized phone search, deletion, health endpoints, and client-side parsers. ### Changed @@ -16,6 +17,7 @@ All notable changes to this project will be documented in this file. - Expanded profile and phone detail views with provider metadata, provenance, freshness, confidence, metrics, and raw source records. - Added the dataset service to Docker Compose, nginx routing, persistent storage, and desktop readiness probes. - Increased the desktop window size and stopped opening DevTools in normal runs. +- Kept stored provider secrets out of the renderer and applied credential changes through atomic, owner-only `.env` updates. ### Documentation - Updated the platform, environment, desktop UI, dataset service, and organization profile documentation for the current architecture. diff --git a/README.md b/README.md index d1bbc9c..4cf2a54 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ npm install npm start ``` -The UI checks service readiness on launch and can attempt to start the Compose stack when Docker is available. Packaged builds are created with `npm run package`. +The UI checks service readiness on launch and can attempt to start the Compose stack when Docker is available. Its Integrations workspace shows the configured and reachable state for X/Tweepy, Twilio Lookup, and local datasets. Users can replace provider credentials there without exposing existing values to the renderer. Packaged builds are created with `npm run package`. ## Environment values diff --git a/whoisit b/whoisit index e847a17..80a4a00 160000 --- a/whoisit +++ b/whoisit @@ -1 +1 @@ -Subproject commit e847a1728fc805c20e34bc4370d7cf0e5f4ea5c6 +Subproject commit 80a4a00566e592043023055876ff0bb9c9f6a428 From ee2d0d6f9ede823646c6dad83688035f1740bfac Mon Sep 17 00:00:00 2001 From: "Akeem T. L. King" Date: Fri, 17 Jul 2026 10:57:49 -0400 Subject: [PATCH 03/10] Update integration settings UI revision --- whoisit | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/whoisit b/whoisit index 80a4a00..1e9dc16 160000 --- a/whoisit +++ b/whoisit @@ -1 +1 @@ -Subproject commit 80a4a00566e592043023055876ff0bb9c9f6a428 +Subproject commit 1e9dc16dd5c75acecacb59e2dabfacccc75f4de3 From bd8903fdbd102cfc598fb4361e88cc9e706b0cce Mon Sep 17 00:00:00 2001 From: "Akeem T. L. King" Date: Fri, 17 Jul 2026 10:58:47 -0400 Subject: [PATCH 04/10] Record integration settings release notes --- CHANGELOG-2026-07-17.md | 1 + CHANGELOG.md | 1 + 2 files changed, 2 insertions(+) diff --git a/CHANGELOG-2026-07-17.md b/CHANGELOG-2026-07-17.md index 9c01844..50ba646 100644 --- a/CHANGELOG-2026-07-17.md +++ b/CHANGELOG-2026-07-17.md @@ -16,6 +16,7 @@ - Kept API failures scoped to their own source and search workflow. - Added dataset routing, persistence, and readiness checks to the deployed architecture. - Restricted saved provider credentials to the operating-system user and prevented existing secrets from being returned to the UI. +- Fixed platform-root discovery in packaged Electron layouts so integration settings use the correct `.env` file. ## Documentation diff --git a/CHANGELOG.md b/CHANGELOG.md index b9b5edc..42e54b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ All notable changes to this project will be documented in this file. - Added the dataset service to Docker Compose, nginx routing, persistent storage, and desktop readiness probes. - Increased the desktop window size and stopped opening DevTools in normal runs. - Kept stored provider secrets out of the renderer and applied credential changes through atomic, owner-only `.env` updates. +- Fixed packaged Electron builds so they locate the parent platform configuration and report the real integration state. ### Documentation - Updated the platform, environment, desktop UI, dataset service, and organization profile documentation for the current architecture. From f59cb13fad93850b62d31034e6e1eae3ea0fd747 Mon Sep 17 00:00:00 2001 From: "Akeem T. L. King" Date: Fri, 17 Jul 2026 11:10:12 -0400 Subject: [PATCH 05/10] Surface X provider failures --- CHANGELOG-2026-07-17.md | 1 + CHANGELOG.md | 1 + profile_search | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG-2026-07-17.md b/CHANGELOG-2026-07-17.md index 50ba646..bcfa159 100644 --- a/CHANGELOG-2026-07-17.md +++ b/CHANGELOG-2026-07-17.md @@ -17,6 +17,7 @@ - Added dataset routing, persistence, and readiness checks to the deployed architecture. - Restricted saved provider credentials to the operating-system user and prevented existing secrets from being returned to the UI. - Fixed platform-root discovery in packaged Electron layouts so integration settings use the correct `.env` file. +- Mapped X API failures to explicit profile-inspection responses, including `402` for depleted credits and `429` for rate limits. ## Documentation diff --git a/CHANGELOG.md b/CHANGELOG.md index 42e54b8..83680d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ All notable changes to this project will be documented in this file. - Increased the desktop window size and stopped opening DevTools in normal runs. - Kept stored provider secrets out of the renderer and applied credential changes through atomic, owner-only `.env` updates. - Fixed packaged Electron builds so they locate the parent platform configuration and report the real integration state. +- Replaced opaque profile-inspection 500 errors with actionable X provider responses for depleted credits, rate limits, invalid access, and missing profiles. ### Documentation - Updated the platform, environment, desktop UI, dataset service, and organization profile documentation for the current architecture. diff --git a/profile_search b/profile_search index d134f8e..0d58dbb 160000 --- a/profile_search +++ b/profile_search @@ -1 +1 @@ -Subproject commit d134f8e8a30bdb643b4f2050d16306cce6fa04fd +Subproject commit 0d58dbb9f96533c509f938605c53642fcb8194c2 From 7fa5f212b8c2469b98dbf9240082cd9adc0aa5bf Mon Sep 17 00:00:00 2001 From: "Akeem T. L. King" Date: Fri, 17 Jul 2026 11:23:12 -0400 Subject: [PATCH 06/10] Resolve Compose services dynamically --- CHANGELOG-2026-07-17.md | 1 + CHANGELOG.md | 1 + nginx.conf | 22 ++++++++++++++-------- tests/test_nginx_routes.py | 31 +++++++++++++++++++++++++++++++ 4 files changed, 47 insertions(+), 8 deletions(-) create mode 100644 tests/test_nginx_routes.py diff --git a/CHANGELOG-2026-07-17.md b/CHANGELOG-2026-07-17.md index bcfa159..ee158ea 100644 --- a/CHANGELOG-2026-07-17.md +++ b/CHANGELOG-2026-07-17.md @@ -18,6 +18,7 @@ - Restricted saved provider credentials to the operating-system user and prevented existing secrets from being returned to the UI. - Fixed platform-root discovery in packaged Electron layouts so integration settings use the correct `.env` file. - Mapped X API failures to explicit profile-inspection responses, including `402` for depleted credits and `429` for rate limits. +- Made nginx resolve Compose service names at request time, preventing cross-service 404s after containers receive new IP addresses. ## Documentation diff --git a/CHANGELOG.md b/CHANGELOG.md index 83680d9..61b3e12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ All notable changes to this project will be documented in this file. - Kept stored provider secrets out of the renderer and applied credential changes through atomic, owner-only `.env` updates. - Fixed packaged Electron builds so they locate the parent platform configuration and report the real integration state. - Replaced opaque profile-inspection 500 errors with actionable X provider responses for depleted credits, rate limits, invalid access, and missing profiles. +- Switched nginx service routing to Docker's runtime DNS resolver so container recreation cannot silently send requests to reassigned service IPs. ### Documentation - Updated the platform, environment, desktop UI, dataset service, and organization profile documentation for the current architecture. diff --git a/nginx.conf b/nginx.conf index a1dd81b..5d4028b 100644 --- a/nginx.conf +++ b/nginx.conf @@ -5,9 +5,15 @@ events { http { server { listen 80; + resolver 127.0.0.11 valid=10s ipv6=off; + + set $profile_checker_upstream profile_checker:8000; + set $profile_search_upstream profile_search:8000; + set $phone_search_upstream phone_search:8000; + set $dataset_service_upstream dataset_service:8000; location = /scan/healthz { - proxy_pass http://profile_checker:8000/healthz; + proxy_pass http://$profile_checker_upstream/healthz; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -15,7 +21,7 @@ http { } location /scan/ { - proxy_pass http://profile_checker:8000; + proxy_pass http://$profile_checker_upstream; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -23,7 +29,7 @@ http { } location = /focus/readyz { - proxy_pass http://profile_search:8000/readyz; + proxy_pass http://$profile_search_upstream/readyz; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -31,7 +37,7 @@ http { } location /focus { - proxy_pass http://profile_search:8000; + proxy_pass http://$profile_search_upstream; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -39,7 +45,7 @@ http { } location = /phone_search/healthz { - proxy_pass http://phone_search:8000/healthz; + proxy_pass http://$phone_search_upstream/healthz; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -47,7 +53,7 @@ http { } location /phone_search { - proxy_pass http://phone_search:8000/lookup/caller-name; + proxy_pass http://$phone_search_upstream/lookup/caller-name; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -55,7 +61,7 @@ http { } location = /datasets/healthz { - proxy_pass http://dataset_service:8000/healthz; + proxy_pass http://$dataset_service_upstream/healthz; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -63,7 +69,7 @@ http { } location /datasets { - proxy_pass http://dataset_service:8000; + proxy_pass http://$dataset_service_upstream; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; diff --git a/tests/test_nginx_routes.py b/tests/test_nginx_routes.py new file mode 100644 index 0000000..24a318c --- /dev/null +++ b/tests/test_nginx_routes.py @@ -0,0 +1,31 @@ +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] + + +def test_nginx_uses_docker_dns_for_runtime_service_resolution(): + config = (ROOT / "nginx.conf").read_text(encoding="utf-8") + + assert "resolver 127.0.0.11" in config + for service in ( + "profile_checker", + "profile_search", + "phone_search", + "dataset_service", + ): + assert f"set ${service}_upstream {service}:8000;" in config + assert f"proxy_pass http://${service}_upstream" in config + + +def test_nginx_keeps_health_and_application_routes_separate(): + config = (ROOT / "nginx.conf").read_text(encoding="utf-8") + + assert "location = /scan/healthz" in config + assert "location /scan/" in config + assert "location = /focus/readyz" in config + assert "location /focus" in config + assert "location = /phone_search/healthz" in config + assert "location /phone_search" in config + assert "location = /datasets/healthz" in config + assert "location /datasets" in config From f8df1451afe5a8994cc08beee1c2998011965830 Mon Sep 17 00:00:00 2001 From: "Akeem T. L. King" Date: Fri, 17 Jul 2026 11:32:17 -0400 Subject: [PATCH 07/10] Preserve username search evidence --- CHANGELOG-2026-07-17.md | 1 + CHANGELOG.md | 1 + README.md | 2 +- profile_checker | 2 +- whoisit | 2 +- 5 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG-2026-07-17.md b/CHANGELOG-2026-07-17.md index ee158ea..066840d 100644 --- a/CHANGELOG-2026-07-17.md +++ b/CHANGELOG-2026-07-17.md @@ -7,6 +7,7 @@ - Per-row validation, rejected-row reporting, provenance, observation time, confidence, and raw source retention. - Search history and combined live/local results in the Electron application. - Integration status and credential management for X/Tweepy and Twilio Lookup. +- Rich username result cards that remain informative when paid X profile inspection is unavailable. - Backend and renderer parser regression tests. ## Changed diff --git a/CHANGELOG.md b/CHANGELOG.md index 61b3e12..37bd315 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ All notable changes to this project will be documented in this file. - Added CSV, JSON, JSONL, and NDJSON parsing, previews, automatic field suggestions, per-row validation, provenance, and raw-record retention. - Added combined live and imported-source searching, local search history, raw JSON inspection, and dataset management to the Electron UI. - Added an Integrations workspace for X/Tweepy, Twilio Lookup, and local dataset connection status with protected credential replacement. +- Expanded username result cards with public validation evidence, direct profile links, dataset provenance, observation time, and confidence. - Expanded regression coverage for dataset import, fuzzy profile search, normalized phone search, deletion, health endpoints, and client-side parsers. ### Changed diff --git a/README.md b/README.md index 4cf2a54..46353aa 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ Do not commit `.env`. Imported datasets may contain sensitive or licensed inform | `GET /datasets/search/phones?phone_number=...` | Search imported phone records | | `DELETE /datasets/{dataset_id}` | Delete a dataset and its records | -The import API accepts JSON rows after the UI parses CSV, JSON, JSONL, or NDJSON. A single import is limited to 10,000 records and the UI limits files to 10 MB. Canonical records preserve their source row, dataset provenance, observation time, and optional confidence score. Phone search uses normalized E.164 values; profile search supports exact or fuzzy matching. +The import API accepts JSON rows after the UI parses CSV, JSON, JSONL, or NDJSON. A single import is limited to 10,000 records and the UI limits files to 10 MB. Canonical records preserve their source row, dataset provenance, observation time, and optional confidence score. Phone search uses normalized E.164 values; profile search supports exact or fuzzy matching. Username result cards also retain public validation evidence, profile links, dataset provenance, observation time, and confidence so searches remain informative when expanded X API inspection has no credits. ## Operations and tests diff --git a/profile_checker b/profile_checker index 4cb13a2..ee78272 160000 --- a/profile_checker +++ b/profile_checker @@ -1 +1 @@ -Subproject commit 4cb13a2622fdaa5b2a30770ea7c2356b0784afdb +Subproject commit ee782728492c4f2f4af506f7af66edba1f400333 diff --git a/whoisit b/whoisit index 1e9dc16..15bc680 160000 --- a/whoisit +++ b/whoisit @@ -1 +1 @@ -Subproject commit 1e9dc16dd5c75acecacb59e2dabfacccc75f4de3 +Subproject commit 15bc68059c092a7530d7d2f35738f0ec5c58bc19 From c867868e3ac7a518769c68f9fe42005472255eda Mon Sep 17 00:00:00 2001 From: "Akeem T. L. King" Date: Fri, 17 Jul 2026 11:35:56 -0400 Subject: [PATCH 08/10] Add fictional dataset samples --- CHANGELOG-2026-07-17.md | 1 + CHANGELOG.md | 1 + README.md | 2 + dataset_service/README.md | 9 ++++ examples/datasets/sample_phone_records.jsonl | 4 ++ examples/datasets/sample_profiles.csv | 5 ++ tests/test_dataset_service.py | 53 ++++++++++++++++++++ 7 files changed, 75 insertions(+) create mode 100644 examples/datasets/sample_phone_records.jsonl create mode 100644 examples/datasets/sample_profiles.csv diff --git a/CHANGELOG-2026-07-17.md b/CHANGELOG-2026-07-17.md index 066840d..f37dacd 100644 --- a/CHANGELOG-2026-07-17.md +++ b/CHANGELOG-2026-07-17.md @@ -8,6 +8,7 @@ - Search history and combined live/local results in the Electron application. - Integration status and credential management for X/Tweepy and Twilio Lookup. - Rich username result cards that remain informative when paid X profile inspection is unavailable. +- Fictional sample profile and phone files for testing the dataset workflow without real personal data. - Backend and renderer parser regression tests. ## Changed diff --git a/CHANGELOG.md b/CHANGELOG.md index 37bd315..a2059b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ All notable changes to this project will be documented in this file. - Added combined live and imported-source searching, local search history, raw JSON inspection, and dataset management to the Electron UI. - Added an Integrations workspace for X/Tweepy, Twilio Lookup, and local dataset connection status with protected credential replacement. - Expanded username result cards with public validation evidence, direct profile links, dataset provenance, observation time, and confidence. +- Added fictional profile CSV and phone JSONL datasets for testing imports, mapping, local search, and rich result rendering. - Expanded regression coverage for dataset import, fuzzy profile search, normalized phone search, deletion, health endpoints, and client-side parsers. ### Changed diff --git a/README.md b/README.md index 46353aa..4ce3721 100644 --- a/README.md +++ b/README.md @@ -90,5 +90,7 @@ Liveness endpoints are available at `/scan/healthz`, `/phone_search/healthz`, an - [Desktop UI guide](whoisit/README.md) - [Dataset service guide](dataset_service/README.md) +- [Sample profile dataset](examples/datasets/sample_profiles.csv) +- [Sample phone dataset](examples/datasets/sample_phone_records.jsonl) - [Changelog](CHANGELOG.md) - [2026-07-17 release notes](CHANGELOG-2026-07-17.md) diff --git a/dataset_service/README.md b/dataset_service/README.md index adb9c27..43b617b 100644 --- a/dataset_service/README.md +++ b/dataset_service/README.md @@ -27,6 +27,15 @@ Compose stores `/data/datasets.db` in the named `dataset_data` volume. Deleting The API is intentionally local and currently has no authentication layer. Do not expose the nginx proxy to an untrusted network without adding access control. +## Sample data + +The platform repository includes two fictional datasets for exercising the complete import and search workflow: + +- [`examples/datasets/sample_profiles.csv`](../examples/datasets/sample_profiles.csv) +- [`examples/datasets/sample_phone_records.jsonl`](../examples/datasets/sample_phone_records.jsonl) + +Import them from the desktop Datasets workspace. The canonical headers are auto-mapped. Search for `demo_ada_1843` or `grace_demo_1906` after importing the profile file, and `+12025550101` through `+12025550104` after importing the phone file. The phone values use the reserved North American `555-01xx` fictional range and do not identify real subscribers. + ## Development ```bash diff --git a/examples/datasets/sample_phone_records.jsonl b/examples/datasets/sample_phone_records.jsonl new file mode 100644 index 0000000..b6a96a4 --- /dev/null +++ b/examples/datasets/sample_phone_records.jsonl @@ -0,0 +1,4 @@ +{"phone_number":"+12025550101","valid":true,"national_format":"(202) 555-0101","country_code":"US","caller_name":"Ada Example Labs","caller_type":"BUSINESS","carrier_name":"Example Wireless","line_type":"mobile","location":"Washington, DC","observed_at":"2026-07-15T14:30:00Z","confidence":0.96,"source":"Fictional QA directory","case_note":"Reserved 555-01xx test number; no real subscriber."} +{"phone_number":"+12025550102","valid":true,"national_format":"(202) 555-0102","country_code":"US","caller_name":"Grace Demo Consulting","caller_type":"BUSINESS","carrier_name":"Example Fiber","line_type":"landline","location":"Washington, DC","observed_at":"2026-07-16T10:15:00Z","confidence":0.91,"source":"Fictional QA directory","case_note":"Reserved 555-01xx test number; no real subscriber."} +{"phone_number":"+12025550103","valid":true,"national_format":"(202) 555-0103","country_code":"US","caller_name":"River Sample","caller_type":"CONSUMER","carrier_name":"Example Voice","line_type":"voip","location":"Washington, DC","observed_at":"2026-07-14T22:10:00Z","confidence":0.79,"source":"Synthetic research export","case_note":"Reserved 555-01xx test number; no real subscriber."} +{"phone_number":"+12025550104","valid":false,"national_format":"(202) 555-0104","country_code":"US","caller_name":null,"caller_type":null,"carrier_name":"Example Carrier","line_type":"unknown","location":"Washington, DC","observed_at":"2026-07-17T08:00:00Z","confidence":0.55,"source":"Synthetic research export","case_note":"Reserved 555-01xx test number used to exercise false and null values."} diff --git a/examples/datasets/sample_profiles.csv b/examples/datasets/sample_profiles.csv new file mode 100644 index 0000000..5f488f2 --- /dev/null +++ b/examples/datasets/sample_profiles.csv @@ -0,0 +1,5 @@ +username,platform,profile_url,display_name,bio,location,website,verified,protected,created_at,profile_image_url,profile_banner_url,followers_count,following_count,post_count,listed_count,like_count,media_count,observed_at,confidence,source +demo_ada_1843,DemoNet,https://profiles.example/demo_ada_1843,Ada Example,"Builds analytical engines, documentation, and excellent tea.",London,https://ada.example,yes,no,2021-03-14T09:26:00Z,https://images.example/ada-avatar.png,https://images.example/ada-banner.png,"12,450",312,1840,87,9310,244,2026-07-15T14:30:00Z,0.97,Fictional QA directory +grace_demo_1906,CodeCommons,https://profiles.example/grace_demo_1906,Grace Demo,"Compiler enthusiast and systems educator.",Arlington VA,https://grace.example,true,false,2020-12-09T18:05:00Z,https://images.example/grace-avatar.png,,8830,541,2265,42,7100,118,2026-07-16T10:15:00Z,0.94,Fictional QA directory +river_sample_42,PhotoSquare,https://profiles.example/river_sample_42,River Sample,"Landscape photographer. All locations and identities in this row are fictional.","Portland, OR",https://river.example,false,true,2023-06-01T07:45:00Z,https://images.example/river-avatar.png,https://images.example/river-banner.png,642,389,211,3,1870,196,2026-07-14T22:10:00Z,0.81,Synthetic research export +orbit_test_lab,ForumHub,https://profiles.example/orbit_test_lab,Orbit Test Lab,"A synthetic organization record for testing imported profile search.",Remote,https://orbit.example,false,false,2024-01-20T12:00:00Z,,,154,28,76,1,220,12,2026-07-17T08:00:00Z,0.76,Synthetic research export diff --git a/tests/test_dataset_service.py b/tests/test_dataset_service.py index 1a5ab92..a118719 100644 --- a/tests/test_dataset_service.py +++ b/tests/test_dataset_service.py @@ -1,3 +1,5 @@ +import csv +import json import sys from pathlib import Path @@ -143,3 +145,54 @@ def test_phone_import_search_records_and_delete(tmp_path): ).json()["records"] == [] ) + + +def test_documented_sample_files_import_and_search(tmp_path): + client = make_client(tmp_path) + samples = ROOT / "examples" / "datasets" + + with (samples / "sample_profiles.csv").open(newline="", encoding="utf-8") as file: + profile_rows = list(csv.DictReader(file)) + profile_import = client.post( + "/datasets/import", + json={ + "name": "Fictional profile samples", + "record_type": "profile", + "filename": "sample_profiles.csv", + "rows": profile_rows, + }, + ) + assert profile_import.status_code == 201 + assert profile_import.json()["imported"] == 4 + assert profile_import.json()["rejected"] == 0 + + profiles = client.get( + "/datasets/search/profiles", params={"query": "demo_ada_1843"} + ).json()["records"] + assert len(profiles) == 1 + assert profiles[0]["name"] == "Ada Example" + assert profiles[0]["metrics"]["followers_count"] == 12450 + + with (samples / "sample_phone_records.jsonl").open(encoding="utf-8") as file: + phone_rows = [json.loads(line) for line in file if line.strip()] + phone_import = client.post( + "/datasets/import", + json={ + "name": "Fictional phone samples", + "record_type": "phone", + "filename": "sample_phone_records.jsonl", + "rows": phone_rows, + }, + ) + assert phone_import.status_code == 201 + assert phone_import.json()["imported"] == 4 + assert phone_import.json()["rejected"] == 0 + + phones = client.get( + "/datasets/search/phones", params={"phone_number": "+12025550104"} + ).json()["records"] + assert len(phones) == 1 + assert phones[0]["valid"] is False + assert phones[0]["raw"]["case_note"].endswith( + "used to exercise false and null values." + ) From 3af5fec26611d079cd09d051fc3944dede7c484d Mon Sep 17 00:00:00 2001 From: "Akeem T. L. King" Date: Fri, 17 Jul 2026 11:43:31 -0400 Subject: [PATCH 09/10] Present one filtered search workspace --- CHANGELOG-2026-07-17.md | 3 ++- CHANGELOG.md | 3 ++- README.md | 2 +- whoisit | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG-2026-07-17.md b/CHANGELOG-2026-07-17.md index f37dacd..99fd165 100644 --- a/CHANGELOG-2026-07-17.md +++ b/CHANGELOG-2026-07-17.md @@ -13,7 +13,8 @@ ## Changed -- Reorganized the desktop application into Username, Phone, Datasets, and History workspaces. +- Unified username and phone lookup into a Search workspace with Auto, Profile, and Phone modes plus source filters that avoid unnecessary provider calls. +- Reorganized the desktop application into Search, Datasets, Integrations, and History workspaces. - Expanded profile and caller details and made source provenance visible throughout the UI. - Kept API failures scoped to their own source and search workflow. - Added dataset routing, persistence, and readiness checks to the deployed architecture. diff --git a/CHANGELOG.md b/CHANGELOG.md index a2059b8..e81d623 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,8 @@ All notable changes to this project will be documented in this file. - Expanded regression coverage for dataset import, fuzzy profile search, normalized phone search, deletion, health endpoints, and client-side parsers. ### Changed -- Reworked the Electron interface into cohesive Username, Phone, Datasets, and History workspaces with master/detail results. +- Unified username and phone lookup into one Search workspace with conservative auto-detection, explicit Profile and Phone modes, and source filters that control which APIs and datasets are queried. +- Reworked the Electron interface into cohesive Search, Datasets, Integrations, and History workspaces with type-specific master/detail results. - Isolated live-provider failures so other sources can still return results and errors remain in the correct search workspace. - Expanded profile and phone detail views with provider metadata, provenance, freshness, confidence, metrics, and raw source records. - Added the dataset service to Docker Compose, nginx routing, persistent storage, and desktop readiness probes. diff --git a/README.md b/README.md index 4ce3721..6e4178c 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ nginx :80 `-- /datasets* -> dataset_service (SQLite imports and local search) ``` -Docker Compose runs the four FastAPI services, nginx, and a persistent `dataset_data` volume. The Electron UI queries live and local sources independently and combines successful results, so one unavailable provider does not prevent other sources from being searched. +Docker Compose runs the four FastAPI services, nginx, and a persistent `dataset_data` volume. The Electron UI provides one search workspace with Auto, Profile, and Phone modes plus All, Live APIs, and Imported datasets source filters. It queries enabled sources independently and combines successful results, so one unavailable provider does not prevent other sources from being searched. ## Requirements diff --git a/whoisit b/whoisit index 15bc680..acf6867 160000 --- a/whoisit +++ b/whoisit @@ -1 +1 @@ -Subproject commit 15bc68059c092a7530d7d2f35738f0ec5c58bc19 +Subproject commit acf6867ebf8bad79d7a41d8f8ad5fae770f80b3b From fb168671cd6100724d894db9efeb4f1287fae254 Mon Sep 17 00:00:00 2001 From: "Akeem T. L. King" Date: Fri, 17 Jul 2026 11:52:14 -0400 Subject: [PATCH 10/10] Unify imported identity records --- CHANGELOG-2026-07-17.md | 3 + CHANGELOG.md | 2 + README.md | 9 +- dataset_service/README.md | 20 +- dataset_service/models.py | 6 +- dataset_service/repository.py | 429 ++++++++++++++++++++------ dataset_service/server.py | 50 +++ examples/datasets/sample_entities.csv | 4 + tests/test_dataset_service.py | 100 +++++- whoisit | 2 +- 10 files changed, 515 insertions(+), 110 deletions(-) create mode 100644 examples/datasets/sample_entities.csv diff --git a/CHANGELOG-2026-07-17.md b/CHANGELOG-2026-07-17.md index 99fd165..f4cffb2 100644 --- a/CHANGELOG-2026-07-17.md +++ b/CHANGELOG-2026-07-17.md @@ -2,6 +2,7 @@ ## Added +- Sparse entities that can contain a profile identifier, a phone identifier, or both, with associations visible from either search. - Persistent SQLite-backed profile and phone datasets with mapped imports and local search. - CSV, JSON, JSONL, and NDJSON parsing with automatic field suggestions and previews. - Per-row validation, rejected-row reporting, provenance, observation time, confidence, and raw source retention. @@ -13,6 +14,8 @@ ## Changed +- Replaced the desktop record-type selector with one entity ingestion and field-mapping pipeline. +- Added an in-place SQLite schema migration while retaining legacy profile and phone import compatibility. - Unified username and phone lookup into a Search workspace with Auto, Profile, and Phone modes plus source filters that avoid unnecessary provider calls. - Reorganized the desktop application into Search, Datasets, Integrations, and History workspaces. - Expanded profile and caller details and made source provenance visible throughout the UI. diff --git a/CHANGELOG.md b/CHANGELOG.md index e81d623..c35ee3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ All notable changes to this project will be documented in this file. ## [Unreleased] ### Added +- Added a sparse entity model that associates optional profile and phone identifiers while preserving their type-specific metadata. - Added a persistent SQLite dataset service for mapped profile and phone imports. - Added CSV, JSON, JSONL, and NDJSON parsing, previews, automatic field suggestions, per-row validation, provenance, and raw-record retention. - Added combined live and imported-source searching, local search history, raw JSON inspection, and dataset management to the Electron UI. @@ -14,6 +15,7 @@ All notable changes to this project will be documented in this file. - Expanded regression coverage for dataset import, fuzzy profile search, normalized phone search, deletion, health endpoints, and client-side parsers. ### Changed +- Replaced separate profile and phone dataset ingestion with one entity mapping pipeline; legacy API imports remain compatible and existing SQLite databases migrate in place. - Unified username and phone lookup into one Search workspace with conservative auto-detection, explicit Profile and Phone modes, and source filters that control which APIs and datasets are queried. - Reworked the Electron interface into cohesive Search, Datasets, Integrations, and History workspaces with type-specific master/detail results. - Isolated live-provider failures so other sources can still return results and errors remain in the correct search workspace. diff --git a/README.md b/README.md index 6e4178c..037715e 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ nginx :80 |-- /scan/* -> profile_checker (public username availability) |-- /focus* -> profile_search (X profile metadata via Tweepy) |-- /phone_search* -> phone_search (caller-name data via Twilio) - `-- /datasets* -> dataset_service (SQLite imports and local search) + `-- /datasets* -> dataset_service (SQLite entity imports and local search) ``` Docker Compose runs the four FastAPI services, nginx, and a persistent `dataset_data` volume. The Electron UI provides one search workspace with Auto, Profile, and Phone modes plus All, Live APIs, and Imported datasets source filters. It queries enabled sources independently and combines successful results, so one unavailable provider does not prevent other sources from being searched. @@ -66,13 +66,13 @@ Do not commit `.env`. Imported datasets may contain sensitive or licensed inform | `GET /focus?url=https://x.com/{username}` | Retrieve expanded X profile metadata | | `GET /phone_search?phone_number=+18135551212` | Retrieve live caller-name metadata | | `GET /datasets` | List imported datasets | -| `GET /datasets/schema/{profile\|phone}` | Get canonical import fields and a sample | -| `POST /datasets/import` | Import mapped profile or phone rows | +| `GET /datasets/schema/entity` | Get the unified sparse entity schema and sample | +| `POST /datasets/import` | Import mapped entities containing profile identifiers, phone identifiers, or both | | `GET /datasets/search/profiles?query=...&fuzzy=true` | Search imported profiles | | `GET /datasets/search/phones?phone_number=...` | Search imported phone records | | `DELETE /datasets/{dataset_id}` | Delete a dataset and its records | -The import API accepts JSON rows after the UI parses CSV, JSON, JSONL, or NDJSON. A single import is limited to 10,000 records and the UI limits files to 10 MB. Canonical records preserve their source row, dataset provenance, observation time, and optional confidence score. Phone search uses normalized E.164 values; profile search supports exact or fuzzy matching. Username result cards also retain public validation evidence, profile links, dataset provenance, observation time, and confidence so searches remain informative when expanded X API inspection has no credits. +The import API accepts sparse entity rows after the UI parses CSV, JSON, JSONL, or NDJSON. Each row needs at least one username, profile URL, or E.164 phone number; profile and phone fields can coexist and remain associated in search results. A single import is limited to 10,000 records and the UI limits files to 10 MB. Canonical records preserve their source row, dataset provenance, observation time, and optional confidence score. Legacy typed imports remain API-compatible and existing databases migrate in place. ## Operations and tests @@ -90,6 +90,7 @@ Liveness endpoints are available at `/scan/healthz`, `/phone_search/healthz`, an - [Desktop UI guide](whoisit/README.md) - [Dataset service guide](dataset_service/README.md) +- [Sample combined entity dataset](examples/datasets/sample_entities.csv) - [Sample profile dataset](examples/datasets/sample_profiles.csv) - [Sample phone dataset](examples/datasets/sample_phone_records.jsonl) - [Changelog](CHANGELOG.md) diff --git a/dataset_service/README.md b/dataset_service/README.md index 43b617b..5104616 100644 --- a/dataset_service/README.md +++ b/dataset_service/README.md @@ -1,19 +1,20 @@ # Dataset service -The dataset service provides normalized, persistent local sources for the platform's profile and phone searches. It uses SQLite and has no external provider dependency. +The dataset service provides normalized, persistent identity sources for the platform's profile and phone searches. It uses SQLite and has no external provider dependency. ## Supported input -The Electron UI parses `.csv`, `.json`, `.jsonl`, and `.ndjson` files, then maps source columns to the canonical fields returned by: +The Electron UI parses `.csv`, `.json`, `.jsonl`, and `.ndjson` files, then maps every import through the entity schema returned by: ```http -GET /datasets/schema/profile -GET /datasets/schema/phone +GET /datasets/schema/entity ``` -Profile records require either a username or profile URL. Phone records require an E.164 number such as `+18135551212`. Invalid booleans, negative metrics, confidence outside `0..1`, and malformed identifiers are rejected per row without discarding valid rows. +Each sparse entity row requires at least one searchable identifier: a username, profile URL, or E.164 phone number such as `+12025550101`. All other fields are optional. When both profile and phone fields exist, the service stores typed identifiers under one entity so a match for either identifier exposes the association. Invalid booleans, negative metrics, confidence outside `0..1`, and malformed identifiers are rejected per row without discarding valid rows. -Each accepted record includes: +The legacy `profile` and `phone` schemas and import values remain accepted for API compatibility, but the desktop application uses only the entity pipeline. Existing SQLite databases are migrated in place when the service starts. + +Each accepted entity includes: - its dataset and source; - an observation timestamp; @@ -23,18 +24,19 @@ Each accepted record includes: ## Storage and limits -Compose stores `/data/datasets.db` in the named `dataset_data` volume. Deleting a dataset cascades to its records. Each import accepts 1–10,000 JSON object rows; the desktop UI additionally enforces a 10 MB file limit. +Compose stores `/data/datasets.db` in the named `dataset_data` volume. Deleting a dataset cascades to its entities and typed identifiers. Each import accepts 1–10,000 JSON object rows; the desktop UI additionally enforces a 10 MB file limit. The API is intentionally local and currently has no authentication layer. Do not expose the nginx proxy to an untrusted network without adding access control. ## Sample data -The platform repository includes two fictional datasets for exercising the complete import and search workflow: +The platform repository includes a combined fictional entity dataset plus separate fixtures for exercising compatibility and source-specific fields: +- [`examples/datasets/sample_entities.csv`](../examples/datasets/sample_entities.csv) - [`examples/datasets/sample_profiles.csv`](../examples/datasets/sample_profiles.csv) - [`examples/datasets/sample_phone_records.jsonl`](../examples/datasets/sample_phone_records.jsonl) -Import them from the desktop Datasets workspace. The canonical headers are auto-mapped. Search for `demo_ada_1843` or `grace_demo_1906` after importing the profile file, and `+12025550101` through `+12025550104` after importing the phone file. The phone values use the reserved North American `555-01xx` fictional range and do not identify real subscribers. +Import any of them from the same desktop Datasets workflow; canonical headers are auto-mapped. The combined file demonstrates a username and phone on one entity, a profile-only entity, and a phone-only entity. Search for `demo_ada_1843` or `+12025550101` to see the association in both directions. The phone values use the reserved North American `555-01xx` fictional range and do not identify real subscribers. ## Development diff --git a/dataset_service/models.py b/dataset_service/models.py index d07f09c..ee385c5 100644 --- a/dataset_service/models.py +++ b/dataset_service/models.py @@ -5,13 +5,14 @@ class RecordType(str, Enum): + entity = "entity" profile = "profile" phone = "phone" class DatasetImportRequest(BaseModel): name: str = Field(min_length=1, max_length=120) - record_type: RecordType + record_type: RecordType = RecordType.entity filename: str | None = Field(default=None, max_length=255) mapping: dict[str, str] = Field(default_factory=dict) rows: list[dict[str, Any]] = Field(min_items=1, max_items=10_000) @@ -27,6 +28,8 @@ class DatasetImportResult(BaseModel): imported: int rejected: int rejected_rows: list[RejectedRow] + profile_identifiers: int = 0 + phone_identifiers: int = 0 class DatasetSummary(BaseModel): @@ -50,3 +53,4 @@ class DatasetSchema(BaseModel): record_type: RecordType fields: list[FieldDefinition] sample: dict[str, Any] + identifier_fields: list[str] = Field(default_factory=list) diff --git a/dataset_service/repository.py b/dataset_service/repository.py index eedd01a..a3f0b33 100644 --- a/dataset_service/repository.py +++ b/dataset_service/repository.py @@ -49,6 +49,88 @@ "source", ) +ENTITY_FIELDS = tuple(dict.fromkeys(PROFILE_FIELDS + PHONE_FIELDS)) + +SCHEMA_SQL = """ +CREATE TABLE IF NOT EXISTS datasets ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + record_type TEXT NOT NULL CHECK(record_type IN ('entity', 'profile', 'phone')), + filename TEXT, + imported_at TEXT NOT NULL, + row_count INTEGER NOT NULL, + rejected_count INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS entities ( + id TEXT PRIMARY KEY, + dataset_id TEXT NOT NULL REFERENCES datasets(id) ON DELETE CASCADE, + display_name TEXT, + bio TEXT, + location TEXT, + website TEXT, + observed_at TEXT, + confidence REAL, + source TEXT, + raw_json TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS profile_records ( + id TEXT PRIMARY KEY, + dataset_id TEXT NOT NULL REFERENCES datasets(id) ON DELETE CASCADE, + entity_id TEXT REFERENCES entities(id) ON DELETE CASCADE, + platform TEXT, + username TEXT, + username_normalized TEXT, + profile_url TEXT, + display_name TEXT, + bio TEXT, + location TEXT, + website TEXT, + verified INTEGER, + protected INTEGER, + created_at TEXT, + profile_image_url TEXT, + profile_banner_url TEXT, + followers_count INTEGER, + following_count INTEGER, + post_count INTEGER, + listed_count INTEGER, + like_count INTEGER, + media_count INTEGER, + observed_at TEXT, + confidence REAL, + source TEXT, + raw_json TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS phone_records ( + id TEXT PRIMARY KEY, + dataset_id TEXT NOT NULL REFERENCES datasets(id) ON DELETE CASCADE, + entity_id TEXT REFERENCES entities(id) ON DELETE CASCADE, + phone_number TEXT NOT NULL, + valid INTEGER, + national_format TEXT, + country_code TEXT, + caller_name TEXT, + caller_type TEXT, + carrier_name TEXT, + line_type TEXT, + location TEXT, + observed_at TEXT, + confidence REAL, + source TEXT, + raw_json TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_entity_dataset ON entities(dataset_id); +CREATE INDEX IF NOT EXISTS idx_profile_username ON profile_records(username_normalized); +CREATE INDEX IF NOT EXISTS idx_profile_entity ON profile_records(entity_id); +CREATE INDEX IF NOT EXISTS idx_profile_display_name ON profile_records(display_name); +CREATE INDEX IF NOT EXISTS idx_phone_number ON phone_records(phone_number); +CREATE INDEX IF NOT EXISTS idx_phone_entity ON phone_records(entity_id); +""" + def utc_now() -> str: return datetime.now(timezone.utc).isoformat() @@ -130,72 +212,70 @@ def connect(self) -> sqlite3.Connection: def initialize(self) -> None: with self.connect() as connection: - connection.executescript( - """ - CREATE TABLE IF NOT EXISTS datasets ( - id TEXT PRIMARY KEY, - name TEXT NOT NULL, - record_type TEXT NOT NULL CHECK(record_type IN ('profile', 'phone')), - filename TEXT, - imported_at TEXT NOT NULL, - row_count INTEGER NOT NULL, - rejected_count INTEGER NOT NULL - ); - - CREATE TABLE IF NOT EXISTS profile_records ( - id TEXT PRIMARY KEY, - dataset_id TEXT NOT NULL REFERENCES datasets(id) ON DELETE CASCADE, - platform TEXT, - username TEXT, - username_normalized TEXT, - profile_url TEXT, - display_name TEXT, - bio TEXT, - location TEXT, - website TEXT, - verified INTEGER, - protected INTEGER, - created_at TEXT, - profile_image_url TEXT, - profile_banner_url TEXT, - followers_count INTEGER, - following_count INTEGER, - post_count INTEGER, - listed_count INTEGER, - like_count INTEGER, - media_count INTEGER, - observed_at TEXT, - confidence REAL, - source TEXT, - raw_json TEXT NOT NULL - ); - - CREATE TABLE IF NOT EXISTS phone_records ( - id TEXT PRIMARY KEY, - dataset_id TEXT NOT NULL REFERENCES datasets(id) ON DELETE CASCADE, - phone_number TEXT NOT NULL, - valid INTEGER, - national_format TEXT, - country_code TEXT, - caller_name TEXT, - caller_type TEXT, - carrier_name TEXT, - line_type TEXT, - location TEXT, - observed_at TEXT, - confidence REAL, - source TEXT, - raw_json TEXT NOT NULL - ); - - CREATE INDEX IF NOT EXISTS idx_profile_username - ON profile_records(username_normalized); - CREATE INDEX IF NOT EXISTS idx_profile_display_name - ON profile_records(display_name); - CREATE INDEX IF NOT EXISTS idx_phone_number - ON phone_records(phone_number); - """ + datasets_sql = connection.execute( + "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'datasets'" + ).fetchone() + profile_columns = { + row["name"] + for row in connection.execute("PRAGMA table_info(profile_records)") + } + if datasets_sql and ( + "'entity'" not in datasets_sql["sql"] or "entity_id" not in profile_columns + ): + self.migrate_legacy_schema(connection) + else: + connection.executescript(SCHEMA_SQL) + + @staticmethod + def migrate_legacy_schema(connection: sqlite3.Connection) -> None: + connection.execute("PRAGMA foreign_keys = OFF") + connection.executescript( + """ + ALTER TABLE datasets RENAME TO legacy_datasets; + ALTER TABLE profile_records RENAME TO legacy_profile_records; + ALTER TABLE phone_records RENAME TO legacy_phone_records; + """ + ) + connection.executescript(SCHEMA_SQL) + connection.executescript( + """ + INSERT INTO datasets + (id, name, record_type, filename, imported_at, row_count, rejected_count) + SELECT id, name, record_type, filename, imported_at, row_count, rejected_count + FROM legacy_datasets; + + INSERT INTO profile_records ( + id, dataset_id, entity_id, platform, username, username_normalized, + profile_url, display_name, bio, location, website, verified, + protected, created_at, profile_image_url, profile_banner_url, + followers_count, following_count, post_count, listed_count, + like_count, media_count, observed_at, confidence, source, raw_json + ) + SELECT + id, dataset_id, NULL, platform, username, username_normalized, + profile_url, display_name, bio, location, website, verified, + protected, created_at, profile_image_url, profile_banner_url, + followers_count, following_count, post_count, listed_count, + like_count, media_count, observed_at, confidence, source, raw_json + FROM legacy_profile_records; + + INSERT INTO phone_records ( + id, dataset_id, entity_id, phone_number, valid, national_format, + country_code, caller_name, caller_type, carrier_name, + line_type, location, observed_at, confidence, source, raw_json ) + SELECT + id, dataset_id, NULL, phone_number, valid, national_format, + country_code, caller_name, caller_type, carrier_name, + line_type, location, observed_at, confidence, source, raw_json + FROM legacy_phone_records; + + DROP TABLE legacy_profile_records; + DROP TABLE legacy_phone_records; + DROP TABLE legacy_datasets; + """ + ) + connection.executescript(SCHEMA_SQL) @staticmethod def mapped_row( @@ -208,19 +288,32 @@ def mapped_row( def import_dataset(self, request: DatasetImportRequest) -> DatasetImportResult: dataset_id = str(uuid4()) - accepted: list[dict[str, Any]] = [] + entities: list[dict[str, Any]] = [] + profiles: list[dict[str, Any]] = [] + phones: list[dict[str, Any]] = [] rejected: list[RejectedRow] = [] + imported = 0 for row_number, raw_row in enumerate(request.rows, start=1): try: - if request.record_type.value == "profile": - accepted.append( + if request.record_type.value == "entity": + entity, profile, phone = self.prepare_entity( + raw_row, request.mapping, request.name + ) + entities.append(entity) + if profile: + profiles.append(profile) + if phone: + phones.append(phone) + elif request.record_type.value == "profile": + profiles.append( self.prepare_profile(raw_row, request.mapping, request.name) ) else: - accepted.append( + phones.append( self.prepare_phone(raw_row, request.mapping, request.name) ) + imported += 1 except (TypeError, ValueError) as exc: rejected.append(RejectedRow(row_number=row_number, reason=str(exc))) @@ -238,53 +331,116 @@ def import_dataset(self, request: DatasetImportRequest) -> DatasetImportResult: request.record_type.value, request.filename, imported_at, - len(accepted), + imported, len(rejected), ), ) - if request.record_type.value == "profile": + if entities: + connection.executemany( + """ + INSERT INTO entities ( + id, dataset_id, display_name, bio, location, website, + observed_at, confidence, source, raw_json + ) VALUES ( + :id, :dataset_id, :display_name, :bio, :location, :website, + :observed_at, :confidence, :source, :raw_json + ) + """, + [dict(record, dataset_id=dataset_id) for record in entities], + ) + if profiles: connection.executemany( """ INSERT INTO profile_records ( - id, dataset_id, platform, username, username_normalized, + id, dataset_id, entity_id, platform, username, username_normalized, profile_url, display_name, bio, location, website, verified, protected, created_at, profile_image_url, profile_banner_url, followers_count, following_count, post_count, listed_count, like_count, media_count, observed_at, confidence, source, raw_json ) VALUES ( - :id, :dataset_id, :platform, :username, :username_normalized, + :id, :dataset_id, :entity_id, :platform, :username, :username_normalized, :profile_url, :display_name, :bio, :location, :website, :verified, :protected, :created_at, :profile_image_url, :profile_banner_url, :followers_count, :following_count, :post_count, :listed_count, :like_count, :media_count, :observed_at, :confidence, :source, :raw_json ) """, - [dict(record, dataset_id=dataset_id) for record in accepted], + [ + dict(record, dataset_id=dataset_id, entity_id=record.get("entity_id")) + for record in profiles + ], ) - else: + if phones: connection.executemany( """ INSERT INTO phone_records ( - id, dataset_id, phone_number, valid, national_format, + id, dataset_id, entity_id, phone_number, valid, national_format, country_code, caller_name, caller_type, carrier_name, line_type, location, observed_at, confidence, source, raw_json ) VALUES ( - :id, :dataset_id, :phone_number, :valid, :national_format, + :id, :dataset_id, :entity_id, :phone_number, :valid, :national_format, :country_code, :caller_name, :caller_type, :carrier_name, :line_type, :location, :observed_at, :confidence, :source, :raw_json ) """, - [dict(record, dataset_id=dataset_id) for record in accepted], + [ + dict(record, dataset_id=dataset_id, entity_id=record.get("entity_id")) + for record in phones + ], ) return DatasetImportResult( dataset_id=dataset_id, - imported=len(accepted), + imported=imported, rejected=len(rejected), rejected_rows=rejected[:100], + profile_identifiers=len(profiles), + phone_identifiers=len(phones), ) + def prepare_entity( + self, raw_row: dict[str, Any], mapping: dict[str, str], dataset_name: str + ) -> tuple[dict[str, Any], dict[str, Any] | None, dict[str, Any] | None]: + values = self.mapped_row(raw_row, mapping, ENTITY_FIELDS) + has_profile = bool( + normalize_username(values["username"]) or username_from_url(values["profile_url"]) + ) + has_phone = bool(clean_text(values["phone_number"])) + if not has_profile and not has_phone: + raise ValueError("username, profile_url, or phone_number is required") + + entity_id = str(uuid4()) + observed_at = clean_text(values["observed_at"]) or utc_now() + confidence = parse_confidence(values["confidence"]) + source = clean_text(values["source"]) or dataset_name + display_name = clean_text(values["display_name"]) or clean_text( + values["caller_name"] + ) + entity = { + "id": entity_id, + "display_name": display_name, + "bio": clean_text(values["bio"]), + "location": clean_text(values["location"]), + "website": clean_text(values["website"]), + "observed_at": observed_at, + "confidence": confidence, + "source": source, + "raw_json": json.dumps(raw_row, default=str), + } + + profile = ( + self.prepare_profile(raw_row, mapping, dataset_name) if has_profile else None + ) + phone = self.prepare_phone(raw_row, mapping, dataset_name) if has_phone else None + if profile: + profile["entity_id"] = entity_id + profile["display_name"] = profile["display_name"] or display_name + if phone: + phone["entity_id"] = entity_id + phone["caller_name"] = phone["caller_name"] or display_name + return entity, profile, phone + def prepare_profile( self, raw_row: dict[str, Any], mapping: dict[str, str], dataset_name: str ) -> dict[str, Any]: @@ -391,7 +547,13 @@ def search_profiles(self, query: str, fuzzy: bool = False) -> list[dict[str, Any """, (normalized,), ).fetchall() - return [self.profile_response(row) for row in rows] + records = [] + for row in rows: + record = self.profile_response(row) + if row["entity_id"]: + record["entity"] = self.entity_summary(connection, row["entity_id"]) + records.append(record) + return records def search_phones(self, phone_number: str) -> list[dict[str, Any]]: normalized = normalize_phone(phone_number) @@ -407,7 +569,13 @@ def search_phones(self, phone_number: str) -> list[dict[str, Any]]: """, (normalized,), ).fetchall() - return [self.phone_response(row) for row in rows] + records = [] + for row in rows: + record = self.phone_response(row) + if row["entity_id"]: + record["entity"] = self.entity_summary(connection, row["entity_id"]) + records.append(record) + return records def dataset_records( self, dataset_id: str, limit: int, offset: int @@ -418,21 +586,98 @@ def dataset_records( ).fetchone() if not dataset: return None - table = ( - "profile_records" - if dataset["record_type"] == "profile" - else "phone_records" - ) + if dataset["record_type"] == "entity": + rows = connection.execute( + "SELECT * FROM entities WHERE dataset_id = ? LIMIT ? OFFSET ?", + (dataset_id, limit, offset), + ).fetchall() + return "entity", [ + self.entity_response(connection, row) for row in rows + ] + table = "profile_records" if dataset["record_type"] == "profile" else "phone_records" rows = connection.execute( f"SELECT * FROM {table} WHERE dataset_id = ? LIMIT ? OFFSET ?", (dataset_id, limit, offset), ).fetchall() - return dataset["record_type"], [ - self.profile_response(row) - if dataset["record_type"] == "profile" - else self.phone_response(row) - for row in rows - ] + records = [] + for row in rows: + record = ( + self.profile_response(row) + if dataset["record_type"] == "profile" + else self.phone_response(row) + ) + if row["entity_id"]: + record["entity"] = self.entity_summary(connection, row["entity_id"]) + records.append(record) + return dataset["record_type"], records + + def entity_summary( + self, connection: sqlite3.Connection, entity_id: str + ) -> dict[str, Any] | None: + entity = connection.execute( + "SELECT * FROM entities WHERE id = ?", (entity_id,) + ).fetchone() + if not entity: + return None + profiles = connection.execute( + """ + SELECT id, platform, username, profile_url + FROM profile_records + WHERE entity_id = ? + ORDER BY platform, username + """, + (entity_id,), + ).fetchall() + phones = connection.execute( + """ + SELECT id, phone_number, caller_name, carrier_name, line_type + FROM phone_records + WHERE entity_id = ? + ORDER BY phone_number + """, + (entity_id,), + ).fetchall() + return { + "id": entity["id"], + "name": entity["display_name"], + "bio": entity["bio"], + "location": entity["location"], + "website": entity["website"], + "profiles": [ + { + "id": row["id"], + "platform": row["platform"], + "username": row["username"], + "profile_uri": row["profile_url"], + } + for row in profiles + ], + "phone_numbers": [ + { + "id": row["id"], + "phone_number": row["phone_number"], + "caller_name": row["caller_name"], + "carrier_name": row["carrier_name"], + "line_type": row["line_type"], + } + for row in phones + ], + } + + def entity_response( + self, connection: sqlite3.Connection, row: sqlite3.Row + ) -> dict[str, Any]: + summary = self.entity_summary(connection, row["id"]) or {} + return { + **summary, + "record_type": "entity", + "source_type": "dataset", + "dataset_id": row["dataset_id"], + "source": row["source"], + "observed_at": row["observed_at"], + "confidence": row["confidence"], + "raw": json.loads(row["raw_json"]), + } @staticmethod def profile_response(row: sqlite3.Row) -> dict[str, Any]: diff --git a/dataset_service/server.py b/dataset_service/server.py index 2d27b33..2307f1e 100644 --- a/dataset_service/server.py +++ b/dataset_service/server.py @@ -83,6 +83,54 @@ }, ) +ENTITY_SCHEMA = DatasetSchema( + record_type=RecordType.entity, + identifier_fields=["username", "profile_url", "phone_number"], + fields=[ + FieldDefinition(key="username", label="Username", description="Account handle, with or without @."), + FieldDefinition(key="platform", label="Platform", description="Network or site name, for example X or GitHub."), + FieldDefinition(key="profile_url", label="Profile URL", description="Canonical public profile URL; can identify a profile when username is absent."), + FieldDefinition(key="phone_number", label="Phone number", description="Number in E.164 format, for example +12025550101."), + FieldDefinition(key="display_name", label="Display name", description="Person, organization, or account name."), + FieldDefinition(key="bio", label="Bio", description="Source-provided identity or profile description."), + FieldDefinition(key="location", label="Location", description="Public or source-provided location."), + FieldDefinition(key="website", label="Website", description="Website associated with the entity or profile."), + FieldDefinition(key="verified", label="Verified", description="Boolean profile verification status."), + FieldDefinition(key="protected", label="Protected", description="Boolean protected/private profile status."), + FieldDefinition(key="created_at", label="Profile created at", description="Account creation timestamp."), + FieldDefinition(key="profile_image_url", label="Profile image URL", description="Avatar URL."), + FieldDefinition(key="profile_banner_url", label="Profile banner URL", description="Header/banner image URL."), + FieldDefinition(key="followers_count", label="Followers", description="Follower count."), + FieldDefinition(key="following_count", label="Following", description="Following count."), + FieldDefinition(key="post_count", label="Posts", description="Post or status count."), + FieldDefinition(key="listed_count", label="Listed", description="List membership count."), + FieldDefinition(key="like_count", label="Likes", description="Like or favorite count."), + FieldDefinition(key="media_count", label="Media", description="Media post count."), + FieldDefinition(key="valid", label="Phone valid", description="Boolean phone validity status."), + FieldDefinition(key="national_format", label="National phone format", description="Locally formatted phone number."), + FieldDefinition(key="country_code", label="Country code", description="ISO country code."), + FieldDefinition(key="caller_name", label="Caller name", description="Person or business caller name; used as the entity name when display name is absent."), + FieldDefinition(key="caller_type", label="Caller type", description="Caller-name classification."), + FieldDefinition(key="carrier_name", label="Carrier", description="Network carrier name."), + FieldDefinition(key="line_type", label="Line type", description="Mobile, landline, VoIP, or other classification."), + FieldDefinition(key="observed_at", label="Observed at", description="Timestamp when the source observed this record."), + FieldDefinition(key="confidence", label="Confidence", description="Optional confidence score from 0 to 1."), + FieldDefinition(key="source", label="Source", description="Original source name or URL."), + ], + sample={ + "username": "demo_ada_1843", + "platform": "Example Network", + "profile_url": "https://profiles.example/demo_ada_1843", + "phone_number": "+12025550101", + "display_name": "Ada Example", + "carrier_name": "Example Wireless", + "line_type": "mobile", + "observed_at": "2026-07-17T12:00:00Z", + "confidence": 0.9, + "source": "Fictional research export", + }, +) + def create_app(database_path: str | Path | None = None) -> FastAPI: fallback = Path(tempfile.gettempdir()) / "whoisit-datasets.db" @@ -105,6 +153,8 @@ def healthz() -> dict[str, str]: @app.get("/datasets/schema/{record_type}", response_model=DatasetSchema) def dataset_schema(record_type: RecordType) -> DatasetSchema: + if record_type is RecordType.entity: + return ENTITY_SCHEMA return PROFILE_SCHEMA if record_type is RecordType.profile else PHONE_SCHEMA @app.post( diff --git a/examples/datasets/sample_entities.csv b/examples/datasets/sample_entities.csv new file mode 100644 index 0000000..5afadd6 --- /dev/null +++ b/examples/datasets/sample_entities.csv @@ -0,0 +1,4 @@ +username,platform,profile_url,phone_number,display_name,bio,location,website,verified,followers_count,valid,national_format,country_code,caller_name,caller_type,carrier_name,line_type,observed_at,confidence,source +demo_ada_1843,DemoNet,https://profiles.example/demo_ada_1843,+12025550101,Ada Example,"A fictional combined profile and phone entity.",London,https://ada.example,true,12450,true,"(202) 555-0101",US,Ada Example Labs,BUSINESS,Example Wireless,mobile,2026-07-15T14:30:00Z,0.97,Fictional QA directory +grace_demo_1906,CodeCommons,https://profiles.example/grace_demo_1906,,Grace Demo,"A fictional profile-only entity.",Arlington VA,https://grace.example,true,8830,,,,,,,,2026-07-16T10:15:00Z,0.94,Fictional QA directory +,,,+12025550102,Phone Only Demo,,,,,,true,"(202) 555-0102",US,Phone Only Demo,CONSUMER,Example Fiber,landline,2026-07-16T11:00:00Z,0.88,Fictional QA directory diff --git a/tests/test_dataset_service.py b/tests/test_dataset_service.py index a118719..5f086ef 100644 --- a/tests/test_dataset_service.py +++ b/tests/test_dataset_service.py @@ -20,10 +20,80 @@ def test_health_and_schema_are_available(tmp_path): client = make_client(tmp_path) assert client.get("/healthz").json() == {"status": "ok"} - schema = client.get("/datasets/schema/profile") + schema = client.get("/datasets/schema/entity") assert schema.status_code == 200 - assert schema.json()["record_type"] == "profile" + assert schema.json()["record_type"] == "entity" assert any(field["key"] == "username" for field in schema.json()["fields"]) + assert schema.json()["identifier_fields"] == [ + "username", + "profile_url", + "phone_number", + ] + + +def test_entity_import_accepts_sparse_and_combined_identifiers(tmp_path): + client = make_client(tmp_path) + response = client.post( + "/datasets/import", + json={ + "name": "Combined identity export", + "filename": "identities.csv", + "mapping": { + "username": "handle", + "phone_number": "mobile", + "display_name": "name", + "platform": "network", + "carrier_name": "carrier", + }, + "rows": [ + { + "handle": "@combined_demo", + "mobile": "+1 (202) 555-0101", + "name": "Combined Demo", + "network": "ExampleNet", + "carrier": "Example Wireless", + }, + { + "handle": "", + "mobile": "+12025550102", + "name": "Phone Only Demo", + "carrier": "Example Fiber", + }, + {"handle": "", "mobile": "", "name": "No identifier"}, + ], + }, + ) + + assert response.status_code == 201 + result = response.json() + assert result["imported"] == 2 + assert result["rejected"] == 1 + assert result["profile_identifiers"] == 1 + assert result["phone_identifiers"] == 2 + assert "username, profile_url, or phone_number" in result["rejected_rows"][0]["reason"] + + profile = client.get( + "/datasets/search/profiles", params={"query": "combined_demo"} + ).json()["records"][0] + assert profile["name"] == "Combined Demo" + assert profile["entity"]["phone_numbers"][0]["phone_number"] == "+12025550101" + + phone = client.get( + "/datasets/search/phones", params={"phone_number": "+12025550101"} + ).json()["records"][0] + assert phone["caller_name"] == "Combined Demo" + assert phone["entity"]["profiles"][0]["username"] == "combined_demo" + + phone_only = client.get( + "/datasets/search/phones", params={"phone_number": "+12025550102"} + ).json()["records"][0] + assert phone_only["entity"]["profiles"] == [] + + datasets = client.get("/datasets").json() + assert datasets[0]["record_type"] == "entity" + records = client.get(f"/datasets/{result['dataset_id']}/records").json() + assert records["record_type"] == "entity" + assert len(records["records"]) == 2 def test_profile_import_preserves_provenance_and_supports_fuzzy_search(tmp_path): @@ -151,6 +221,30 @@ def test_documented_sample_files_import_and_search(tmp_path): client = make_client(tmp_path) samples = ROOT / "examples" / "datasets" + with (samples / "sample_entities.csv").open(newline="", encoding="utf-8") as file: + entity_rows = list(csv.DictReader(file)) + entity_import = client.post( + "/datasets/import", + json={ + "name": "Fictional entity samples", + "filename": "sample_entities.csv", + "rows": entity_rows, + }, + ) + assert entity_import.status_code == 201 + assert entity_import.json()["imported"] == 3 + assert entity_import.json()["profile_identifiers"] == 2 + assert entity_import.json()["phone_identifiers"] == 2 + combined = client.get( + "/datasets/search/profiles", params={"query": "demo_ada_1843"} + ).json()["records"] + assert any( + record.get("entity", {}).get("phone_numbers", [{}])[0].get("phone_number") + == "+12025550101" + for record in combined + if record.get("entity", {}).get("phone_numbers") + ) + with (samples / "sample_profiles.csv").open(newline="", encoding="utf-8") as file: profile_rows = list(csv.DictReader(file)) profile_import = client.post( @@ -169,7 +263,7 @@ def test_documented_sample_files_import_and_search(tmp_path): profiles = client.get( "/datasets/search/profiles", params={"query": "demo_ada_1843"} ).json()["records"] - assert len(profiles) == 1 + assert len(profiles) == 2 assert profiles[0]["name"] == "Ada Example" assert profiles[0]["metrics"]["followers_count"] == 12450 diff --git a/whoisit b/whoisit index acf6867..6ff8ada 160000 --- a/whoisit +++ b/whoisit @@ -1 +1 @@ -Subproject commit acf6867ebf8bad79d7a41d8f8ad5fae770f80b3b +Subproject commit 6ff8ada4bb6550ed9b61816c5c852d70ce519bc0