diff --git a/.env.example b/.env.example index f31c356..a1fa4f5 100644 --- a/.env.example +++ b/.env.example @@ -1,8 +1,14 @@ # Copy this file to .env and fill in the required values before running docker compose. -# profile_search +# profile_search: X API v2 bearer token used by the Tweepy client. +# Create it in an X Developer Portal project/app. Keep it secret. TWEEPY_BEARER_TOKEN=your_tweepy_bearer_token -# phone_search +# phone_search: Twilio API credentials from the Twilio Console. +# The Account SID normally starts with AC. Keep the auth token secret. TWILIO_ACCOUNT_SID=your_twilio_account_sid TWILIO_AUTH_TOKEN=your_twilio_auth_token + +# dataset_service: Compose already persists this path in the dataset_data volume. +# Override only when you intentionally want a different in-container location. +# DATASET_DB_PATH=/data/datasets.db diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f02ff5d --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.env +__pycache__/ +*.py[cod] +finder.log diff --git a/.gitmodules b/.gitmodules index 12dca78..faca96f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,3 +7,6 @@ [submodule "phone_search"] path = phone_search url = git@github.com:osint-services/phone_search +[submodule "whoisit"] + path = whoisit + url = git@github.com:osint-services/whoisit.git diff --git a/CHANGELOG-2026-07-17.md b/CHANGELOG-2026-07-17.md new file mode 100644 index 0000000..f4cffb2 --- /dev/null +++ b/CHANGELOG-2026-07-17.md @@ -0,0 +1,34 @@ +# Changelog for 2026-07-17 + +## Added + +- Sparse entities that can contain a profile identifier, a phone identifier, or both, with associations visible from either search. +- Persistent SQLite-backed profile and phone datasets with mapped imports and local search. +- CSV, JSON, JSONL, and NDJSON parsing with automatic field suggestions and previews. +- Per-row validation, rejected-row reporting, provenance, observation time, confidence, and raw source retention. +- Search history and combined live/local results in the Electron application. +- Integration status and credential management for X/Tweepy and Twilio Lookup. +- Rich username result cards that remain informative when paid X profile inspection is unavailable. +- Fictional sample profile and phone files for testing the dataset workflow without real personal data. +- Backend and renderer parser regression tests. + +## Changed + +- Replaced the desktop record-type selector with one entity ingestion and field-mapping pipeline. +- Added an in-place SQLite schema migration while retaining legacy profile and phone import compatibility. +- Unified username and phone lookup into a Search workspace with Auto, Profile, and Phone modes plus source filters that avoid unnecessary provider calls. +- Reorganized the desktop application into Search, Datasets, Integrations, and History workspaces. +- Expanded profile and caller details and made source provenance visible throughout the UI. +- Kept API failures scoped to their own source and search workflow. +- Added dataset routing, persistence, and readiness checks to the deployed architecture. +- Restricted saved provider credentials to the operating-system user and prevented existing secrets from being returned to the UI. +- Fixed platform-root discovery in packaged Electron layouts so integration settings use the correct `.env` file. +- Mapped X API failures to explicit profile-inspection responses, including `402` for depleted credits and `429` for rate limits. +- Made nginx resolve Compose service names at request time, preventing cross-service 404s after containers receive new IP addresses. + +## Documentation + +- Rewrote the root architecture and setup guide. +- Expanded environment-variable guidance. +- Rewrote the desktop UI guide and added a dataset service guide. +- Updated the OSINT Services organization profile README. diff --git a/CHANGELOG.md b/CHANGELOG.md index d0056f0..c35ee3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,12 +5,28 @@ All notable changes to this project will be documented in this file. ## [Unreleased] ### Added -- Added lightweight `/healthz` endpoints to the FastAPI services for smoke checks and container health probes. -- Added an initial regression test suite covering the new health endpoints. +- Added a sparse entity model that associates optional profile and phone identifiers while preserving their type-specific metadata. +- Added a persistent SQLite dataset service for mapped profile and phone imports. +- Added CSV, JSON, JSONL, and NDJSON parsing, previews, automatic field suggestions, per-row validation, provenance, and raw-record retention. +- Added combined live and imported-source searching, local search history, raw JSON inspection, and dataset management to the Electron UI. +- Added an Integrations workspace for X/Tweepy, Twilio Lookup, and local dataset connection status with protected credential replacement. +- Expanded username result cards with public validation evidence, direct profile links, dataset provenance, observation time, and confidence. +- Added fictional profile CSV and phone JSONL datasets for testing imports, mapping, local search, and rich result rendering. +- Expanded regression coverage for dataset import, fuzzy profile search, normalized phone search, deletion, health endpoints, and client-side parsers. ### Changed -- Parallelized profile checker requests with bounded concurrency to reduce scan latency across site checks. -- Removed the obsolete Compose `version` field from the deployment configuration to avoid startup warnings. +- Replaced separate profile and phone dataset ingestion with one entity mapping pipeline; legacy API imports remain compatible and existing SQLite databases migrate in place. +- Unified username and phone lookup into one Search workspace with conservative auto-detection, explicit Profile and Phone modes, and source filters that control which APIs and datasets are queried. +- Reworked the Electron interface into cohesive Search, Datasets, Integrations, and History workspaces with type-specific master/detail results. +- Isolated live-provider failures so other sources can still return results and errors remain in the correct search workspace. +- Expanded profile and phone detail views with provider metadata, provenance, freshness, confidence, metrics, and raw source records. +- Added the dataset service to Docker Compose, nginx routing, persistent storage, and desktop readiness probes. +- Increased the desktop window size and stopped opening DevTools in normal runs. +- Kept stored provider secrets out of the renderer and applied credential changes through atomic, owner-only `.env` updates. +- Fixed packaged Electron builds so they locate the parent platform configuration and report the real integration state. +- Replaced opaque profile-inspection 500 errors with actionable X provider responses for depleted credits, rate limits, invalid access, and missing profiles. +- Switched nginx service routing to Docker's runtime DNS resolver so container recreation cannot silently send requests to reassigned service IPs. ### Documentation -- Updated the platform README to describe the new health checks and operational improvements. +- Updated the platform, environment, desktop UI, dataset service, and organization profile documentation for the current architecture. +- Added [2026-07-17 release notes](CHANGELOG-2026-07-17.md). diff --git a/Dockerfile b/Dockerfile index 82f3f8f..897b2cc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,6 +8,7 @@ WORKDIR /app COPY ./profile_checker /app/profile_checker COPY ./profile_search /app/profile_search COPY ./phone_search /app/phone_search +COPY ./dataset_service /app/dataset_service # Install the selected service dependencies ARG APP_DIR=profile_checker diff --git a/README.md b/README.md index 3ca93be..037715e 100644 --- a/README.md +++ b/README.md @@ -1,54 +1,97 @@ -# platform +# OSINT Services Platform -Stands up multiple microservices so they can be contacted using a single URI with `nginx` and `docker compose`. -Each service is stored as a git submodule; see each service directory for service-specific details. +The platform combines live public-data APIs with user-supplied datasets behind one nginx endpoint and one Electron investigation workspace. + +## Architecture + +```text +whoisit desktop app + | + v +nginx :80 + |-- /scan/* -> profile_checker (public username availability) + |-- /focus* -> profile_search (X profile metadata via Tweepy) + |-- /phone_search* -> phone_search (caller-name data via Twilio) + `-- /datasets* -> dataset_service (SQLite entity imports and local search) +``` + +Docker Compose runs the four FastAPI services, nginx, and a persistent `dataset_data` volume. The Electron UI provides one search workspace with Auto, Profile, and Phone modes plus All, Live APIs, and Imported datasets source filters. It queries enabled sources independently and combines successful results, so one unavailable provider does not prevent other sources from being searched. ## Requirements -- [`docker`](https://www.docker.com/) -- Docker Compose support (`docker compose`) +- Docker with `docker compose` +- Node.js 18 or newer to build or run the Electron UI +- An X API bearer token for profile inspection +- Twilio Account SID and Auth Token for live phone lookup ## Setup -1. Initialize submodules if needed: - ```bash git submodule update --init --recursive +cp .env.example .env ``` -2. Build and start all services: +Add your provider credentials to `.env`, then start the API stack: ```bash ./scripts/start.sh ``` -3. Stop the stack: +Run the desktop UI from a second terminal: ```bash -./scripts/stop.sh +cd whoisit +npm install +npm start ``` -4. Rebuild the services: +The UI checks service readiness on launch and can attempt to start the Compose stack when Docker is available. Its Integrations workspace shows the configured and reachable state for X/Tweepy, Twilio Lookup, and local datasets. Users can replace provider credentials there without exposing existing values to the renderer. Packaged builds are created with `npm run package`. -```bash -./scripts/build.sh -``` +## Environment values -5. Stream logs: +| Variable | Required | Used by | Meaning | +| --- | --- | --- | --- | +| `TWEEPY_BEARER_TOKEN` | For live profile inspection | `profile_search` | X API v2 application bearer token. Despite the variable name, the credential is issued by X; Tweepy is the Python client. | +| `TWILIO_ACCOUNT_SID` | For live phone lookup | `phone_search` | Twilio account identifier, normally beginning with `AC`. | +| `TWILIO_AUTH_TOKEN` | For live phone lookup | `phone_search` | Secret used with the Account SID to authenticate Twilio API calls. | +| `DATASET_DB_PATH` | No | `dataset_service` | SQLite file location. Compose sets this to `/data/datasets.db` inside its persistent volume. | + +Do not commit `.env`. Imported datasets may contain sensitive or licensed information; only import data you are authorized to retain and use. + +## API routes + +| Route | Purpose | +| --- | --- | +| `GET /scan/{username}` | Check supported sites for a public username | +| `GET /focus?url=https://x.com/{username}` | Retrieve expanded X profile metadata | +| `GET /phone_search?phone_number=+18135551212` | Retrieve live caller-name metadata | +| `GET /datasets` | List imported datasets | +| `GET /datasets/schema/entity` | Get the unified sparse entity schema and sample | +| `POST /datasets/import` | Import mapped entities containing profile identifiers, phone identifiers, or both | +| `GET /datasets/search/profiles?query=...&fuzzy=true` | Search imported profiles | +| `GET /datasets/search/phones?phone_number=...` | Search imported phone records | +| `DELETE /datasets/{dataset_id}` | Delete a dataset and its records | + +The import API accepts sparse entity rows after the UI parses CSV, JSON, JSONL, or NDJSON. Each row needs at least one username, profile URL, or E.164 phone number; profile and phone fields can coexist and remain associated in search results. A single import is limited to 10,000 records and the UI limits files to 10 MB. Canonical records preserve their source row, dataset provenance, observation time, and optional confidence score. Legacy typed imports remain API-compatible and existing databases migrate in place. + +## Operations and tests ```bash -./scripts/logs.sh +./scripts/build.sh # rebuild images +./scripts/logs.sh # stream Compose logs +./scripts/stop.sh # stop the stack +.venv/bin/pytest -q # backend tests +cd whoisit && npm test # importer parser tests ``` -## Notes - -- The proxy listens on port `80` and routes `/scan/...` to `profile_checker`, `/focus` to `profile_search`, and `/phone_search` to `phone_search`. -- `profile_search` requires `TWEEPY_BEARER_TOKEN` to be available in the environment when running the Compose stack. -- `phone_search` requires `TWILIO_ACCOUNT_SID` and `TWILIO_AUTH_TOKEN` to be set. -- Copy `.env.example` to `.env` and fill in the required values before starting Compose. -- Each service now exposes a lightweight `/healthz` endpoint for smoke checks and container health probes. +Liveness endpoints are available at `/scan/healthz`, `/phone_search/healthz`, and `/datasets/healthz`. Profile inspection also exposes `/focus/readyz`, which verifies that its X credential is configured. -## Changelog +## Documentation -- See [CHANGELOG.md](CHANGELOG.md) for the current release notes. -- See [CHANGELOG-2026-07-16.md](CHANGELOG-2026-07-16.md) for the latest dated entry. +- [Desktop UI guide](whoisit/README.md) +- [Dataset service guide](dataset_service/README.md) +- [Sample combined entity dataset](examples/datasets/sample_entities.csv) +- [Sample profile dataset](examples/datasets/sample_profiles.csv) +- [Sample phone dataset](examples/datasets/sample_phone_records.jsonl) +- [Changelog](CHANGELOG.md) +- [2026-07-17 release notes](CHANGELOG-2026-07-17.md) diff --git a/dataset_service/README.md b/dataset_service/README.md new file mode 100644 index 0000000..5104616 --- /dev/null +++ b/dataset_service/README.md @@ -0,0 +1,46 @@ +# Dataset service + +The dataset service provides normalized, persistent identity sources for the platform's profile and phone searches. It uses SQLite and has no external provider dependency. + +## Supported input + +The Electron UI parses `.csv`, `.json`, `.jsonl`, and `.ndjson` files, then maps every import through the entity schema returned by: + +```http +GET /datasets/schema/entity +``` + +Each sparse entity row requires at least one searchable identifier: a username, profile URL, or E.164 phone number such as `+12025550101`. All other fields are optional. When both profile and phone fields exist, the service stores typed identifiers under one entity so a match for either identifier exposes the association. Invalid booleans, negative metrics, confidence outside `0..1`, and malformed identifiers are rejected per row without discarding valid rows. + +The legacy `profile` and `phone` schemas and import values remain accepted for API compatibility, but the desktop application uses only the entity pipeline. Existing SQLite databases are migrated in place when the service starts. + +Each accepted entity includes: + +- its dataset and source; +- an observation timestamp; +- optional confidence; +- normalized searchable identifiers; and +- the complete original source row for audit. + +## Storage and limits + +Compose stores `/data/datasets.db` in the named `dataset_data` volume. Deleting a dataset cascades to its entities and typed identifiers. Each import accepts 1–10,000 JSON object rows; the desktop UI additionally enforces a 10 MB file limit. + +The API is intentionally local and currently has no authentication layer. Do not expose the nginx proxy to an untrusted network without adding access control. + +## Sample data + +The platform repository includes a combined fictional entity dataset plus separate fixtures for exercising compatibility and source-specific fields: + +- [`examples/datasets/sample_entities.csv`](../examples/datasets/sample_entities.csv) +- [`examples/datasets/sample_profiles.csv`](../examples/datasets/sample_profiles.csv) +- [`examples/datasets/sample_phone_records.jsonl`](../examples/datasets/sample_phone_records.jsonl) + +Import any of them from the same desktop Datasets workflow; canonical headers are auto-mapped. The combined file demonstrates a username and phone on one entity, a profile-only entity, and a phone-only entity. Search for `demo_ada_1843` or `+12025550101` to see the association in both directions. The phone values use the reserved North American `555-01xx` fictional range and do not identify real subscribers. + +## Development + +```bash +uvicorn dataset_service.server:app --reload --port 8000 +.venv/bin/pytest -q tests/test_dataset_service.py +``` diff --git a/dataset_service/__init__.py b/dataset_service/__init__.py new file mode 100644 index 0000000..d8a5d3b --- /dev/null +++ b/dataset_service/__init__.py @@ -0,0 +1 @@ +"""Local dataset import and search service.""" diff --git a/dataset_service/models.py b/dataset_service/models.py new file mode 100644 index 0000000..ee385c5 --- /dev/null +++ b/dataset_service/models.py @@ -0,0 +1,56 @@ +from enum import Enum +from typing import Any + +from pydantic import BaseModel, Field + + +class RecordType(str, Enum): + entity = "entity" + profile = "profile" + phone = "phone" + + +class DatasetImportRequest(BaseModel): + name: str = Field(min_length=1, max_length=120) + record_type: RecordType = RecordType.entity + filename: str | None = Field(default=None, max_length=255) + mapping: dict[str, str] = Field(default_factory=dict) + rows: list[dict[str, Any]] = Field(min_items=1, max_items=10_000) + + +class RejectedRow(BaseModel): + row_number: int + reason: str + + +class DatasetImportResult(BaseModel): + dataset_id: str + imported: int + rejected: int + rejected_rows: list[RejectedRow] + profile_identifiers: int = 0 + phone_identifiers: int = 0 + + +class DatasetSummary(BaseModel): + id: str + name: str + record_type: RecordType + filename: str | None + imported_at: str + row_count: int + rejected_count: int + + +class FieldDefinition(BaseModel): + key: str + label: str + required: bool = False + description: str + + +class DatasetSchema(BaseModel): + record_type: RecordType + fields: list[FieldDefinition] + sample: dict[str, Any] + identifier_fields: list[str] = Field(default_factory=list) diff --git a/dataset_service/repository.py b/dataset_service/repository.py new file mode 100644 index 0000000..a3f0b33 --- /dev/null +++ b/dataset_service/repository.py @@ -0,0 +1,746 @@ +import json +import re +import sqlite3 +from datetime import datetime, timezone +from pathlib import Path +from typing import Any +from urllib.parse import urlparse +from uuid import uuid4 + +from .models import DatasetImportRequest, DatasetImportResult, RejectedRow + + +PROFILE_FIELDS = ( + "username", + "platform", + "profile_url", + "display_name", + "bio", + "location", + "website", + "verified", + "protected", + "created_at", + "profile_image_url", + "profile_banner_url", + "followers_count", + "following_count", + "post_count", + "listed_count", + "like_count", + "media_count", + "observed_at", + "confidence", + "source", +) + +PHONE_FIELDS = ( + "phone_number", + "valid", + "national_format", + "country_code", + "caller_name", + "caller_type", + "carrier_name", + "line_type", + "location", + "observed_at", + "confidence", + "source", +) + +ENTITY_FIELDS = tuple(dict.fromkeys(PROFILE_FIELDS + PHONE_FIELDS)) + +SCHEMA_SQL = """ +CREATE TABLE IF NOT EXISTS datasets ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + record_type TEXT NOT NULL CHECK(record_type IN ('entity', 'profile', 'phone')), + filename TEXT, + imported_at TEXT NOT NULL, + row_count INTEGER NOT NULL, + rejected_count INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS entities ( + id TEXT PRIMARY KEY, + dataset_id TEXT NOT NULL REFERENCES datasets(id) ON DELETE CASCADE, + display_name TEXT, + bio TEXT, + location TEXT, + website TEXT, + observed_at TEXT, + confidence REAL, + source TEXT, + raw_json TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS profile_records ( + id TEXT PRIMARY KEY, + dataset_id TEXT NOT NULL REFERENCES datasets(id) ON DELETE CASCADE, + entity_id TEXT REFERENCES entities(id) ON DELETE CASCADE, + platform TEXT, + username TEXT, + username_normalized TEXT, + profile_url TEXT, + display_name TEXT, + bio TEXT, + location TEXT, + website TEXT, + verified INTEGER, + protected INTEGER, + created_at TEXT, + profile_image_url TEXT, + profile_banner_url TEXT, + followers_count INTEGER, + following_count INTEGER, + post_count INTEGER, + listed_count INTEGER, + like_count INTEGER, + media_count INTEGER, + observed_at TEXT, + confidence REAL, + source TEXT, + raw_json TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS phone_records ( + id TEXT PRIMARY KEY, + dataset_id TEXT NOT NULL REFERENCES datasets(id) ON DELETE CASCADE, + entity_id TEXT REFERENCES entities(id) ON DELETE CASCADE, + phone_number TEXT NOT NULL, + valid INTEGER, + national_format TEXT, + country_code TEXT, + caller_name TEXT, + caller_type TEXT, + carrier_name TEXT, + line_type TEXT, + location TEXT, + observed_at TEXT, + confidence REAL, + source TEXT, + raw_json TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_entity_dataset ON entities(dataset_id); +CREATE INDEX IF NOT EXISTS idx_profile_username ON profile_records(username_normalized); +CREATE INDEX IF NOT EXISTS idx_profile_entity ON profile_records(entity_id); +CREATE INDEX IF NOT EXISTS idx_profile_display_name ON profile_records(display_name); +CREATE INDEX IF NOT EXISTS idx_phone_number ON phone_records(phone_number); +CREATE INDEX IF NOT EXISTS idx_phone_entity ON phone_records(entity_id); +""" + + +def utc_now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def clean_text(value: Any) -> str | None: + if value is None: + return None + text = str(value).strip() + return text or None + + +def parse_bool(value: Any) -> bool | None: + if value is None or value == "": + return None + if isinstance(value, bool): + return value + normalized = str(value).strip().lower() + if normalized in {"true", "1", "yes", "y"}: + return True + if normalized in {"false", "0", "no", "n"}: + return False + raise ValueError(f"'{value}' is not a boolean") + + +def parse_int(value: Any) -> int | None: + if value is None or value == "": + return None + number = int(str(value).replace(",", "").strip()) + if number < 0: + raise ValueError("metric values cannot be negative") + return number + + +def parse_confidence(value: Any) -> float | None: + if value is None or value == "": + return None + confidence = float(value) + if confidence < 0 or confidence > 1: + raise ValueError("confidence must be between 0 and 1") + return confidence + + +def normalize_username(value: Any) -> str | None: + username = clean_text(value) + return username.lstrip("@").lower() if username else None + + +def username_from_url(value: Any) -> str | None: + url = clean_text(value) + if not url: + return None + path = urlparse(url).path.strip("/") + return normalize_username(path.split("/")[0]) if path else None + + +def normalize_phone(value: Any) -> str: + phone = clean_text(value) + if not phone: + raise ValueError("phone_number is required") + has_plus = phone.startswith("+") + digits = re.sub(r"\D", "", phone) + if not has_plus or not 8 <= len(digits) <= 15: + raise ValueError("phone_number must use E.164 format, for example +18135551212") + return f"+{digits}" + + +class DatasetRepository: + def __init__(self, database_path: str | Path): + self.database_path = str(database_path) + Path(self.database_path).parent.mkdir(parents=True, exist_ok=True) + self.initialize() + + def connect(self) -> sqlite3.Connection: + connection = sqlite3.connect(self.database_path) + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + return connection + + def initialize(self) -> None: + with self.connect() as connection: + datasets_sql = connection.execute( + "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'datasets'" + ).fetchone() + profile_columns = { + row["name"] + for row in connection.execute("PRAGMA table_info(profile_records)") + } + if datasets_sql and ( + "'entity'" not in datasets_sql["sql"] or "entity_id" not in profile_columns + ): + self.migrate_legacy_schema(connection) + else: + connection.executescript(SCHEMA_SQL) + + @staticmethod + def migrate_legacy_schema(connection: sqlite3.Connection) -> None: + connection.execute("PRAGMA foreign_keys = OFF") + connection.executescript( + """ + ALTER TABLE datasets RENAME TO legacy_datasets; + ALTER TABLE profile_records RENAME TO legacy_profile_records; + ALTER TABLE phone_records RENAME TO legacy_phone_records; + """ + ) + connection.executescript(SCHEMA_SQL) + connection.executescript( + """ + INSERT INTO datasets + (id, name, record_type, filename, imported_at, row_count, rejected_count) + SELECT id, name, record_type, filename, imported_at, row_count, rejected_count + FROM legacy_datasets; + + INSERT INTO profile_records ( + id, dataset_id, entity_id, platform, username, username_normalized, + profile_url, display_name, bio, location, website, verified, + protected, created_at, profile_image_url, profile_banner_url, + followers_count, following_count, post_count, listed_count, + like_count, media_count, observed_at, confidence, source, raw_json + ) + SELECT + id, dataset_id, NULL, platform, username, username_normalized, + profile_url, display_name, bio, location, website, verified, + protected, created_at, profile_image_url, profile_banner_url, + followers_count, following_count, post_count, listed_count, + like_count, media_count, observed_at, confidence, source, raw_json + FROM legacy_profile_records; + + INSERT INTO phone_records ( + id, dataset_id, entity_id, phone_number, valid, national_format, + country_code, caller_name, caller_type, carrier_name, + line_type, location, observed_at, confidence, source, raw_json + ) + SELECT + id, dataset_id, NULL, phone_number, valid, national_format, + country_code, caller_name, caller_type, carrier_name, + line_type, location, observed_at, confidence, source, raw_json + FROM legacy_phone_records; + + DROP TABLE legacy_profile_records; + DROP TABLE legacy_phone_records; + DROP TABLE legacy_datasets; + """ + ) + connection.executescript(SCHEMA_SQL) + + @staticmethod + def mapped_row( + row: dict[str, Any], mapping: dict[str, str], canonical_fields: tuple[str, ...] + ) -> dict[str, Any]: + return { + field: row.get(mapping.get(field, field)) + for field in canonical_fields + } + + def import_dataset(self, request: DatasetImportRequest) -> DatasetImportResult: + dataset_id = str(uuid4()) + entities: list[dict[str, Any]] = [] + profiles: list[dict[str, Any]] = [] + phones: list[dict[str, Any]] = [] + rejected: list[RejectedRow] = [] + imported = 0 + + for row_number, raw_row in enumerate(request.rows, start=1): + try: + if request.record_type.value == "entity": + entity, profile, phone = self.prepare_entity( + raw_row, request.mapping, request.name + ) + entities.append(entity) + if profile: + profiles.append(profile) + if phone: + phones.append(phone) + elif request.record_type.value == "profile": + profiles.append( + self.prepare_profile(raw_row, request.mapping, request.name) + ) + else: + phones.append( + self.prepare_phone(raw_row, request.mapping, request.name) + ) + imported += 1 + except (TypeError, ValueError) as exc: + rejected.append(RejectedRow(row_number=row_number, reason=str(exc))) + + imported_at = utc_now() + with self.connect() as connection: + connection.execute( + """ + INSERT INTO datasets + (id, name, record_type, filename, imported_at, row_count, rejected_count) + VALUES (?, ?, ?, ?, ?, ?, ?) + """, + ( + dataset_id, + request.name.strip(), + request.record_type.value, + request.filename, + imported_at, + imported, + len(rejected), + ), + ) + + if entities: + connection.executemany( + """ + INSERT INTO entities ( + id, dataset_id, display_name, bio, location, website, + observed_at, confidence, source, raw_json + ) VALUES ( + :id, :dataset_id, :display_name, :bio, :location, :website, + :observed_at, :confidence, :source, :raw_json + ) + """, + [dict(record, dataset_id=dataset_id) for record in entities], + ) + if profiles: + connection.executemany( + """ + INSERT INTO profile_records ( + id, dataset_id, entity_id, platform, username, username_normalized, + profile_url, display_name, bio, location, website, verified, + protected, created_at, profile_image_url, profile_banner_url, + followers_count, following_count, post_count, listed_count, + like_count, media_count, observed_at, confidence, source, raw_json + ) VALUES ( + :id, :dataset_id, :entity_id, :platform, :username, :username_normalized, + :profile_url, :display_name, :bio, :location, :website, :verified, + :protected, :created_at, :profile_image_url, :profile_banner_url, + :followers_count, :following_count, :post_count, :listed_count, + :like_count, :media_count, :observed_at, :confidence, :source, :raw_json + ) + """, + [ + dict(record, dataset_id=dataset_id, entity_id=record.get("entity_id")) + for record in profiles + ], + ) + if phones: + connection.executemany( + """ + INSERT INTO phone_records ( + id, dataset_id, entity_id, phone_number, valid, national_format, + country_code, caller_name, caller_type, carrier_name, + line_type, location, observed_at, confidence, source, raw_json + ) VALUES ( + :id, :dataset_id, :entity_id, :phone_number, :valid, :national_format, + :country_code, :caller_name, :caller_type, :carrier_name, + :line_type, :location, :observed_at, :confidence, :source, :raw_json + ) + """, + [ + dict(record, dataset_id=dataset_id, entity_id=record.get("entity_id")) + for record in phones + ], + ) + + return DatasetImportResult( + dataset_id=dataset_id, + imported=imported, + rejected=len(rejected), + rejected_rows=rejected[:100], + profile_identifiers=len(profiles), + phone_identifiers=len(phones), + ) + + def prepare_entity( + self, raw_row: dict[str, Any], mapping: dict[str, str], dataset_name: str + ) -> tuple[dict[str, Any], dict[str, Any] | None, dict[str, Any] | None]: + values = self.mapped_row(raw_row, mapping, ENTITY_FIELDS) + has_profile = bool( + normalize_username(values["username"]) or username_from_url(values["profile_url"]) + ) + has_phone = bool(clean_text(values["phone_number"])) + if not has_profile and not has_phone: + raise ValueError("username, profile_url, or phone_number is required") + + entity_id = str(uuid4()) + observed_at = clean_text(values["observed_at"]) or utc_now() + confidence = parse_confidence(values["confidence"]) + source = clean_text(values["source"]) or dataset_name + display_name = clean_text(values["display_name"]) or clean_text( + values["caller_name"] + ) + entity = { + "id": entity_id, + "display_name": display_name, + "bio": clean_text(values["bio"]), + "location": clean_text(values["location"]), + "website": clean_text(values["website"]), + "observed_at": observed_at, + "confidence": confidence, + "source": source, + "raw_json": json.dumps(raw_row, default=str), + } + + profile = ( + self.prepare_profile(raw_row, mapping, dataset_name) if has_profile else None + ) + phone = self.prepare_phone(raw_row, mapping, dataset_name) if has_phone else None + if profile: + profile["entity_id"] = entity_id + profile["display_name"] = profile["display_name"] or display_name + if phone: + phone["entity_id"] = entity_id + phone["caller_name"] = phone["caller_name"] or display_name + return entity, profile, phone + + def prepare_profile( + self, raw_row: dict[str, Any], mapping: dict[str, str], dataset_name: str + ) -> dict[str, Any]: + values = self.mapped_row(raw_row, mapping, PROFILE_FIELDS) + username = clean_text(values["username"]) + username_normalized = normalize_username(username) or username_from_url( + values["profile_url"] + ) + if not username_normalized: + raise ValueError("username or profile_url is required") + username = username or username_normalized + + return { + "id": str(uuid4()), + "platform": clean_text(values["platform"]) or "Imported", + "username": username.lstrip("@"), + "username_normalized": username_normalized, + "profile_url": clean_text(values["profile_url"]), + "display_name": clean_text(values["display_name"]), + "bio": clean_text(values["bio"]), + "location": clean_text(values["location"]), + "website": clean_text(values["website"]), + "verified": parse_bool(values["verified"]), + "protected": parse_bool(values["protected"]), + "created_at": clean_text(values["created_at"]), + "profile_image_url": clean_text(values["profile_image_url"]), + "profile_banner_url": clean_text(values["profile_banner_url"]), + "followers_count": parse_int(values["followers_count"]), + "following_count": parse_int(values["following_count"]), + "post_count": parse_int(values["post_count"]), + "listed_count": parse_int(values["listed_count"]), + "like_count": parse_int(values["like_count"]), + "media_count": parse_int(values["media_count"]), + "observed_at": clean_text(values["observed_at"]) or utc_now(), + "confidence": parse_confidence(values["confidence"]), + "source": clean_text(values["source"]) or dataset_name, + "raw_json": json.dumps(raw_row, default=str), + } + + def prepare_phone( + self, raw_row: dict[str, Any], mapping: dict[str, str], dataset_name: str + ) -> dict[str, Any]: + values = self.mapped_row(raw_row, mapping, PHONE_FIELDS) + return { + "id": str(uuid4()), + "phone_number": normalize_phone(values["phone_number"]), + "valid": parse_bool(values["valid"]), + "national_format": clean_text(values["national_format"]), + "country_code": clean_text(values["country_code"]), + "caller_name": clean_text(values["caller_name"]), + "caller_type": clean_text(values["caller_type"]), + "carrier_name": clean_text(values["carrier_name"]), + "line_type": clean_text(values["line_type"]), + "location": clean_text(values["location"]), + "observed_at": clean_text(values["observed_at"]) or utc_now(), + "confidence": parse_confidence(values["confidence"]), + "source": clean_text(values["source"]) or dataset_name, + "raw_json": json.dumps(raw_row, default=str), + } + + def list_datasets(self) -> list[dict[str, Any]]: + with self.connect() as connection: + rows = connection.execute( + "SELECT * FROM datasets ORDER BY imported_at DESC" + ).fetchall() + return [dict(row) for row in rows] + + def delete_dataset(self, dataset_id: str) -> bool: + with self.connect() as connection: + cursor = connection.execute( + "DELETE FROM datasets WHERE id = ?", (dataset_id,) + ) + return cursor.rowcount > 0 + + def search_profiles(self, query: str, fuzzy: bool = False) -> list[dict[str, Any]]: + normalized = normalize_username(query) + if not normalized: + return [] + with self.connect() as connection: + if fuzzy: + pattern = f"%{normalized}%" + rows = connection.execute( + """ + SELECT profile_records.*, datasets.name AS dataset_name + FROM profile_records + JOIN datasets ON datasets.id = profile_records.dataset_id + WHERE username_normalized LIKE ? + OR LOWER(COALESCE(display_name, '')) LIKE ? + OR LOWER(COALESCE(profile_url, '')) LIKE ? + ORDER BY confidence DESC, observed_at DESC + LIMIT 100 + """, + (pattern, pattern, pattern), + ).fetchall() + else: + rows = connection.execute( + """ + SELECT profile_records.*, datasets.name AS dataset_name + FROM profile_records + JOIN datasets ON datasets.id = profile_records.dataset_id + WHERE username_normalized = ? + ORDER BY confidence DESC, observed_at DESC + LIMIT 100 + """, + (normalized,), + ).fetchall() + records = [] + for row in rows: + record = self.profile_response(row) + if row["entity_id"]: + record["entity"] = self.entity_summary(connection, row["entity_id"]) + records.append(record) + return records + + def search_phones(self, phone_number: str) -> list[dict[str, Any]]: + normalized = normalize_phone(phone_number) + with self.connect() as connection: + rows = connection.execute( + """ + SELECT phone_records.*, datasets.name AS dataset_name + FROM phone_records + JOIN datasets ON datasets.id = phone_records.dataset_id + WHERE phone_number = ? + ORDER BY confidence DESC, observed_at DESC + LIMIT 100 + """, + (normalized,), + ).fetchall() + records = [] + for row in rows: + record = self.phone_response(row) + if row["entity_id"]: + record["entity"] = self.entity_summary(connection, row["entity_id"]) + records.append(record) + return records + + def dataset_records( + self, dataset_id: str, limit: int, offset: int + ) -> tuple[str, list[dict[str, Any]]] | None: + with self.connect() as connection: + dataset = connection.execute( + "SELECT record_type FROM datasets WHERE id = ?", (dataset_id,) + ).fetchone() + if not dataset: + return None + if dataset["record_type"] == "entity": + rows = connection.execute( + "SELECT * FROM entities WHERE dataset_id = ? LIMIT ? OFFSET ?", + (dataset_id, limit, offset), + ).fetchall() + return "entity", [ + self.entity_response(connection, row) for row in rows + ] + table = "profile_records" if dataset["record_type"] == "profile" else "phone_records" + rows = connection.execute( + f"SELECT * FROM {table} WHERE dataset_id = ? LIMIT ? OFFSET ?", + (dataset_id, limit, offset), + ).fetchall() + records = [] + for row in rows: + record = ( + self.profile_response(row) + if dataset["record_type"] == "profile" + else self.phone_response(row) + ) + if row["entity_id"]: + record["entity"] = self.entity_summary(connection, row["entity_id"]) + records.append(record) + return dataset["record_type"], records + + def entity_summary( + self, connection: sqlite3.Connection, entity_id: str + ) -> dict[str, Any] | None: + entity = connection.execute( + "SELECT * FROM entities WHERE id = ?", (entity_id,) + ).fetchone() + if not entity: + return None + profiles = connection.execute( + """ + SELECT id, platform, username, profile_url + FROM profile_records + WHERE entity_id = ? + ORDER BY platform, username + """, + (entity_id,), + ).fetchall() + phones = connection.execute( + """ + SELECT id, phone_number, caller_name, carrier_name, line_type + FROM phone_records + WHERE entity_id = ? + ORDER BY phone_number + """, + (entity_id,), + ).fetchall() + return { + "id": entity["id"], + "name": entity["display_name"], + "bio": entity["bio"], + "location": entity["location"], + "website": entity["website"], + "profiles": [ + { + "id": row["id"], + "platform": row["platform"], + "username": row["username"], + "profile_uri": row["profile_url"], + } + for row in profiles + ], + "phone_numbers": [ + { + "id": row["id"], + "phone_number": row["phone_number"], + "caller_name": row["caller_name"], + "carrier_name": row["carrier_name"], + "line_type": row["line_type"], + } + for row in phones + ], + } + + def entity_response( + self, connection: sqlite3.Connection, row: sqlite3.Row + ) -> dict[str, Any]: + summary = self.entity_summary(connection, row["id"]) or {} + return { + **summary, + "record_type": "entity", + "source_type": "dataset", + "dataset_id": row["dataset_id"], + "source": row["source"], + "observed_at": row["observed_at"], + "confidence": row["confidence"], + "raw": json.loads(row["raw_json"]), + } + + @staticmethod + def profile_response(row: sqlite3.Row) -> dict[str, Any]: + metrics = { + key: row[key] + for key in ( + "followers_count", + "following_count", + "post_count", + "listed_count", + "like_count", + "media_count", + ) + if row[key] is not None + } + return { + "id": row["id"], + "record_type": "profile", + "source_type": "dataset", + "dataset_id": row["dataset_id"], + "dataset_name": row["dataset_name"] if "dataset_name" in row.keys() else None, + "source": row["source"], + "observed_at": row["observed_at"], + "confidence": row["confidence"], + "platform": row["platform"], + "username": row["username"], + "profile_uri": row["profile_url"], + "name": row["display_name"], + "bio": row["bio"], + "location": row["location"], + "website": row["website"], + "verified": ( + bool(row["verified"]) if row["verified"] is not None else None + ), + "protected": ( + bool(row["protected"]) if row["protected"] is not None else None + ), + "created_at": row["created_at"], + "profile_image_url": row["profile_image_url"], + "profile_banner_url": row["profile_banner_url"], + "metrics": metrics, + "raw": json.loads(row["raw_json"]), + } + + @staticmethod + def phone_response(row: sqlite3.Row) -> dict[str, Any]: + return { + "id": row["id"], + "record_type": "phone", + "source_type": "dataset", + "dataset_id": row["dataset_id"], + "dataset_name": row["dataset_name"] if "dataset_name" in row.keys() else None, + "source": row["source"], + "observed_at": row["observed_at"], + "confidence": row["confidence"], + "phone_number": row["phone_number"], + "valid": bool(row["valid"]) if row["valid"] is not None else None, + "national_format": row["national_format"], + "country_code": row["country_code"], + "caller_name": row["caller_name"], + "caller_type": row["caller_type"], + "carrier_name": row["carrier_name"], + "line_type": row["line_type"], + "location": row["location"], + "raw": json.loads(row["raw_json"]), + } diff --git a/dataset_service/requirements.txt b/dataset_service/requirements.txt new file mode 100644 index 0000000..cb9becc --- /dev/null +++ b/dataset_service/requirements.txt @@ -0,0 +1,2 @@ +fastapi[standard]==0.115.0 +pydantic==2.9.2 diff --git a/dataset_service/server.py b/dataset_service/server.py new file mode 100644 index 0000000..2307f1e --- /dev/null +++ b/dataset_service/server.py @@ -0,0 +1,217 @@ +import os +import tempfile +from pathlib import Path + +from fastapi import FastAPI, HTTPException, Query, Response, status +from fastapi.middleware.cors import CORSMiddleware + +from .models import ( + DatasetImportRequest, + DatasetImportResult, + DatasetSchema, + DatasetSummary, + FieldDefinition, + RecordType, +) +from .repository import DatasetRepository + + +PROFILE_SCHEMA = DatasetSchema( + record_type=RecordType.profile, + fields=[ + FieldDefinition(key="username", label="Username", required=True, description="Account handle, with or without @. A profile URL can be used instead."), + FieldDefinition(key="platform", label="Platform", description="Network or site name, for example X or GitHub."), + FieldDefinition(key="profile_url", label="Profile URL", description="Canonical public profile URL."), + FieldDefinition(key="display_name", label="Display name", description="Human-readable account name."), + FieldDefinition(key="bio", label="Bio", description="Public profile description."), + FieldDefinition(key="location", label="Location", description="Public or source-provided location."), + FieldDefinition(key="website", label="Website", description="Website associated with the account."), + FieldDefinition(key="verified", label="Verified", description="Boolean verification status."), + FieldDefinition(key="protected", label="Protected", description="Boolean protected/private status."), + FieldDefinition(key="created_at", label="Created at", description="Account creation timestamp."), + FieldDefinition(key="profile_image_url", label="Profile image URL", description="Avatar URL."), + FieldDefinition(key="profile_banner_url", label="Profile banner URL", description="Header/banner image URL."), + FieldDefinition(key="followers_count", label="Followers", description="Follower count."), + FieldDefinition(key="following_count", label="Following", description="Following count."), + FieldDefinition(key="post_count", label="Posts", description="Post or status count."), + FieldDefinition(key="listed_count", label="Listed", description="List membership count."), + FieldDefinition(key="like_count", label="Likes", description="Like or favorite count."), + FieldDefinition(key="media_count", label="Media", description="Media post count."), + FieldDefinition(key="observed_at", label="Observed at", description="Timestamp when the source observed this record."), + FieldDefinition(key="confidence", label="Confidence", description="Optional confidence score from 0 to 1."), + FieldDefinition(key="source", label="Source", description="Original source name or URL."), + ], + sample={ + "username": "example", + "platform": "X", + "profile_url": "https://x.com/example", + "display_name": "Example Person", + "verified": False, + "followers_count": 1250, + "observed_at": "2026-07-17T12:00:00Z", + "confidence": 0.9, + "source": "Public research export", + }, +) + +PHONE_SCHEMA = DatasetSchema( + record_type=RecordType.phone, + fields=[ + FieldDefinition(key="phone_number", label="Phone number", required=True, description="Number in E.164 format, for example +18135551212."), + FieldDefinition(key="valid", label="Valid", description="Boolean validity status."), + FieldDefinition(key="national_format", label="National format", description="Locally formatted phone number."), + FieldDefinition(key="country_code", label="Country code", description="ISO country code."), + FieldDefinition(key="caller_name", label="Caller name", description="Person or business caller name."), + FieldDefinition(key="caller_type", label="Caller type", description="Caller-name classification."), + FieldDefinition(key="carrier_name", label="Carrier", description="Network carrier name."), + FieldDefinition(key="line_type", label="Line type", description="Mobile, landline, VoIP, or other classification."), + FieldDefinition(key="location", label="Location", description="Source-provided phone location."), + FieldDefinition(key="observed_at", label="Observed at", description="Timestamp when the source observed this record."), + FieldDefinition(key="confidence", label="Confidence", description="Optional confidence score from 0 to 1."), + FieldDefinition(key="source", label="Source", description="Original source name or URL."), + ], + sample={ + "phone_number": "+18135551212", + "valid": True, + "national_format": "(813) 555-1212", + "country_code": "US", + "caller_name": "Example Business", + "line_type": "landline", + "observed_at": "2026-07-17T12:00:00Z", + "confidence": 0.85, + "source": "Licensed directory", + }, +) + +ENTITY_SCHEMA = DatasetSchema( + record_type=RecordType.entity, + identifier_fields=["username", "profile_url", "phone_number"], + fields=[ + FieldDefinition(key="username", label="Username", description="Account handle, with or without @."), + FieldDefinition(key="platform", label="Platform", description="Network or site name, for example X or GitHub."), + FieldDefinition(key="profile_url", label="Profile URL", description="Canonical public profile URL; can identify a profile when username is absent."), + FieldDefinition(key="phone_number", label="Phone number", description="Number in E.164 format, for example +12025550101."), + FieldDefinition(key="display_name", label="Display name", description="Person, organization, or account name."), + FieldDefinition(key="bio", label="Bio", description="Source-provided identity or profile description."), + FieldDefinition(key="location", label="Location", description="Public or source-provided location."), + FieldDefinition(key="website", label="Website", description="Website associated with the entity or profile."), + FieldDefinition(key="verified", label="Verified", description="Boolean profile verification status."), + FieldDefinition(key="protected", label="Protected", description="Boolean protected/private profile status."), + FieldDefinition(key="created_at", label="Profile created at", description="Account creation timestamp."), + FieldDefinition(key="profile_image_url", label="Profile image URL", description="Avatar URL."), + FieldDefinition(key="profile_banner_url", label="Profile banner URL", description="Header/banner image URL."), + FieldDefinition(key="followers_count", label="Followers", description="Follower count."), + FieldDefinition(key="following_count", label="Following", description="Following count."), + FieldDefinition(key="post_count", label="Posts", description="Post or status count."), + FieldDefinition(key="listed_count", label="Listed", description="List membership count."), + FieldDefinition(key="like_count", label="Likes", description="Like or favorite count."), + FieldDefinition(key="media_count", label="Media", description="Media post count."), + FieldDefinition(key="valid", label="Phone valid", description="Boolean phone validity status."), + FieldDefinition(key="national_format", label="National phone format", description="Locally formatted phone number."), + FieldDefinition(key="country_code", label="Country code", description="ISO country code."), + FieldDefinition(key="caller_name", label="Caller name", description="Person or business caller name; used as the entity name when display name is absent."), + FieldDefinition(key="caller_type", label="Caller type", description="Caller-name classification."), + FieldDefinition(key="carrier_name", label="Carrier", description="Network carrier name."), + FieldDefinition(key="line_type", label="Line type", description="Mobile, landline, VoIP, or other classification."), + FieldDefinition(key="observed_at", label="Observed at", description="Timestamp when the source observed this record."), + FieldDefinition(key="confidence", label="Confidence", description="Optional confidence score from 0 to 1."), + FieldDefinition(key="source", label="Source", description="Original source name or URL."), + ], + sample={ + "username": "demo_ada_1843", + "platform": "Example Network", + "profile_url": "https://profiles.example/demo_ada_1843", + "phone_number": "+12025550101", + "display_name": "Ada Example", + "carrier_name": "Example Wireless", + "line_type": "mobile", + "observed_at": "2026-07-17T12:00:00Z", + "confidence": 0.9, + "source": "Fictional research export", + }, +) + + +def create_app(database_path: str | Path | None = None) -> FastAPI: + fallback = Path(tempfile.gettempdir()) / "whoisit-datasets.db" + path = database_path or os.getenv("DATASET_DB_PATH", str(fallback)) + repository = DatasetRepository(path) + app = FastAPI(title="OSINT Dataset Service") + app.state.repository = repository + + app.add_middleware( + CORSMiddleware, + allow_origins=["http://localhost:3000"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], + ) + + @app.get("/healthz") + def healthz() -> dict[str, str]: + return {"status": "ok"} + + @app.get("/datasets/schema/{record_type}", response_model=DatasetSchema) + def dataset_schema(record_type: RecordType) -> DatasetSchema: + if record_type is RecordType.entity: + return ENTITY_SCHEMA + return PROFILE_SCHEMA if record_type is RecordType.profile else PHONE_SCHEMA + + @app.post( + "/datasets/import", + response_model=DatasetImportResult, + status_code=status.HTTP_201_CREATED, + ) + def import_dataset(request: DatasetImportRequest) -> DatasetImportResult: + return repository.import_dataset(request) + + @app.get("/datasets", response_model=list[DatasetSummary]) + def list_datasets() -> list[dict]: + return repository.list_datasets() + + @app.delete("/datasets/{dataset_id}", status_code=status.HTTP_204_NO_CONTENT) + def delete_dataset(dataset_id: str) -> Response: + if not repository.delete_dataset(dataset_id): + raise HTTPException(status_code=404, detail="Dataset not found") + return Response(status_code=status.HTTP_204_NO_CONTENT) + + @app.get("/datasets/{dataset_id}/records") + def dataset_records( + dataset_id: str, + limit: int = Query(default=100, ge=1, le=500), + offset: int = Query(default=0, ge=0), + ) -> dict: + result = repository.dataset_records(dataset_id, limit, offset) + if result is None: + raise HTTPException(status_code=404, detail="Dataset not found") + record_type, records = result + return { + "dataset_id": dataset_id, + "record_type": record_type, + "records": records, + "limit": limit, + "offset": offset, + } + + @app.get("/datasets/search/profiles") + def search_profiles( + query: str = Query(min_length=1), + fuzzy: bool = False, + ) -> dict: + return { + "query": query, + "records": repository.search_profiles(query, fuzzy=fuzzy), + } + + @app.get("/datasets/search/phones") + def search_phones(phone_number: str = Query(min_length=1)) -> dict: + try: + records = repository.search_phones(phone_number) + except ValueError as exc: + raise HTTPException(status_code=422, detail=str(exc)) from exc + return {"query": phone_number, "records": records} + + return app + + +app = create_app() diff --git a/docker-compose.yml b/docker-compose.yml index 1be542a..e9cae25 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -32,7 +32,21 @@ services: environment: - TWILIO_ACCOUNT_SID=${TWILIO_ACCOUNT_SID} - TWILIO_AUTH_TOKEN=${TWILIO_AUTH_TOKEN} - command: uvicorn server:app --host 0.0.0.0 --port 8000 + command: uvicorn phone_search.server:app --host 0.0.0.0 --port 8000 + networks: + - platform_net + + dataset_service: + build: + context: . + dockerfile: Dockerfile + args: + APP_DIR: dataset_service + environment: + - DATASET_DB_PATH=/data/datasets.db + command: uvicorn dataset_service.server:app --host 0.0.0.0 --port 8000 + volumes: + - dataset_data:/data networks: - platform_net @@ -42,6 +56,7 @@ services: - profile_checker - profile_search - phone_search + - dataset_service ports: - "80:80" volumes: @@ -52,3 +67,6 @@ services: networks: platform_net: driver: bridge + +volumes: + dataset_data: diff --git a/examples/datasets/sample_entities.csv b/examples/datasets/sample_entities.csv new file mode 100644 index 0000000..5afadd6 --- /dev/null +++ b/examples/datasets/sample_entities.csv @@ -0,0 +1,4 @@ +username,platform,profile_url,phone_number,display_name,bio,location,website,verified,followers_count,valid,national_format,country_code,caller_name,caller_type,carrier_name,line_type,observed_at,confidence,source +demo_ada_1843,DemoNet,https://profiles.example/demo_ada_1843,+12025550101,Ada Example,"A fictional combined profile and phone entity.",London,https://ada.example,true,12450,true,"(202) 555-0101",US,Ada Example Labs,BUSINESS,Example Wireless,mobile,2026-07-15T14:30:00Z,0.97,Fictional QA directory +grace_demo_1906,CodeCommons,https://profiles.example/grace_demo_1906,,Grace Demo,"A fictional profile-only entity.",Arlington VA,https://grace.example,true,8830,,,,,,,,2026-07-16T10:15:00Z,0.94,Fictional QA directory +,,,+12025550102,Phone Only Demo,,,,,,true,"(202) 555-0102",US,Phone Only Demo,CONSUMER,Example Fiber,landline,2026-07-16T11:00:00Z,0.88,Fictional QA directory diff --git a/examples/datasets/sample_phone_records.jsonl b/examples/datasets/sample_phone_records.jsonl new file mode 100644 index 0000000..b6a96a4 --- /dev/null +++ b/examples/datasets/sample_phone_records.jsonl @@ -0,0 +1,4 @@ +{"phone_number":"+12025550101","valid":true,"national_format":"(202) 555-0101","country_code":"US","caller_name":"Ada Example Labs","caller_type":"BUSINESS","carrier_name":"Example Wireless","line_type":"mobile","location":"Washington, DC","observed_at":"2026-07-15T14:30:00Z","confidence":0.96,"source":"Fictional QA directory","case_note":"Reserved 555-01xx test number; no real subscriber."} +{"phone_number":"+12025550102","valid":true,"national_format":"(202) 555-0102","country_code":"US","caller_name":"Grace Demo Consulting","caller_type":"BUSINESS","carrier_name":"Example Fiber","line_type":"landline","location":"Washington, DC","observed_at":"2026-07-16T10:15:00Z","confidence":0.91,"source":"Fictional QA directory","case_note":"Reserved 555-01xx test number; no real subscriber."} +{"phone_number":"+12025550103","valid":true,"national_format":"(202) 555-0103","country_code":"US","caller_name":"River Sample","caller_type":"CONSUMER","carrier_name":"Example Voice","line_type":"voip","location":"Washington, DC","observed_at":"2026-07-14T22:10:00Z","confidence":0.79,"source":"Synthetic research export","case_note":"Reserved 555-01xx test number; no real subscriber."} +{"phone_number":"+12025550104","valid":false,"national_format":"(202) 555-0104","country_code":"US","caller_name":null,"caller_type":null,"carrier_name":"Example Carrier","line_type":"unknown","location":"Washington, DC","observed_at":"2026-07-17T08:00:00Z","confidence":0.55,"source":"Synthetic research export","case_note":"Reserved 555-01xx test number used to exercise false and null values."} diff --git a/examples/datasets/sample_profiles.csv b/examples/datasets/sample_profiles.csv new file mode 100644 index 0000000..5f488f2 --- /dev/null +++ b/examples/datasets/sample_profiles.csv @@ -0,0 +1,5 @@ +username,platform,profile_url,display_name,bio,location,website,verified,protected,created_at,profile_image_url,profile_banner_url,followers_count,following_count,post_count,listed_count,like_count,media_count,observed_at,confidence,source +demo_ada_1843,DemoNet,https://profiles.example/demo_ada_1843,Ada Example,"Builds analytical engines, documentation, and excellent tea.",London,https://ada.example,yes,no,2021-03-14T09:26:00Z,https://images.example/ada-avatar.png,https://images.example/ada-banner.png,"12,450",312,1840,87,9310,244,2026-07-15T14:30:00Z,0.97,Fictional QA directory +grace_demo_1906,CodeCommons,https://profiles.example/grace_demo_1906,Grace Demo,"Compiler enthusiast and systems educator.",Arlington VA,https://grace.example,true,false,2020-12-09T18:05:00Z,https://images.example/grace-avatar.png,,8830,541,2265,42,7100,118,2026-07-16T10:15:00Z,0.94,Fictional QA directory +river_sample_42,PhotoSquare,https://profiles.example/river_sample_42,River Sample,"Landscape photographer. All locations and identities in this row are fictional.","Portland, OR",https://river.example,false,true,2023-06-01T07:45:00Z,https://images.example/river-avatar.png,https://images.example/river-banner.png,642,389,211,3,1870,196,2026-07-14T22:10:00Z,0.81,Synthetic research export +orbit_test_lab,ForumHub,https://profiles.example/orbit_test_lab,Orbit Test Lab,"A synthetic organization record for testing imported profile search.",Remote,https://orbit.example,false,false,2024-01-20T12:00:00Z,,,154,28,76,1,220,12,2026-07-17T08:00:00Z,0.76,Synthetic research export diff --git a/nginx.conf b/nginx.conf index 1810a2f..5d4028b 100644 --- a/nginx.conf +++ b/nginx.conf @@ -5,9 +5,31 @@ events { http { server { listen 80; + resolver 127.0.0.11 valid=10s ipv6=off; + + set $profile_checker_upstream profile_checker:8000; + set $profile_search_upstream profile_search:8000; + set $phone_search_upstream phone_search:8000; + set $dataset_service_upstream dataset_service:8000; + + location = /scan/healthz { + proxy_pass http://$profile_checker_upstream/healthz; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } location /scan/ { - proxy_pass http://profile_checker:8000; + proxy_pass http://$profile_checker_upstream; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location = /focus/readyz { + proxy_pass http://$profile_search_upstream/readyz; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -15,7 +37,15 @@ http { } location /focus { - proxy_pass http://profile_search:8000; + proxy_pass http://$profile_search_upstream; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location = /phone_search/healthz { + proxy_pass http://$phone_search_upstream/healthz; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -23,7 +53,23 @@ http { } location /phone_search { - proxy_pass http://phone_search:8000/lookup/caller-name; + proxy_pass http://$phone_search_upstream/lookup/caller-name; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location = /datasets/healthz { + proxy_pass http://$dataset_service_upstream/healthz; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location /datasets { + proxy_pass http://$dataset_service_upstream; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -31,4 +77,4 @@ http { } } -} \ No newline at end of file +} diff --git a/profile_checker b/profile_checker index 4cb13a2..ee78272 160000 --- a/profile_checker +++ b/profile_checker @@ -1 +1 @@ -Subproject commit 4cb13a2622fdaa5b2a30770ea7c2356b0784afdb +Subproject commit ee782728492c4f2f4af506f7af66edba1f400333 diff --git a/profile_search b/profile_search index 5710370..0d58dbb 160000 --- a/profile_search +++ b/profile_search @@ -1 +1 @@ -Subproject commit 5710370a6f83e99ec558754b36ddfad5b3e63650 +Subproject commit 0d58dbb9f96533c509f938605c53642fcb8194c2 diff --git a/tests/test_dataset_service.py b/tests/test_dataset_service.py new file mode 100644 index 0000000..5f086ef --- /dev/null +++ b/tests/test_dataset_service.py @@ -0,0 +1,292 @@ +import csv +import json +import sys +from pathlib import Path + +from fastapi.testclient import TestClient + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from dataset_service.server import create_app + + +def make_client(tmp_path): + return TestClient(create_app(tmp_path / "datasets.db")) + + +def test_health_and_schema_are_available(tmp_path): + client = make_client(tmp_path) + + assert client.get("/healthz").json() == {"status": "ok"} + schema = client.get("/datasets/schema/entity") + assert schema.status_code == 200 + assert schema.json()["record_type"] == "entity" + assert any(field["key"] == "username" for field in schema.json()["fields"]) + assert schema.json()["identifier_fields"] == [ + "username", + "profile_url", + "phone_number", + ] + + +def test_entity_import_accepts_sparse_and_combined_identifiers(tmp_path): + client = make_client(tmp_path) + response = client.post( + "/datasets/import", + json={ + "name": "Combined identity export", + "filename": "identities.csv", + "mapping": { + "username": "handle", + "phone_number": "mobile", + "display_name": "name", + "platform": "network", + "carrier_name": "carrier", + }, + "rows": [ + { + "handle": "@combined_demo", + "mobile": "+1 (202) 555-0101", + "name": "Combined Demo", + "network": "ExampleNet", + "carrier": "Example Wireless", + }, + { + "handle": "", + "mobile": "+12025550102", + "name": "Phone Only Demo", + "carrier": "Example Fiber", + }, + {"handle": "", "mobile": "", "name": "No identifier"}, + ], + }, + ) + + assert response.status_code == 201 + result = response.json() + assert result["imported"] == 2 + assert result["rejected"] == 1 + assert result["profile_identifiers"] == 1 + assert result["phone_identifiers"] == 2 + assert "username, profile_url, or phone_number" in result["rejected_rows"][0]["reason"] + + profile = client.get( + "/datasets/search/profiles", params={"query": "combined_demo"} + ).json()["records"][0] + assert profile["name"] == "Combined Demo" + assert profile["entity"]["phone_numbers"][0]["phone_number"] == "+12025550101" + + phone = client.get( + "/datasets/search/phones", params={"phone_number": "+12025550101"} + ).json()["records"][0] + assert phone["caller_name"] == "Combined Demo" + assert phone["entity"]["profiles"][0]["username"] == "combined_demo" + + phone_only = client.get( + "/datasets/search/phones", params={"phone_number": "+12025550102"} + ).json()["records"][0] + assert phone_only["entity"]["profiles"] == [] + + datasets = client.get("/datasets").json() + assert datasets[0]["record_type"] == "entity" + records = client.get(f"/datasets/{result['dataset_id']}/records").json() + assert records["record_type"] == "entity" + assert len(records["records"]) == 2 + + +def test_profile_import_preserves_provenance_and_supports_fuzzy_search(tmp_path): + client = make_client(tmp_path) + response = client.post( + "/datasets/import", + json={ + "name": "Conference profiles", + "record_type": "profile", + "filename": "profiles.csv", + "mapping": { + "username": "handle", + "platform": "network", + "display_name": "full_name", + "bio": "description", + "followers_count": "followers", + "verified": "is_verified", + "confidence": "score", + }, + "rows": [ + { + "handle": "@AdaLovelace", + "network": "X", + "full_name": "Ada Lovelace", + "description": "Computing pioneer", + "followers": "1200", + "is_verified": "yes", + "score": "0.92", + "unmapped_note": "preserve me", + }, + { + "handle": "", + "network": "X", + "full_name": "No identifier", + }, + ], + }, + ) + + assert response.status_code == 201 + result = response.json() + assert result["imported"] == 1 + assert result["rejected"] == 1 + assert result["rejected_rows"][0]["row_number"] == 2 + + exact = client.get( + "/datasets/search/profiles", params={"query": "adalovelace"} + ).json()["records"] + assert len(exact) == 1 + record = exact[0] + assert record["username"] == "AdaLovelace" + assert record["verified"] is True + assert record["metrics"]["followers_count"] == 1200 + assert record["dataset_name"] == "Conference profiles" + assert record["confidence"] == 0.92 + assert record["raw"]["unmapped_note"] == "preserve me" + + fuzzy = client.get( + "/datasets/search/profiles", + params={"query": "lovelac", "fuzzy": "true"}, + ).json()["records"] + assert len(fuzzy) == 1 + + datasets = client.get("/datasets").json() + assert datasets[0]["row_count"] == 1 + assert datasets[0]["rejected_count"] == 1 + + +def test_phone_import_search_records_and_delete(tmp_path): + client = make_client(tmp_path) + imported = client.post( + "/datasets/import", + json={ + "name": "Subscriber export", + "record_type": "phone", + "filename": "subscribers.jsonl", + "mapping": { + "phone_number": "number", + "caller_name": "name", + "country_code": "country", + "carrier_name": "carrier", + "line_type": "kind", + }, + "rows": [ + { + "number": "+1 (813) 555-1212", + "name": "Example Subscriber", + "country": "US", + "carrier": "Example Wireless", + "kind": "mobile", + }, + {"number": "not-a-number", "name": "Invalid"}, + ], + }, + ) + + assert imported.status_code == 201 + result = imported.json() + assert result["imported"] == 1 + dataset_id = result["dataset_id"] + + matches = client.get( + "/datasets/search/phones", params={"phone_number": "+18135551212"} + ).json()["records"] + assert len(matches) == 1 + assert matches[0]["phone_number"] == "+18135551212" + assert matches[0]["caller_name"] == "Example Subscriber" + assert matches[0]["carrier_name"] == "Example Wireless" + + records = client.get(f"/datasets/{dataset_id}/records") + assert records.status_code == 200 + assert len(records.json()["records"]) == 1 + + assert client.delete(f"/datasets/{dataset_id}").status_code == 204 + assert client.get("/datasets").json() == [] + assert ( + client.get( + "/datasets/search/phones", params={"phone_number": "+18135551212"} + ).json()["records"] + == [] + ) + + +def test_documented_sample_files_import_and_search(tmp_path): + client = make_client(tmp_path) + samples = ROOT / "examples" / "datasets" + + with (samples / "sample_entities.csv").open(newline="", encoding="utf-8") as file: + entity_rows = list(csv.DictReader(file)) + entity_import = client.post( + "/datasets/import", + json={ + "name": "Fictional entity samples", + "filename": "sample_entities.csv", + "rows": entity_rows, + }, + ) + assert entity_import.status_code == 201 + assert entity_import.json()["imported"] == 3 + assert entity_import.json()["profile_identifiers"] == 2 + assert entity_import.json()["phone_identifiers"] == 2 + combined = client.get( + "/datasets/search/profiles", params={"query": "demo_ada_1843"} + ).json()["records"] + assert any( + record.get("entity", {}).get("phone_numbers", [{}])[0].get("phone_number") + == "+12025550101" + for record in combined + if record.get("entity", {}).get("phone_numbers") + ) + + with (samples / "sample_profiles.csv").open(newline="", encoding="utf-8") as file: + profile_rows = list(csv.DictReader(file)) + profile_import = client.post( + "/datasets/import", + json={ + "name": "Fictional profile samples", + "record_type": "profile", + "filename": "sample_profiles.csv", + "rows": profile_rows, + }, + ) + assert profile_import.status_code == 201 + assert profile_import.json()["imported"] == 4 + assert profile_import.json()["rejected"] == 0 + + profiles = client.get( + "/datasets/search/profiles", params={"query": "demo_ada_1843"} + ).json()["records"] + assert len(profiles) == 2 + assert profiles[0]["name"] == "Ada Example" + assert profiles[0]["metrics"]["followers_count"] == 12450 + + with (samples / "sample_phone_records.jsonl").open(encoding="utf-8") as file: + phone_rows = [json.loads(line) for line in file if line.strip()] + phone_import = client.post( + "/datasets/import", + json={ + "name": "Fictional phone samples", + "record_type": "phone", + "filename": "sample_phone_records.jsonl", + "rows": phone_rows, + }, + ) + assert phone_import.status_code == 201 + assert phone_import.json()["imported"] == 4 + assert phone_import.json()["rejected"] == 0 + + phones = client.get( + "/datasets/search/phones", params={"phone_number": "+12025550104"} + ).json()["records"] + assert len(phones) == 1 + assert phones[0]["valid"] is False + assert phones[0]["raw"]["case_note"].endswith( + "used to exercise false and null values." + ) diff --git a/tests/test_health_endpoints.py b/tests/test_health_endpoints.py index d0ee796..5b4e234 100644 --- a/tests/test_health_endpoints.py +++ b/tests/test_health_endpoints.py @@ -11,7 +11,7 @@ @pytest.fixture -def apps(monkeypatch): +def apps(monkeypatch, tmp_path): monkeypatch.setenv("TWILIO_ACCOUNT_SID", "AC123") monkeypatch.setenv("TWILIO_AUTH_TOKEN", "secret") monkeypatch.setenv("TWEEPY_BEARER_TOKEN", "token") @@ -19,11 +19,13 @@ def apps(monkeypatch): phone_server = importlib.import_module("phone_search.server") profile_search_server = importlib.import_module("profile_search.server") profile_checker_main = importlib.import_module("profile_checker.main") + from dataset_service.server import create_app as create_dataset_app return { "phone": phone_server.app, "profile_search": profile_search_server.app, "profile_checker": profile_checker_main.app, + "dataset": create_dataset_app(tmp_path / "health-datasets.db"), } @@ -33,3 +35,30 @@ def test_health_endpoint_is_available(apps): response = client.get("/healthz") assert response.status_code == 200, f"{app_name} did not expose /healthz" assert response.json() == {"status": "ok"} + + +def test_profile_search_readiness_requires_bearer_token(apps, monkeypatch): + with TestClient(apps["profile_search"]) as client: + monkeypatch.delenv("TWEEPY_BEARER_TOKEN", raising=False) + monkeypatch.delenv("BEARER_TOKEN", raising=False) + + response = client.get("/readyz") + + assert response.status_code == 503 + assert "TWEEPY_BEARER_TOKEN is missing" in response.json()["detail"] + + +def test_profile_search_returns_configuration_error(apps, monkeypatch): + with TestClient(apps["profile_search"]) as client: + monkeypatch.delenv("TWEEPY_BEARER_TOKEN", raising=False) + monkeypatch.delenv("BEARER_TOKEN", raising=False) + + response = client.get( + "/focus", + params={"url": "https://x.com/example"}, + headers={"Origin": "http://localhost:3000"}, + ) + + assert response.status_code == 503 + assert "Missing TWEEPY_BEARER_TOKEN" in response.json()["detail"] + assert response.headers["access-control-allow-origin"] == "http://localhost:3000" diff --git a/tests/test_nginx_routes.py b/tests/test_nginx_routes.py new file mode 100644 index 0000000..24a318c --- /dev/null +++ b/tests/test_nginx_routes.py @@ -0,0 +1,31 @@ +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] + + +def test_nginx_uses_docker_dns_for_runtime_service_resolution(): + config = (ROOT / "nginx.conf").read_text(encoding="utf-8") + + assert "resolver 127.0.0.11" in config + for service in ( + "profile_checker", + "profile_search", + "phone_search", + "dataset_service", + ): + assert f"set ${service}_upstream {service}:8000;" in config + assert f"proxy_pass http://${service}_upstream" in config + + +def test_nginx_keeps_health_and_application_routes_separate(): + config = (ROOT / "nginx.conf").read_text(encoding="utf-8") + + assert "location = /scan/healthz" in config + assert "location /scan/" in config + assert "location = /focus/readyz" in config + assert "location /focus" in config + assert "location = /phone_search/healthz" in config + assert "location /phone_search" in config + assert "location = /datasets/healthz" in config + assert "location /datasets" in config diff --git a/whoisit b/whoisit new file mode 160000 index 0000000..6ff8ada --- /dev/null +++ b/whoisit @@ -0,0 +1 @@ +Subproject commit 6ff8ada4bb6550ed9b61816c5c852d70ce519bc0