diff --git a/beacon-api/src/axum/admin/mod.rs b/beacon-api/src/axum/admin/mod.rs index 3d88a551..1b369e9b 100644 --- a/beacon-api/src/axum/admin/mod.rs +++ b/beacon-api/src/axum/admin/mod.rs @@ -15,13 +15,14 @@ use crate::axum::auth::basic_auth; mod check; mod crawlers; mod external_tables; +mod tables; /// OpenAPI document marker for the admin surface. #[derive(OpenApi)] #[openapi( modifiers(&SecurityAddon), tags( - (name = "admin", description = "Authenticated administrative endpoints (HTTP Basic auth) for managing crawlers and external tables.") + (name = "admin", description = "Authenticated administrative endpoints (HTTP Basic auth) for managing crawlers and external tables, and inspecting table configuration.") ) )] pub struct AdminApiDoc; @@ -37,6 +38,7 @@ pub(crate) fn setup_admin_router() -> (Router>, utoipa::openapi::Op )) .routes(routes!(crawlers::run_crawler)) .routes(routes!(external_tables::create_external_table)) + .routes(routes!(tables::list_table_config)) .layer(::axum::middleware::from_fn(basic_auth)) .split_for_parts(); diff --git a/beacon-api/src/axum/admin/tables.rs b/beacon-api/src/axum/admin/tables.rs new file mode 100644 index 00000000..40f69fd2 --- /dev/null +++ b/beacon-api/src/axum/admin/tables.rs @@ -0,0 +1,50 @@ +//! Admin endpoint for inspecting a registered table's configuration. + +use std::sync::Arc; + +use ::axum::{ + extract::{Query, State}, + http::StatusCode, + Json, +}; +use beacon_core::api::TableConfigView; +use beacon_core::runtime::Runtime; +use utoipa::{IntoParams, ToSchema}; + +/// Query parameters for [`list_table_config`]. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, ToSchema, IntoParams)] +pub struct ListTableConfigQuery { + /// Name of the registered table whose configuration to return. + pub table_name: String, +} + +/// Returns the storage format and configuration of the named table. +#[tracing::instrument(level = "info", skip(state))] +#[utoipa::path( + tag = "admin", + get, + path = "/api/admin/table-config", + params(ListTableConfigQuery), + responses( + (status = 200, description = "The storage format and configuration of the table", body = TableConfigView), + (status = 404, description = "Table not found"), + ), + security(("basic-auth" = [])) +)] +pub(crate) async fn list_table_config( + State(state): State>, + Query(query): Query, +) -> Result, (StatusCode, String)> { + let result = state.list_table_config(query.table_name.clone()).await; + + match result { + Some(config) => Ok(Json(config)), + None => { + tracing::error!("Error listing table config: table not found"); + Err(( + StatusCode::NOT_FOUND, + format!("Table {} not found", query.table_name), + )) + } + } +} diff --git a/beacon-api/src/axum/client/mod.rs b/beacon-api/src/axum/client/mod.rs index 2d1b4a74..b0324dc8 100644 --- a/beacon-api/src/axum/client/mod.rs +++ b/beacon-api/src/axum/client/mod.rs @@ -18,7 +18,7 @@ mod tables; #[openapi(tags( (name = "query", description = "Execute, validate, explain, and inspect metrics of queries."), (name = "datasets", description = "Discover dataset files in the datasets store and inspect their schemas."), - (name = "tables", description = "List registered tables and inspect their schemas and configuration."), + (name = "tables", description = "List registered tables and inspect their schemas."), (name = "functions", description = "Browse the scalar, aggregate, and table-valued functions available in queries."), (name = "system", description = "Beacon runtime version and host information.") ))] @@ -41,7 +41,6 @@ pub(crate) fn setup_client_router() -> (Router>, utoipa::openapi::O .routes(routes!(tables::list_tables_with_schema)) .routes(routes!(tables::default_table)) .routes(routes!(tables::list_table_schema)) - .routes(routes!(tables::list_table_config)) .routes(routes!(tables::default_table_schema)) .routes(routes!(functions::list_functions)) .routes(routes!(functions::list_table_functions)) diff --git a/beacon-api/src/axum/client/tables.rs b/beacon-api/src/axum/client/tables.rs index 5947909e..715b2c40 100644 --- a/beacon-api/src/axum/client/tables.rs +++ b/beacon-api/src/axum/client/tables.rs @@ -7,7 +7,7 @@ use ::axum::{ http::StatusCode, Json, }; -use beacon_core::api::{SchemaFieldView, SchemaView, TableConfigView}; +use beacon_core::api::{SchemaFieldView, SchemaView}; use beacon_core::runtime::Runtime; use utoipa::{IntoParams, ToSchema}; @@ -110,48 +110,6 @@ pub(crate) async fn list_table_schema( } } -/// Query parameters for [`list_table_config`]. -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, ToSchema, IntoParams)] -pub struct ListTableConfigQuery { - /// Name of the registered table whose configuration to return. - pub table_name: String, -} - -/// Returns the storage format and configuration of the named table. -#[tracing::instrument(level = "info", skip(state))] -#[utoipa::path( - tag = "tables", - get, - path = "/api/table-config", - params(ListTableConfigQuery), - responses( - (status = 200, description = "The storage format and configuration of the table", body = TableConfigView), - (status = 404, description = "Table not found"), - ), - security( - (), - ("basic-auth" = []), - ("bearer" = []) - ) -)] -pub(crate) async fn list_table_config( - State(state): State>, - Query(query): Query, -) -> Result, (StatusCode, String)> { - let result = state.list_table_config(query.table_name.clone()).await; - - match result { - Some(config) => Ok(Json(config)), - None => { - tracing::error!("Error listing table config: table not found"); - Err(( - StatusCode::NOT_FOUND, - format!("Table {} not found", query.table_name), - )) - } - } -} - /// Returns the Arrow schema of the runtime's default table. #[tracing::instrument(level = "info", skip(state))] #[utoipa::path( diff --git a/beacon-core/src/api.rs b/beacon-core/src/api.rs index 5d5e283c..a22ce966 100644 --- a/beacon-core/src/api.rs +++ b/beacon-core/src/api.rs @@ -219,7 +219,7 @@ impl TryFrom> for TableConfigView { options.retain(|key, _| !key.starts_with("__")); } // Never expose a persisted credential — even encrypted — through - // the public table-config endpoint (external SQL-database tables + // the admin table-config endpoint (external SQL-database tables // carry one in `secret`). if config.contains_key("secret") { config.insert("secret".to_string(), Value::String("***".to_string())); @@ -441,7 +441,7 @@ mod table_config_redaction_tests { use super::*; use beacon_sql_databases::{EncryptedSecret, SqlDatabaseTableDefinition, SqlEngine}; - /// The public table-config view must never expose a persisted credential, + /// The admin table-config view must never expose a persisted credential, /// even in its encrypted form — the `secret` field is replaced with `***`. #[test] fn sql_database_secret_is_redacted_in_config_view() { diff --git a/docs/docs/1.7.3/api/exploring-data-lake.md b/docs/docs/1.7.3/api/exploring-data-lake.md index 467a177c..23a9ccaf 100644 --- a/docs/docs/1.7.3/api/exploring-data-lake.md +++ b/docs/docs/1.7.3/api/exploring-data-lake.md @@ -94,6 +94,12 @@ The Arrow schema of the default table (the one queried when a request omits GET /api/default-table-schema ``` +::: info Deprecated alias +`GET /api/query/available-columns` is a deprecated endpoint that returns only the +column names of the default table schema. Use `/api/default-table-schema` in new +code. +::: + ### All tables with schemas Convenient for UI discovery, but can be slow on large installations: @@ -104,10 +110,14 @@ GET /api/tables-with-schema ### Table configuration -Shows how a table was constructed — paths, file format, statistics settings, etc.: +Shows how a table was constructed — paths, file format, statistics settings, etc. +This endpoint is **admin-only** (see [Admin](#admin)) and requires HTTP Basic +auth; unauthenticated requests get `401`. Sensitive options such as SQL-database +passwords are redacted (the `secret` field is returned as `***`). ```http -GET /api/table-config?table_name=default +GET /api/admin/table-config?table_name=default +Authorization: Basic ``` ## Functions @@ -160,6 +170,7 @@ addition, these dedicated, JSON-typed admin endpoints are available: | Method | Path | Purpose | | ------ | ---- | ------- | | `GET` | `/api/admin/check` | Connectivity check; returns `{ "is_admin": true }` | +| `GET` | `/api/admin/table-config` | Inspect a table's storage format and configuration | | `POST` | `/api/admin/crawlers` | Define (or replace) a crawler | | `GET` | `/api/admin/crawlers` | List defined crawlers | | `GET` | `/api/admin/crawlers/{name}` | Get one crawler (or `404`) | @@ -167,6 +178,24 @@ addition, these dedicated, JSON-typed admin endpoints are available: | `DELETE` | `/api/admin/crawlers/{name}` | Drop a crawler (crawled tables are left in place) | | `POST` | `/api/admin/external-tables` | Create an external table from structured fields | +Every example below sends the credentials via HTTP Basic auth +(`Authorization: Basic `); the header is omitted from +the snippets after the first for brevity. + +Check that your credentials are accepted: + +```http +GET /api/admin/check +Authorization: Basic +``` + +Inspect a table's storage format and configuration (sensitive options such as +SQL-database passwords are returned as `***`): + +```http +GET /api/admin/table-config?table_name=default +``` + Create a crawler (the structured equivalent of [`CREATE CRAWLER`](../data-lake/crawlers.md)): ```http @@ -190,6 +219,28 @@ Content-Type: application/json "partition_cols": ["year", "month"] } ``` +List the defined crawlers, or fetch a single one by name: + +```http +GET /api/admin/crawlers +``` + +```http +GET /api/admin/crawlers/argo +``` + +Run a crawler once on demand (returns its crawl report): + +```http +POST /api/admin/crawlers/argo/run +``` + +Drop a crawler (its already-crawled tables are left in place): + +```http +DELETE /api/admin/crawlers/argo +``` + ## OpenAPI This page is a curated subset. The complete, always-current request and response diff --git a/docs/docs/1.7.3/data-lake/sql-databases.md b/docs/docs/1.7.3/data-lake/sql-databases.md index 2467abd7..44b0cf3c 100644 --- a/docs/docs/1.7.3/data-lake/sql-databases.md +++ b/docs/docs/1.7.3/data-lake/sql-databases.md @@ -52,7 +52,7 @@ export BEACON_SECRETS_KEY="$(openssl rand -base64 32)" :::warning Key management - The persisted credential can only be decrypted with the **same** key. If you lose or rotate `BEACON_SECRETS_KEY`, existing SQL database tables can no longer be queried — drop and recreate them with the new key. -- The password is **never** returned by the API. `GET /api/table-config` shows the `secret` field as `***`, and the encrypted material never appears in logs. +- The password is **never** returned by the API. `GET /api/admin/table-config` (admin basic-auth required) shows the `secret` field as `***`, and the encrypted material never appears in logs. ::: ## Defining a SQL database table @@ -123,7 +123,7 @@ A SQL database table behaves like any other table: ```http GET /api/tables GET /api/table-schema?table_name=orders -GET /api/table-config?table_name=orders # the `secret` field is shown as *** +GET /api/admin/table-config?table_name=orders # admin basic-auth; the `secret` field is shown as *** ``` ## Removing a SQL database table diff --git a/integration-tests/test_sql_databases.py b/integration-tests/test_sql_databases.py index 3b14a375..b59c6020 100644 --- a/integration-tests/test_sql_databases.py +++ b/integration-tests/test_sql_databases.py @@ -142,8 +142,14 @@ def test_postgres_table_listed(client, pg_table): def test_postgres_credentials_redacted_in_config(client, pg_table): - """The unauthenticated table-config endpoint must not expose the password.""" - resp = client.get(f"/api/table-config?table_name={pg_table}") + """The admin-only table-config endpoint must not expose the password, and + must reject unauthenticated callers.""" + # Unauthenticated callers are rejected (the endpoint now lives behind the + # admin basic-auth gate). + unauth = client.admin_get(f"/api/admin/table-config?table_name={pg_table}", admin=False) + assert unauth.status_code == 401 + + resp = client.admin_get(f"/api/admin/table-config?table_name={pg_table}") assert resp.status_code == 200 body = resp.text assert PG_PASSWORD not in body