From d7d1daa993dfd80f0f1383793ce3d1bdb191d9b9 Mon Sep 17 00:00:00 2001 From: samlamont Date: Mon, 1 Jun 2026 16:24:05 -0400 Subject: [PATCH 01/62] local polaris testing wip --- ingress/garden.yaml | 16 + ingress/manifests/polaris.yaml.tpl | 21 + .../manifests/bootstrap-job.yaml | 10 + .../manifests/realm-configmap.yaml.tpl | 75 +++- polaris-bootstrap/garden.yaml | 32 ++ polaris-bootstrap/manifests/acl-config.yaml | 47 +++ .../manifests/acl-config.yaml.tpl | 48 +++ .../manifests/bootstrap-job.yaml | 167 ++++++++ polaris-migration-plan.md | 390 ++++++++++++++++++ polaris-pg/garden.yaml | 12 + polaris-pg/manifests/polaris-pg.yaml | 102 +++++ polaris/garden.yaml | 15 + polaris/manifests/polaris-config.yaml.tpl | 42 ++ polaris/manifests/polaris.yaml.tpl | 290 +++++++++++++ secrets/secrets.local.yaml | 18 +- trino/garden.yaml | 46 ++- 16 files changed, 1321 insertions(+), 10 deletions(-) create mode 100644 ingress/manifests/polaris.yaml.tpl create mode 100644 polaris-bootstrap/garden.yaml create mode 100644 polaris-bootstrap/manifests/acl-config.yaml create mode 100644 polaris-bootstrap/manifests/acl-config.yaml.tpl create mode 100644 polaris-bootstrap/manifests/bootstrap-job.yaml create mode 100644 polaris-migration-plan.md create mode 100644 polaris-pg/garden.yaml create mode 100644 polaris-pg/manifests/polaris-pg.yaml create mode 100644 polaris/garden.yaml create mode 100644 polaris/manifests/polaris-config.yaml.tpl create mode 100644 polaris/manifests/polaris.yaml.tpl diff --git a/ingress/garden.yaml b/ingress/garden.yaml index e8f4017..fd0487e 100644 --- a/ingress/garden.yaml +++ b/ingress/garden.yaml @@ -70,6 +70,22 @@ dependencies: environments: - local - remote +--- +kind: Deploy +type: kubernetes +name: polaris-ingress +spec: + manifestTemplates: + - manifests/polaris.yaml.tpl +dependencies: + - deploy.cert-manager + - deploy.letsencrypt + - deploy.cert + - deploy.contour + - deploy.polaris +environments: + - local + - remote # --- # kind: Deploy # type: kubernetes diff --git a/ingress/manifests/polaris.yaml.tpl b/ingress/manifests/polaris.yaml.tpl new file mode 100644 index 0000000..98495f2 --- /dev/null +++ b/ingress/manifests/polaris.yaml.tpl @@ -0,0 +1,21 @@ +apiVersion: projectcontour.io/v1 +kind: HTTPProxy +metadata: + name: polaris-httpproxy + namespace: ${environment.namespace} +spec: + virtualhost: + fqdn: polaris.${var.hostname} + tls: + secretName: polaris.${var.hostname}-tls + routes: + - services: + - name: polaris + port: 8181 + conditions: + - prefix: /api/catalog + - services: + - name: polaris + port: 8181 + conditions: + - prefix: /api/management diff --git a/keycloak-bootstrap/manifests/bootstrap-job.yaml b/keycloak-bootstrap/manifests/bootstrap-job.yaml index 8fd8522..a9731fd 100644 --- a/keycloak-bootstrap/manifests/bootstrap-job.yaml +++ b/keycloak-bootstrap/manifests/bootstrap-job.yaml @@ -43,6 +43,16 @@ spec: secretKeyRef: name: teehr-api-secrets key: client-secret + - name: TRINO_POLARIS_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: trino-polaris-secrets + key: client-secret + - name: SPARK_POLARIS_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: spark-polaris-secrets + key: client-secret - name: SMTP_HOST valueFrom: configMapKeyRef: diff --git a/keycloak-bootstrap/manifests/realm-configmap.yaml.tpl b/keycloak-bootstrap/manifests/realm-configmap.yaml.tpl index 6e70ed0..1bdfa9f 100644 --- a/keycloak-bootstrap/manifests/realm-configmap.yaml.tpl +++ b/keycloak-bootstrap/manifests/realm-configmap.yaml.tpl @@ -33,7 +33,12 @@ data: { "name": "admin" }, { "name": "basic-user" }, { "name": "jupyter-user" }, - { "name": "iceberg-user" } + { "name": "iceberg-user" }, + { "name": "iceberg-catalog-admin" }, + { "name": "iceberg-namespace-public-read" }, + { "name": "iceberg-namespace-public-write" }, + { "name": "iceberg-namespace-restricted-read" }, + { "name": "iceberg-namespace-restricted-write" } ] }, "groups": [ @@ -73,6 +78,26 @@ data: "view-realm" ] } + }, + { + "name": "iceberg-public-readers", + "realmRoles": ["iceberg-namespace-public-read"] + }, + { + "name": "iceberg-public-writers", + "realmRoles": ["iceberg-namespace-public-write", "iceberg-namespace-public-read"] + }, + { + "name": "iceberg-restricted-readers", + "realmRoles": ["iceberg-namespace-restricted-read"] + }, + { + "name": "iceberg-restricted-writers", + "realmRoles": ["iceberg-namespace-restricted-write", "iceberg-namespace-restricted-read"] + }, + { + "name": "iceberg-catalog-admins", + "realmRoles": ["iceberg-catalog-admin"] } ], "defaultGroups": [ @@ -165,6 +190,54 @@ data: "webOrigins": [ "https://prefect.${var.hostname}" ] + }, + { + "clientId": "trino-polaris", + "enabled": true, + "protocol": "openid-connect", + "publicClient": false, + "serviceAccountsEnabled": true, + "secret": "$(env:TRINO_POLARIS_CLIENT_SECRET)", + "protocolMappers": [ + { + "name": "realm-roles", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-realm-role-mapper", + "consentRequired": false, + "config": { + "multivalued": "true", + "id.token.claim": "true", + "access.token.claim": "true", + "userinfo.token.claim": "true", + "claim.name": "realm_access.roles", + "jsonType.label": "String" + } + } + ] + }, + { + "clientId": "spark-polaris", + "enabled": true, + "protocol": "openid-connect", + "publicClient": false, + "serviceAccountsEnabled": true, + "secret": "$(env:SPARK_POLARIS_CLIENT_SECRET)", + "protocolMappers": [ + { + "name": "realm-roles", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-realm-role-mapper", + "consentRequired": false, + "config": { + "multivalued": "true", + "id.token.claim": "true", + "access.token.claim": "true", + "userinfo.token.claim": "true", + "claim.name": "realm_access.roles", + "jsonType.label": "String" + } + } + ] } ] } diff --git a/polaris-bootstrap/garden.yaml b/polaris-bootstrap/garden.yaml new file mode 100644 index 0000000..e7a44ed --- /dev/null +++ b/polaris-bootstrap/garden.yaml @@ -0,0 +1,32 @@ +kind: Deploy +type: kubernetes +name: polaris-acl-config +description: K8s Deploy Polaris ACL ConfigMap +dependencies: + - deploy.secrets +environments: + - local + - remote +spec: + manifestTemplates: + - ./manifests/acl-config.yaml.tpl +--- +kind: Deploy +type: exec +name: polaris-bootstrap +description: Run Polaris bootstrap job to create catalog, namespaces, and ACL grants +dependencies: + - deploy.polaris + - deploy.polaris-acl-config + - deploy.keycloak-bootstrap +environments: + - local + - remote +spec: + deployCommand: + - bash + - -c + - >- + kubectl -n "${environment.namespace}" delete job polaris-bootstrap --ignore-not-found=true && + kubectl -n "${environment.namespace}" apply -f ./manifests/bootstrap-job.yaml && + kubectl -n "${environment.namespace}" wait --for=condition=Complete job/polaris-bootstrap --timeout=120s diff --git a/polaris-bootstrap/manifests/acl-config.yaml b/polaris-bootstrap/manifests/acl-config.yaml new file mode 100644 index 0000000..4dbd2fd --- /dev/null +++ b/polaris-bootstrap/manifests/acl-config.yaml @@ -0,0 +1,47 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: polaris-acl-config +data: + acl-config.json: | + { + "realm": "teehr", + "catalog": "teehr", + "catalog_admin_keycloak_role": "iceberg-catalog-admin", + "namespaces": [ + { + "name": "public", + "roles": [ + { + "keycloak_role": "iceberg-namespace-public-read", + "polaris_principal_role": "public_reader", + "polaris_catalog_role": "public_read_role", + "privileges": ["TABLE_READ_DATA", "TABLE_LIST", "NAMESPACE_LIST"] + }, + { + "keycloak_role": "iceberg-namespace-public-write", + "polaris_principal_role": "public_writer", + "polaris_catalog_role": "public_write_role", + "privileges": ["TABLE_WRITE_DATA", "TABLE_READ_DATA", "TABLE_LIST", "NAMESPACE_LIST", "CREATE_TABLE"] + } + ] + }, + { + "name": "restricted", + "roles": [ + { + "keycloak_role": "iceberg-namespace-restricted-read", + "polaris_principal_role": "restricted_reader", + "polaris_catalog_role": "restricted_read_role", + "privileges": ["TABLE_READ_DATA", "TABLE_LIST", "NAMESPACE_LIST"] + }, + { + "keycloak_role": "iceberg-namespace-restricted-write", + "polaris_principal_role": "restricted_writer", + "polaris_catalog_role": "restricted_write_role", + "privileges": ["TABLE_WRITE_DATA", "TABLE_READ_DATA", "TABLE_LIST", "NAMESPACE_LIST", "CREATE_TABLE"] + } + ] + } + ] + } diff --git a/polaris-bootstrap/manifests/acl-config.yaml.tpl b/polaris-bootstrap/manifests/acl-config.yaml.tpl new file mode 100644 index 0000000..a95e5a6 --- /dev/null +++ b/polaris-bootstrap/manifests/acl-config.yaml.tpl @@ -0,0 +1,48 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: polaris-acl-config +data: + warehouse: "${var.polaris.catalogWarehouse}" + acl-config.json: | + { + "realm": "teehr", + "catalog": "teehr", + "catalog_admin_keycloak_role": "iceberg-catalog-admin", + "namespaces": [ + { + "name": "public", + "roles": [ + { + "keycloak_role": "iceberg-namespace-public-read", + "polaris_principal_role": "public_reader", + "polaris_catalog_role": "public_read_role", + "privileges": ["TABLE_READ_DATA", "TABLE_LIST", "NAMESPACE_LIST"] + }, + { + "keycloak_role": "iceberg-namespace-public-write", + "polaris_principal_role": "public_writer", + "polaris_catalog_role": "public_write_role", + "privileges": ["TABLE_WRITE_DATA", "TABLE_READ_DATA", "TABLE_LIST", "NAMESPACE_LIST", "CREATE_TABLE"] + } + ] + }, + { + "name": "restricted", + "roles": [ + { + "keycloak_role": "iceberg-namespace-restricted-read", + "polaris_principal_role": "restricted_reader", + "polaris_catalog_role": "restricted_read_role", + "privileges": ["TABLE_READ_DATA", "TABLE_LIST", "NAMESPACE_LIST"] + }, + { + "keycloak_role": "iceberg-namespace-restricted-write", + "polaris_principal_role": "restricted_writer", + "polaris_catalog_role": "restricted_write_role", + "privileges": ["TABLE_WRITE_DATA", "TABLE_READ_DATA", "TABLE_LIST", "NAMESPACE_LIST", "CREATE_TABLE"] + } + ] + } + ] + } diff --git a/polaris-bootstrap/manifests/bootstrap-job.yaml b/polaris-bootstrap/manifests/bootstrap-job.yaml new file mode 100644 index 0000000..1f6697c --- /dev/null +++ b/polaris-bootstrap/manifests/bootstrap-job.yaml @@ -0,0 +1,167 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: polaris-bootstrap + labels: + app: polaris-bootstrap +spec: + ttlSecondsAfterFinished: 100 + template: + metadata: + labels: + app: polaris-bootstrap + spec: + restartPolicy: Never + containers: + - name: polaris-bootstrap + image: prefecthq/prefect:3.2.0-python3.10 + env: + - name: POLARIS_MANAGEMENT_URL + value: "http://polaris:8181" + - name: POLARIS_ROOT_CREDENTIALS + valueFrom: + secretKeyRef: + name: polaris-secrets + key: root-credentials + - name: POLARIS_REALM_NAME + value: "teehr" + command: + - python + - -c + - | + import json + import os + import sys + import requests + + MGMT_URL = os.environ["POLARIS_MANAGEMENT_URL"] + ROOT_CREDS = os.environ["POLARIS_ROOT_CREDENTIALS"] + REALM = os.environ.get("POLARIS_REALM_NAME", "teehr") + + # Read warehouse path from ConfigMap volume mount + with open("/config/warehouse") as f: + WAREHOUSE = f.read().strip() + + username, password = ROOT_CREDS.split(":", 1) + + print(f"[polaris-bootstrap] Authenticating with Polaris management API at {MGMT_URL}") + + # Obtain initial token from Polaris internal token service + token_resp = requests.post( + f"{MGMT_URL}/api/catalog/v1/oauth/tokens", + headers={"X-Polaris-Realm": REALM}, + data={ + "grant_type": "client_credentials", + "client_id": username, + "client_secret": password, + "scope": "PRINCIPAL_ROLE:ALL", + }, + ) + token_resp.raise_for_status() + token = token_resp.json()["access_token"] + print("[polaris-bootstrap] Token obtained successfully") + + headers = { + "Authorization": f"Bearer {token}", + "X-Polaris-Realm": REALM, + "Content-Type": "application/json", + } + + def mgmt(method, path, **kwargs): + url = f"{MGMT_URL}/api/management/v1{path}" + resp = getattr(requests, method)(url, headers=headers, **kwargs) + if resp.status_code not in (200, 201, 204, 409): + print(f" ERROR {resp.status_code}: {resp.text}", file=sys.stderr) + resp.raise_for_status() + return resp + + # Load ACL config + with open("/config/acl-config.json") as f: + config = json.load(f) + + catalog_name = config["catalog"] + print(f"[polaris-bootstrap] Creating catalog: {catalog_name}") + + # Create catalog + mgmt("post", "/catalogs", json={ + "name": catalog_name, + "type": "INTERNAL", + "properties": {"default-base-location": WAREHOUSE}, + "storageConfigInfo": { + "storageType": "S3", + "allowedLocations": [WAREHOUSE], + }, + }) + print(f" catalog '{catalog_name}' created (or already exists)") + + # Create catalog-level admin principal role + admin_pr = "catalog_admin" + mgmt("post", "/principal-roles", json={"name": admin_pr}) + print(f" principal role '{admin_pr}' created") + + # Grant catalog admin privileges + mgmt("post", f"/catalogs/{catalog_name}/catalog-roles", json={ + "name": "catalog_admin_role" + }) + for priv in ["CATALOG_MANAGE_CONTENT", "CATALOG_MANAGE_METADATA"]: + mgmt("put", f"/catalogs/{catalog_name}/catalog-roles/catalog_admin_role/grants", json={ + "type": "catalog", + "privilege": priv, + }) + mgmt("put", f"/principal-roles/{admin_pr}/catalog-roles/{catalog_name}", json={ + "name": "catalog_admin_role" + }) + print(f" catalog admin grants assigned") + + # Process namespaces and roles from ACL config + for ns_config in config["namespaces"]: + ns_name = ns_config["name"] + print(f"[polaris-bootstrap] Processing namespace: {ns_name}") + + # Create namespace in catalog via catalog REST API + ns_resp = requests.post( + f"{MGMT_URL}/api/catalog/v1/{catalog_name}/namespaces", + headers={**headers}, + json={"namespace": [ns_name]}, + ) + if ns_resp.status_code not in (200, 201, 409): + ns_resp.raise_for_status() + print(f" namespace '{ns_name}' created (or already exists)") + + for role_config in ns_config["roles"]: + pr_name = role_config["polaris_principal_role"] + cr_name = role_config["polaris_catalog_role"] + privileges = role_config["privileges"] + + print(f" creating principal role: {pr_name}, catalog role: {cr_name}") + + # Create principal role + mgmt("post", "/principal-roles", json={"name": pr_name}) + + # Create catalog role + mgmt("post", f"/catalogs/{catalog_name}/catalog-roles", json={"name": cr_name}) + + # Grant namespace privileges to catalog role + for priv in privileges: + mgmt("put", f"/catalogs/{catalog_name}/catalog-roles/{cr_name}/grants", json={ + "type": "namespace", + "namespace": [ns_name], + "privilege": priv, + }) + + # Assign catalog role to principal role + mgmt("put", f"/principal-roles/{pr_name}/catalog-roles/{catalog_name}", json={ + "name": cr_name + }) + + print(f" privileges and catalog role assigned") + + print("[polaris-bootstrap] Bootstrap complete!") + volumeMounts: + - name: acl-config + mountPath: /config + readOnly: true + volumes: + - name: acl-config + configMap: + name: polaris-acl-config diff --git a/polaris-migration-plan.md b/polaris-migration-plan.md new file mode 100644 index 0000000..30cbf75 --- /dev/null +++ b/polaris-migration-plan.md @@ -0,0 +1,390 @@ +# Plan: Replace iceberg-rest with Apache Polaris (Keycloak-integrated) + +## TL;DR + +Replace `tabulario/iceberg-rest` with `apache/polaris`. Tight Keycloak integration via JWKS + a `PrincipalRoleMapper` that reads `realm_access.roles` from JWTs, mapping Keycloak roles directly to Polaris principal roles at runtime — no per-user Polaris registration needed. A data-driven bootstrap job creates the full namespace × privilege ACL structure from a ConfigMap. Polaris and Keycloak are the control plane; all other services are the data plane. + +--- + +> **Implementation order for `garden deploy` to work**: Phase 7 (Garden variables) and the secrets additions from Phase 1 must be done first — before any other phase — because Garden resolves `${var.polaris.*}` template variables at render time and pods reference the new secrets at startup. + +## Phase 1: Database — `polaris-pg` + +1. Create `polaris-pg/manifests/polaris-pg.yaml` (static, not `.tpl` — matches `iceberg-pg` which uses `manifestFiles`, no Garden templating needed). Mirror `iceberg-pg/manifests/iceberg-pg.yaml` exactly with these substitutions: + - All `iceberg-pg` → `polaris-pg`, all `iceberg` → `polaris` (names, labels, PVC claim) + - `POSTGRES_DB`/`POSTGRES_USER`/`POSTGRES_PASSWORD` env vars read from `polaris-db-secrets` + - Readiness/liveness `pg_isready` probes use `-U polaris -d polaris` + - PVC storage: 10Gi (same as iceberg-pg) + +2. Create `polaris-pg/garden.yaml` — `type: kubernetes`, `name: polaris-database`, `spec.manifestFiles: [./manifests/polaris-pg.yaml]`, depends on `deploy.secrets` — mirrors `iceberg-pg/garden.yaml` exactly with name substitution. + +3. Add `polaris-db-secrets` to `secrets/secrets.local.yaml` and `secrets/secrets.remote.yaml`. **Secrets are NOT standalone K8s manifests** — the `secrets/garden.yaml` `$forEach` loop creates K8s Secrets automatically from these varfiles. Add: + ```yaml + polaris-db-secrets: + data: + database: polaris + username: polaris + password: polaris123 # local; use strong password in remote + polaris-secrets: + data: + root-credentials: "root:secret123" # POLARIS_ROOT_CREDENTIALS format (:); use strong value in remote + trino-polaris-secrets: + data: + client-secret: local-trino-polaris-client-secret # plain secret — used by keycloak-bootstrap env var + credential: "trino-polaris:local-trino-polaris-client-secret" # full credential string — mounted as Trino credential file + spark-polaris-secrets: + data: + client-secret: local-spark-polaris-client-secret + ``` + > **Note**: `trino-polaris-secrets` needs **two keys** because the Keycloak bootstrap job needs the plain secret value (`client-secret`) to set as the Keycloak client secret, while Trino's credential file mount needs the full `trino-polaris:` string (`credential`). Using the `credential` key for the Keycloak env var would inject the wrong value. + +--- + +## Phase 2: Polaris Deployment — `polaris/` + +4. Create `polaris/manifests/polaris-config.yaml.tpl` — ConfigMap with `polaris-server.yml`: + - JDBC persistence → `polaris-pg` + - OIDC issuers as a **list** (multi-realm extensible); JWKS URI internal for `local`, external for `remote` + - **`PrincipalRoleMapper`** pointing at `realm_access.roles` in the JWT — maps Keycloak role names to Polaris principal role names at runtime. This is the critical link; without it user token pass-through grants zero privileges. Exact config key must be verified against the pinned Polaris release. + +5. Create `polaris/manifests/polaris.yaml.tpl` — Deployment + Service + SA: + - Image: `apache/polaris` pinned to a specific release tag + - Ports 8182 (catalog REST API), port 8183 (management API); Service exposes both + - **No IRSA annotation** — Polaris is pure metadata; S3 I/O stays with Trino/Spark + - ServiceAccount metadata must use `${environment.namespace}` — do NOT copy the hardcoded `namespace: teehr-hub` from `iceberg-rest.yaml.tpl` + - Local env gets MinIO credentials (same `${if environment.name == "local"}` pattern as `iceberg-rest.yaml.tpl`) + - `POLARIS_ROOT_CREDENTIALS` from `polaris-secrets` key `root-credentials` + - JDBC credentials from `polaris-db-secrets` (Quarkus datasource env var names must be verified against the pinned release — typically `QUARKUS_DATASOURCE_JDBC_URL`, `QUARKUS_DATASOURCE_USERNAME`, `QUARKUS_DATASOURCE_PASSWORD`) + - **Readiness and liveness probes are required** — the `polaris-bootstrap` exec deploy depends on `deploy.polaris` and will fire immediately when Polaris is marked ready; without probes the Quarkus app may still be starting: + ```yaml + readinessProbe: + httpGet: + path: /q/health/ready + port: 8182 + initialDelaySeconds: 20 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /q/health/live + port: 8182 + initialDelaySeconds: 30 + periodSeconds: 10 + ``` + +6. Create `polaris/garden.yaml` — `type: kubernetes`, `spec.manifestTemplates`, depends on `deploy.secrets`, `deploy.polaris-database`, `deploy.keycloak` + +--- + +## Phase 3: Keycloak Bootstrap Updates + +7. Update `keycloak-bootstrap/manifests/realm-configmap.yaml.tpl` — inside the existing `teehr-realm.json` data key: + - Add to `roles.realm` array: `iceberg-catalog-admin`, `iceberg-namespace-public-read`, `iceberg-namespace-public-write`, `iceberg-namespace-restricted-read`, `iceberg-namespace-restricted-write` + - Add to `groups` array: + - `iceberg-public-readers` (realmRoles: `iceberg-namespace-public-read`) + - `iceberg-public-writers` (realmRoles: `iceberg-namespace-public-write`, `iceberg-namespace-public-read`) + - `iceberg-restricted-readers` (realmRoles: `iceberg-namespace-restricted-read`) + - `iceberg-restricted-writers` (realmRoles: `iceberg-namespace-restricted-write`, `iceberg-namespace-restricted-read`) + - `iceberg-catalog-admins` (realmRoles: `iceberg-catalog-admin`) + - Retain existing `iceberg-user` role and group during transition + - Add to `clients` array — **two** new confidential service account clients (not three — Polaris does not need its own Keycloak client; it validates tokens via JWKS only and does not perform token introspection or act as an OAuth2 client itself): + - `trino-polaris` client — `serviceAccountsEnabled: true`, `secret: $(env:TRINO_POLARIS_CLIENT_SECRET)`, add `realm_access` protocol mapper to include roles in access token + - `spark-polaris` client — `serviceAccountsEnabled: true`, `secret: $(env:SPARK_POLARIS_CLIENT_SECRET)`, same `realm_access` mapper + - **Note on `realm_access.roles` claim**: Keycloak includes realm roles in access tokens by default, but verify against the running Keycloak version. If the Polaris `PrincipalRoleMapper` needs roles under a custom claim path, add an explicit `oidc-usermodel-realm-role-mapper` protocolMapper to the Polaris-facing clients. + +8. Update `keycloak-bootstrap/manifests/bootstrap-job.yaml` — add two new `env` entries matching the existing pattern: + - `TRINO_POLARIS_CLIENT_SECRET` from `trino-polaris-secrets` key `client-secret` + - `SPARK_POLARIS_CLIENT_SECRET` from `spark-polaris-secrets` key `client-secret` + +--- + +## Phase 4: Polaris Bootstrap Job — `polaris-bootstrap/` + +9. Create `polaris-bootstrap/manifests/acl-config.yaml` — a ConfigMap containing a declarative ACL definition (JSON) that the bootstrap job consumes. This makes the job **data-driven**: adding a new namespace = edit this file + re-run bootstrap job, no code changes: + ```json + { + "realm": "teehr", + "catalog": "teehr", + "catalog_admin_keycloak_role": "iceberg-catalog-admin", + "namespaces": [ + { + "name": "public", + "roles": [ + { + "keycloak_role": "iceberg-namespace-public-read", + "polaris_principal_role": "public_reader", + "polaris_catalog_role": "public_read_role", + "privileges": ["TABLE_READ_DATA", "TABLE_LIST", "NAMESPACE_LIST"] + }, + { + "keycloak_role": "iceberg-namespace-public-write", + "polaris_principal_role": "public_writer", + "polaris_catalog_role": "public_write_role", + "privileges": ["TABLE_WRITE_DATA", "TABLE_READ_DATA", "TABLE_LIST", "NAMESPACE_LIST", "CREATE_TABLE"] + } + ] + }, + { + "name": "restricted", + "roles": [ + { + "keycloak_role": "iceberg-namespace-restricted-read", + "polaris_principal_role": "restricted_reader", + "polaris_catalog_role": "restricted_read_role", + "privileges": ["TABLE_READ_DATA", "TABLE_LIST", "NAMESPACE_LIST"] + }, + { + "keycloak_role": "iceberg-namespace-restricted-write", + "polaris_principal_role": "restricted_writer", + "polaris_catalog_role": "restricted_write_role", + "privileges": ["TABLE_WRITE_DATA", "TABLE_READ_DATA", "TABLE_LIST", "NAMESPACE_LIST", "CREATE_TABLE"] + } + ] + } + ] + } + ``` + +10. Create `polaris-bootstrap/manifests/bootstrap-job.yaml` — reuse a Prefect image (same pattern as `prefect-workflows/manifests/load-secrets.yaml`, which uses `prefecthq/prefect:3.2.0-python3.10`; use whichever tag is current in the codebase — no new image or Dockerfile needed) running an inline Python script (`python -c |`) matching that pattern. The script: + - Accepts `POLARIS_REALM_NAME` env var (default `teehr`); sends `X-Polaris-Realm` header on every Management API call + - Uses stdlib (`json`, `os`) + `requests` (available in the Prefect image) — no custom image build + - Reads ACL config from mounted `acl-config.yaml` ConfigMap + - Creates the `teehr` catalog pointing at the warehouse (S3/MinIO path from env) + - Iterates config to create: namespaces, catalog roles with privilege grants, principal roles, catalog role → principal role assignments + - Creates `iceberg-catalog-admin` principal role with full catalog-level grants + - `ttlSecondsAfterFinished: 100` — matches existing Job pattern in `load-secrets.yaml` + - Required env vars in the Job manifest: + - `POLARIS_MANAGEMENT_URL: http://polaris:8183` — base URL for the management API + - `POLARIS_ROOT_CREDENTIALS` from `polaris-secrets` key `root-credentials` — used to obtain an initial Bearer token from the management API + - `POLARIS_REALM_NAME: teehr` (or from env) + - `CATALOG_WAREHOUSE` from `${var.polaris.catalogWarehouse}` (or hardcoded for local/remote) + +11. Create `polaris-bootstrap/garden.yaml` — mirrors `keycloak-bootstrap/garden.yaml` with **two deploys in one file** (this is required so the ConfigMap exists in Kubernetes before the Job pod tries to mount it): + ```yaml + kind: Deploy + type: kubernetes + name: polaris-acl-config + dependencies: + - deploy.secrets + environments: + - local + - remote + spec: + manifestFiles: + - ./manifests/acl-config.yaml + --- + kind: Deploy + type: exec + name: polaris-bootstrap + dependencies: + - deploy.polaris + - deploy.polaris-acl-config + - deploy.keycloak-bootstrap # logical ordering: Keycloak roles must exist before bootstrap maps them + environments: + - local + - remote + spec: + deployCommand: + - bash + - -c + - >- + kubectl -n "${environment.namespace}" delete job polaris-bootstrap --ignore-not-found=true && + kubectl -n "${environment.namespace}" apply -f ./manifests/bootstrap-job.yaml && + kubectl -n "${environment.namespace}" wait --for=condition=Complete job/polaris-bootstrap --timeout=120s + ``` + +--- + +## Phase 5: Trino Catalog Config Update + +12. Update `trino/garden.yaml` — **two separate Deploy blocks exist (local and remote); both must be updated**. Trino is a **Helm chart deployment** (`type: helm`) with no separate manifests directory — all config goes in Helm values. Changes per block: + + **Catalog properties** (in `catalogs.iceberg` multiline string): + - Replace `iceberg.rest-catalog.uri` value with `${var.polaris.catalogUri}` + - Replace `iceberg.rest-catalog.warehouse` value with `${var.polaris.catalogWarehouse}` + - Add `iceberg.rest-catalog.security=OAUTH2` + - Add `iceberg.rest-catalog.oauth2.server-uri=${var.polaris.oauthServerUri}` + - Add `iceberg.rest-catalog.oauth2.credential-file=/etc/trino/polaris-credential` + - Add `iceberg.rest-catalog.oauth2.scope=openid` + - Add `iceberg.rest-catalog.header.X-Polaris-Realm=teehr` + - Keep `iceberg.catalog.type=rest` (unchanged) + + **Credential file delivery** — Trino catalog properties do not support env var interpolation, so the OAuth2 credential must be a mounted file. The `trino-polaris-secrets` key `credential` stores the full `trino-polaris:` string. Mount it via Helm values: + ```yaml + coordinator: + extraVolumes: + - name: polaris-credential + secret: + secretName: trino-polaris-secrets + items: + - key: credential + path: polaris-credential + extraVolumeMounts: + - name: polaris-credential + mountPath: /etc/trino/polaris-credential + subPath: polaris-credential + readOnly: true + ``` + No init container needed. + + **Per-block constraints:** + - Local block: keep existing MinIO `env` entries; keep `s3.path-style-access` and `s3.endpoint` catalog properties + - Remote block: keep existing `serviceAccount.annotations` IRSA entry + - Both blocks: keep all existing `accessControl` configmap rules unchanged + +--- + +## Phase 6: Spark Session Updates + +13. Update `spark_session_utils.py` — changes are **minimal and additive**, preserving all existing function signatures and behavior: + - Add an optional `oauth2_token: str = None` parameter to `create_spark_session()` — passed through to `_configure_iceberg_catalogs()` + - In `_configure_iceberg_catalogs()`: add OAuth2 conf.set calls at the end of the existing function body: + - If `oauth2_token` provided (JupyterHub user token pass-through): set `rest.auth.type=oauth2` + `rest.auth.oauth2.token=` + - If absent (Prefect batch): set `rest.auth.type=oauth2`, `rest.auth.oauth2.server-uri` (from `POLARIS_OAUTH2_SERVER_URI` env), `rest.auth.oauth2.credential=spark-polaris:` (from `SPARK_POLARIS_CLIENT_SECRET` env), `rest.auth.oauth2.scope=openid` + - Both paths: set `rest.transport.header.X-Polaris-Realm=teehr` + - The existing `update_configs: Dict[str, str]` parameter on `create_spark_session()` remains available as an override escape hatch — no structural change needed + - **No changes** to `_create_spark_base_session`, `_set_spark_cluster_configuration`, `_set_aws_credentials_in_spark`, `_update_configs_and_packages`, `_set_catalog_metadata`, or any other existing functions + +14. Update `teehr/src/teehr/const.py` — add `POLARIS_OAUTH2_SERVER_URI` and `SPARK_POLARIS_CLIENT_SECRET` env var reads alongside existing constants. + +15. Update `prefect-workflows/manifests/prefect-deployer-job.yaml` — change `REMOTE_CATALOG_REST_URI` value from `${var.iceberg.catalogUri}` to `${var.polaris.catalogUri}`. + +--- + +## Phase 7: Garden Variables & Wiring + +16. Update `project.garden.yml` — add `polaris` variable group with **local/remote divergence** (same pattern as existing `iceberg.*` divergence): + ```yaml + # local: + polaris: + catalogUri: http://polaris:8182/api/catalog + oauthServerUri: http://keycloak-service:8080/realms/teehr/protocol/openid-connect/token + catalogWarehouse: s3://warehouse/ + catalogType: rest + inCluster: "true" + catalogS3PathStyleAccess: "true" + catalogS3Endpoint: "http://minio:9000" + + # remote: + polaris: + catalogUri: https://polaris.${var.hostname}/api/catalog + oauthServerUri: https://auth.${var.hostname}/realms/teehr/protocol/openid-connect/token + catalogWarehouse: s3://dev-teehr-iceberg-warehouse/ + catalogType: rest + inCluster: "false" + catalogS3PathStyleAccess: "false" + catalogS3Endpoint: "" + ``` + Keep existing `iceberg.*` variables during transition. The `polaris` variable group is a complete superset — once all consumers are migrated, `iceberg.*` can be removed entirely. + +17. Add comment blocks in `project.garden.yml` marking **control plane** (`keycloak*`, `polaris*`, `cert-manager`) vs **data plane** modules to document the intended future cluster boundary. All cross-plane references go through `polaris.*` variable group entries — no hardcoded in-cluster hostnames in data plane configs. + +--- + +## Phase 8: Polaris Ingress — Required + +18. Update `ingress/garden.yaml` — add a new `kind: Deploy` entry `name: polaris-ingress` following the exact pattern of existing entries: `type: kubernetes`, `spec.manifestTemplates: [manifests/polaris.yaml.tpl]`, dependencies on `deploy.cert-manager`, `deploy.letsencrypt`, `deploy.cert`, `deploy.contour`, `deploy.polaris`. + +19. Create `ingress/manifests/polaris.yaml.tpl` — Contour `HTTPProxy` following the exact pattern of existing manifests: + - `fqdn: polaris.${var.hostname}`, TLS `secretName: polaris.${var.hostname}-tls` + - Route `/api/catalog` → `polaris:8182` (catalog REST API) + - Route `/api/management` → `polaris:8183` (management API) + +--- + +## Phase 9: Retire iceberg-rest + +20. Disable `iceberg-rest` Garden deployment after all consumers (Trino, Spark, Prefect workflows) are verified connected to Polaris. Keep `iceberg-pg` until catalog data migration is confirmed complete. + +--- + +## Files — New + +| File | Notes | +|---|---| +| `polaris-pg/garden.yaml` | Mirror `iceberg-pg/garden.yaml`; uses `manifestFiles` | +| `polaris-pg/manifests/polaris-pg.yaml` | Static manifest (no `.tpl`); mirror `iceberg-pg/manifests/iceberg-pg.yaml` | +| `polaris/garden.yaml` | Depends on `polaris-database`, `keycloak`, `secrets` | +| `polaris/manifests/polaris.yaml.tpl` | Deployment + Service + SA; no IRSA annotation | +| `polaris/manifests/polaris-config.yaml.tpl` | ConfigMap with multi-issuer `polaris-server.yml` + `PrincipalRoleMapper` | +| `polaris-bootstrap/garden.yaml` | Two deploys: `type: kubernetes` (acl-config ConfigMap) + `type: exec` (job); both with `environments: [local, remote]` | +| `polaris-bootstrap/manifests/bootstrap-job.yaml` | Reuses `prefecthq/prefect:3.4.24-python3.12`; inline Python; parameterized on `POLARIS_REALM_NAME` | +| `polaris-bootstrap/manifests/acl-config.yaml` | Declarative namespace × privilege ACL ConfigMap | +| `ingress/manifests/polaris.yaml.tpl` | Contour HTTPProxy; routes for ports 8182 and 8183 | + +## Files — Modified + +| File | Change | +|---|---| +| `secrets/secrets.local.yaml` | Add `polaris-db-secrets`, `polaris-secrets`, `trino-polaris-secrets`, `spark-polaris-secrets` | +| `secrets/secrets.remote.yaml` | Same four secrets with production-grade values | +| `keycloak-bootstrap/manifests/realm-configmap.yaml.tpl` | Add 5 realm roles, 5 groups, 2 confidential clients (`trino-polaris`, `spark-polaris`) | +| `keycloak-bootstrap/manifests/bootstrap-job.yaml` | Add 2 new secret env vars (`TRINO_POLARIS_CLIENT_SECRET`, `SPARK_POLARIS_CLIENT_SECRET`) | +| `trino/garden.yaml` | Both local + remote Deploy blocks: OAuth2 catalog auth + credential-file volume mount via Helm values | +| `spark_session_utils.py` | Additive: dual-path OAuth2 + `X-Polaris-Realm` header; no existing signatures changed | +| `teehr/src/teehr/const.py` | Add `POLARIS_OAUTH2_SERVER_URI`, `SPARK_POLARIS_CLIENT_SECRET` | +| `prefect-workflows/manifests/prefect-deployer-job.yaml` | Update `REMOTE_CATALOG_REST_URI` to `${var.polaris.catalogUri}` | +| `project.garden.yml` | Add `polaris` variable group + control/data plane comments | +| `ingress/garden.yaml` | Add `polaris-ingress` Deploy entry | + +--- + +## Verification + +1. `kubectl get pods` — `polaris` and `polaris-pg` both Running +2. `kubectl logs deployment/polaris` — OIDC + `PrincipalRoleMapper` config loaded; no startup errors +3. Fetch `client_credentials` token for `trino-polaris` from Keycloak → `GET https://polaris.${var.hostname}/api/catalog/v1/config` with `X-Polaris-Realm: teehr` → expect 200; confirm `realm_access.roles` present in decoded token +4. `trino --execute "SHOW SCHEMAS IN iceberg"` → teehr schema visible +5. JupyterHub Spark with user token injected → user with no namespace role gets 403 from Polaris +6. Prefect batch job → `spark-polaris` service client token accepted; `REMOTE_CATALOG_REST_URI` resolves to Polaris +7. **ACL matrix**: + - `iceberg-namespace-public-read` member → read `public` ✓, write `public` ✗, read `restricted` ✗ + - `iceberg-namespace-public-write` member → read+write `public` ✓, `restricted` ✗ + - `iceberg-namespace-restricted-read` member → read `restricted` ✓, write `public` ✗ + - `iceberg-catalog-admin` member → full access to all namespaces ✓ +8. Edit `acl-config.yaml` to add a new namespace, re-run bootstrap job → new namespace ACLs applied; no code changes required +9. Run existing `teehr/tests/` catalog operation tests + +--- + +## Decisions + +- `apache/polaris` image (Apache incubator), pinned to a specific release tag — not `latest` +- `polaris.yaml.tpl` ServiceAccount uses `${environment.namespace}` — not hardcoded `teehr-hub` like `iceberg-rest.yaml.tpl` +- Readiness/liveness probes on `/q/health/ready` and `/q/health/live` (port 8182) required so Garden waits for Polaris to be truly ready before firing `polaris-bootstrap` +- `polaris` Keycloak client removed — Polaris validates tokens via JWKS only and needs no Keycloak service account +- New dedicated `polaris-pg` PostgreSQL instance (not reusing `iceberg-pg`) +- `polaris-pg` uses `manifestFiles` (static, no `.tpl`) — matching `iceberg-pg` pattern exactly +- Secrets via `secrets/secrets.local.yaml` + `secrets/secrets.remote.yaml` varfiles — consistent with existing `$forEach` pattern; no standalone K8s Secret manifests +- Trino OAuth2 credential delivered via mounted credential-file (`iceberg.rest-catalog.oauth2.credential-file`) — avoids env var interpolation limitations in Trino catalog properties; stored as full `trino-polaris:` string, mounted via `subPath`, no init container +- Trino: `client_credentials` — access control enforced at Trino layer; Polaris sees service identity +- Spark in Prefect: `client_credentials` (`spark-polaris`) — headless batch, no user context +- Spark in JupyterHub: user token pass-through — Polaris enforces per-user namespace/table ACLs +- Polaris has **no IRSA annotation** — pure metadata service on the control plane +- Control plane: Polaris + Keycloak (future: dedicated cluster); Data plane: all other services +- All cross-plane URLs go through `polaris.*` Garden variable group — no hardcoded in-cluster hostnames in data plane configs +- `polaris-server.yml` uses an issuer allow-list (not single issuer) from day one for multi-realm extensibility +- `polaris-bootstrap` reuses `prefecthq/prefect:3.4.24-python3.12` image (already in codebase via `load-secrets.yaml`) — no new Dockerfile or image build +- `polaris-bootstrap/garden.yaml` uses **two deploys**: `type: kubernetes` (deploys `acl-config.yaml` ConfigMap) + `type: exec` (runs the job) — matches `keycloak-bootstrap` pattern exactly; required because the Job pod mounts the ConfigMap as a volume +- `trino-polaris-secrets` has **two keys**: `client-secret` (plain value, used by `keycloak-bootstrap` env var) and `credential` (full `trino-polaris:` string, mounted as Trino credential file) — the same secret provides both without duplication +- `polaris` variable group is a complete superset of `iceberg` variable group, enabling full future removal of `iceberg.*` after migration +- `polaris-bootstrap` job env vars include `POLARIS_MANAGEMENT_URL` (http://polaris:8183) and `POLARIS_ROOT_CREDENTIALS` from `polaris-secrets` — required for the script to authenticate and call the management API +- `polaris-bootstrap` exec deploy depends on `deploy.keycloak-bootstrap` for logical ordering (Keycloak roles must exist before the bootstrap job maps them) +- Trino `iceberg.rest-catalog.warehouse` updated to `${var.polaris.catalogWarehouse}` alongside the URI change +- `spark_session_utils.py` changes are additive only — new `oauth2_token: str = None` parameter; all existing callers unaffected; `update_configs` escape hatch unchanged +- `X-Polaris-Realm` header explicit in all client configs from day one +- Keycloak role taxonomy: namespace × privilege; group-based assignment; coarse `iceberg-user` retained during transition +- `PrincipalRoleMapper` in `polaris-server.yml` resolves JWT `realm_access.roles` → Polaris principal roles at runtime; no per-user Polaris principal registration needed + +--- + +## Further Considerations + +1. **Polaris image tag**: Pin to a specific release (e.g., `0.9.0`) — `apache/polaris` is under active development and `latest` may break between deployments. + +2. **DB schema init**: Confirm whether Polaris auto-migrates its PostgreSQL schema on first start or requires a separate init job — check release notes for the pinned version before implementing Phase 1. + +3. **Catalog data migration**: Existing tables registered in `iceberg-pg`'s JDBC catalog will not auto-appear in Polaris. The `polaris-bootstrap` job needs a migration step to re-register existing namespaces/tables, or plan for a re-ingest window before retiring `iceberg-rest` in Phase 9. + +4. **OPA for Trino access control (future)**: Trino uses a single `trino-polaris` service identity so Polaris cannot enforce per-user namespace/table ACLs for Trino queries. Open Policy Agent (OPA) — a lightweight Go service on the control plane — can fill this gap. Trino's native OPA system access control plugin receives full query context (user identity, Keycloak groups, target catalog/schema/table) and evaluates Rego policies that mirror the Keycloak role taxonomy. Policy changes hot-reload via ConfigMap without Trino restarts. Would require: new `opa/` Garden module + `access-control.name=opa` in Trino config + policy ConfigMap mirroring the Phase 3 role taxonomy. + +5. **Per-workflow Prefect clients (future)**: Replace single `spark-polaris` client with per-category Keycloak clients (`prefect-ingest`, `prefect-metrics`, etc.), each granted only the namespace roles it needs. Prefect deployment job templates inject credentials via workflow-specific K8s Secrets. `spark_session_utils.py` reads `SPARK_POLARIS_CLIENT_ID` from env rather than hardcoding. No changes needed to Polaris bootstrap or `acl-config.yaml`. diff --git a/polaris-pg/garden.yaml b/polaris-pg/garden.yaml new file mode 100644 index 0000000..b1d7783 --- /dev/null +++ b/polaris-pg/garden.yaml @@ -0,0 +1,12 @@ +kind: Deploy +type: kubernetes +name: polaris-database +description: K8s Deploy Polaris Catalog Database +dependencies: + - deploy.secrets +environments: + - local + - remote +spec: + manifestFiles: + - ./manifests/polaris-pg.yaml diff --git a/polaris-pg/manifests/polaris-pg.yaml b/polaris-pg/manifests/polaris-pg.yaml new file mode 100644 index 0000000..09d1f4d --- /dev/null +++ b/polaris-pg/manifests/polaris-pg.yaml @@ -0,0 +1,102 @@ +apiVersion: v1 +kind: Service +metadata: + name: polaris-pg +spec: + type: ClusterIP + ports: + - port: 5432 + targetPort: 5432 + selector: + app: polaris-pg + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: polaris-pg +spec: + replicas: 1 + selector: + matchLabels: + app: polaris-pg + template: + metadata: + labels: + app: polaris-pg + spec: + # nodeSelector: + # teehr-hub/nodegroup-name: core-a + containers: + - name: postgres + image: postgres:15 + env: + - name: POSTGRES_DB + valueFrom: + secretKeyRef: + name: polaris-db-secrets + key: database + - name: POSTGRES_USER + valueFrom: + secretKeyRef: + name: polaris-db-secrets + key: username + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: polaris-db-secrets + key: password + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + ports: + - containerPort: 5432 + readinessProbe: + exec: + command: + - pg_isready + - -U + - polaris + - -d + - polaris + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + livenessProbe: + exec: + command: + - pg_isready + - -U + - polaris + - -d + - polaris + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + volumeMounts: + - name: pgdata + mountPath: /var/lib/postgresql/data + volumes: + - name: pgdata + persistentVolumeClaim: + claimName: polaris-pg-data + +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: polaris-pg-data +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi diff --git a/polaris/garden.yaml b/polaris/garden.yaml new file mode 100644 index 0000000..3159fe0 --- /dev/null +++ b/polaris/garden.yaml @@ -0,0 +1,15 @@ +kind: Deploy +type: kubernetes +name: polaris +description: K8s Deploy Apache Polaris Iceberg Catalog +dependencies: + - deploy.secrets + - deploy.polaris-database + - deploy.keycloak +environments: + - local + - remote +spec: + manifestTemplates: + - ./manifests/polaris-config.yaml.tpl + - ./manifests/polaris.yaml.tpl diff --git a/polaris/manifests/polaris-config.yaml.tpl b/polaris/manifests/polaris-config.yaml.tpl new file mode 100644 index 0000000..7e30d0f --- /dev/null +++ b/polaris/manifests/polaris-config.yaml.tpl @@ -0,0 +1,42 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: polaris-config +data: + application.properties: | + # Persistence: PostgreSQL via JDBC + polaris.persistence.type=relational-jdbc + polaris.persistence.auto-bootstrap-types=relational-jdbc + polaris.persistence.relational.jdbc.database-type=postgresql + polaris.persistence.relational.jdbc.max-retries=5 + polaris.persistence.relational.jdbc.initial-delay-in-ms=100 + polaris.persistence.relational.jdbc.max-duration-in-ms=5000 + + # Realm context + polaris.realm-context.realms=teehr + + # Authentication: MIXED — internal token service available for bootstrap, + # external JWT (Keycloak) accepted for runtime clients. + polaris.authentication.type=mixed + + # Map JWT preferred_username → Polaris principal name + polaris.oidc.principal-mapper.name-claim-path=preferred_username + + # Map Keycloak realm roles → Polaris PRINCIPAL_ROLE: + polaris.oidc.principal-roles-mapper.mappings[0].regex=(.+) + polaris.oidc.principal-roles-mapper.mappings[0].replacement=PRINCIPAL_ROLE:$1 + + # Quarkus OIDC: use Keycloak for JWT JWKS validation (bearer-only resource server) + quarkus.oidc.application-type=service + # auth-server-url is injected via environment variable per environment + + # Extract Keycloak realm roles from realm_access/roles JWT claim + quarkus.oidc.roles.role-claim-path=realm_access/roles + + # Disable OTEL by default (enable separately if monitoring is wired) + quarkus.otel.sdk.disabled=true + + # Allow S3 and MinIO storage types + polaris.features."SUPPORTED_CATALOG_STORAGE_TYPES"=["S3","GCS","AZURE","FILE"] + polaris.features."ALLOW_INSECURE_STORAGE_TYPES"=true + polaris.readiness.ignore-severe-issues=true diff --git a/polaris/manifests/polaris.yaml.tpl b/polaris/manifests/polaris.yaml.tpl new file mode 100644 index 0000000..aa3567e --- /dev/null +++ b/polaris/manifests/polaris.yaml.tpl @@ -0,0 +1,290 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: polaris-schema +data: + init.sql: | + -- Idempotent schema initialization for Apache Polaris 1.5.0 (PostgreSQL) + -- Safe to run on fresh or already-initialized databases. + CREATE SCHEMA IF NOT EXISTS POLARIS_SCHEMA; + SET search_path TO POLARIS_SCHEMA; + + CREATE TABLE IF NOT EXISTS version ( + version_key TEXT PRIMARY KEY, + version_value INTEGER NOT NULL + ); + INSERT INTO version (version_key, version_value) + VALUES ('version', 4) + ON CONFLICT (version_key) DO UPDATE SET version_value = EXCLUDED.version_value; + COMMENT ON TABLE version IS 'the version of the JDBC schema in use'; + + CREATE TABLE IF NOT EXISTS entities ( + realm_id TEXT NOT NULL, + catalog_id BIGINT NOT NULL, + id BIGINT NOT NULL, + parent_id BIGINT NOT NULL, + name TEXT NOT NULL, + entity_version INT NOT NULL, + type_code INT NOT NULL, + sub_type_code INT NOT NULL, + create_timestamp BIGINT NOT NULL, + drop_timestamp BIGINT NOT NULL, + purge_timestamp BIGINT NOT NULL, + to_purge_timestamp BIGINT NOT NULL, + last_update_timestamp BIGINT NOT NULL, + properties JSONB NOT NULL DEFAULT '{}'::JSONB, + internal_properties JSONB NOT NULL DEFAULT '{}'::JSONB, + grant_records_version INT NOT NULL, + location TEXT, + location_without_scheme TEXT, + PRIMARY KEY (realm_id, id), + CONSTRAINT constraint_name UNIQUE (realm_id, catalog_id, parent_id, type_code, name) + ); + -- Idempotent column migrations for databases created from older schema versions + ALTER TABLE entities ADD COLUMN IF NOT EXISTS location TEXT; + ALTER TABLE entities ADD COLUMN IF NOT EXISTS location_without_scheme TEXT; + + CREATE INDEX IF NOT EXISTS idx_entities ON entities (realm_id, catalog_id, id); + CREATE INDEX IF NOT EXISTS idx_entities_catalog_id_id ON entities (catalog_id, id); + CREATE INDEX IF NOT EXISTS idx_locations ON entities (realm_id, parent_id, location); + + CREATE TABLE IF NOT EXISTS grant_records ( + catalog_id BIGINT NOT NULL, + privilege_type TEXT NOT NULL, + securable_type TEXT NOT NULL, + securable_id BIGINT NOT NULL, + grantee_type TEXT NOT NULL, + grantee_id BIGINT NOT NULL, + realm_id TEXT NOT NULL, + PRIMARY KEY (realm_id, securable_id, grantee_id, privilege_type) + ); + CREATE INDEX IF NOT EXISTS idx_grant_records ON grant_records (realm_id, catalog_id, securable_id); + CREATE INDEX IF NOT EXISTS idx_grant_records_grantee ON grant_records (realm_id, grantee_id); + + CREATE TABLE IF NOT EXISTS principal_authentication_data ( + realm_id TEXT NOT NULL, + principal_id BIGINT NOT NULL, + main_secret_hash TEXT NOT NULL, + secondary_secret_hash TEXT, + created_at BIGINT NOT NULL, + PRIMARY KEY (realm_id, principal_id) + ); + + CREATE TABLE IF NOT EXISTS policy_mapping_record ( + realm_id TEXT NOT NULL, + catalog_id BIGINT NOT NULL, + target_type_code INT NOT NULL, + target_id BIGINT NOT NULL, + policy_type_code INT NOT NULL, + policy_id BIGINT NOT NULL, + parameters JSONB, + PRIMARY KEY (realm_id, target_id, policy_id) + ); + COMMENT ON TABLE policy_mapping_record IS 'stores attachments of policies to targets'; + CREATE INDEX IF NOT EXISTS idx_policy_mapping_record ON policy_mapping_record (realm_id, catalog_id, target_type_code, target_id); + + CREATE TABLE IF NOT EXISTS events ( + id BIGSERIAL PRIMARY KEY, + realm_id TEXT NOT NULL, + timestamp BIGINT NOT NULL, + event_type TEXT NOT NULL, + event_data JSONB NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_events ON events (realm_id, timestamp); + + CREATE TABLE IF NOT EXISTS idempotency_records ( + realm_id TEXT NOT NULL, + request_id TEXT NOT NULL, + response_code INT NOT NULL, + response_body TEXT, + created_at BIGINT NOT NULL, + expires_at BIGINT NOT NULL, + PRIMARY KEY (realm_id, request_id) + ); + COMMENT ON TABLE idempotency_records IS 'stores idempotency records for REST requests'; + CREATE INDEX IF NOT EXISTS idx_idempotency_records ON idempotency_records (realm_id, request_id); + CREATE INDEX IF NOT EXISTS idx_idempotency_records_expiry ON idempotency_records (expires_at); + + CREATE TABLE IF NOT EXISTS commit_metrics_report ( + id BIGSERIAL PRIMARY KEY, + realm_id TEXT NOT NULL, + catalog_id BIGINT NOT NULL, + timestamp BIGINT NOT NULL, + metrics JSONB NOT NULL + ); + COMMENT ON TABLE commit_metrics_report IS 'stores commit metrics reports'; + CREATE INDEX IF NOT EXISTS idx_commit_metrics_report ON commit_metrics_report (realm_id, catalog_id, timestamp); + + CREATE TABLE IF NOT EXISTS scan_metrics_report ( + id BIGSERIAL PRIMARY KEY, + realm_id TEXT NOT NULL, + catalog_id BIGINT NOT NULL, + timestamp BIGINT NOT NULL, + metrics JSONB NOT NULL + ); + COMMENT ON TABLE scan_metrics_report IS 'stores scan metrics reports'; + CREATE INDEX IF NOT EXISTS idx_scan_metrics_report ON scan_metrics_report (realm_id, catalog_id, timestamp); +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: polaris + namespace: ${environment.namespace} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: polaris + name: polaris +spec: + replicas: 1 + selector: + matchLabels: + app: polaris + template: + metadata: + labels: + app: polaris + spec: + serviceAccountName: polaris + # nodeSelector: + # teehr-hub/nodegroup-name: core-a + initContainers: + - name: schema-init + image: postgres:15 + env: + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: polaris-db-secrets + key: password + - name: PGUSER + valueFrom: + secretKeyRef: + name: polaris-db-secrets + key: username + command: + - sh + - -c + - | + until pg_isready -h polaris-pg -U "$PGUSER"; do + echo "Waiting for polaris-pg..."; sleep 2; + done + psql -h polaris-pg -U "$PGUSER" -d polaris -f /schema/init.sql + volumeMounts: + - name: polaris-schema + mountPath: /schema + containers: + - name: polaris + # TODO: pin to a specific release tag before production use (e.g., apache/polaris:1.0.0) + image: apache/polaris:latest + imagePullPolicy: IfNotPresent + env: + # Bootstrap root credentials — format: realm,clientId,clientSecret + - name: POLARIS_BOOTSTRAP_CREDENTIALS + valueFrom: + secretKeyRef: + name: polaris-secrets + key: bootstrap-credentials + # Quarkus datasource (JDBC connection to polaris-pg) + - name: QUARKUS_DATASOURCE_JDBC_URL + value: jdbc:postgresql://polaris-pg:5432/polaris + - name: QUARKUS_DATASOURCE_USERNAME + valueFrom: + secretKeyRef: + name: polaris-db-secrets + key: username + - name: QUARKUS_DATASOURCE_PASSWORD + valueFrom: + secretKeyRef: + name: polaris-db-secrets + key: password + # OIDC: Keycloak JWKS endpoint for JWT validation (environment-specific) + ${if environment.name == "local"} + - name: QUARKUS_OIDC_AUTH_SERVER_URL + value: ${var.polaris.oauthServerUri} + # Local MinIO S3 credentials + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: minio-secrets + key: accesskey + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: minio-secrets + key: secretkey + - name: AWS_S3_ENDPOINT + value: ${var.polaris.catalogS3Endpoint} + - name: AWS_S3_PATH_STYLE_ACCESS + value: "${var.polaris.catalogS3PathStyleAccess}" + ${endif} + ${if environment.name != "local"} + - name: QUARKUS_OIDC_AUTH_SERVER_URL + value: ${var.polaris.oauthServerUri} + ${endif} + - name: AWS_REGION + value: us-east-2 + ports: + - name: api + containerPort: 8181 + protocol: TCP + - name: management + containerPort: 8182 + protocol: TCP + readinessProbe: + httpGet: + path: /q/health/ready + port: 8182 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + livenessProbe: + httpGet: + path: /q/health/live + port: 8182 + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + resources: + requests: + cpu: 200m + memory: 512Mi + limits: + cpu: 1000m + memory: 1Gi + volumeMounts: + - name: polaris-config + mountPath: /deployments/config/application.properties + subPath: application.properties + readOnly: true + volumes: + - name: polaris-config + configMap: + name: polaris-config + - name: polaris-schema + configMap: + name: polaris-schema + +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: polaris + name: polaris +spec: + ports: + - name: api + protocol: TCP + port: 8181 + targetPort: 8181 + - name: management + protocol: TCP + port: 8182 + targetPort: 8182 + selector: + app: polaris diff --git a/secrets/secrets.local.yaml b/secrets/secrets.local.yaml index 5e806e8..1d7e1cd 100644 --- a/secrets/secrets.local.yaml +++ b/secrets/secrets.local.yaml @@ -43,4 +43,20 @@ secrets: secretkey: minioadmin123 prefect-workflow-secrets: data: - api-usgs-pat: CHANGE_ME_API_USGS_PAT \ No newline at end of file + api-usgs-pat: CHANGE_ME_API_USGS_PAT + polaris-db-secrets: + data: + database: polaris + username: polaris + password: polaris123 + polaris-secrets: + data: + root-credentials: "root:secret123" + bootstrap-credentials: "teehr,root,secret123" + trino-polaris-secrets: + data: + client-secret: local-trino-polaris-client-secret + credential: "trino-polaris:local-trino-polaris-client-secret" + spark-polaris-secrets: + data: + client-secret: local-spark-polaris-client-secret \ No newline at end of file diff --git a/trino/garden.yaml b/trino/garden.yaml index abcdfd3..56c7e7b 100644 --- a/trino/garden.yaml +++ b/trino/garden.yaml @@ -12,7 +12,7 @@ spec: name: trino repo: https://trinodb.github.io/charts version: 1.41.0 - + values: server: workers: 1 @@ -26,9 +26,13 @@ spec: catalogs: iceberg: |- connector.name=iceberg - iceberg.catalog.type=${var.iceberg.catalogType} - iceberg.rest-catalog.uri=${var.iceberg.catalogUri} - iceberg.rest-catalog.warehouse=${var.iceberg.catalogWarehouse} + iceberg.catalog.type=${var.polaris.catalogType} + iceberg.rest-catalog.uri=${var.polaris.catalogUri} + iceberg.rest-catalog.warehouse=${var.polaris.catalogWarehouse} + iceberg.rest-catalog.security=OAUTH2 + iceberg.rest-catalog.oauth2.server-uri=${var.polaris.oauthServerUri} + iceberg.rest-catalog.oauth2.credential=$${ENV:POLARIS_CREDENTIAL} + iceberg.rest-catalog.oauth2.scope=openid # S3 Configuration fs.native-s3.enabled=true s3.path-style-access=${var.iceberg.catalogS3PathStyleAccess} @@ -61,6 +65,11 @@ spec: secretKeyRef: name: minio-secrets key: secretkey + - name: POLARIS_CREDENTIAL + valueFrom: + secretKeyRef: + name: trino-polaris-secrets + key: credential --- # Remote environment deployment kind: Deploy @@ -76,7 +85,7 @@ spec: name: trino repo: https://trinodb.github.io/charts version: 1.41.0 - + values: server: workers: "${environment.name == 'local' ? 1 : 2}" @@ -106,9 +115,13 @@ spec: catalogs: iceberg: |- connector.name=iceberg - iceberg.catalog.type=${var.iceberg.catalogType} - iceberg.rest-catalog.uri=${var.iceberg.catalogUri} - iceberg.rest-catalog.warehouse=${var.iceberg.catalogWarehouse} + iceberg.catalog.type=${var.polaris.catalogType} + iceberg.rest-catalog.uri=${var.polaris.catalogUri} + iceberg.rest-catalog.warehouse=${var.polaris.catalogWarehouse} + iceberg.rest-catalog.security=OAUTH2 + iceberg.rest-catalog.oauth2.server-uri=${var.polaris.oauthServerUri} + iceberg.rest-catalog.oauth2.credential=$${ENV:POLARIS_CREDENTIAL} + iceberg.rest-catalog.oauth2.scope=openid # S3 Configuration fs.native-s3.enabled=true s3.region=${var.aws.region} @@ -128,6 +141,23 @@ spec: ] } + coordinator: + extraVolumeMounts: [] + resources: + requests: + cpu: 1 + memory: 4Gi + limits: + cpu: 2 + memory: 8Gi + + env: + - name: POLARIS_CREDENTIAL + valueFrom: + secretKeyRef: + name: trino-polaris-secrets + key: credential + serviceAccount: create: true name: trino From bba57e9ef6c3ab718aa603a9b2c9f61efc433dbf Mon Sep 17 00:00:00 2001 From: samlamont Date: Thu, 4 Jun 2026 08:41:56 -0400 Subject: [PATCH 02/62] wip --- .../manifests/bootstrap-job.yaml | 4 +- polaris/manifests/polaris-config.yaml.tpl | 27 +-- polaris/manifests/polaris.yaml.tpl | 199 ++++-------------- 3 files changed, 50 insertions(+), 180 deletions(-) diff --git a/polaris-bootstrap/manifests/bootstrap-job.yaml b/polaris-bootstrap/manifests/bootstrap-job.yaml index 1f6697c..772e0c9 100644 --- a/polaris-bootstrap/manifests/bootstrap-job.yaml +++ b/polaris-bootstrap/manifests/bootstrap-job.yaml @@ -49,7 +49,7 @@ spec: # Obtain initial token from Polaris internal token service token_resp = requests.post( f"{MGMT_URL}/api/catalog/v1/oauth/tokens", - headers={"X-Polaris-Realm": REALM}, + headers={"X-Iceberg-Access-Realm": REALM}, data={ "grant_type": "client_credentials", "client_id": username, @@ -63,7 +63,7 @@ spec: headers = { "Authorization": f"Bearer {token}", - "X-Polaris-Realm": REALM, + "X-Iceberg-Access-Realm": REALM, "Content-Type": "application/json", } diff --git a/polaris/manifests/polaris-config.yaml.tpl b/polaris/manifests/polaris-config.yaml.tpl index 7e30d0f..b9bbc55 100644 --- a/polaris/manifests/polaris-config.yaml.tpl +++ b/polaris/manifests/polaris-config.yaml.tpl @@ -4,39 +4,32 @@ metadata: name: polaris-config data: application.properties: | - # Persistence: PostgreSQL via JDBC + # Persistence: PostgreSQL Configuration polaris.persistence.type=relational-jdbc - polaris.persistence.auto-bootstrap-types=relational-jdbc polaris.persistence.relational.jdbc.database-type=postgresql + + quarkus.datasource.db-kind=postgresql + quarkus.datasource.jdbc.url=jdbc:postgresql://polaris-pg:5432/polaris + quarkus.datasource.username=polaris + polaris.persistence.relational.jdbc.max-retries=5 polaris.persistence.relational.jdbc.initial-delay-in-ms=100 polaris.persistence.relational.jdbc.max-duration-in-ms=5000 - # Realm context - polaris.realm-context.realms=teehr - - # Authentication: MIXED — internal token service available for bootstrap, - # external JWT (Keycloak) accepted for runtime clients. + # Authentication Context Configuration polaris.authentication.type=mixed - - # Map JWT preferred_username → Polaris principal name polaris.oidc.principal-mapper.name-claim-path=preferred_username - # Map Keycloak realm roles → Polaris PRINCIPAL_ROLE: + # In Polaris 1.5.0, mappings are structured as direct lists polaris.oidc.principal-roles-mapper.mappings[0].regex=(.+) polaris.oidc.principal-roles-mapper.mappings[0].replacement=PRINCIPAL_ROLE:$1 - # Quarkus OIDC: use Keycloak for JWT JWKS validation (bearer-only resource server) + # Quarkus OIDC Service Identity Configuration quarkus.oidc.application-type=service - # auth-server-url is injected via environment variable per environment - - # Extract Keycloak realm roles from realm_access/roles JWT claim quarkus.oidc.roles.role-claim-path=realm_access/roles - - # Disable OTEL by default (enable separately if monitoring is wired) quarkus.otel.sdk.disabled=true - # Allow S3 and MinIO storage types + # Storage Properties Integration polaris.features."SUPPORTED_CATALOG_STORAGE_TYPES"=["S3","GCS","AZURE","FILE"] polaris.features."ALLOW_INSECURE_STORAGE_TYPES"=true polaris.readiness.ignore-severe-issues=true diff --git a/polaris/manifests/polaris.yaml.tpl b/polaris/manifests/polaris.yaml.tpl index aa3567e..2439079 100644 --- a/polaris/manifests/polaris.yaml.tpl +++ b/polaris/manifests/polaris.yaml.tpl @@ -1,131 +1,4 @@ apiVersion: v1 -kind: ConfigMap -metadata: - name: polaris-schema -data: - init.sql: | - -- Idempotent schema initialization for Apache Polaris 1.5.0 (PostgreSQL) - -- Safe to run on fresh or already-initialized databases. - CREATE SCHEMA IF NOT EXISTS POLARIS_SCHEMA; - SET search_path TO POLARIS_SCHEMA; - - CREATE TABLE IF NOT EXISTS version ( - version_key TEXT PRIMARY KEY, - version_value INTEGER NOT NULL - ); - INSERT INTO version (version_key, version_value) - VALUES ('version', 4) - ON CONFLICT (version_key) DO UPDATE SET version_value = EXCLUDED.version_value; - COMMENT ON TABLE version IS 'the version of the JDBC schema in use'; - - CREATE TABLE IF NOT EXISTS entities ( - realm_id TEXT NOT NULL, - catalog_id BIGINT NOT NULL, - id BIGINT NOT NULL, - parent_id BIGINT NOT NULL, - name TEXT NOT NULL, - entity_version INT NOT NULL, - type_code INT NOT NULL, - sub_type_code INT NOT NULL, - create_timestamp BIGINT NOT NULL, - drop_timestamp BIGINT NOT NULL, - purge_timestamp BIGINT NOT NULL, - to_purge_timestamp BIGINT NOT NULL, - last_update_timestamp BIGINT NOT NULL, - properties JSONB NOT NULL DEFAULT '{}'::JSONB, - internal_properties JSONB NOT NULL DEFAULT '{}'::JSONB, - grant_records_version INT NOT NULL, - location TEXT, - location_without_scheme TEXT, - PRIMARY KEY (realm_id, id), - CONSTRAINT constraint_name UNIQUE (realm_id, catalog_id, parent_id, type_code, name) - ); - -- Idempotent column migrations for databases created from older schema versions - ALTER TABLE entities ADD COLUMN IF NOT EXISTS location TEXT; - ALTER TABLE entities ADD COLUMN IF NOT EXISTS location_without_scheme TEXT; - - CREATE INDEX IF NOT EXISTS idx_entities ON entities (realm_id, catalog_id, id); - CREATE INDEX IF NOT EXISTS idx_entities_catalog_id_id ON entities (catalog_id, id); - CREATE INDEX IF NOT EXISTS idx_locations ON entities (realm_id, parent_id, location); - - CREATE TABLE IF NOT EXISTS grant_records ( - catalog_id BIGINT NOT NULL, - privilege_type TEXT NOT NULL, - securable_type TEXT NOT NULL, - securable_id BIGINT NOT NULL, - grantee_type TEXT NOT NULL, - grantee_id BIGINT NOT NULL, - realm_id TEXT NOT NULL, - PRIMARY KEY (realm_id, securable_id, grantee_id, privilege_type) - ); - CREATE INDEX IF NOT EXISTS idx_grant_records ON grant_records (realm_id, catalog_id, securable_id); - CREATE INDEX IF NOT EXISTS idx_grant_records_grantee ON grant_records (realm_id, grantee_id); - - CREATE TABLE IF NOT EXISTS principal_authentication_data ( - realm_id TEXT NOT NULL, - principal_id BIGINT NOT NULL, - main_secret_hash TEXT NOT NULL, - secondary_secret_hash TEXT, - created_at BIGINT NOT NULL, - PRIMARY KEY (realm_id, principal_id) - ); - - CREATE TABLE IF NOT EXISTS policy_mapping_record ( - realm_id TEXT NOT NULL, - catalog_id BIGINT NOT NULL, - target_type_code INT NOT NULL, - target_id BIGINT NOT NULL, - policy_type_code INT NOT NULL, - policy_id BIGINT NOT NULL, - parameters JSONB, - PRIMARY KEY (realm_id, target_id, policy_id) - ); - COMMENT ON TABLE policy_mapping_record IS 'stores attachments of policies to targets'; - CREATE INDEX IF NOT EXISTS idx_policy_mapping_record ON policy_mapping_record (realm_id, catalog_id, target_type_code, target_id); - - CREATE TABLE IF NOT EXISTS events ( - id BIGSERIAL PRIMARY KEY, - realm_id TEXT NOT NULL, - timestamp BIGINT NOT NULL, - event_type TEXT NOT NULL, - event_data JSONB NOT NULL - ); - CREATE INDEX IF NOT EXISTS idx_events ON events (realm_id, timestamp); - - CREATE TABLE IF NOT EXISTS idempotency_records ( - realm_id TEXT NOT NULL, - request_id TEXT NOT NULL, - response_code INT NOT NULL, - response_body TEXT, - created_at BIGINT NOT NULL, - expires_at BIGINT NOT NULL, - PRIMARY KEY (realm_id, request_id) - ); - COMMENT ON TABLE idempotency_records IS 'stores idempotency records for REST requests'; - CREATE INDEX IF NOT EXISTS idx_idempotency_records ON idempotency_records (realm_id, request_id); - CREATE INDEX IF NOT EXISTS idx_idempotency_records_expiry ON idempotency_records (expires_at); - - CREATE TABLE IF NOT EXISTS commit_metrics_report ( - id BIGSERIAL PRIMARY KEY, - realm_id TEXT NOT NULL, - catalog_id BIGINT NOT NULL, - timestamp BIGINT NOT NULL, - metrics JSONB NOT NULL - ); - COMMENT ON TABLE commit_metrics_report IS 'stores commit metrics reports'; - CREATE INDEX IF NOT EXISTS idx_commit_metrics_report ON commit_metrics_report (realm_id, catalog_id, timestamp); - - CREATE TABLE IF NOT EXISTS scan_metrics_report ( - id BIGSERIAL PRIMARY KEY, - realm_id TEXT NOT NULL, - catalog_id BIGINT NOT NULL, - timestamp BIGINT NOT NULL, - metrics JSONB NOT NULL - ); - COMMENT ON TABLE scan_metrics_report IS 'stores scan metrics reports'; - CREATE INDEX IF NOT EXISTS idx_scan_metrics_report ON scan_metrics_report (realm_id, catalog_id, timestamp); ---- -apiVersion: v1 kind: ServiceAccount metadata: name: polaris @@ -148,46 +21,55 @@ spec: app: polaris spec: serviceAccountName: polaris - # nodeSelector: - # teehr-hub/nodegroup-name: core-a initContainers: - - name: schema-init - image: postgres:15 + - name: schema-bootstrap + image: apache/polaris-admin-tool:1.5.0 + imagePullPolicy: IfNotPresent + args: + - bootstrap + - --realm=teehr + - -c=teehr,root,secret123 + - -p env: - - name: PGPASSWORD + # Core persistence assignment + - name: POLARIS_PERSISTENCE_TYPE + value: relational-jdbc + - name: POLARIS_PERSISTENCE_AUTO_BOOTSTRAP_TYPES + value: relational-jdbc + - name: POLARIS_REALM_CONTEXT_REALMS + value: teehr + + # FIXED: Explicit lowercase system translations to force Agroal activation + - name: quarkus_datasource_db-kind + value: postgresql + - name: quarkus_datasource_jdbc_url + value: jdbc:postgresql://polaris-pg:5432/polaris + - name: quarkus_datasource_username + value: polaris + + # Fetch secret credentials cleanly + - name: quarkus_datasource_password valueFrom: secretKeyRef: name: polaris-db-secrets key: password - - name: PGUSER - valueFrom: - secretKeyRef: - name: polaris-db-secrets - key: username - command: - - sh - - -c - - | - until pg_isready -h polaris-pg -U "$PGUSER"; do - echo "Waiting for polaris-pg..."; sleep 2; - done - psql -h polaris-pg -U "$PGUSER" -d polaris -f /schema/init.sql volumeMounts: - - name: polaris-schema - mountPath: /schema + - name: polaris-config + mountPath: /deployments/config/application.properties + subPath: application.properties + readOnly: true + containers: - name: polaris - # TODO: pin to a specific release tag before production use (e.g., apache/polaris:1.0.0) - image: apache/polaris:latest + image: apache/polaris:1.5.0 imagePullPolicy: IfNotPresent env: - # Bootstrap root credentials — format: realm,clientId,clientSecret - - name: POLARIS_BOOTSTRAP_CREDENTIALS - valueFrom: - secretKeyRef: - name: polaris-secrets - key: bootstrap-credentials - # Quarkus datasource (JDBC connection to polaris-pg) + # FIX 1: Force the server runtime to recognize your specific realm! + - name: POLARIS_REALM_CONTEXT_REALMS + value: teehr + # Keep main container aligned to the relational-jdbc metastore + - name: POLARIS_PERSISTENCE_TYPE + value: relational-jdbc - name: QUARKUS_DATASOURCE_JDBC_URL value: jdbc:postgresql://polaris-pg:5432/polaris - name: QUARKUS_DATASOURCE_USERNAME @@ -200,11 +82,9 @@ spec: secretKeyRef: name: polaris-db-secrets key: password - # OIDC: Keycloak JWKS endpoint for JWT validation (environment-specific) ${if environment.name == "local"} - name: QUARKUS_OIDC_AUTH_SERVER_URL value: ${var.polaris.oauthServerUri} - # Local MinIO S3 credentials - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: @@ -265,9 +145,6 @@ spec: - name: polaris-config configMap: name: polaris-config - - name: polaris-schema - configMap: - name: polaris-schema --- apiVersion: v1 From d82cfe7c3376bdc4c2d2c1a14fa10807a85e5cb8 Mon Sep 17 00:00:00 2001 From: samlamont Date: Fri, 5 Jun 2026 08:39:30 -0400 Subject: [PATCH 03/62] copilot updates for multi-realm generalization --- jupyterhub/garden.yaml | 26 +- polaris-bootstrap/garden.yaml | 2 +- polaris-bootstrap/manifests/acl-config.yaml | 111 +++++-- .../manifests/acl-config.yaml.tpl | 112 +++++-- .../manifests/bootstrap-job.yaml | 314 ++++++++++++------ polaris/manifests/polaris-config.yaml.tpl | 4 - polaris/manifests/polaris.yaml.tpl | 28 +- trino/garden.yaml | 2 + 8 files changed, 400 insertions(+), 199 deletions(-) diff --git a/jupyterhub/garden.yaml b/jupyterhub/garden.yaml index 12d119a..bdcdcd0 100644 --- a/jupyterhub/garden.yaml +++ b/jupyterhub/garden.yaml @@ -150,12 +150,13 @@ spec: secretKeyRef: name: jupyter-user-secrets key: api-key - REMOTE_CATALOG_TYPE: ${var.iceberg.catalogType} - REMOTE_CATALOG_REST_URI: ${var.iceberg.catalogUri} - REMOTE_WAREHOUSE_S3_PATH: ${var.iceberg.catalogWarehouse} - REMOTE_CATALOG_S3_ENDPOINT: ${var.iceberg.catalogS3Endpoint} - REMOTE_CATALOG_S3_PATH_STYLE_ACCESS: ${var.iceberg.catalogS3PathStyleAccess} - IN_CLUSTER: ${var.iceberg.inCluster} + REMOTE_CATALOG_TYPE: ${var.polaris.catalogType} + REMOTE_CATALOG_REST_URI: ${var.polaris.catalogUri} + REMOTE_WAREHOUSE_S3_PATH: ${var.polaris.catalogWarehouse} + REMOTE_CATALOG_S3_ENDPOINT: ${var.polaris.catalogS3Endpoint} + REMOTE_CATALOG_S3_PATH_STYLE_ACCESS: ${var.polaris.catalogS3PathStyleAccess} + IN_CLUSTER: ${var.polaris.inCluster} + POLARIS_DEFAULT_REALM: ${var.polaris.defaultRealm} extraFiles: jupyter_config: mountPath: /etc/jupyter/jupyter_notebook_config.py @@ -332,12 +333,13 @@ spec: TEEHR_SPARK_IMAGE: ${actions.build.teehr-spark-executor-image.outputs.deploymentImageId} TEEHR_NAMESPACE: ${environment.namespace} AWS_REGION: ${var.aws.region} - REMOTE_CATALOG_TYPE: ${var.iceberg.catalogType} - REMOTE_CATALOG_REST_URI: ${var.iceberg.catalogUri} - REMOTE_WAREHOUSE_S3_PATH: ${var.iceberg.catalogWarehouse} - REMOTE_CATALOG_S3_ENDPOINT: ${var.iceberg.catalogS3Endpoint} - REMOTE_CATALOG_S3_PATH_STYLE_ACCESS: ${var.iceberg.catalogS3PathStyleAccess} - IN_CLUSTER: ${var.iceberg.inCluster} + REMOTE_CATALOG_TYPE: ${var.polaris.catalogType} + REMOTE_CATALOG_REST_URI: ${var.polaris.catalogUri} + REMOTE_WAREHOUSE_S3_PATH: ${var.polaris.catalogWarehouse} + REMOTE_CATALOG_S3_ENDPOINT: ${var.polaris.catalogS3Endpoint} + REMOTE_CATALOG_S3_PATH_STYLE_ACCESS: ${var.polaris.catalogS3PathStyleAccess} + IN_CLUSTER: ${var.polaris.inCluster} + POLARIS_DEFAULT_REALM: ${var.polaris.defaultRealm} TRINO_HOST: ${var.trino.host} TRINO_PORT: ${var.trino.port} TRINO_CATALOG: ${var.trino.catalog} diff --git a/polaris-bootstrap/garden.yaml b/polaris-bootstrap/garden.yaml index e7a44ed..27364a4 100644 --- a/polaris-bootstrap/garden.yaml +++ b/polaris-bootstrap/garden.yaml @@ -29,4 +29,4 @@ spec: - >- kubectl -n "${environment.namespace}" delete job polaris-bootstrap --ignore-not-found=true && kubectl -n "${environment.namespace}" apply -f ./manifests/bootstrap-job.yaml && - kubectl -n "${environment.namespace}" wait --for=condition=Complete job/polaris-bootstrap --timeout=120s + kubectl -n "${environment.namespace}" wait --for=condition=Complete job/polaris-bootstrap --timeout=600s diff --git a/polaris-bootstrap/manifests/acl-config.yaml b/polaris-bootstrap/manifests/acl-config.yaml index 4dbd2fd..a73a8ec 100644 --- a/polaris-bootstrap/manifests/acl-config.yaml +++ b/polaris-bootstrap/manifests/acl-config.yaml @@ -5,43 +5,90 @@ metadata: data: acl-config.json: | { - "realm": "teehr", - "catalog": "teehr", - "catalog_admin_keycloak_role": "iceberg-catalog-admin", - "namespaces": [ + "realms": [ { - "name": "public", - "roles": [ + "realm": "teehr", + "catalog": "teehr", + "warehouse": "s3a://teehr-warehouse/polaris", + "storage_type": "S3", + "allowed_locations": [ + "s3a://teehr-warehouse/polaris" + ], + "admin": { + "principal_role": "iceberg-catalog-admin", + "catalog_role": "catalog_admin_role", + "privileges": [ + "CATALOG_MANAGE_CONTENT", + "CATALOG_MANAGE_METADATA" + ] + }, + "namespace_policies": [ { - "keycloak_role": "iceberg-namespace-public-read", - "polaris_principal_role": "public_reader", - "polaris_catalog_role": "public_read_role", - "privileges": ["TABLE_READ_DATA", "TABLE_LIST", "NAMESPACE_LIST"] + "namespace": "public", + "roles": [ + { + "principal_role": "iceberg-namespace-public-read", + "catalog_role": "public_read_role", + "grants": [ + { + "type": "namespace", + "privileges": [ + "NAMESPACE_READ_PROPERTIES" + ] + } + ] + }, + { + "principal_role": "iceberg-namespace-public-write", + "catalog_role": "public_write_role", + "grants": [ + { + "type": "namespace", + "privileges": [ + "NAMESPACE_READ_PROPERTIES", + "NAMESPACE_WRITE_PROPERTIES", + "TABLE_CREATE", + "TABLE_DROP" + ] + } + ] + } + ] }, { - "keycloak_role": "iceberg-namespace-public-write", - "polaris_principal_role": "public_writer", - "polaris_catalog_role": "public_write_role", - "privileges": ["TABLE_WRITE_DATA", "TABLE_READ_DATA", "TABLE_LIST", "NAMESPACE_LIST", "CREATE_TABLE"] + "namespace": "restricted", + "roles": [ + { + "principal_role": "iceberg-namespace-restricted-read", + "catalog_role": "restricted_read_role", + "grants": [ + { + "type": "namespace", + "privileges": [ + "NAMESPACE_READ_PROPERTIES" + ] + } + ] + }, + { + "principal_role": "iceberg-namespace-restricted-write", + "catalog_role": "restricted_write_role", + "grants": [ + { + "type": "namespace", + "privileges": [ + "NAMESPACE_READ_PROPERTIES", + "NAMESPACE_WRITE_PROPERTIES", + "TABLE_CREATE", + "TABLE_DROP" + ] + } + ] + } + ] } - ] - }, - { - "name": "restricted", - "roles": [ - { - "keycloak_role": "iceberg-namespace-restricted-read", - "polaris_principal_role": "restricted_reader", - "polaris_catalog_role": "restricted_read_role", - "privileges": ["TABLE_READ_DATA", "TABLE_LIST", "NAMESPACE_LIST"] - }, - { - "keycloak_role": "iceberg-namespace-restricted-write", - "polaris_principal_role": "restricted_writer", - "polaris_catalog_role": "restricted_write_role", - "privileges": ["TABLE_WRITE_DATA", "TABLE_READ_DATA", "TABLE_LIST", "NAMESPACE_LIST", "CREATE_TABLE"] - } - ] + ], + "table_policies": [] } ] } diff --git a/polaris-bootstrap/manifests/acl-config.yaml.tpl b/polaris-bootstrap/manifests/acl-config.yaml.tpl index a95e5a6..ca3c167 100644 --- a/polaris-bootstrap/manifests/acl-config.yaml.tpl +++ b/polaris-bootstrap/manifests/acl-config.yaml.tpl @@ -3,46 +3,92 @@ kind: ConfigMap metadata: name: polaris-acl-config data: - warehouse: "${var.polaris.catalogWarehouse}" acl-config.json: | { - "realm": "teehr", - "catalog": "teehr", - "catalog_admin_keycloak_role": "iceberg-catalog-admin", - "namespaces": [ + "realms": [ { - "name": "public", - "roles": [ + "realm": "${var.polaris.defaultRealm}", + "catalog": "${var.polaris.defaultRealm}", + "warehouse": "${var.polaris.catalogWarehouse}", + "storage_type": "S3", + "allowed_locations": [ + "${var.polaris.catalogWarehouse}" + ], + "admin": { + "principal_role": "iceberg-catalog-admin", + "catalog_role": "catalog_admin_role", + "privileges": [ + "CATALOG_MANAGE_CONTENT", + "CATALOG_MANAGE_METADATA" + ] + }, + "namespace_policies": [ { - "keycloak_role": "iceberg-namespace-public-read", - "polaris_principal_role": "public_reader", - "polaris_catalog_role": "public_read_role", - "privileges": ["TABLE_READ_DATA", "TABLE_LIST", "NAMESPACE_LIST"] + "namespace": "public", + "roles": [ + { + "principal_role": "iceberg-namespace-public-read", + "catalog_role": "public_read_role", + "grants": [ + { + "type": "namespace", + "privileges": [ + "NAMESPACE_READ_PROPERTIES" + ] + } + ] + }, + { + "principal_role": "iceberg-namespace-public-write", + "catalog_role": "public_write_role", + "grants": [ + { + "type": "namespace", + "privileges": [ + "NAMESPACE_READ_PROPERTIES", + "NAMESPACE_WRITE_PROPERTIES", + "TABLE_CREATE", + "TABLE_DROP" + ] + } + ] + } + ] }, { - "keycloak_role": "iceberg-namespace-public-write", - "polaris_principal_role": "public_writer", - "polaris_catalog_role": "public_write_role", - "privileges": ["TABLE_WRITE_DATA", "TABLE_READ_DATA", "TABLE_LIST", "NAMESPACE_LIST", "CREATE_TABLE"] + "namespace": "restricted", + "roles": [ + { + "principal_role": "iceberg-namespace-restricted-read", + "catalog_role": "restricted_read_role", + "grants": [ + { + "type": "namespace", + "privileges": [ + "NAMESPACE_READ_PROPERTIES" + ] + } + ] + }, + { + "principal_role": "iceberg-namespace-restricted-write", + "catalog_role": "restricted_write_role", + "grants": [ + { + "type": "namespace", + "privileges": [ + "NAMESPACE_READ_PROPERTIES", + "NAMESPACE_WRITE_PROPERTIES", + "TABLE_CREATE", + "TABLE_DROP" + ] + } + ] + } + ] } - ] - }, - { - "name": "restricted", - "roles": [ - { - "keycloak_role": "iceberg-namespace-restricted-read", - "polaris_principal_role": "restricted_reader", - "polaris_catalog_role": "restricted_read_role", - "privileges": ["TABLE_READ_DATA", "TABLE_LIST", "NAMESPACE_LIST"] - }, - { - "keycloak_role": "iceberg-namespace-restricted-write", - "polaris_principal_role": "restricted_writer", - "polaris_catalog_role": "restricted_write_role", - "privileges": ["TABLE_WRITE_DATA", "TABLE_READ_DATA", "TABLE_LIST", "NAMESPACE_LIST", "CREATE_TABLE"] - } - ] + ], + "table_policies": [] } ] } diff --git a/polaris-bootstrap/manifests/bootstrap-job.yaml b/polaris-bootstrap/manifests/bootstrap-job.yaml index 772e0c9..bae5429 100644 --- a/polaris-bootstrap/manifests/bootstrap-job.yaml +++ b/polaris-bootstrap/manifests/bootstrap-job.yaml @@ -23,8 +23,6 @@ spec: secretKeyRef: name: polaris-secrets key: root-credentials - - name: POLARIS_REALM_NAME - value: "teehr" command: - python - -c @@ -32,136 +30,236 @@ spec: import json import os import sys + import requests - MGMT_URL = os.environ["POLARIS_MANAGEMENT_URL"] + MGMT_URL = os.environ.get("POLARIS_MANAGEMENT_URL", "http://polaris:8181") ROOT_CREDS = os.environ["POLARIS_ROOT_CREDENTIALS"] - REALM = os.environ.get("POLARIS_REALM_NAME", "teehr") + username, password = ROOT_CREDS.split(":", 1) - # Read warehouse path from ConfigMap volume mount - with open("/config/warehouse") as f: - WAREHOUSE = f.read().strip() + SUCCESS_CODES = (200, 201, 204, 409) - username, password = ROOT_CREDS.split(":", 1) + def is_duplicate_error(resp): + return resp.status_code == 500 and "duplicate key" in resp.text.lower() + + def realm_headers(realm, token=None, content_type="application/json"): + headers = { + "X-Polaris-Realm": realm, + } + if token: + headers["Authorization"] = f"Bearer {token}" + if content_type: + headers["Content-Type"] = content_type + return headers + + def request_token(realm): + token_resp = requests.post( + f"{MGMT_URL}/api/catalog/v1/oauth/tokens", + headers=realm_headers(realm, content_type="application/x-www-form-urlencoded"), + data={ + "grant_type": "client_credentials", + "client_id": username, + "client_secret": password, + "scope": "PRINCIPAL_ROLE:ALL", + "realm": realm, + }, + ) + token_resp.raise_for_status() + return token_resp.json()["access_token"] - print(f"[polaris-bootstrap] Authenticating with Polaris management API at {MGMT_URL}") - - # Obtain initial token from Polaris internal token service - token_resp = requests.post( - f"{MGMT_URL}/api/catalog/v1/oauth/tokens", - headers={"X-Iceberg-Access-Realm": REALM}, - data={ - "grant_type": "client_credentials", - "client_id": username, - "client_secret": password, - "scope": "PRINCIPAL_ROLE:ALL", - }, - ) - token_resp.raise_for_status() - token = token_resp.json()["access_token"] - print("[polaris-bootstrap] Token obtained successfully") - - headers = { - "Authorization": f"Bearer {token}", - "X-Iceberg-Access-Realm": REALM, - "Content-Type": "application/json", - } - - def mgmt(method, path, **kwargs): + def mgmt(method, realm, token, path, **kwargs): url = f"{MGMT_URL}/api/management/v1{path}" - resp = getattr(requests, method)(url, headers=headers, **kwargs) - if resp.status_code not in (200, 201, 204, 409): - print(f" ERROR {resp.status_code}: {resp.text}", file=sys.stderr) - resp.raise_for_status() + resp = getattr(requests, method)(url, headers=realm_headers(realm, token=token), **kwargs) + + if resp.status_code in SUCCESS_CODES or is_duplicate_error(resp): + return resp + + print(f"ERROR {resp.status_code}: {resp.text}", file=sys.stderr) + resp.raise_for_status() return resp - # Load ACL config - with open("/config/acl-config.json") as f: + def ensure_namespace(realm, token, catalog_name, namespace): + resp = requests.post( + f"{MGMT_URL}/api/catalog/v1/{catalog_name}/namespaces", + headers=realm_headers(realm, token=token), + json={"namespace": [namespace]}, + ) + if resp.status_code in SUCCESS_CODES or is_duplicate_error(resp): + return + resp.raise_for_status() + + def grant_to_catalog_role(realm, token, catalog_name, catalog_role, grant, fallback_namespace=None): + payload = { + "type": grant["type"], + "privilege": grant["privilege"], + } + + if grant["type"] in ("namespace", "table"): + namespace = grant.get("namespace") or fallback_namespace + if not namespace: + raise ValueError("Namespace/table grant missing namespace") + payload["namespace"] = [namespace] + + if grant["type"] == "table": + table_name = grant.get("table") + if not table_name: + raise ValueError("Table grant missing table") + payload["table"] = table_name + + mgmt( + "put", + realm, + token, + f"/catalogs/{catalog_name}/catalog-roles/{catalog_role}/grants", + json=payload, + ) + + def create_role_bindings(realm, token, catalog_name, role_bindings, namespace=None): + for role_cfg in role_bindings: + principal_role = role_cfg["principal_role"] + catalog_role = role_cfg["catalog_role"] + + mgmt("post", realm, token, "/principal-roles", json={"name": principal_role}) + mgmt("post", realm, token, f"/catalogs/{catalog_name}/catalog-roles", json={"name": catalog_role}) + + for grant_cfg in role_cfg.get("grants", []): + for privilege in grant_cfg.get("privileges", []): + grant_to_catalog_role( + realm, + token, + catalog_name, + catalog_role, + { + "type": grant_cfg["type"], + "namespace": grant_cfg.get("namespace"), + "table": grant_cfg.get("table"), + "privilege": privilege, + }, + fallback_namespace=namespace, + ) + + mgmt( + "put", + realm, + token, + f"/principal-roles/{principal_role}/catalog-roles/{catalog_name}", + json={"name": catalog_role}, + ) + + with open("/config/acl-config.json", encoding="utf-8") as f: config = json.load(f) - catalog_name = config["catalog"] - print(f"[polaris-bootstrap] Creating catalog: {catalog_name}") - - # Create catalog - mgmt("post", "/catalogs", json={ - "name": catalog_name, - "type": "INTERNAL", - "properties": {"default-base-location": WAREHOUSE}, - "storageConfigInfo": { - "storageType": "S3", - "allowedLocations": [WAREHOUSE], - }, - }) - print(f" catalog '{catalog_name}' created (or already exists)") - - # Create catalog-level admin principal role - admin_pr = "catalog_admin" - mgmt("post", "/principal-roles", json={"name": admin_pr}) - print(f" principal role '{admin_pr}' created") - - # Grant catalog admin privileges - mgmt("post", f"/catalogs/{catalog_name}/catalog-roles", json={ - "name": "catalog_admin_role" - }) - for priv in ["CATALOG_MANAGE_CONTENT", "CATALOG_MANAGE_METADATA"]: - mgmt("put", f"/catalogs/{catalog_name}/catalog-roles/catalog_admin_role/grants", json={ - "type": "catalog", - "privilege": priv, - }) - mgmt("put", f"/principal-roles/{admin_pr}/catalog-roles/{catalog_name}", json={ - "name": "catalog_admin_role" - }) - print(f" catalog admin grants assigned") - - # Process namespaces and roles from ACL config - for ns_config in config["namespaces"]: - ns_name = ns_config["name"] - print(f"[polaris-bootstrap] Processing namespace: {ns_name}") - - # Create namespace in catalog via catalog REST API - ns_resp = requests.post( - f"{MGMT_URL}/api/catalog/v1/{catalog_name}/namespaces", - headers={**headers}, - json={"namespace": [ns_name]}, + realms_cfg = config.get("realms", []) + if not realms_cfg: + raise ValueError("acl-config.json must define a non-empty 'realms' list") + + for realm_cfg in realms_cfg: + realm = realm_cfg["realm"] + catalog_name = realm_cfg["catalog"] + warehouse = realm_cfg["warehouse"] + storage_type = realm_cfg.get("storage_type", "S3") + allowed_locations = realm_cfg.get("allowed_locations") or [warehouse] + + print(f"[polaris-bootstrap] realm={realm} catalog={catalog_name}") + token = request_token(realm) + + mgmt( + "post", + realm, + token, + "/catalogs", + json={ + "name": catalog_name, + "type": "INTERNAL", + "properties": {"default-base-location": warehouse}, + "storageConfigInfo": { + "storageType": storage_type, + "allowedLocations": allowed_locations, + }, + }, ) - if ns_resp.status_code not in (200, 201, 409): - ns_resp.raise_for_status() - print(f" namespace '{ns_name}' created (or already exists)") - for role_config in ns_config["roles"]: - pr_name = role_config["polaris_principal_role"] - cr_name = role_config["polaris_catalog_role"] - privileges = role_config["privileges"] + admin_cfg = realm_cfg.get("admin") + if admin_cfg: + create_role_bindings( + realm, + token, + catalog_name, + [ + { + "principal_role": admin_cfg["principal_role"], + "catalog_role": admin_cfg.get("catalog_role", "catalog_admin_role"), + "grants": [ + { + "type": "catalog", + "privileges": admin_cfg.get( + "privileges", + ["CATALOG_MANAGE_CONTENT", "CATALOG_MANAGE_METADATA"], + ), + } + ], + } + ], + ) - print(f" creating principal role: {pr_name}, catalog role: {cr_name}") + namespace_set = set() + for ns_policy in realm_cfg.get("namespace_policies", []): + namespace_set.add(ns_policy["namespace"]) + for table_policy in realm_cfg.get("table_policies", []): + namespace_set.add(table_policy["namespace"]) - # Create principal role - mgmt("post", "/principal-roles", json={"name": pr_name}) + for namespace in sorted(namespace_set): + ensure_namespace(realm, token, catalog_name, namespace) - # Create catalog role - mgmt("post", f"/catalogs/{catalog_name}/catalog-roles", json={"name": cr_name}) + for ns_policy in realm_cfg.get("namespace_policies", []): + create_role_bindings( + realm, + token, + catalog_name, + ns_policy.get("roles", []), + namespace=ns_policy["namespace"], + ) - # Grant namespace privileges to catalog role - for priv in privileges: - mgmt("put", f"/catalogs/{catalog_name}/catalog-roles/{cr_name}/grants", json={ - "type": "namespace", - "namespace": [ns_name], - "privilege": priv, - }) + for table_policy in realm_cfg.get("table_policies", []): + table_namespace = table_policy["namespace"] + table_name = table_policy["table"] + normalized_roles = [] - # Assign catalog role to principal role - mgmt("put", f"/principal-roles/{pr_name}/catalog-roles/{catalog_name}", json={ - "name": cr_name - }) + for role_cfg in table_policy.get("roles", []): + normalized_roles.append( + { + "principal_role": role_cfg["principal_role"], + "catalog_role": role_cfg["catalog_role"], + "grants": [ + { + "type": "table", + "namespace": table_namespace, + "table": table_name, + "privileges": grant_cfg.get("privileges", []), + } + for grant_cfg in role_cfg.get("grants", []) + ], + } + ) - print(f" privileges and catalog role assigned") + create_role_bindings( + realm, + token, + catalog_name, + normalized_roles, + namespace=table_namespace, + ) - print("[polaris-bootstrap] Bootstrap complete!") + print("[polaris-bootstrap] Bootstrap complete") volumeMounts: - name: acl-config - mountPath: /config + mountPath: /config/acl-config.json + subPath: acl-config.json readOnly: true volumes: - name: acl-config configMap: name: polaris-acl-config + items: + - key: acl-config.json + path: acl-config.json diff --git a/polaris/manifests/polaris-config.yaml.tpl b/polaris/manifests/polaris-config.yaml.tpl index b9bbc55..daa70bf 100644 --- a/polaris/manifests/polaris-config.yaml.tpl +++ b/polaris/manifests/polaris-config.yaml.tpl @@ -20,10 +20,6 @@ data: polaris.authentication.type=mixed polaris.oidc.principal-mapper.name-claim-path=preferred_username - # In Polaris 1.5.0, mappings are structured as direct lists - polaris.oidc.principal-roles-mapper.mappings[0].regex=(.+) - polaris.oidc.principal-roles-mapper.mappings[0].replacement=PRINCIPAL_ROLE:$1 - # Quarkus OIDC Service Identity Configuration quarkus.oidc.application-type=service quarkus.oidc.roles.role-claim-path=realm_access/roles diff --git a/polaris/manifests/polaris.yaml.tpl b/polaris/manifests/polaris.yaml.tpl index 2439079..ddca38e 100644 --- a/polaris/manifests/polaris.yaml.tpl +++ b/polaris/manifests/polaris.yaml.tpl @@ -25,11 +25,16 @@ spec: - name: schema-bootstrap image: apache/polaris-admin-tool:1.5.0 imagePullPolicy: IfNotPresent - args: - - bootstrap - - --realm=teehr - - -c=teehr,root,secret123 - - -p + command: + - /bin/sh + - -ec + - | + ROOT_USER="${ROOT_CREDENTIALS%%:*}" + ROOT_PASSWORD="${ROOT_CREDENTIALS#*:}" + exec bootstrap \ + --realm="${POLARIS_BOOTSTRAP_REALM}" \ + -c="${POLARIS_BOOTSTRAP_REALM},${ROOT_USER},${ROOT_PASSWORD}" \ + -p env: # Core persistence assignment - name: POLARIS_PERSISTENCE_TYPE @@ -37,7 +42,14 @@ spec: - name: POLARIS_PERSISTENCE_AUTO_BOOTSTRAP_TYPES value: relational-jdbc - name: POLARIS_REALM_CONTEXT_REALMS - value: teehr + value: ${var.polaris.realmsCsv} + - name: POLARIS_BOOTSTRAP_REALM + value: ${var.polaris.defaultRealm} + - name: ROOT_CREDENTIALS + valueFrom: + secretKeyRef: + name: polaris-secrets + key: root-credentials # FIXED: Explicit lowercase system translations to force Agroal activation - name: quarkus_datasource_db-kind @@ -64,10 +76,8 @@ spec: image: apache/polaris:1.5.0 imagePullPolicy: IfNotPresent env: - # FIX 1: Force the server runtime to recognize your specific realm! - name: POLARIS_REALM_CONTEXT_REALMS - value: teehr - # Keep main container aligned to the relational-jdbc metastore + value: ${var.polaris.realmsCsv} - name: POLARIS_PERSISTENCE_TYPE value: relational-jdbc - name: QUARKUS_DATASOURCE_JDBC_URL diff --git a/trino/garden.yaml b/trino/garden.yaml index 56c7e7b..5f33e03 100644 --- a/trino/garden.yaml +++ b/trino/garden.yaml @@ -33,6 +33,7 @@ spec: iceberg.rest-catalog.oauth2.server-uri=${var.polaris.oauthServerUri} iceberg.rest-catalog.oauth2.credential=$${ENV:POLARIS_CREDENTIAL} iceberg.rest-catalog.oauth2.scope=openid + iceberg.rest-catalog.header.X-Polaris-Realm=${var.polaris.defaultRealm} # S3 Configuration fs.native-s3.enabled=true s3.path-style-access=${var.iceberg.catalogS3PathStyleAccess} @@ -122,6 +123,7 @@ spec: iceberg.rest-catalog.oauth2.server-uri=${var.polaris.oauthServerUri} iceberg.rest-catalog.oauth2.credential=$${ENV:POLARIS_CREDENTIAL} iceberg.rest-catalog.oauth2.scope=openid + iceberg.rest-catalog.header.X-Polaris-Realm=${var.polaris.defaultRealm} # S3 Configuration fs.native-s3.enabled=true s3.region=${var.aws.region} From d505bab95abe88d51ae485b44f78fb381dda3301 Mon Sep 17 00:00:00 2001 From: samlamont Date: Wed, 10 Jun 2026 08:47:01 -0400 Subject: [PATCH 04/62] fixing local deployment --- polaris/manifests/polaris.yaml.tpl | 26 +++++++++++++------------- secrets/secrets.local.yaml | 2 ++ trino/garden.yaml | 3 +-- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/polaris/manifests/polaris.yaml.tpl b/polaris/manifests/polaris.yaml.tpl index ddca38e..334641b 100644 --- a/polaris/manifests/polaris.yaml.tpl +++ b/polaris/manifests/polaris.yaml.tpl @@ -25,16 +25,11 @@ spec: - name: schema-bootstrap image: apache/polaris-admin-tool:1.5.0 imagePullPolicy: IfNotPresent - command: - - /bin/sh - - -ec - - | - ROOT_USER="${ROOT_CREDENTIALS%%:*}" - ROOT_PASSWORD="${ROOT_CREDENTIALS#*:}" - exec bootstrap \ - --realm="${POLARIS_BOOTSTRAP_REALM}" \ - -c="${POLARIS_BOOTSTRAP_REALM},${ROOT_USER},${ROOT_PASSWORD}" \ - -p + args: + - bootstrap + - --realm=$(POLARIS_BOOTSTRAP_REALM) + - -c=$(POLARIS_BOOTSTRAP_REALM),$(ROOT_USERNAME),$(ROOT_PASSWORD) + - -p env: # Core persistence assignment - name: POLARIS_PERSISTENCE_TYPE @@ -45,13 +40,18 @@ spec: value: ${var.polaris.realmsCsv} - name: POLARIS_BOOTSTRAP_REALM value: ${var.polaris.defaultRealm} - - name: ROOT_CREDENTIALS + - name: ROOT_USERNAME valueFrom: secretKeyRef: name: polaris-secrets - key: root-credentials + key: root-username + - name: ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: polaris-secrets + key: root-password - # FIXED: Explicit lowercase system translations to force Agroal activation + # Explicit lowercase system translations to force Agroal activation - name: quarkus_datasource_db-kind value: postgresql - name: quarkus_datasource_jdbc_url diff --git a/secrets/secrets.local.yaml b/secrets/secrets.local.yaml index 1d7e1cd..990a32c 100644 --- a/secrets/secrets.local.yaml +++ b/secrets/secrets.local.yaml @@ -53,6 +53,8 @@ secrets: data: root-credentials: "root:secret123" bootstrap-credentials: "teehr,root,secret123" + root-username: "root" + root-password: "secret123" trino-polaris-secrets: data: client-secret: local-trino-polaris-client-secret diff --git a/trino/garden.yaml b/trino/garden.yaml index 5f33e03..51aafe0 100644 --- a/trino/garden.yaml +++ b/trino/garden.yaml @@ -23,6 +23,7 @@ spec: # worker: # nodeSelector: # teehr-hub/nodegroup-name: core-a + # iceberg.rest-catalog.header.X-Polaris-Realm=${var.polaris.defaultRealm} # removed from below. Needed for multi-realm support? catalogs: iceberg: |- connector.name=iceberg @@ -33,7 +34,6 @@ spec: iceberg.rest-catalog.oauth2.server-uri=${var.polaris.oauthServerUri} iceberg.rest-catalog.oauth2.credential=$${ENV:POLARIS_CREDENTIAL} iceberg.rest-catalog.oauth2.scope=openid - iceberg.rest-catalog.header.X-Polaris-Realm=${var.polaris.defaultRealm} # S3 Configuration fs.native-s3.enabled=true s3.path-style-access=${var.iceberg.catalogS3PathStyleAccess} @@ -123,7 +123,6 @@ spec: iceberg.rest-catalog.oauth2.server-uri=${var.polaris.oauthServerUri} iceberg.rest-catalog.oauth2.credential=$${ENV:POLARIS_CREDENTIAL} iceberg.rest-catalog.oauth2.scope=openid - iceberg.rest-catalog.header.X-Polaris-Realm=${var.polaris.defaultRealm} # S3 Configuration fs.native-s3.enabled=true s3.region=${var.aws.region} From c80e96805eb9657d8135c65508743a09b6c95bb6 Mon Sep 17 00:00:00 2001 From: samlamont Date: Tue, 16 Jun 2026 14:28:04 -0400 Subject: [PATCH 05/62] remove unused manifest --- polaris-bootstrap/manifests/acl-config.yaml | 94 --------------------- 1 file changed, 94 deletions(-) delete mode 100644 polaris-bootstrap/manifests/acl-config.yaml diff --git a/polaris-bootstrap/manifests/acl-config.yaml b/polaris-bootstrap/manifests/acl-config.yaml deleted file mode 100644 index a73a8ec..0000000 --- a/polaris-bootstrap/manifests/acl-config.yaml +++ /dev/null @@ -1,94 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: polaris-acl-config -data: - acl-config.json: | - { - "realms": [ - { - "realm": "teehr", - "catalog": "teehr", - "warehouse": "s3a://teehr-warehouse/polaris", - "storage_type": "S3", - "allowed_locations": [ - "s3a://teehr-warehouse/polaris" - ], - "admin": { - "principal_role": "iceberg-catalog-admin", - "catalog_role": "catalog_admin_role", - "privileges": [ - "CATALOG_MANAGE_CONTENT", - "CATALOG_MANAGE_METADATA" - ] - }, - "namespace_policies": [ - { - "namespace": "public", - "roles": [ - { - "principal_role": "iceberg-namespace-public-read", - "catalog_role": "public_read_role", - "grants": [ - { - "type": "namespace", - "privileges": [ - "NAMESPACE_READ_PROPERTIES" - ] - } - ] - }, - { - "principal_role": "iceberg-namespace-public-write", - "catalog_role": "public_write_role", - "grants": [ - { - "type": "namespace", - "privileges": [ - "NAMESPACE_READ_PROPERTIES", - "NAMESPACE_WRITE_PROPERTIES", - "TABLE_CREATE", - "TABLE_DROP" - ] - } - ] - } - ] - }, - { - "namespace": "restricted", - "roles": [ - { - "principal_role": "iceberg-namespace-restricted-read", - "catalog_role": "restricted_read_role", - "grants": [ - { - "type": "namespace", - "privileges": [ - "NAMESPACE_READ_PROPERTIES" - ] - } - ] - }, - { - "principal_role": "iceberg-namespace-restricted-write", - "catalog_role": "restricted_write_role", - "grants": [ - { - "type": "namespace", - "privileges": [ - "NAMESPACE_READ_PROPERTIES", - "NAMESPACE_WRITE_PROPERTIES", - "TABLE_CREATE", - "TABLE_DROP" - ] - } - ] - } - ] - } - ], - "table_policies": [] - } - ] - } From 720d1f758ca566a709f79ae15678d001e3844905 Mon Sep 17 00:00:00 2001 From: samlamont Date: Thu, 18 Jun 2026 14:12:18 -0400 Subject: [PATCH 06/62] [wip] creating polaris principals from keycloak users; enabling jupyterhub token encryption; fixing catalog access --- jupyterhub/garden.yaml | 38 ++++++- .../manifests/realm-configmap.yaml.tpl | 14 +++ polaris-bootstrap/garden.yaml | 17 ++- .../manifests/acl-config.yaml.tpl | 8 +- .../manifests/bootstrap-job.yaml | 40 +++++++ .../polaris-principal-sync-cronjob.yaml.tpl | 50 ++++++++ .../polaris-sync-principals-script.yaml | 107 ++++++++++++++++++ polaris/manifests/polaris-config.yaml.tpl | 21 +++- polaris/manifests/polaris.yaml.tpl | 37 ++++-- secrets/secrets.local.yaml | 1 + 10 files changed, 320 insertions(+), 13 deletions(-) create mode 100644 polaris-bootstrap/manifests/polaris-principal-sync-cronjob.yaml.tpl create mode 100644 polaris-bootstrap/manifests/polaris-sync-principals-script.yaml diff --git a/jupyterhub/garden.yaml b/jupyterhub/garden.yaml index bdcdcd0..53f52ed 100644 --- a/jupyterhub/garden.yaml +++ b/jupyterhub/garden.yaml @@ -85,6 +85,11 @@ spec: name: jupyterhub-profile-list key: profile-list.json optional: true + JUPYTERHUB_CRYPT_KEY: + valueFrom: + secretKeyRef: + name: jupyterhub + key: JUPYTERHUB_CRYPT_KEY extraConfig: spark-config: | @@ -120,6 +125,12 @@ spec: raise RuntimeError(f"Profile at index {index} is missing required key: display_name") c.KubeSpawner.profile_list = profile_list + auth-state: | + c.Authenticator.enable_auth_state = True + async def auth_state_hook(spawner, auth_state): + if auth_state and "access_token" in auth_state: + spawner.environment["POLARIS_USER_TOKEN"] = auth_state["access_token"] + c.Spawner.auth_state_hook = auth_state_hook cull-kernels: | c.MappingKernelManager.cull_idle_timeout = 3600 c.MappingKernelManager.cull_connected = True @@ -152,11 +163,17 @@ spec: key: api-key REMOTE_CATALOG_TYPE: ${var.polaris.catalogType} REMOTE_CATALOG_REST_URI: ${var.polaris.catalogUri} - REMOTE_WAREHOUSE_S3_PATH: ${var.polaris.catalogWarehouse} + REMOTE_WAREHOUSE_S3_PATH: ${var.polaris.defaultRealm} REMOTE_CATALOG_S3_ENDPOINT: ${var.polaris.catalogS3Endpoint} REMOTE_CATALOG_S3_PATH_STYLE_ACCESS: ${var.polaris.catalogS3PathStyleAccess} IN_CLUSTER: ${var.polaris.inCluster} POLARIS_DEFAULT_REALM: ${var.polaris.defaultRealm} + POLARIS_OAUTH2_SERVER_URI: ${var.polaris.oauthServerUri} + SPARK_POLARIS_CLIENT_SECRET: + valueFrom: + secretKeyRef: + name: spark-polaris-secrets + key: client-secret extraFiles: jupyter_config: mountPath: /etc/jupyter/jupyter_notebook_config.py @@ -273,6 +290,11 @@ spec: name: jupyterhub-profile-list key: profile-list.json optional: true + JUPYTERHUB_CRYPT_KEY: + valueFrom: + secretKeyRef: + name: jupyterhub + key: JUPYTERHUB_CRYPT_KEY extraConfig: spark-config: | # Allow users to create Spark sessions @@ -307,6 +329,12 @@ spec: raise RuntimeError(f"Profile at index {index} is missing required key: display_name") c.KubeSpawner.profile_list = profile_list + auth-state: | + c.Authenticator.enable_auth_state = True + async def auth_state_hook(spawner, auth_state): + if auth_state and "access_token" in auth_state: + spawner.environment["POLARIS_USER_TOKEN"] = auth_state["access_token"] + c.Spawner.auth_state_hook = auth_state_hook cull-kernels: | c.MappingKernelManager.cull_idle_timeout = 3600 c.MappingKernelManager.cull_connected = True @@ -335,11 +363,17 @@ spec: AWS_REGION: ${var.aws.region} REMOTE_CATALOG_TYPE: ${var.polaris.catalogType} REMOTE_CATALOG_REST_URI: ${var.polaris.catalogUri} - REMOTE_WAREHOUSE_S3_PATH: ${var.polaris.catalogWarehouse} + REMOTE_WAREHOUSE_S3_PATH: ${var.polaris.defaultRealm} REMOTE_CATALOG_S3_ENDPOINT: ${var.polaris.catalogS3Endpoint} REMOTE_CATALOG_S3_PATH_STYLE_ACCESS: ${var.polaris.catalogS3PathStyleAccess} IN_CLUSTER: ${var.polaris.inCluster} POLARIS_DEFAULT_REALM: ${var.polaris.defaultRealm} + POLARIS_OAUTH2_SERVER_URI: ${var.polaris.oauthServerUri} + SPARK_POLARIS_CLIENT_SECRET: + valueFrom: + secretKeyRef: + name: spark-polaris-secrets + key: client-secret TRINO_HOST: ${var.trino.host} TRINO_PORT: ${var.trino.port} TRINO_CATALOG: ${var.trino.catalog} diff --git a/keycloak-bootstrap/manifests/realm-configmap.yaml.tpl b/keycloak-bootstrap/manifests/realm-configmap.yaml.tpl index 1bdfa9f..9f8b605 100644 --- a/keycloak-bootstrap/manifests/realm-configmap.yaml.tpl +++ b/keycloak-bootstrap/manifests/realm-configmap.yaml.tpl @@ -151,6 +151,20 @@ data: "userinfo.token.claim": "true", "claim.name": "groups" } + }, + { + "name": "realm-roles", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-realm-role-mapper", + "consentRequired": false, + "config": { + "multivalued": "true", + "id.token.claim": "true", + "access.token.claim": "true", + "userinfo.token.claim": "true", + "claim.name": "realm_access.roles", + "jsonType.label": "String" + } } ], "redirectUris": [ diff --git a/polaris-bootstrap/garden.yaml b/polaris-bootstrap/garden.yaml index 27364a4..96c1149 100644 --- a/polaris-bootstrap/garden.yaml +++ b/polaris-bootstrap/garden.yaml @@ -1,7 +1,7 @@ kind: Deploy type: kubernetes name: polaris-acl-config -description: K8s Deploy Polaris ACL ConfigMap +description: K8s Deploy Polaris ACL ConfigMap and shared sync script dependencies: - deploy.secrets environments: @@ -10,6 +10,21 @@ environments: spec: manifestTemplates: - ./manifests/acl-config.yaml.tpl + - ./manifests/polaris-sync-principals-script.yaml +--- +kind: Deploy +type: kubernetes +name: polaris-principal-sync +description: CronJob that syncs enabled Keycloak users to Polaris principals every 5 minutes +dependencies: + - deploy.secrets + - deploy.polaris-bootstrap +environments: + - local + - remote +spec: + manifestTemplates: + - ./manifests/polaris-principal-sync-cronjob.yaml.tpl --- kind: Deploy type: exec diff --git a/polaris-bootstrap/manifests/acl-config.yaml.tpl b/polaris-bootstrap/manifests/acl-config.yaml.tpl index ca3c167..f67851b 100644 --- a/polaris-bootstrap/manifests/acl-config.yaml.tpl +++ b/polaris-bootstrap/manifests/acl-config.yaml.tpl @@ -88,7 +88,13 @@ data: ] } ], - "table_policies": [] + "table_policies": [], + "principals": [ + { + "name": "spark-polaris", + "principal_role": "iceberg-catalog-admin" + } + ] } ] } diff --git a/polaris-bootstrap/manifests/bootstrap-job.yaml b/polaris-bootstrap/manifests/bootstrap-job.yaml index bae5429..2172e3e 100644 --- a/polaris-bootstrap/manifests/bootstrap-job.yaml +++ b/polaris-bootstrap/manifests/bootstrap-job.yaml @@ -23,12 +23,25 @@ spec: secretKeyRef: name: polaris-secrets key: root-credentials + - name: KEYCLOAK_URL + value: "http://keycloak-service.teehr-hub.svc.cluster.local:8080" + - name: KEYCLOAK_ADMIN_USER + valueFrom: + secretKeyRef: + name: keycloak-admin-secrets + key: username + - name: KEYCLOAK_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: keycloak-admin-secrets + key: password command: - python - -c - | import json import os + import subprocess import sys import requests @@ -250,12 +263,36 @@ spec: namespace=table_namespace, ) + # Sync Keycloak users to Polaris principals via shared script. + # Role assignment is omitted — the OIDC role mapper handles it at auth time. + subprocess.run( + ["python", "/scripts/sync_principals.py"], + check=True, + env={**os.environ, "POLARIS_REALM": realm}, + ) + + # Service account principals (static list from acl-config). + # These authenticate via client credentials (not OIDC user tokens), + # so their principal role must be assigned explicitly here. + for principal_cfg in realm_cfg.get("principals", []): + principal_name = principal_cfg["name"] + principal_role = principal_cfg["principal_role"] + print(f"[polaris-bootstrap] Ensuring service principal: {principal_name}") + mgmt("post", realm, token, "/principals", + json={"name": principal_name, "type": "USER", "properties": {}}) + mgmt("put", realm, token, + f"/principals/{principal_name}/principal-roles", + json={"name": principal_role}) + print("[polaris-bootstrap] Bootstrap complete") volumeMounts: - name: acl-config mountPath: /config/acl-config.json subPath: acl-config.json readOnly: true + - name: sync-script + mountPath: /scripts + readOnly: true volumes: - name: acl-config configMap: @@ -263,3 +300,6 @@ spec: items: - key: acl-config.json path: acl-config.json + - name: sync-script + configMap: + name: polaris-sync-principals-script diff --git a/polaris-bootstrap/manifests/polaris-principal-sync-cronjob.yaml.tpl b/polaris-bootstrap/manifests/polaris-principal-sync-cronjob.yaml.tpl new file mode 100644 index 0000000..8008358 --- /dev/null +++ b/polaris-bootstrap/manifests/polaris-principal-sync-cronjob.yaml.tpl @@ -0,0 +1,50 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: polaris-principal-sync +spec: + schedule: "*/5 * * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + template: + spec: + restartPolicy: Never + containers: + - name: polaris-principal-sync + image: prefecthq/prefect:3.2.0-python3.10 + env: + - name: POLARIS_MANAGEMENT_URL + value: "http://polaris:8181" + - name: POLARIS_ROOT_CREDENTIALS + valueFrom: + secretKeyRef: + name: polaris-secrets + key: root-credentials + - name: POLARIS_REALM + value: "${var.polaris.defaultRealm}" + - name: KEYCLOAK_URL + value: "http://keycloak-service.teehr-hub.svc.cluster.local:8080" + - name: KEYCLOAK_ADMIN_USER + valueFrom: + secretKeyRef: + name: keycloak-admin-secrets + key: username + - name: KEYCLOAK_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: keycloak-admin-secrets + key: password + command: + - python + - /scripts/sync_principals.py + volumeMounts: + - name: sync-script + mountPath: /scripts + readOnly: true + volumes: + - name: sync-script + configMap: + name: polaris-sync-principals-script diff --git a/polaris-bootstrap/manifests/polaris-sync-principals-script.yaml b/polaris-bootstrap/manifests/polaris-sync-principals-script.yaml new file mode 100644 index 0000000..01b686f --- /dev/null +++ b/polaris-bootstrap/manifests/polaris-sync-principals-script.yaml @@ -0,0 +1,107 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: polaris-sync-principals-script +data: + sync_principals.py: | + """ + Syncs all enabled Keycloak realm users into Polaris as principal entities. + Role assignment is intentionally omitted: the Polaris OIDC role mapper maps + realm_access.roles claims to PRINCIPAL_ROLE: at authentication time. + + Required environment variables: + POLARIS_MANAGEMENT_URL e.g. http://polaris:8181 + POLARIS_ROOT_CREDENTIALS e.g. root:secret (used only to obtain a management token) + POLARIS_REALM e.g. teehr + KEYCLOAK_URL e.g. http://keycloak-service:8080 + KEYCLOAK_ADMIN_USER Keycloak admin username + KEYCLOAK_ADMIN_PASSWORD Keycloak admin password + """ + import os + import sys + import requests + + MGMT_URL = os.environ["POLARIS_MANAGEMENT_URL"] + REALM = os.environ["POLARIS_REALM"] + ROOT_CREDS = os.environ["POLARIS_ROOT_CREDENTIALS"] + KC_URL = os.environ["KEYCLOAK_URL"] + KC_ADMIN_USER = os.environ["KEYCLOAK_ADMIN_USER"] + KC_ADMIN_PASSWORD = os.environ["KEYCLOAK_ADMIN_PASSWORD"] + _root_user, _root_password = ROOT_CREDS.split(":", 1) + + + def get_polaris_token(): + resp = requests.post( + f"{MGMT_URL}/api/catalog/v1/oauth/tokens", + headers={"X-Polaris-Realm": REALM, "Content-Type": "application/x-www-form-urlencoded"}, + data={ + "grant_type": "client_credentials", + "client_id": _root_user, + "client_secret": _root_password, + "scope": "PRINCIPAL_ROLE:ALL", + }, + ) + resp.raise_for_status() + return resp.json()["access_token"] + + + def get_keycloak_token(): + resp = requests.post( + f"{KC_URL}/realms/master/protocol/openid-connect/token", + data={ + "grant_type": "password", + "client_id": "admin-cli", + "username": KC_ADMIN_USER, + "password": KC_ADMIN_PASSWORD, + }, + ) + resp.raise_for_status() + return resp.json()["access_token"] + + + def iter_keycloak_users(kc_token): + first = 0 + max_per_page = 100 + while True: + resp = requests.get( + f"{KC_URL}/admin/realms/{REALM}/users", + headers={"Authorization": f"Bearer {kc_token}"}, + params={"first": first, "max": max_per_page, "enabled": "true"}, + ) + resp.raise_for_status() + page = resp.json() + if not page: + break + yield from page + if len(page) < max_per_page: + break + first += max_per_page + + + def ensure_principal(polaris_token, principal_name): + resp = requests.post( + f"{MGMT_URL}/api/management/v1/principals", + headers={ + "X-Polaris-Realm": REALM, + "Authorization": f"Bearer {polaris_token}", + "Content-Type": "application/json", + }, + json={"name": principal_name, "type": "USER", "properties": {}}, + ) + if resp.status_code in (200, 201, 204, 409): + return + print(f"ERROR creating principal '{principal_name}': {resp.status_code} {resp.text}", file=sys.stderr) + resp.raise_for_status() + + + polaris_token = get_polaris_token() + kc_token = get_keycloak_token() + + synced = 0 + for user in iter_keycloak_users(kc_token): + name = user.get("username") + if name: + ensure_principal(polaris_token, name) + synced += 1 + + print(f"[polaris-principal-sync] Synced {synced} principals from Keycloak realm '{REALM}'") diff --git a/polaris/manifests/polaris-config.yaml.tpl b/polaris/manifests/polaris-config.yaml.tpl index daa70bf..f0b160e 100644 --- a/polaris/manifests/polaris-config.yaml.tpl +++ b/polaris/manifests/polaris-config.yaml.tpl @@ -17,13 +17,30 @@ data: polaris.persistence.relational.jdbc.max-duration-in-ms=5000 # Authentication Context Configuration + # This realm is configured to use an external identity provider (IDP) for authentication only. + # It accepts tokens issued by Keycloak only. + # polaris.authentication.type=external + # This realm is configured to use both the internal and external authentication. + # It accepts tokens issued by both Polaris and Keycloak. polaris.authentication.type=mixed + # These are global. You can also set per realm like: + # polaris.authentication.realm1.type=external polaris.oidc.principal-mapper.name-claim-path=preferred_username - # Quarkus OIDC Service Identity Configuration + # Quarkus OIDC — tenant-enabled=true is required; without it Quarkus disables the + # Default tenant and rejects all Bearer tokens with 401 regardless of other config. + quarkus.oidc.tenant-enabled=true quarkus.oidc.application-type=service + quarkus.oidc.client-id=jupyterhub + quarkus.tls.trust-all=true + quarkus.oidc.connection-delay=PT10S + quarkus.oidc.connection-retry-count=5 + quarkus.oidc.token.audience=account + quarkus.oidc.token.issuer=any quarkus.oidc.roles.role-claim-path=realm_access/roles - quarkus.otel.sdk.disabled=true + polaris.oidc.principal-roles-mapper.type=default + polaris.oidc.principal-roles-mapper.mappings[0].regex=^(.*)$ + polaris.oidc.principal-roles-mapper.mappings[0].replacement=PRINCIPAL_ROLE:$1 # Storage Properties Integration polaris.features."SUPPORTED_CATALOG_STORAGE_TYPES"=["S3","GCS","AZURE","FILE"] diff --git a/polaris/manifests/polaris.yaml.tpl b/polaris/manifests/polaris.yaml.tpl index 334641b..3baf78e 100644 --- a/polaris/manifests/polaris.yaml.tpl +++ b/polaris/manifests/polaris.yaml.tpl @@ -21,15 +21,38 @@ spec: app: polaris spec: serviceAccountName: polaris + ${if environment.name == "local"} + hostAliases: + - ip: "${var.polaris.keycloakIngressIp}" + hostnames: + - "auth.${var.hostname}" + ${endif} initContainers: - name: schema-bootstrap image: apache/polaris-admin-tool:1.5.0 imagePullPolicy: IfNotPresent - args: - - bootstrap - - --realm=$(POLARIS_BOOTSTRAP_REALM) - - -c=$(POLARIS_BOOTSTRAP_REALM),$(ROOT_USERNAME),$(ROOT_PASSWORD) - - -p + command: + - /bin/sh + - -c + - | + JAR=/deployments/polaris-admin-tool.jar + echo "Found jar: $JAR" + OUTPUT=$(java $JAVA_OPTS -jar "$JAR" bootstrap \ + --realm="$POLARIS_BOOTSTRAP_REALM" \ + -c "$POLARIS_BOOTSTRAP_REALM,$ROOT_USERNAME,$ROOT_PASSWORD" \ + -p 2>&1) + EXIT_CODE=$? + echo "$OUTPUT" + if [ $EXIT_CODE -eq 0 ]; then + echo "Bootstrap succeeded." + exit 0 + fi + if echo "$OUTPUT" | grep -q "already been bootstrapped"; then + echo "Metastore already bootstrapped — skipping." + exit 0 + fi + echo "Bootstrap failed with unexpected error (exit code $EXIT_CODE)." + exit $EXIT_CODE env: # Core persistence assignment - name: POLARIS_PERSISTENCE_TYPE @@ -94,7 +117,7 @@ spec: key: password ${if environment.name == "local"} - name: QUARKUS_OIDC_AUTH_SERVER_URL - value: ${var.polaris.oauthServerUri} + value: ${var.polaris.oidcIssuerUri} - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: @@ -112,7 +135,7 @@ spec: ${endif} ${if environment.name != "local"} - name: QUARKUS_OIDC_AUTH_SERVER_URL - value: ${var.polaris.oauthServerUri} + value: ${var.polaris.oidcIssuerUri} ${endif} - name: AWS_REGION value: us-east-2 diff --git a/secrets/secrets.local.yaml b/secrets/secrets.local.yaml index 990a32c..c89d81b 100644 --- a/secrets/secrets.local.yaml +++ b/secrets/secrets.local.yaml @@ -3,6 +3,7 @@ secrets: data: OAUTH_CLIENT_ID: jupyterhub OAUTH_CLIENT_SECRET: local-jupyterhub-client-secret + JUPYTERHUB_CRYPT_KEY: 55b6f7f4ead8f8fec6fe024c2ecdaff76cb024916876f50d668437e6f8d0e051 prefect-db-secrets: data: database: prefect From fb52e3b4f4f9e334e2b181a3700d96f6c263ad29 Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Tue, 14 Jul 2026 09:59:30 -0400 Subject: [PATCH 07/62] add some carry-over planning docs --- docs/polaris-identity-propagation-plan.md | 649 ++++++++++++++++++++++ docs/polaris-migration-plan.md | 390 +++++++++++++ 2 files changed, 1039 insertions(+) create mode 100644 docs/polaris-identity-propagation-plan.md create mode 100644 docs/polaris-migration-plan.md diff --git a/docs/polaris-identity-propagation-plan.md b/docs/polaris-identity-propagation-plan.md new file mode 100644 index 0000000..0c9baf2 --- /dev/null +++ b/docs/polaris-identity-propagation-plan.md @@ -0,0 +1,649 @@ +# TEEHR Hub Identity Propagation and Fine-Grained Data Authorization Plan + +Last updated: 2026-06-17 + +## Goals + +Establish a unified identity and authorization architecture for all TEEHR Hub services that access Iceberg-backed data so that: + +- Keycloak remains the single source of truth for user identity and group membership. +- JupyterHub, FastAPI, and other user-facing entry points propagate end-user identity forward instead of collapsing users into a shared technical principal. +- Apache Polaris becomes the primary enforcement point for fine-grained catalog and table permissions. +- Spark, Trino, and other compute/query services access Iceberg through Polaris in a way that preserves user-level authorization context. +- Existing services continue to work during migration, with clear phases and fallback paths. + +## Principles + +1. **Identity source of truth:** Keycloak owns users, groups, and role/group assignments. +2. **Data authorization source of truth:** Polaris owns fine-grained data authorization for Iceberg catalogs, namespaces, tables, and write operations. +3. **Policy enforcement at the data plane:** JupyterHub and FastAPI should not be the final authority for what data a user can read or write. They should authenticate the user, propagate identity, and rely on downstream systems to enforce. +4. **End-user identity propagation:** Where a user initiates an action, downstream data access should occur with a credential or context representing that user. +5. **Coarse vs fine-grained separation:** Keycloak groups remain coarse platform entitlements; Polaris policies hold fine-grained data access rules. +6. **Hybrid authorization model:** Policies are assigned primarily to groups, but requests should always carry the individual user identity for auditability, traceability, and exception handling. +7. **Least privilege:** Shared Kubernetes, AWS, or service identities must not undermine per-user data authorization. +8. **Short-lived credentials:** Prefer short-lived, revocable credentials/tokens over long-lived shared secrets inside notebook or app runtimes. + +## Target Architecture + +### Flow A: Keycloak → JupyterHub → Polaris → Iceberg/Spark + +1. User authenticates to JupyterHub with Keycloak OIDC. +2. JupyterHub stores auth state and extracts a minimal set of claims/groups needed for spawn-time decisions and downstream identity propagation. +3. Jupyter single-user server receives either: + - a short-lived access token for the logged-in user, or + - a short-lived exchanged token/credential derived from the user identity. +4. Notebook clients (Spark, PyIceberg, direct REST catalog clients) present that per-user credential to Polaris. +5. Polaris evaluates both the individual user principal and the user’s Keycloak-derived groups. +6. Iceberg operations proceed only if Polaris authorizes them. +7. Underlying object storage access is constrained so users cannot bypass Polaris with broad direct bucket permissions. + +### Flow B: Keycloak → FastAPI → Trino → Polaris → Iceberg + +1. User authenticates to FastAPI with a Keycloak JWT. +2. FastAPI validates the JWT and extracts user identity plus coarse claims. +3. For data-plane calls, FastAPI forwards end-user identity to Trino using a supported secure mechanism. +4. Trino uses that end-user context when resolving catalog access through Polaris. +5. Polaris evaluates both the individual user principal and the user’s Keycloak-derived groups. +6. FastAPI remains responsible for app-level capabilities and route access, but not for warehouse data authorization beyond coarse gatekeeping. + +## Identity Model + +### Keycloak responsibilities + +Keycloak should continue to manage: + +- users +- default baseline access +- group membership +- OIDC clients for frontend, JupyterHub, FastAPI, and any service-to-service integrations + +The existing coarse group model is a strong starting point: + +- `basic-user` +- `jupyter-user` +- `jupyter-admin` +- `iceberg-user` +- `key-management-admin` +- `prefect-admin` +- `webapi-admin` + +### Recommended group strategy + +Keep Keycloak groups coarse and human-manageable. Suggested semantics: + +- `basic-user`: baseline authenticated application access +- `jupyter-user`: allowed to access JupyterHub +- `jupyter-admin`: JupyterHub admin rights only +- `iceberg-user`: allowed to use catalog-backed data tools +- optional team/domain groups: e.g. `hydrology-team`, `forecast-team`, `operations-team` + +Avoid expressing every dataset/table permission directly in Keycloak groups. Instead, use Keycloak groups as the default policy-assignment mechanism in Polaris and reserve direct per-user grants for exceptional cases. + +### Hybrid authorization model + +The target authorization model should be hybrid: + +- Every human-originated request carries the individual Keycloak user identity end to end. +- Polaris policy assignment is primarily group-based. +- Individual user grants are exceptions, not the default. +- Non-human workloads use distinct service principals with narrowly scoped rights. +- Audit logs should preserve both the individual principal and the effective groups used for authorization. + +Rule of thumb: + +> Policies are assigned mostly to groups, but requests always carry the individual user identity. + +### Claims to propagate + +At minimum, downstream systems should have access to: + +- subject (`sub`) +- preferred username or stable user identifier +- groups +- optionally realm roles if needed for coarse app behavior +- issuer (`iss`) +- audience/client context as needed for validation + +## Polaris Authorization Model + +### Polaris as the fine-grained authority + +Polaris should become the primary location for: + +- catalog-level permissions +- namespace/schema permissions +- table/view permissions +- read vs write separation +- admin/management operations +- team/project-specific access + +### Mapping strategy + +Prefer these patterns, in order: + +1. group-based policy mapping for team/domain entitlements +2. direct user principal mapping for limited exceptions +3. distinct service principals for non-human automation paths + +Examples: + +- `iceberg-user` grants general eligibility to use catalog-backed services +- `hydrology-team` maps to read access on selected namespaces/tables +- `forecast-team` maps to write access on forecast-derived tables only +- a specific user may receive a temporary direct grant for a narrow exception case +- automation service accounts get narrowly scoped non-human privileges + +### Project namespace model + +Eligible human users should be able to create and manage project namespaces for exploratory, collaborative, and intermediate work. + +Recommended rules: + +- Project namespace creation is allowed only for users with the required coarse entitlement, such as `iceberg-user`. +- Project namespaces live under the `projects` prefix and use a user-provided name: `projects.`. +- Namespace names must satisfy validation and uniqueness rules. +- Project namespaces are created lazily on first use. +- The creating user becomes the initial owner and receives read, write, and manage permissions for that namespace. +- New project namespaces default to `private` visibility. +- The owner may optionally configure the namespace to allow read-only access for all eligible Iceberg users. +- Shared/team/production namespaces remain governed primarily by group-based Polaris policies and separate governance. + +Recommended sharing modes for the initial implementation: + +- `private`: only the owner may access the namespace, except for narrowly scoped admin or maintenance access +- `all-iceberg-users-read`: all eligible Iceberg users may read objects in the namespace, while only the owner may write or manage objects + +This model intentionally uses project namespaces as the initial self-service workspace primitive. A separate `users.*` personal-namespace model is deferred unless later usage demonstrates a clear need for a distinct personal workspace class. + +### Project namespace naming and governance rules + +Recommended v1 naming rules for `projects.`: + +- use lowercase letters, numbers, and hyphens only +- must start with a letter +- should be globally unique under `projects.*` +- names that differ only by case should be treated as the same name +- reserve selected names and prefixes such as `admin`, `system`, `default`, `prod`, `production`, and `shared` +- apply a reasonable maximum length to the provided namespace component, such as 50 characters + +Recommended v1 governance rules: + +- the creating user becomes the initial owner +- the owner receives read, write, and manage permissions for the namespace +- new namespaces default to `private` +- the owner may switch visibility between `private` and `all-iceberg-users-read` +- arbitrary custom ACLs, explicit collaborators, and self-service ownership transfer are out of scope for v1 +- ownership transfer is admin-controlled in the initial implementation +- admins may recover, reassign, archive, or otherwise govern orphaned namespaces when an owner leaves or loses entitlement +- apply an initial per-user limit on the number of project namespaces, such as 10, with admin override if needed + +### Human vs automation separation + +Polaris policy should explicitly separate: + +- human interactive access from JupyterHub and FastAPI +- delegated execution on behalf of a human user +- non-human automation such as Prefect, ingestion jobs, and maintenance workflows + +Non-human automation should use distinct service principals and should not impersonate human users by default. + +## JupyterHub Design + +### JupyterHub responsibilities + +JupyterHub should: + +- authenticate users with Keycloak +- authorize JupyterHub access using coarse groups +- persist auth state securely +- pass minimal downstream identity into notebook runtimes +- optionally shape spawn behavior based on groups + +JupyterHub should not be the final authority for Iceberg permissions. + +### Recommended implementation pattern + +1. Enable and secure `auth_state` persistence. +2. At login or pre-spawn time, read: + - `sub` + - username + - groups + - token expiry metadata + - access token only if needed +3. Pass into notebook pods: + - minimal identity env vars for UX and telemetry + - a short-lived token or exchanged credential for Polaris access +4. Avoid passing refresh tokens into notebook environments unless absolutely necessary. +5. Use spawn hooks to gate profiles/features, not to implement table-level authorization. + +### Notebook runtime behavior + +Notebook runtimes should: + +- authenticate to Polaris as the end user +- use catalog-aware clients for Iceberg access +- avoid direct object-store access patterns that bypass Polaris +- allow creation and management of project namespaces for users with the required entitlement + +### Spark integration + +Spark jobs launched from Jupyter should preserve the originating end-user identity when accessing Polaris. This likely means: + +- Spark driver receives user credential/context from the notebook environment +- Spark catalog configuration uses Polaris endpoints and auth settings +- executor-side access follows the driver’s authenticated catalog interactions or other supported user-context mechanism +- audit context should preserve both the initiating user identity and the effective groups used for authorization + +Exact mechanics depend on the Spark + Iceberg + Polaris auth model you select, but the design goal is unchanged: no shared “all notebooks are the same person” catalog identity. + +### Kubernetes and AWS identity caution + +The current shared `jupyter` service account and shared IRSA role are acceptable for platform operations only if they do not grant blanket data access that bypasses Polaris. + +Recommended direction: + +- keep shared pod identity narrow +- do not rely on shared IRSA for warehouse authorization +- ensure direct S3/object-store permissions are minimized relative to Polaris-mediated access + +## FastAPI Design + +### FastAPI responsibilities + +FastAPI should: + +- validate Keycloak JWTs +- enforce application-level route permissions and coarse feature gates +- propagate end-user identity to Trino/data clients +- not substitute a shared privileged warehouse identity for end-user requests + +### Existing repo alignment + +The API already validates Keycloak JWTs and extracts realm roles. This is a good foundation for: + +- app-level authorization +- request identity extraction +- future downstream identity propagation + +### Forwarding identity to Trino + +The exact Trino integration should be chosen based on supported secure mechanisms, but the plan should require: + +- preserving a stable end-user identity into the Trino session +- preserving effective Keycloak groups or equivalent authorization context where supported +- preventing FastAPI from always querying as a single technical user for end-user traffic +- aligning Trino catalog access with Polaris-enforced permissions + +Candidate approaches to evaluate: + +1. user identity forwarded as authenticated session principal +2. OAuth/OIDC-aware Trino integration if supported by chosen deployment +3. trusted proxy/service pattern only if it still preserves distinguishable end-user principals and auditable enforcement + +## Service-by-Service Policy Split + +### Keycloak + +Owns: + +- authentication +- user lifecycle +- groups/roles +- client registration + +Does not own: + +- fine-grained Iceberg table permissions + +### JupyterHub + +Owns: + +- notebook login authorization +- admin access to JupyterHub +- spawn-time feature gating + +Does not own: + +- final warehouse data authorization + +### FastAPI + +Owns: + +- API authentication +- app feature authorization +- rate limiting / route protection / business rules + +Does not own: + +- final Iceberg table authorization for end-user data access + +### Polaris + +Owns: + +- fine-grained catalog and table authorization +- group-based policy evaluation as the default mechanism +- limited direct user grants for exceptional cases +- project namespace ownership and namespace-level self-service rules +- data-access decisions for Iceberg-aware clients/services + +### Trino / Spark + +Owns: + +- execution under propagated user identity +- honoring Polaris-backed catalog authorization +- preserving auditability of the human initiator where applicable + +### Automation services + +Own: + +- non-human scheduled or background execution under distinct service principals + +Do not own: + +- human-interactive identity or authorization decisions + +## Implementation Checklist + +### Repository and application changes + +#### JupyterHub + +- Locate current JupyterHub auth configuration. +- Confirm Keycloak OIDC integration path. +- Verify whether `auth_state` is enabled and persisted securely. +- Identify where pre-spawn hooks can extract username, `sub`, groups, and token expiry metadata. +- Decide what minimal identity context should be injected into notebook runtimes. +- Decide whether notebook runtimes receive a direct short-lived user token or an exchanged credential. +- Prototype notebook-side access to Polaris as the end user. +- Identify where project namespace create-on-first-use logic should live for Jupyter-driven workflows. + +#### FastAPI + +- Locate JWT validation and auth dependency code. +- Confirm where username, subject, roles, and groups are extracted today. +- Add or refine a canonical request identity object. +- Trace every FastAPI path that triggers Trino or Iceberg-backed access. +- Identify where end-user identity must be forwarded downstream. +- Determine whether project namespace operations will be exposed through API endpoints. +- If project namespace operations are exposed through the API, define behavior for create namespace, get namespace visibility, set namespace visibility, and list owned namespaces. + +#### Trino integration + +- Locate current Trino client or session creation code. +- Identify how user identity is currently represented in Trino sessions. +- Determine where group or authorization context could be forwarded. +- Document whether current behavior uses a shared technical principal. +- Define required code and configuration changes for per-user session context. + +#### Spark and notebook data access + +- Locate Spark catalog configuration used by notebooks. +- Determine how Polaris would be configured as the Iceberg catalog. +- Verify where Spark receives user auth context. +- Determine whether executor behavior preserves user-context semantics. +- Prototype read and write in a user-created project namespace. + +#### Project namespace workflow + +- Define the canonical create-on-first-use workflow. +- Define validation for `projects.`. +- Implement or prototype name validation rules. +- Define the visibility enum for v1: `private` and `all-iceberg-users-read`. +- Define owner permissions. +- Define admin-only ownership transfer handling. +- Define behavior when a namespace already exists. +- Define behavior when the creating user lacks `iceberg-user`. + +#### Audit and observability + +- Identify where to log initiating user identity. +- Identify where to log effective groups. +- Identify where to log namespace creation and visibility changes. +- Define correlation points between Keycloak user, Jupyter session or API request, Trino or Spark execution, and Polaris authorization decisions. + +### Platform and infrastructure changes + +#### Keycloak + +- Confirm the final group model for v1. +- Confirm that groups claims are present in tokens where needed. +- Confirm username stability expectations. +- Confirm whether `preferred_username` is sufficient for namespace naming. +- Confirm whether immutable `sub` should also be logged for ownership and audit. + +#### Polaris + +- Confirm the supported authentication method for end-user principals. +- Confirm how Polaris consumes user and group information. +- Confirm namespace creation APIs or workflow. +- Confirm the grant model for namespace owner read/write/manage, all-iceberg-users read-only, and admin recovery access. +- Confirm whether visibility toggling maps cleanly to grant changes. +- Confirm how to model project namespace ownership operationally. + +#### Object storage, AWS, and IAM + +- Inventory current object-store access paths. +- Identify any direct bucket permissions that bypass Polaris. +- Narrow shared IRSA or service-account access where needed. +- Define operational and admin exceptions. +- Confirm whether project namespace creation requires additional storage-side setup. + +#### Kubernetes and deployment + +- Locate JupyterHub deployment configuration. +- Locate FastAPI deployment configuration. +- Identify the secret and token handling mechanism. +- Confirm how short-lived credentials would be passed and rotated. +- Identify configuration surfaces for Polaris endpoints and auth settings. + +#### Admin and governance operations + +- Define the admin process for namespace reassignment. +- Define the admin process for orphaned namespaces. +- Define the namespace quota override process. +- Define the reserved-name management process. + +### Validation spikes and unknowns + +- Validate whether Polaris can directly authenticate Keycloak-issued user tokens. +- Validate whether Polaris can evaluate group-based policy from those tokens. +- Validate the exact grant model needed for namespace ownership and read-only sharing. +- Validate whether a notebook can authenticate to Polaris as the actual user. +- Validate the safest token propagation pattern for notebooks. +- Validate how long-lived notebook sessions behave when tokens expire. +- Validate whether Spark can access Polaris with preserved user context. +- Validate whether user attribution is maintained only at the driver level or throughout execution. +- Validate whether Trino can preserve end-user identity in the way Polaris needs. +- Validate whether group context can be propagated or reconstructed for Trino-driven requests. +- Validate whether project namespaces can be created lazily without fragile race conditions. +- Validate how duplicate creation attempts should behave. +- Validate how visibility changes will be represented in Polaris grants. +- Validate which current credentials or access paths still allow storage bypass. + +### Recommended implementation order + +1. Validate the Polaris authentication and policy model. +2. Validate the Jupyter-to-Polaris end-user flow. +3. Validate the FastAPI-to-Trino-to-Polaris flow. +4. Inventory current storage bypass paths. +5. Implement the first vertical slice around project namespace creation and default private access. +6. Validate owner read/write access, denied access for a second user, and read-only access after switching visibility to `all-iceberg-users-read`. +7. Add audit logging, admin recovery flows, quotas, and broader Spark/API parity. + +## Migration Phases + +### Phase 0: Discovery and capability validation + +Validate product capabilities and constraints before committing implementation details: + +- how Polaris authenticates principals and consumes OIDC/user identity +- how Iceberg clients authenticate to Polaris +- how Trino integrates with Polaris and preserves user identity +- how Spark integrates with Polaris and preserves user identity +- how groups/claims can be surfaced to Polaris, directly or indirectly +- how project namespace creation, sharing, and ownership can be represented in Polaris policy +- whether token exchange, service delegation, or direct bearer-token auth is preferred +- what object-store permissions are still required beneath Polaris + +Deliverables: + +- architecture decision record +- supported auth flow matrix for Jupyter, Spark, FastAPI, Trino +- gap list for unsupported assumptions + +### Phase 1: Identity inventory and policy model + +Define the canonical identity and policy model: + +- inventory existing Keycloak groups and roles +- identify coarse platform groups to keep +- define any new team/domain groups +- define how Keycloak groups map into Polaris principals/policies +- define criteria for when direct user grants are allowed +- define naming conventions for users, groups, namespaces, catalogs +- define the project namespace naming, entitlement, and visibility model + +Deliverables: + +- identity map +- group-to-Polaris policy map +- per-user exception policy +- project namespace policy +- example access-control matrix for target state + +### Phase 2: JupyterHub identity propagation foundation + +Implement the JupyterHub foundation for user-context propagation: + +- enable auth state persistence +- add pre-spawn logic to extract minimal claims +- inject minimal identity metadata into notebook sessions +- evaluate secure handling for short-lived user token or exchanged credential +- validate notebook-to-Polaris authentication path +- validate audit visibility of user identity and effective groups +- validate project namespace create/read/write behavior for eligible users + +Deliverables: + +- JupyterHub configuration changes +- secret-handling model +- proof of concept notebook access path +- audit-context validation notes +- project namespace proof of concept + +### Phase 3: Spark + Iceberg + Polaris user-context path + +Implement Spark access through Polaris under end-user identity: + +- configure Spark Iceberg catalog for Polaris +- validate read/write behavior by user/group +- validate create/read/write behavior in user-created project namespaces +- ensure job submissions launched from Jupyter preserve user context +- confirm executor/runtime behavior does not collapse to a shared catalog principal +- confirm how delegated execution is attributed in logs and policy evaluation + +Deliverables: + +- Spark catalog configuration +- end-to-end auth test cases +- operational notes for debugging and token expiry +- delegated execution audit model +- project namespace Spark validation cases + +### Phase 4: FastAPI → Trino user-context propagation + +Implement user-context propagation for API-driven data access: + +- formalize request identity object in FastAPI +- define Trino session principal propagation strategy +- validate per-user access behavior through Polaris +- validate group-based authorization behavior through Polaris +- validate create/read/write behavior for project namespaces where API workflows support it +- keep existing app-level route authorization intact + +Deliverables: + +- FastAPI integration design +- Trino integration configuration +- end-to-end API authz test cases +- request-to-query audit mapping +- project namespace API validation cases + +### Phase 5: Storage hardening / bypass prevention + +Reduce or eliminate bypass paths that would undermine Polaris: + +- audit direct S3/object-store permissions for Jupyter, Spark, Trino, and service accounts +- minimize shared credentials with broad warehouse access +- ensure intended clients access warehouse data through Polaris-mediated paths +- define exceptions explicitly for admin/maintenance automation +- ensure automation principals are separated from human interactive access + +Deliverables: + +- credential inventory +- least-privilege policy changes +- documented exception list +- human-vs-automation access boundary documentation + +### Phase 6: Rollout, observability, and migration cleanup + +Roll out incrementally and verify behavior: + +- pilot with a small set of users/groups +- compare current vs target behavior +- add audit logging and request tracing where possible +- deprecate old shared-identity assumptions +- update docs and developer notebooks/examples +- document the user experience for project namespace creation and visibility configuration + +Deliverables: + +- rollout checklist +- audit/observability plan +- migration completion checklist +- project namespace user guidance + +## Open Design Questions + +1. What auth mechanism does the selected Polaris deployment support for end-user principals? +2. Can Polaris directly evaluate Keycloak-issued JWTs, or is an intermediate exchange/delegation layer needed? +3. How are Keycloak groups or equivalent authorization attributes surfaced to Polaris policy evaluation? +4. What naming validation and uniqueness rules should apply to `projects.`? +5. Should project namespace visibility initially support only `private` and `all-iceberg-users-read`, or additional sharing modes? +6. What is the supported user-identity propagation mechanism from Trino to Polaris? +7. What is the supported user-identity propagation mechanism from Spark to Polaris? +8. Do PyIceberg and any direct notebook clients need separate auth handling from Spark? +9. How will token refresh work for long-lived notebook sessions and Spark jobs? +10. What direct object-store permissions are still needed, and how do we prevent them from bypassing Polaris? +11. Which existing services besides Jupyter, FastAPI, Spark, and Trino also need user-context-aware Iceberg access? +12. How should non-human automation be separated from human end-user access? +13. What audit trail is required to correlate Keycloak user, notebook/API request, Trino/Spark execution, effective groups, project namespace ownership, and Polaris decision? +14. What governance process should control direct per-user exceptions? +15. What quotas, lifecycle rules, or cleanup policies should apply to project namespaces? + +## Immediate Next Steps + +1. Inspect current JupyterHub config and identify where to enable `auth_state` and pre-spawn claim handling. +2. Document current FastAPI → Trino call paths and whether requests already preserve end-user context. +3. Inventory all Iceberg-accessing services in the repo and classify them as human-initiated vs automation. +4. Research and document the chosen Polaris auth capabilities and Trino/Spark integration constraints. +5. Draft the target group-to-policy mapping for current TEEHR personas and datasets. +6. Define the exception policy for direct user grants and the boundary for automation principals. +7. Define the project namespace naming, entitlement, and visibility model. + +## Non-Goals + +This plan does not attempt to: + +- move all fine-grained authorization into Keycloak +- use notebook-side Python logic as the primary enforcement point +- keep broad shared storage credentials as the long-term authorization model +- make direct per-user grants the default authorization strategy +- allow arbitrary self-created shared namespaces without governance +- finalize product-specific config syntax before validating supported auth paths diff --git a/docs/polaris-migration-plan.md b/docs/polaris-migration-plan.md new file mode 100644 index 0000000..30cbf75 --- /dev/null +++ b/docs/polaris-migration-plan.md @@ -0,0 +1,390 @@ +# Plan: Replace iceberg-rest with Apache Polaris (Keycloak-integrated) + +## TL;DR + +Replace `tabulario/iceberg-rest` with `apache/polaris`. Tight Keycloak integration via JWKS + a `PrincipalRoleMapper` that reads `realm_access.roles` from JWTs, mapping Keycloak roles directly to Polaris principal roles at runtime — no per-user Polaris registration needed. A data-driven bootstrap job creates the full namespace × privilege ACL structure from a ConfigMap. Polaris and Keycloak are the control plane; all other services are the data plane. + +--- + +> **Implementation order for `garden deploy` to work**: Phase 7 (Garden variables) and the secrets additions from Phase 1 must be done first — before any other phase — because Garden resolves `${var.polaris.*}` template variables at render time and pods reference the new secrets at startup. + +## Phase 1: Database — `polaris-pg` + +1. Create `polaris-pg/manifests/polaris-pg.yaml` (static, not `.tpl` — matches `iceberg-pg` which uses `manifestFiles`, no Garden templating needed). Mirror `iceberg-pg/manifests/iceberg-pg.yaml` exactly with these substitutions: + - All `iceberg-pg` → `polaris-pg`, all `iceberg` → `polaris` (names, labels, PVC claim) + - `POSTGRES_DB`/`POSTGRES_USER`/`POSTGRES_PASSWORD` env vars read from `polaris-db-secrets` + - Readiness/liveness `pg_isready` probes use `-U polaris -d polaris` + - PVC storage: 10Gi (same as iceberg-pg) + +2. Create `polaris-pg/garden.yaml` — `type: kubernetes`, `name: polaris-database`, `spec.manifestFiles: [./manifests/polaris-pg.yaml]`, depends on `deploy.secrets` — mirrors `iceberg-pg/garden.yaml` exactly with name substitution. + +3. Add `polaris-db-secrets` to `secrets/secrets.local.yaml` and `secrets/secrets.remote.yaml`. **Secrets are NOT standalone K8s manifests** — the `secrets/garden.yaml` `$forEach` loop creates K8s Secrets automatically from these varfiles. Add: + ```yaml + polaris-db-secrets: + data: + database: polaris + username: polaris + password: polaris123 # local; use strong password in remote + polaris-secrets: + data: + root-credentials: "root:secret123" # POLARIS_ROOT_CREDENTIALS format (:); use strong value in remote + trino-polaris-secrets: + data: + client-secret: local-trino-polaris-client-secret # plain secret — used by keycloak-bootstrap env var + credential: "trino-polaris:local-trino-polaris-client-secret" # full credential string — mounted as Trino credential file + spark-polaris-secrets: + data: + client-secret: local-spark-polaris-client-secret + ``` + > **Note**: `trino-polaris-secrets` needs **two keys** because the Keycloak bootstrap job needs the plain secret value (`client-secret`) to set as the Keycloak client secret, while Trino's credential file mount needs the full `trino-polaris:` string (`credential`). Using the `credential` key for the Keycloak env var would inject the wrong value. + +--- + +## Phase 2: Polaris Deployment — `polaris/` + +4. Create `polaris/manifests/polaris-config.yaml.tpl` — ConfigMap with `polaris-server.yml`: + - JDBC persistence → `polaris-pg` + - OIDC issuers as a **list** (multi-realm extensible); JWKS URI internal for `local`, external for `remote` + - **`PrincipalRoleMapper`** pointing at `realm_access.roles` in the JWT — maps Keycloak role names to Polaris principal role names at runtime. This is the critical link; without it user token pass-through grants zero privileges. Exact config key must be verified against the pinned Polaris release. + +5. Create `polaris/manifests/polaris.yaml.tpl` — Deployment + Service + SA: + - Image: `apache/polaris` pinned to a specific release tag + - Ports 8182 (catalog REST API), port 8183 (management API); Service exposes both + - **No IRSA annotation** — Polaris is pure metadata; S3 I/O stays with Trino/Spark + - ServiceAccount metadata must use `${environment.namespace}` — do NOT copy the hardcoded `namespace: teehr-hub` from `iceberg-rest.yaml.tpl` + - Local env gets MinIO credentials (same `${if environment.name == "local"}` pattern as `iceberg-rest.yaml.tpl`) + - `POLARIS_ROOT_CREDENTIALS` from `polaris-secrets` key `root-credentials` + - JDBC credentials from `polaris-db-secrets` (Quarkus datasource env var names must be verified against the pinned release — typically `QUARKUS_DATASOURCE_JDBC_URL`, `QUARKUS_DATASOURCE_USERNAME`, `QUARKUS_DATASOURCE_PASSWORD`) + - **Readiness and liveness probes are required** — the `polaris-bootstrap` exec deploy depends on `deploy.polaris` and will fire immediately when Polaris is marked ready; without probes the Quarkus app may still be starting: + ```yaml + readinessProbe: + httpGet: + path: /q/health/ready + port: 8182 + initialDelaySeconds: 20 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /q/health/live + port: 8182 + initialDelaySeconds: 30 + periodSeconds: 10 + ``` + +6. Create `polaris/garden.yaml` — `type: kubernetes`, `spec.manifestTemplates`, depends on `deploy.secrets`, `deploy.polaris-database`, `deploy.keycloak` + +--- + +## Phase 3: Keycloak Bootstrap Updates + +7. Update `keycloak-bootstrap/manifests/realm-configmap.yaml.tpl` — inside the existing `teehr-realm.json` data key: + - Add to `roles.realm` array: `iceberg-catalog-admin`, `iceberg-namespace-public-read`, `iceberg-namespace-public-write`, `iceberg-namespace-restricted-read`, `iceberg-namespace-restricted-write` + - Add to `groups` array: + - `iceberg-public-readers` (realmRoles: `iceberg-namespace-public-read`) + - `iceberg-public-writers` (realmRoles: `iceberg-namespace-public-write`, `iceberg-namespace-public-read`) + - `iceberg-restricted-readers` (realmRoles: `iceberg-namespace-restricted-read`) + - `iceberg-restricted-writers` (realmRoles: `iceberg-namespace-restricted-write`, `iceberg-namespace-restricted-read`) + - `iceberg-catalog-admins` (realmRoles: `iceberg-catalog-admin`) + - Retain existing `iceberg-user` role and group during transition + - Add to `clients` array — **two** new confidential service account clients (not three — Polaris does not need its own Keycloak client; it validates tokens via JWKS only and does not perform token introspection or act as an OAuth2 client itself): + - `trino-polaris` client — `serviceAccountsEnabled: true`, `secret: $(env:TRINO_POLARIS_CLIENT_SECRET)`, add `realm_access` protocol mapper to include roles in access token + - `spark-polaris` client — `serviceAccountsEnabled: true`, `secret: $(env:SPARK_POLARIS_CLIENT_SECRET)`, same `realm_access` mapper + - **Note on `realm_access.roles` claim**: Keycloak includes realm roles in access tokens by default, but verify against the running Keycloak version. If the Polaris `PrincipalRoleMapper` needs roles under a custom claim path, add an explicit `oidc-usermodel-realm-role-mapper` protocolMapper to the Polaris-facing clients. + +8. Update `keycloak-bootstrap/manifests/bootstrap-job.yaml` — add two new `env` entries matching the existing pattern: + - `TRINO_POLARIS_CLIENT_SECRET` from `trino-polaris-secrets` key `client-secret` + - `SPARK_POLARIS_CLIENT_SECRET` from `spark-polaris-secrets` key `client-secret` + +--- + +## Phase 4: Polaris Bootstrap Job — `polaris-bootstrap/` + +9. Create `polaris-bootstrap/manifests/acl-config.yaml` — a ConfigMap containing a declarative ACL definition (JSON) that the bootstrap job consumes. This makes the job **data-driven**: adding a new namespace = edit this file + re-run bootstrap job, no code changes: + ```json + { + "realm": "teehr", + "catalog": "teehr", + "catalog_admin_keycloak_role": "iceberg-catalog-admin", + "namespaces": [ + { + "name": "public", + "roles": [ + { + "keycloak_role": "iceberg-namespace-public-read", + "polaris_principal_role": "public_reader", + "polaris_catalog_role": "public_read_role", + "privileges": ["TABLE_READ_DATA", "TABLE_LIST", "NAMESPACE_LIST"] + }, + { + "keycloak_role": "iceberg-namespace-public-write", + "polaris_principal_role": "public_writer", + "polaris_catalog_role": "public_write_role", + "privileges": ["TABLE_WRITE_DATA", "TABLE_READ_DATA", "TABLE_LIST", "NAMESPACE_LIST", "CREATE_TABLE"] + } + ] + }, + { + "name": "restricted", + "roles": [ + { + "keycloak_role": "iceberg-namespace-restricted-read", + "polaris_principal_role": "restricted_reader", + "polaris_catalog_role": "restricted_read_role", + "privileges": ["TABLE_READ_DATA", "TABLE_LIST", "NAMESPACE_LIST"] + }, + { + "keycloak_role": "iceberg-namespace-restricted-write", + "polaris_principal_role": "restricted_writer", + "polaris_catalog_role": "restricted_write_role", + "privileges": ["TABLE_WRITE_DATA", "TABLE_READ_DATA", "TABLE_LIST", "NAMESPACE_LIST", "CREATE_TABLE"] + } + ] + } + ] + } + ``` + +10. Create `polaris-bootstrap/manifests/bootstrap-job.yaml` — reuse a Prefect image (same pattern as `prefect-workflows/manifests/load-secrets.yaml`, which uses `prefecthq/prefect:3.2.0-python3.10`; use whichever tag is current in the codebase — no new image or Dockerfile needed) running an inline Python script (`python -c |`) matching that pattern. The script: + - Accepts `POLARIS_REALM_NAME` env var (default `teehr`); sends `X-Polaris-Realm` header on every Management API call + - Uses stdlib (`json`, `os`) + `requests` (available in the Prefect image) — no custom image build + - Reads ACL config from mounted `acl-config.yaml` ConfigMap + - Creates the `teehr` catalog pointing at the warehouse (S3/MinIO path from env) + - Iterates config to create: namespaces, catalog roles with privilege grants, principal roles, catalog role → principal role assignments + - Creates `iceberg-catalog-admin` principal role with full catalog-level grants + - `ttlSecondsAfterFinished: 100` — matches existing Job pattern in `load-secrets.yaml` + - Required env vars in the Job manifest: + - `POLARIS_MANAGEMENT_URL: http://polaris:8183` — base URL for the management API + - `POLARIS_ROOT_CREDENTIALS` from `polaris-secrets` key `root-credentials` — used to obtain an initial Bearer token from the management API + - `POLARIS_REALM_NAME: teehr` (or from env) + - `CATALOG_WAREHOUSE` from `${var.polaris.catalogWarehouse}` (or hardcoded for local/remote) + +11. Create `polaris-bootstrap/garden.yaml` — mirrors `keycloak-bootstrap/garden.yaml` with **two deploys in one file** (this is required so the ConfigMap exists in Kubernetes before the Job pod tries to mount it): + ```yaml + kind: Deploy + type: kubernetes + name: polaris-acl-config + dependencies: + - deploy.secrets + environments: + - local + - remote + spec: + manifestFiles: + - ./manifests/acl-config.yaml + --- + kind: Deploy + type: exec + name: polaris-bootstrap + dependencies: + - deploy.polaris + - deploy.polaris-acl-config + - deploy.keycloak-bootstrap # logical ordering: Keycloak roles must exist before bootstrap maps them + environments: + - local + - remote + spec: + deployCommand: + - bash + - -c + - >- + kubectl -n "${environment.namespace}" delete job polaris-bootstrap --ignore-not-found=true && + kubectl -n "${environment.namespace}" apply -f ./manifests/bootstrap-job.yaml && + kubectl -n "${environment.namespace}" wait --for=condition=Complete job/polaris-bootstrap --timeout=120s + ``` + +--- + +## Phase 5: Trino Catalog Config Update + +12. Update `trino/garden.yaml` — **two separate Deploy blocks exist (local and remote); both must be updated**. Trino is a **Helm chart deployment** (`type: helm`) with no separate manifests directory — all config goes in Helm values. Changes per block: + + **Catalog properties** (in `catalogs.iceberg` multiline string): + - Replace `iceberg.rest-catalog.uri` value with `${var.polaris.catalogUri}` + - Replace `iceberg.rest-catalog.warehouse` value with `${var.polaris.catalogWarehouse}` + - Add `iceberg.rest-catalog.security=OAUTH2` + - Add `iceberg.rest-catalog.oauth2.server-uri=${var.polaris.oauthServerUri}` + - Add `iceberg.rest-catalog.oauth2.credential-file=/etc/trino/polaris-credential` + - Add `iceberg.rest-catalog.oauth2.scope=openid` + - Add `iceberg.rest-catalog.header.X-Polaris-Realm=teehr` + - Keep `iceberg.catalog.type=rest` (unchanged) + + **Credential file delivery** — Trino catalog properties do not support env var interpolation, so the OAuth2 credential must be a mounted file. The `trino-polaris-secrets` key `credential` stores the full `trino-polaris:` string. Mount it via Helm values: + ```yaml + coordinator: + extraVolumes: + - name: polaris-credential + secret: + secretName: trino-polaris-secrets + items: + - key: credential + path: polaris-credential + extraVolumeMounts: + - name: polaris-credential + mountPath: /etc/trino/polaris-credential + subPath: polaris-credential + readOnly: true + ``` + No init container needed. + + **Per-block constraints:** + - Local block: keep existing MinIO `env` entries; keep `s3.path-style-access` and `s3.endpoint` catalog properties + - Remote block: keep existing `serviceAccount.annotations` IRSA entry + - Both blocks: keep all existing `accessControl` configmap rules unchanged + +--- + +## Phase 6: Spark Session Updates + +13. Update `spark_session_utils.py` — changes are **minimal and additive**, preserving all existing function signatures and behavior: + - Add an optional `oauth2_token: str = None` parameter to `create_spark_session()` — passed through to `_configure_iceberg_catalogs()` + - In `_configure_iceberg_catalogs()`: add OAuth2 conf.set calls at the end of the existing function body: + - If `oauth2_token` provided (JupyterHub user token pass-through): set `rest.auth.type=oauth2` + `rest.auth.oauth2.token=` + - If absent (Prefect batch): set `rest.auth.type=oauth2`, `rest.auth.oauth2.server-uri` (from `POLARIS_OAUTH2_SERVER_URI` env), `rest.auth.oauth2.credential=spark-polaris:` (from `SPARK_POLARIS_CLIENT_SECRET` env), `rest.auth.oauth2.scope=openid` + - Both paths: set `rest.transport.header.X-Polaris-Realm=teehr` + - The existing `update_configs: Dict[str, str]` parameter on `create_spark_session()` remains available as an override escape hatch — no structural change needed + - **No changes** to `_create_spark_base_session`, `_set_spark_cluster_configuration`, `_set_aws_credentials_in_spark`, `_update_configs_and_packages`, `_set_catalog_metadata`, or any other existing functions + +14. Update `teehr/src/teehr/const.py` — add `POLARIS_OAUTH2_SERVER_URI` and `SPARK_POLARIS_CLIENT_SECRET` env var reads alongside existing constants. + +15. Update `prefect-workflows/manifests/prefect-deployer-job.yaml` — change `REMOTE_CATALOG_REST_URI` value from `${var.iceberg.catalogUri}` to `${var.polaris.catalogUri}`. + +--- + +## Phase 7: Garden Variables & Wiring + +16. Update `project.garden.yml` — add `polaris` variable group with **local/remote divergence** (same pattern as existing `iceberg.*` divergence): + ```yaml + # local: + polaris: + catalogUri: http://polaris:8182/api/catalog + oauthServerUri: http://keycloak-service:8080/realms/teehr/protocol/openid-connect/token + catalogWarehouse: s3://warehouse/ + catalogType: rest + inCluster: "true" + catalogS3PathStyleAccess: "true" + catalogS3Endpoint: "http://minio:9000" + + # remote: + polaris: + catalogUri: https://polaris.${var.hostname}/api/catalog + oauthServerUri: https://auth.${var.hostname}/realms/teehr/protocol/openid-connect/token + catalogWarehouse: s3://dev-teehr-iceberg-warehouse/ + catalogType: rest + inCluster: "false" + catalogS3PathStyleAccess: "false" + catalogS3Endpoint: "" + ``` + Keep existing `iceberg.*` variables during transition. The `polaris` variable group is a complete superset — once all consumers are migrated, `iceberg.*` can be removed entirely. + +17. Add comment blocks in `project.garden.yml` marking **control plane** (`keycloak*`, `polaris*`, `cert-manager`) vs **data plane** modules to document the intended future cluster boundary. All cross-plane references go through `polaris.*` variable group entries — no hardcoded in-cluster hostnames in data plane configs. + +--- + +## Phase 8: Polaris Ingress — Required + +18. Update `ingress/garden.yaml` — add a new `kind: Deploy` entry `name: polaris-ingress` following the exact pattern of existing entries: `type: kubernetes`, `spec.manifestTemplates: [manifests/polaris.yaml.tpl]`, dependencies on `deploy.cert-manager`, `deploy.letsencrypt`, `deploy.cert`, `deploy.contour`, `deploy.polaris`. + +19. Create `ingress/manifests/polaris.yaml.tpl` — Contour `HTTPProxy` following the exact pattern of existing manifests: + - `fqdn: polaris.${var.hostname}`, TLS `secretName: polaris.${var.hostname}-tls` + - Route `/api/catalog` → `polaris:8182` (catalog REST API) + - Route `/api/management` → `polaris:8183` (management API) + +--- + +## Phase 9: Retire iceberg-rest + +20. Disable `iceberg-rest` Garden deployment after all consumers (Trino, Spark, Prefect workflows) are verified connected to Polaris. Keep `iceberg-pg` until catalog data migration is confirmed complete. + +--- + +## Files — New + +| File | Notes | +|---|---| +| `polaris-pg/garden.yaml` | Mirror `iceberg-pg/garden.yaml`; uses `manifestFiles` | +| `polaris-pg/manifests/polaris-pg.yaml` | Static manifest (no `.tpl`); mirror `iceberg-pg/manifests/iceberg-pg.yaml` | +| `polaris/garden.yaml` | Depends on `polaris-database`, `keycloak`, `secrets` | +| `polaris/manifests/polaris.yaml.tpl` | Deployment + Service + SA; no IRSA annotation | +| `polaris/manifests/polaris-config.yaml.tpl` | ConfigMap with multi-issuer `polaris-server.yml` + `PrincipalRoleMapper` | +| `polaris-bootstrap/garden.yaml` | Two deploys: `type: kubernetes` (acl-config ConfigMap) + `type: exec` (job); both with `environments: [local, remote]` | +| `polaris-bootstrap/manifests/bootstrap-job.yaml` | Reuses `prefecthq/prefect:3.4.24-python3.12`; inline Python; parameterized on `POLARIS_REALM_NAME` | +| `polaris-bootstrap/manifests/acl-config.yaml` | Declarative namespace × privilege ACL ConfigMap | +| `ingress/manifests/polaris.yaml.tpl` | Contour HTTPProxy; routes for ports 8182 and 8183 | + +## Files — Modified + +| File | Change | +|---|---| +| `secrets/secrets.local.yaml` | Add `polaris-db-secrets`, `polaris-secrets`, `trino-polaris-secrets`, `spark-polaris-secrets` | +| `secrets/secrets.remote.yaml` | Same four secrets with production-grade values | +| `keycloak-bootstrap/manifests/realm-configmap.yaml.tpl` | Add 5 realm roles, 5 groups, 2 confidential clients (`trino-polaris`, `spark-polaris`) | +| `keycloak-bootstrap/manifests/bootstrap-job.yaml` | Add 2 new secret env vars (`TRINO_POLARIS_CLIENT_SECRET`, `SPARK_POLARIS_CLIENT_SECRET`) | +| `trino/garden.yaml` | Both local + remote Deploy blocks: OAuth2 catalog auth + credential-file volume mount via Helm values | +| `spark_session_utils.py` | Additive: dual-path OAuth2 + `X-Polaris-Realm` header; no existing signatures changed | +| `teehr/src/teehr/const.py` | Add `POLARIS_OAUTH2_SERVER_URI`, `SPARK_POLARIS_CLIENT_SECRET` | +| `prefect-workflows/manifests/prefect-deployer-job.yaml` | Update `REMOTE_CATALOG_REST_URI` to `${var.polaris.catalogUri}` | +| `project.garden.yml` | Add `polaris` variable group + control/data plane comments | +| `ingress/garden.yaml` | Add `polaris-ingress` Deploy entry | + +--- + +## Verification + +1. `kubectl get pods` — `polaris` and `polaris-pg` both Running +2. `kubectl logs deployment/polaris` — OIDC + `PrincipalRoleMapper` config loaded; no startup errors +3. Fetch `client_credentials` token for `trino-polaris` from Keycloak → `GET https://polaris.${var.hostname}/api/catalog/v1/config` with `X-Polaris-Realm: teehr` → expect 200; confirm `realm_access.roles` present in decoded token +4. `trino --execute "SHOW SCHEMAS IN iceberg"` → teehr schema visible +5. JupyterHub Spark with user token injected → user with no namespace role gets 403 from Polaris +6. Prefect batch job → `spark-polaris` service client token accepted; `REMOTE_CATALOG_REST_URI` resolves to Polaris +7. **ACL matrix**: + - `iceberg-namespace-public-read` member → read `public` ✓, write `public` ✗, read `restricted` ✗ + - `iceberg-namespace-public-write` member → read+write `public` ✓, `restricted` ✗ + - `iceberg-namespace-restricted-read` member → read `restricted` ✓, write `public` ✗ + - `iceberg-catalog-admin` member → full access to all namespaces ✓ +8. Edit `acl-config.yaml` to add a new namespace, re-run bootstrap job → new namespace ACLs applied; no code changes required +9. Run existing `teehr/tests/` catalog operation tests + +--- + +## Decisions + +- `apache/polaris` image (Apache incubator), pinned to a specific release tag — not `latest` +- `polaris.yaml.tpl` ServiceAccount uses `${environment.namespace}` — not hardcoded `teehr-hub` like `iceberg-rest.yaml.tpl` +- Readiness/liveness probes on `/q/health/ready` and `/q/health/live` (port 8182) required so Garden waits for Polaris to be truly ready before firing `polaris-bootstrap` +- `polaris` Keycloak client removed — Polaris validates tokens via JWKS only and needs no Keycloak service account +- New dedicated `polaris-pg` PostgreSQL instance (not reusing `iceberg-pg`) +- `polaris-pg` uses `manifestFiles` (static, no `.tpl`) — matching `iceberg-pg` pattern exactly +- Secrets via `secrets/secrets.local.yaml` + `secrets/secrets.remote.yaml` varfiles — consistent with existing `$forEach` pattern; no standalone K8s Secret manifests +- Trino OAuth2 credential delivered via mounted credential-file (`iceberg.rest-catalog.oauth2.credential-file`) — avoids env var interpolation limitations in Trino catalog properties; stored as full `trino-polaris:` string, mounted via `subPath`, no init container +- Trino: `client_credentials` — access control enforced at Trino layer; Polaris sees service identity +- Spark in Prefect: `client_credentials` (`spark-polaris`) — headless batch, no user context +- Spark in JupyterHub: user token pass-through — Polaris enforces per-user namespace/table ACLs +- Polaris has **no IRSA annotation** — pure metadata service on the control plane +- Control plane: Polaris + Keycloak (future: dedicated cluster); Data plane: all other services +- All cross-plane URLs go through `polaris.*` Garden variable group — no hardcoded in-cluster hostnames in data plane configs +- `polaris-server.yml` uses an issuer allow-list (not single issuer) from day one for multi-realm extensibility +- `polaris-bootstrap` reuses `prefecthq/prefect:3.4.24-python3.12` image (already in codebase via `load-secrets.yaml`) — no new Dockerfile or image build +- `polaris-bootstrap/garden.yaml` uses **two deploys**: `type: kubernetes` (deploys `acl-config.yaml` ConfigMap) + `type: exec` (runs the job) — matches `keycloak-bootstrap` pattern exactly; required because the Job pod mounts the ConfigMap as a volume +- `trino-polaris-secrets` has **two keys**: `client-secret` (plain value, used by `keycloak-bootstrap` env var) and `credential` (full `trino-polaris:` string, mounted as Trino credential file) — the same secret provides both without duplication +- `polaris` variable group is a complete superset of `iceberg` variable group, enabling full future removal of `iceberg.*` after migration +- `polaris-bootstrap` job env vars include `POLARIS_MANAGEMENT_URL` (http://polaris:8183) and `POLARIS_ROOT_CREDENTIALS` from `polaris-secrets` — required for the script to authenticate and call the management API +- `polaris-bootstrap` exec deploy depends on `deploy.keycloak-bootstrap` for logical ordering (Keycloak roles must exist before the bootstrap job maps them) +- Trino `iceberg.rest-catalog.warehouse` updated to `${var.polaris.catalogWarehouse}` alongside the URI change +- `spark_session_utils.py` changes are additive only — new `oauth2_token: str = None` parameter; all existing callers unaffected; `update_configs` escape hatch unchanged +- `X-Polaris-Realm` header explicit in all client configs from day one +- Keycloak role taxonomy: namespace × privilege; group-based assignment; coarse `iceberg-user` retained during transition +- `PrincipalRoleMapper` in `polaris-server.yml` resolves JWT `realm_access.roles` → Polaris principal roles at runtime; no per-user Polaris principal registration needed + +--- + +## Further Considerations + +1. **Polaris image tag**: Pin to a specific release (e.g., `0.9.0`) — `apache/polaris` is under active development and `latest` may break between deployments. + +2. **DB schema init**: Confirm whether Polaris auto-migrates its PostgreSQL schema on first start or requires a separate init job — check release notes for the pinned version before implementing Phase 1. + +3. **Catalog data migration**: Existing tables registered in `iceberg-pg`'s JDBC catalog will not auto-appear in Polaris. The `polaris-bootstrap` job needs a migration step to re-register existing namespaces/tables, or plan for a re-ingest window before retiring `iceberg-rest` in Phase 9. + +4. **OPA for Trino access control (future)**: Trino uses a single `trino-polaris` service identity so Polaris cannot enforce per-user namespace/table ACLs for Trino queries. Open Policy Agent (OPA) — a lightweight Go service on the control plane — can fill this gap. Trino's native OPA system access control plugin receives full query context (user identity, Keycloak groups, target catalog/schema/table) and evaluates Rego policies that mirror the Keycloak role taxonomy. Policy changes hot-reload via ConfigMap without Trino restarts. Would require: new `opa/` Garden module + `access-control.name=opa` in Trino config + policy ConfigMap mirroring the Phase 3 role taxonomy. + +5. **Per-workflow Prefect clients (future)**: Replace single `spark-polaris` client with per-category Keycloak clients (`prefect-ingest`, `prefect-metrics`, etc.), each granted only the namespace roles it needs. Prefect deployment job templates inject credentials via workflow-specific K8s Secrets. `spark_session_utils.py` reads `SPARK_POLARIS_CLIENT_ID` from env rather than hardcoding. No changes needed to Polaris bootstrap or `acl-config.yaml`. From 46cc1c6d85d67d69d15b3724bd9be97cd8e2793d Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Fri, 17 Jul 2026 14:19:43 -0400 Subject: [PATCH 08/62] cleanup duplicate file --- polaris-migration-plan.md | 390 -------------------------------------- 1 file changed, 390 deletions(-) delete mode 100644 polaris-migration-plan.md diff --git a/polaris-migration-plan.md b/polaris-migration-plan.md deleted file mode 100644 index 30cbf75..0000000 --- a/polaris-migration-plan.md +++ /dev/null @@ -1,390 +0,0 @@ -# Plan: Replace iceberg-rest with Apache Polaris (Keycloak-integrated) - -## TL;DR - -Replace `tabulario/iceberg-rest` with `apache/polaris`. Tight Keycloak integration via JWKS + a `PrincipalRoleMapper` that reads `realm_access.roles` from JWTs, mapping Keycloak roles directly to Polaris principal roles at runtime — no per-user Polaris registration needed. A data-driven bootstrap job creates the full namespace × privilege ACL structure from a ConfigMap. Polaris and Keycloak are the control plane; all other services are the data plane. - ---- - -> **Implementation order for `garden deploy` to work**: Phase 7 (Garden variables) and the secrets additions from Phase 1 must be done first — before any other phase — because Garden resolves `${var.polaris.*}` template variables at render time and pods reference the new secrets at startup. - -## Phase 1: Database — `polaris-pg` - -1. Create `polaris-pg/manifests/polaris-pg.yaml` (static, not `.tpl` — matches `iceberg-pg` which uses `manifestFiles`, no Garden templating needed). Mirror `iceberg-pg/manifests/iceberg-pg.yaml` exactly with these substitutions: - - All `iceberg-pg` → `polaris-pg`, all `iceberg` → `polaris` (names, labels, PVC claim) - - `POSTGRES_DB`/`POSTGRES_USER`/`POSTGRES_PASSWORD` env vars read from `polaris-db-secrets` - - Readiness/liveness `pg_isready` probes use `-U polaris -d polaris` - - PVC storage: 10Gi (same as iceberg-pg) - -2. Create `polaris-pg/garden.yaml` — `type: kubernetes`, `name: polaris-database`, `spec.manifestFiles: [./manifests/polaris-pg.yaml]`, depends on `deploy.secrets` — mirrors `iceberg-pg/garden.yaml` exactly with name substitution. - -3. Add `polaris-db-secrets` to `secrets/secrets.local.yaml` and `secrets/secrets.remote.yaml`. **Secrets are NOT standalone K8s manifests** — the `secrets/garden.yaml` `$forEach` loop creates K8s Secrets automatically from these varfiles. Add: - ```yaml - polaris-db-secrets: - data: - database: polaris - username: polaris - password: polaris123 # local; use strong password in remote - polaris-secrets: - data: - root-credentials: "root:secret123" # POLARIS_ROOT_CREDENTIALS format (:); use strong value in remote - trino-polaris-secrets: - data: - client-secret: local-trino-polaris-client-secret # plain secret — used by keycloak-bootstrap env var - credential: "trino-polaris:local-trino-polaris-client-secret" # full credential string — mounted as Trino credential file - spark-polaris-secrets: - data: - client-secret: local-spark-polaris-client-secret - ``` - > **Note**: `trino-polaris-secrets` needs **two keys** because the Keycloak bootstrap job needs the plain secret value (`client-secret`) to set as the Keycloak client secret, while Trino's credential file mount needs the full `trino-polaris:` string (`credential`). Using the `credential` key for the Keycloak env var would inject the wrong value. - ---- - -## Phase 2: Polaris Deployment — `polaris/` - -4. Create `polaris/manifests/polaris-config.yaml.tpl` — ConfigMap with `polaris-server.yml`: - - JDBC persistence → `polaris-pg` - - OIDC issuers as a **list** (multi-realm extensible); JWKS URI internal for `local`, external for `remote` - - **`PrincipalRoleMapper`** pointing at `realm_access.roles` in the JWT — maps Keycloak role names to Polaris principal role names at runtime. This is the critical link; without it user token pass-through grants zero privileges. Exact config key must be verified against the pinned Polaris release. - -5. Create `polaris/manifests/polaris.yaml.tpl` — Deployment + Service + SA: - - Image: `apache/polaris` pinned to a specific release tag - - Ports 8182 (catalog REST API), port 8183 (management API); Service exposes both - - **No IRSA annotation** — Polaris is pure metadata; S3 I/O stays with Trino/Spark - - ServiceAccount metadata must use `${environment.namespace}` — do NOT copy the hardcoded `namespace: teehr-hub` from `iceberg-rest.yaml.tpl` - - Local env gets MinIO credentials (same `${if environment.name == "local"}` pattern as `iceberg-rest.yaml.tpl`) - - `POLARIS_ROOT_CREDENTIALS` from `polaris-secrets` key `root-credentials` - - JDBC credentials from `polaris-db-secrets` (Quarkus datasource env var names must be verified against the pinned release — typically `QUARKUS_DATASOURCE_JDBC_URL`, `QUARKUS_DATASOURCE_USERNAME`, `QUARKUS_DATASOURCE_PASSWORD`) - - **Readiness and liveness probes are required** — the `polaris-bootstrap` exec deploy depends on `deploy.polaris` and will fire immediately when Polaris is marked ready; without probes the Quarkus app may still be starting: - ```yaml - readinessProbe: - httpGet: - path: /q/health/ready - port: 8182 - initialDelaySeconds: 20 - periodSeconds: 10 - livenessProbe: - httpGet: - path: /q/health/live - port: 8182 - initialDelaySeconds: 30 - periodSeconds: 10 - ``` - -6. Create `polaris/garden.yaml` — `type: kubernetes`, `spec.manifestTemplates`, depends on `deploy.secrets`, `deploy.polaris-database`, `deploy.keycloak` - ---- - -## Phase 3: Keycloak Bootstrap Updates - -7. Update `keycloak-bootstrap/manifests/realm-configmap.yaml.tpl` — inside the existing `teehr-realm.json` data key: - - Add to `roles.realm` array: `iceberg-catalog-admin`, `iceberg-namespace-public-read`, `iceberg-namespace-public-write`, `iceberg-namespace-restricted-read`, `iceberg-namespace-restricted-write` - - Add to `groups` array: - - `iceberg-public-readers` (realmRoles: `iceberg-namespace-public-read`) - - `iceberg-public-writers` (realmRoles: `iceberg-namespace-public-write`, `iceberg-namespace-public-read`) - - `iceberg-restricted-readers` (realmRoles: `iceberg-namespace-restricted-read`) - - `iceberg-restricted-writers` (realmRoles: `iceberg-namespace-restricted-write`, `iceberg-namespace-restricted-read`) - - `iceberg-catalog-admins` (realmRoles: `iceberg-catalog-admin`) - - Retain existing `iceberg-user` role and group during transition - - Add to `clients` array — **two** new confidential service account clients (not three — Polaris does not need its own Keycloak client; it validates tokens via JWKS only and does not perform token introspection or act as an OAuth2 client itself): - - `trino-polaris` client — `serviceAccountsEnabled: true`, `secret: $(env:TRINO_POLARIS_CLIENT_SECRET)`, add `realm_access` protocol mapper to include roles in access token - - `spark-polaris` client — `serviceAccountsEnabled: true`, `secret: $(env:SPARK_POLARIS_CLIENT_SECRET)`, same `realm_access` mapper - - **Note on `realm_access.roles` claim**: Keycloak includes realm roles in access tokens by default, but verify against the running Keycloak version. If the Polaris `PrincipalRoleMapper` needs roles under a custom claim path, add an explicit `oidc-usermodel-realm-role-mapper` protocolMapper to the Polaris-facing clients. - -8. Update `keycloak-bootstrap/manifests/bootstrap-job.yaml` — add two new `env` entries matching the existing pattern: - - `TRINO_POLARIS_CLIENT_SECRET` from `trino-polaris-secrets` key `client-secret` - - `SPARK_POLARIS_CLIENT_SECRET` from `spark-polaris-secrets` key `client-secret` - ---- - -## Phase 4: Polaris Bootstrap Job — `polaris-bootstrap/` - -9. Create `polaris-bootstrap/manifests/acl-config.yaml` — a ConfigMap containing a declarative ACL definition (JSON) that the bootstrap job consumes. This makes the job **data-driven**: adding a new namespace = edit this file + re-run bootstrap job, no code changes: - ```json - { - "realm": "teehr", - "catalog": "teehr", - "catalog_admin_keycloak_role": "iceberg-catalog-admin", - "namespaces": [ - { - "name": "public", - "roles": [ - { - "keycloak_role": "iceberg-namespace-public-read", - "polaris_principal_role": "public_reader", - "polaris_catalog_role": "public_read_role", - "privileges": ["TABLE_READ_DATA", "TABLE_LIST", "NAMESPACE_LIST"] - }, - { - "keycloak_role": "iceberg-namespace-public-write", - "polaris_principal_role": "public_writer", - "polaris_catalog_role": "public_write_role", - "privileges": ["TABLE_WRITE_DATA", "TABLE_READ_DATA", "TABLE_LIST", "NAMESPACE_LIST", "CREATE_TABLE"] - } - ] - }, - { - "name": "restricted", - "roles": [ - { - "keycloak_role": "iceberg-namespace-restricted-read", - "polaris_principal_role": "restricted_reader", - "polaris_catalog_role": "restricted_read_role", - "privileges": ["TABLE_READ_DATA", "TABLE_LIST", "NAMESPACE_LIST"] - }, - { - "keycloak_role": "iceberg-namespace-restricted-write", - "polaris_principal_role": "restricted_writer", - "polaris_catalog_role": "restricted_write_role", - "privileges": ["TABLE_WRITE_DATA", "TABLE_READ_DATA", "TABLE_LIST", "NAMESPACE_LIST", "CREATE_TABLE"] - } - ] - } - ] - } - ``` - -10. Create `polaris-bootstrap/manifests/bootstrap-job.yaml` — reuse a Prefect image (same pattern as `prefect-workflows/manifests/load-secrets.yaml`, which uses `prefecthq/prefect:3.2.0-python3.10`; use whichever tag is current in the codebase — no new image or Dockerfile needed) running an inline Python script (`python -c |`) matching that pattern. The script: - - Accepts `POLARIS_REALM_NAME` env var (default `teehr`); sends `X-Polaris-Realm` header on every Management API call - - Uses stdlib (`json`, `os`) + `requests` (available in the Prefect image) — no custom image build - - Reads ACL config from mounted `acl-config.yaml` ConfigMap - - Creates the `teehr` catalog pointing at the warehouse (S3/MinIO path from env) - - Iterates config to create: namespaces, catalog roles with privilege grants, principal roles, catalog role → principal role assignments - - Creates `iceberg-catalog-admin` principal role with full catalog-level grants - - `ttlSecondsAfterFinished: 100` — matches existing Job pattern in `load-secrets.yaml` - - Required env vars in the Job manifest: - - `POLARIS_MANAGEMENT_URL: http://polaris:8183` — base URL for the management API - - `POLARIS_ROOT_CREDENTIALS` from `polaris-secrets` key `root-credentials` — used to obtain an initial Bearer token from the management API - - `POLARIS_REALM_NAME: teehr` (or from env) - - `CATALOG_WAREHOUSE` from `${var.polaris.catalogWarehouse}` (or hardcoded for local/remote) - -11. Create `polaris-bootstrap/garden.yaml` — mirrors `keycloak-bootstrap/garden.yaml` with **two deploys in one file** (this is required so the ConfigMap exists in Kubernetes before the Job pod tries to mount it): - ```yaml - kind: Deploy - type: kubernetes - name: polaris-acl-config - dependencies: - - deploy.secrets - environments: - - local - - remote - spec: - manifestFiles: - - ./manifests/acl-config.yaml - --- - kind: Deploy - type: exec - name: polaris-bootstrap - dependencies: - - deploy.polaris - - deploy.polaris-acl-config - - deploy.keycloak-bootstrap # logical ordering: Keycloak roles must exist before bootstrap maps them - environments: - - local - - remote - spec: - deployCommand: - - bash - - -c - - >- - kubectl -n "${environment.namespace}" delete job polaris-bootstrap --ignore-not-found=true && - kubectl -n "${environment.namespace}" apply -f ./manifests/bootstrap-job.yaml && - kubectl -n "${environment.namespace}" wait --for=condition=Complete job/polaris-bootstrap --timeout=120s - ``` - ---- - -## Phase 5: Trino Catalog Config Update - -12. Update `trino/garden.yaml` — **two separate Deploy blocks exist (local and remote); both must be updated**. Trino is a **Helm chart deployment** (`type: helm`) with no separate manifests directory — all config goes in Helm values. Changes per block: - - **Catalog properties** (in `catalogs.iceberg` multiline string): - - Replace `iceberg.rest-catalog.uri` value with `${var.polaris.catalogUri}` - - Replace `iceberg.rest-catalog.warehouse` value with `${var.polaris.catalogWarehouse}` - - Add `iceberg.rest-catalog.security=OAUTH2` - - Add `iceberg.rest-catalog.oauth2.server-uri=${var.polaris.oauthServerUri}` - - Add `iceberg.rest-catalog.oauth2.credential-file=/etc/trino/polaris-credential` - - Add `iceberg.rest-catalog.oauth2.scope=openid` - - Add `iceberg.rest-catalog.header.X-Polaris-Realm=teehr` - - Keep `iceberg.catalog.type=rest` (unchanged) - - **Credential file delivery** — Trino catalog properties do not support env var interpolation, so the OAuth2 credential must be a mounted file. The `trino-polaris-secrets` key `credential` stores the full `trino-polaris:` string. Mount it via Helm values: - ```yaml - coordinator: - extraVolumes: - - name: polaris-credential - secret: - secretName: trino-polaris-secrets - items: - - key: credential - path: polaris-credential - extraVolumeMounts: - - name: polaris-credential - mountPath: /etc/trino/polaris-credential - subPath: polaris-credential - readOnly: true - ``` - No init container needed. - - **Per-block constraints:** - - Local block: keep existing MinIO `env` entries; keep `s3.path-style-access` and `s3.endpoint` catalog properties - - Remote block: keep existing `serviceAccount.annotations` IRSA entry - - Both blocks: keep all existing `accessControl` configmap rules unchanged - ---- - -## Phase 6: Spark Session Updates - -13. Update `spark_session_utils.py` — changes are **minimal and additive**, preserving all existing function signatures and behavior: - - Add an optional `oauth2_token: str = None` parameter to `create_spark_session()` — passed through to `_configure_iceberg_catalogs()` - - In `_configure_iceberg_catalogs()`: add OAuth2 conf.set calls at the end of the existing function body: - - If `oauth2_token` provided (JupyterHub user token pass-through): set `rest.auth.type=oauth2` + `rest.auth.oauth2.token=` - - If absent (Prefect batch): set `rest.auth.type=oauth2`, `rest.auth.oauth2.server-uri` (from `POLARIS_OAUTH2_SERVER_URI` env), `rest.auth.oauth2.credential=spark-polaris:` (from `SPARK_POLARIS_CLIENT_SECRET` env), `rest.auth.oauth2.scope=openid` - - Both paths: set `rest.transport.header.X-Polaris-Realm=teehr` - - The existing `update_configs: Dict[str, str]` parameter on `create_spark_session()` remains available as an override escape hatch — no structural change needed - - **No changes** to `_create_spark_base_session`, `_set_spark_cluster_configuration`, `_set_aws_credentials_in_spark`, `_update_configs_and_packages`, `_set_catalog_metadata`, or any other existing functions - -14. Update `teehr/src/teehr/const.py` — add `POLARIS_OAUTH2_SERVER_URI` and `SPARK_POLARIS_CLIENT_SECRET` env var reads alongside existing constants. - -15. Update `prefect-workflows/manifests/prefect-deployer-job.yaml` — change `REMOTE_CATALOG_REST_URI` value from `${var.iceberg.catalogUri}` to `${var.polaris.catalogUri}`. - ---- - -## Phase 7: Garden Variables & Wiring - -16. Update `project.garden.yml` — add `polaris` variable group with **local/remote divergence** (same pattern as existing `iceberg.*` divergence): - ```yaml - # local: - polaris: - catalogUri: http://polaris:8182/api/catalog - oauthServerUri: http://keycloak-service:8080/realms/teehr/protocol/openid-connect/token - catalogWarehouse: s3://warehouse/ - catalogType: rest - inCluster: "true" - catalogS3PathStyleAccess: "true" - catalogS3Endpoint: "http://minio:9000" - - # remote: - polaris: - catalogUri: https://polaris.${var.hostname}/api/catalog - oauthServerUri: https://auth.${var.hostname}/realms/teehr/protocol/openid-connect/token - catalogWarehouse: s3://dev-teehr-iceberg-warehouse/ - catalogType: rest - inCluster: "false" - catalogS3PathStyleAccess: "false" - catalogS3Endpoint: "" - ``` - Keep existing `iceberg.*` variables during transition. The `polaris` variable group is a complete superset — once all consumers are migrated, `iceberg.*` can be removed entirely. - -17. Add comment blocks in `project.garden.yml` marking **control plane** (`keycloak*`, `polaris*`, `cert-manager`) vs **data plane** modules to document the intended future cluster boundary. All cross-plane references go through `polaris.*` variable group entries — no hardcoded in-cluster hostnames in data plane configs. - ---- - -## Phase 8: Polaris Ingress — Required - -18. Update `ingress/garden.yaml` — add a new `kind: Deploy` entry `name: polaris-ingress` following the exact pattern of existing entries: `type: kubernetes`, `spec.manifestTemplates: [manifests/polaris.yaml.tpl]`, dependencies on `deploy.cert-manager`, `deploy.letsencrypt`, `deploy.cert`, `deploy.contour`, `deploy.polaris`. - -19. Create `ingress/manifests/polaris.yaml.tpl` — Contour `HTTPProxy` following the exact pattern of existing manifests: - - `fqdn: polaris.${var.hostname}`, TLS `secretName: polaris.${var.hostname}-tls` - - Route `/api/catalog` → `polaris:8182` (catalog REST API) - - Route `/api/management` → `polaris:8183` (management API) - ---- - -## Phase 9: Retire iceberg-rest - -20. Disable `iceberg-rest` Garden deployment after all consumers (Trino, Spark, Prefect workflows) are verified connected to Polaris. Keep `iceberg-pg` until catalog data migration is confirmed complete. - ---- - -## Files — New - -| File | Notes | -|---|---| -| `polaris-pg/garden.yaml` | Mirror `iceberg-pg/garden.yaml`; uses `manifestFiles` | -| `polaris-pg/manifests/polaris-pg.yaml` | Static manifest (no `.tpl`); mirror `iceberg-pg/manifests/iceberg-pg.yaml` | -| `polaris/garden.yaml` | Depends on `polaris-database`, `keycloak`, `secrets` | -| `polaris/manifests/polaris.yaml.tpl` | Deployment + Service + SA; no IRSA annotation | -| `polaris/manifests/polaris-config.yaml.tpl` | ConfigMap with multi-issuer `polaris-server.yml` + `PrincipalRoleMapper` | -| `polaris-bootstrap/garden.yaml` | Two deploys: `type: kubernetes` (acl-config ConfigMap) + `type: exec` (job); both with `environments: [local, remote]` | -| `polaris-bootstrap/manifests/bootstrap-job.yaml` | Reuses `prefecthq/prefect:3.4.24-python3.12`; inline Python; parameterized on `POLARIS_REALM_NAME` | -| `polaris-bootstrap/manifests/acl-config.yaml` | Declarative namespace × privilege ACL ConfigMap | -| `ingress/manifests/polaris.yaml.tpl` | Contour HTTPProxy; routes for ports 8182 and 8183 | - -## Files — Modified - -| File | Change | -|---|---| -| `secrets/secrets.local.yaml` | Add `polaris-db-secrets`, `polaris-secrets`, `trino-polaris-secrets`, `spark-polaris-secrets` | -| `secrets/secrets.remote.yaml` | Same four secrets with production-grade values | -| `keycloak-bootstrap/manifests/realm-configmap.yaml.tpl` | Add 5 realm roles, 5 groups, 2 confidential clients (`trino-polaris`, `spark-polaris`) | -| `keycloak-bootstrap/manifests/bootstrap-job.yaml` | Add 2 new secret env vars (`TRINO_POLARIS_CLIENT_SECRET`, `SPARK_POLARIS_CLIENT_SECRET`) | -| `trino/garden.yaml` | Both local + remote Deploy blocks: OAuth2 catalog auth + credential-file volume mount via Helm values | -| `spark_session_utils.py` | Additive: dual-path OAuth2 + `X-Polaris-Realm` header; no existing signatures changed | -| `teehr/src/teehr/const.py` | Add `POLARIS_OAUTH2_SERVER_URI`, `SPARK_POLARIS_CLIENT_SECRET` | -| `prefect-workflows/manifests/prefect-deployer-job.yaml` | Update `REMOTE_CATALOG_REST_URI` to `${var.polaris.catalogUri}` | -| `project.garden.yml` | Add `polaris` variable group + control/data plane comments | -| `ingress/garden.yaml` | Add `polaris-ingress` Deploy entry | - ---- - -## Verification - -1. `kubectl get pods` — `polaris` and `polaris-pg` both Running -2. `kubectl logs deployment/polaris` — OIDC + `PrincipalRoleMapper` config loaded; no startup errors -3. Fetch `client_credentials` token for `trino-polaris` from Keycloak → `GET https://polaris.${var.hostname}/api/catalog/v1/config` with `X-Polaris-Realm: teehr` → expect 200; confirm `realm_access.roles` present in decoded token -4. `trino --execute "SHOW SCHEMAS IN iceberg"` → teehr schema visible -5. JupyterHub Spark with user token injected → user with no namespace role gets 403 from Polaris -6. Prefect batch job → `spark-polaris` service client token accepted; `REMOTE_CATALOG_REST_URI` resolves to Polaris -7. **ACL matrix**: - - `iceberg-namespace-public-read` member → read `public` ✓, write `public` ✗, read `restricted` ✗ - - `iceberg-namespace-public-write` member → read+write `public` ✓, `restricted` ✗ - - `iceberg-namespace-restricted-read` member → read `restricted` ✓, write `public` ✗ - - `iceberg-catalog-admin` member → full access to all namespaces ✓ -8. Edit `acl-config.yaml` to add a new namespace, re-run bootstrap job → new namespace ACLs applied; no code changes required -9. Run existing `teehr/tests/` catalog operation tests - ---- - -## Decisions - -- `apache/polaris` image (Apache incubator), pinned to a specific release tag — not `latest` -- `polaris.yaml.tpl` ServiceAccount uses `${environment.namespace}` — not hardcoded `teehr-hub` like `iceberg-rest.yaml.tpl` -- Readiness/liveness probes on `/q/health/ready` and `/q/health/live` (port 8182) required so Garden waits for Polaris to be truly ready before firing `polaris-bootstrap` -- `polaris` Keycloak client removed — Polaris validates tokens via JWKS only and needs no Keycloak service account -- New dedicated `polaris-pg` PostgreSQL instance (not reusing `iceberg-pg`) -- `polaris-pg` uses `manifestFiles` (static, no `.tpl`) — matching `iceberg-pg` pattern exactly -- Secrets via `secrets/secrets.local.yaml` + `secrets/secrets.remote.yaml` varfiles — consistent with existing `$forEach` pattern; no standalone K8s Secret manifests -- Trino OAuth2 credential delivered via mounted credential-file (`iceberg.rest-catalog.oauth2.credential-file`) — avoids env var interpolation limitations in Trino catalog properties; stored as full `trino-polaris:` string, mounted via `subPath`, no init container -- Trino: `client_credentials` — access control enforced at Trino layer; Polaris sees service identity -- Spark in Prefect: `client_credentials` (`spark-polaris`) — headless batch, no user context -- Spark in JupyterHub: user token pass-through — Polaris enforces per-user namespace/table ACLs -- Polaris has **no IRSA annotation** — pure metadata service on the control plane -- Control plane: Polaris + Keycloak (future: dedicated cluster); Data plane: all other services -- All cross-plane URLs go through `polaris.*` Garden variable group — no hardcoded in-cluster hostnames in data plane configs -- `polaris-server.yml` uses an issuer allow-list (not single issuer) from day one for multi-realm extensibility -- `polaris-bootstrap` reuses `prefecthq/prefect:3.4.24-python3.12` image (already in codebase via `load-secrets.yaml`) — no new Dockerfile or image build -- `polaris-bootstrap/garden.yaml` uses **two deploys**: `type: kubernetes` (deploys `acl-config.yaml` ConfigMap) + `type: exec` (runs the job) — matches `keycloak-bootstrap` pattern exactly; required because the Job pod mounts the ConfigMap as a volume -- `trino-polaris-secrets` has **two keys**: `client-secret` (plain value, used by `keycloak-bootstrap` env var) and `credential` (full `trino-polaris:` string, mounted as Trino credential file) — the same secret provides both without duplication -- `polaris` variable group is a complete superset of `iceberg` variable group, enabling full future removal of `iceberg.*` after migration -- `polaris-bootstrap` job env vars include `POLARIS_MANAGEMENT_URL` (http://polaris:8183) and `POLARIS_ROOT_CREDENTIALS` from `polaris-secrets` — required for the script to authenticate and call the management API -- `polaris-bootstrap` exec deploy depends on `deploy.keycloak-bootstrap` for logical ordering (Keycloak roles must exist before the bootstrap job maps them) -- Trino `iceberg.rest-catalog.warehouse` updated to `${var.polaris.catalogWarehouse}` alongside the URI change -- `spark_session_utils.py` changes are additive only — new `oauth2_token: str = None` parameter; all existing callers unaffected; `update_configs` escape hatch unchanged -- `X-Polaris-Realm` header explicit in all client configs from day one -- Keycloak role taxonomy: namespace × privilege; group-based assignment; coarse `iceberg-user` retained during transition -- `PrincipalRoleMapper` in `polaris-server.yml` resolves JWT `realm_access.roles` → Polaris principal roles at runtime; no per-user Polaris principal registration needed - ---- - -## Further Considerations - -1. **Polaris image tag**: Pin to a specific release (e.g., `0.9.0`) — `apache/polaris` is under active development and `latest` may break between deployments. - -2. **DB schema init**: Confirm whether Polaris auto-migrates its PostgreSQL schema on first start or requires a separate init job — check release notes for the pinned version before implementing Phase 1. - -3. **Catalog data migration**: Existing tables registered in `iceberg-pg`'s JDBC catalog will not auto-appear in Polaris. The `polaris-bootstrap` job needs a migration step to re-register existing namespaces/tables, or plan for a re-ingest window before retiring `iceberg-rest` in Phase 9. - -4. **OPA for Trino access control (future)**: Trino uses a single `trino-polaris` service identity so Polaris cannot enforce per-user namespace/table ACLs for Trino queries. Open Policy Agent (OPA) — a lightweight Go service on the control plane — can fill this gap. Trino's native OPA system access control plugin receives full query context (user identity, Keycloak groups, target catalog/schema/table) and evaluates Rego policies that mirror the Keycloak role taxonomy. Policy changes hot-reload via ConfigMap without Trino restarts. Would require: new `opa/` Garden module + `access-control.name=opa` in Trino config + policy ConfigMap mirroring the Phase 3 role taxonomy. - -5. **Per-workflow Prefect clients (future)**: Replace single `spark-polaris` client with per-category Keycloak clients (`prefect-ingest`, `prefect-metrics`, etc.), each granted only the namespace roles it needs. Prefect deployment job templates inject credentials via workflow-specific K8s Secrets. `spark_session_utils.py` reads `SPARK_POLARIS_CLIENT_ID` from env rather than hardcoding. No changes needed to Polaris bootstrap or `acl-config.yaml`. From c58a94e40f8c9248a31e9d5f78159a9dc8a745c2 Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Fri, 17 Jul 2026 14:53:41 -0400 Subject: [PATCH 09/62] remove duplicate sections --- trino/garden.yaml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/trino/garden.yaml b/trino/garden.yaml index 51aafe0..db58019 100644 --- a/trino/garden.yaml +++ b/trino/garden.yaml @@ -142,16 +142,6 @@ spec: ] } - coordinator: - extraVolumeMounts: [] - resources: - requests: - cpu: 1 - memory: 4Gi - limits: - cpu: 2 - memory: 8Gi - env: - name: POLARIS_CREDENTIAL valueFrom: From 5363d2e0f481cbdb17e2462d8e7f0671a6e53612 Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Fri, 17 Jul 2026 14:54:01 -0400 Subject: [PATCH 10/62] fix naming bug in example/template --- example.project.garden.yml => project.garden.yml.template | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename example.project.garden.yml => project.garden.yml.template (100%) diff --git a/example.project.garden.yml b/project.garden.yml.template similarity index 100% rename from example.project.garden.yml rename to project.garden.yml.template From 065d47d03122af37efafb66bf22339844b6412f6 Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Fri, 17 Jul 2026 14:54:11 -0400 Subject: [PATCH 11/62] fix ingress dep --- ingress/garden.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ingress/garden.yaml b/ingress/garden.yaml index 103c03b..2b7f2f2 100644 --- a/ingress/garden.yaml +++ b/ingress/garden.yaml @@ -58,10 +58,6 @@ spec: manifestTemplates: - manifests/polaris.yaml.tpl dependencies: - - deploy.cert-manager - - deploy.letsencrypt - - deploy.cert - - deploy.contour - deploy.polaris environments: - local From 1b8da91696ea593392b92ca425649f4ed48fe992 Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Thu, 23 Jul 2026 17:07:08 -0400 Subject: [PATCH 12/62] Stabilize Jupyter Polaris flow before AuthManager work --- docs/polaris-local-spark-simple-example.md | 107 ++++++ .../developer/01_setup_minio_warehouse.ipynb | 189 ++++++++++- .../polaris_spark_namespace_table_example.py | 311 ++++++++++++++++++ examples/developer/setup_utils.py | 262 ++++++++++++++- iceberg-pg/garden.yaml | 1 + iceberg-rest/garden.yaml | 1 + jupyterhub/garden.yaml | 24 +- .../manifests/local-users-configmap.yaml.tpl | 1 + .../manifests/realm-configmap.yaml.tpl | 1 + .../manifests/acl-config.yaml.tpl | 4 + .../manifests/bootstrap-job.yaml | 117 ++++++- .../polaris-principal-sync-cronjob.yaml.tpl | 8 +- .../polaris-sync-principals-script.yaml | 76 ++++- polaris/manifests/polaris-config.yaml.tpl | 5 + polaris/manifests/polaris.yaml.tpl | 6 - project.garden.yml.template | 21 +- scripts/run_jupyter_polaris_spark_example.sh | 134 ++++++++ trino/garden.yaml | 4 +- 18 files changed, 1219 insertions(+), 53 deletions(-) create mode 100644 docs/polaris-local-spark-simple-example.md create mode 100644 examples/developer/polaris_spark_namespace_table_example.py create mode 100755 scripts/run_jupyter_polaris_spark_example.sh diff --git a/docs/polaris-local-spark-simple-example.md b/docs/polaris-local-spark-simple-example.md new file mode 100644 index 0000000..f3f0e6e --- /dev/null +++ b/docs/polaris-local-spark-simple-example.md @@ -0,0 +1,107 @@ +# Polaris Local Spark Simple Example + +This is a minimal local-development flow for Polaris on KinD with MinIO using Spark. + +## Goal + +- Authenticate as `admin` +- Connect Spark to Polaris REST catalog +- Create a namespace +- Create and query a table + +## Prerequisites + +- Cluster is deployed and healthy: `garden deploy` +- A Jupyter single-user pod is running (for example `jupyter-admin`) +- Local Keycloak users exist (`admin` / `admin` by default) + +## 1. Refresh Polaris bootstrap config + +Run this once after changing Polaris/MinIO/catalog settings: + +```bash +kubectl -n teehr-hub delete job polaris-bootstrap --ignore-not-found=true +kubectl -n teehr-hub apply -f ./polaris-bootstrap/manifests/bootstrap-job.yaml +kubectl -n teehr-hub wait --for=condition=Complete job/polaris-bootstrap --timeout=600s +``` + +## 2. Run the minimal Spark example + +```bash +bash ./scripts/run_jupyter_polaris_spark_example.sh +``` + +Optional: keep created objects instead of dropping them. + +```bash +EXTRA_ARGS="--keep --namespace spark_demo_manual --table demo_table" bash ./scripts/run_jupyter_polaris_spark_example.sh +``` + +## 3. Expected behavior + +The script prints: + +- effective Spark Polaris/MinIO config +- namespace listing +- namespace creation +- table creation, insert, and select + +## 4. Known failure signature and meaning + +If table creation fails with `UnknownHostException` and a host like `warehouse.minio`, then: + +- Spark-to-Polaris auth is working +- namespace operations are working +- Polaris server-side object store write is using virtual-host style DNS +- local MinIO path-style behavior is not being honored for that write path + +Root cause observed in this repo: + +- catalog-level `s3.*` properties were present, but Polaris table-write path required `table-default.s3.*` +- once `table-default.s3.endpoint`, `table-default.s3.path-style-access`, and `table-default.s3.region` were added, Spark table create succeeded + +Check quickly: + +```bash +kubectl -n teehr-hub logs deploy/polaris --since=10m | grep -E 'UnknownHostException|warehouse.minio|Unable to execute HTTP request' +``` + +Verify active catalog settings (from a Jupyter pod): + +```bash +kubectl -n teehr-hub exec jupyter-admin -c notebook -- python -c "import requests; t=requests.post('http://keycloak-service:8080/realms/teehr/protocol/openid-connect/token',data={'grant_type':'password','client_id':'jupyterhub','client_secret':'local-jupyterhub-client-secret','username':'admin','password':'admin','scope':'openid profile email'},timeout=20).json()['access_token']; h={'Authorization':f'Bearer {t}','X-Polaris-Realm':'teehr'}; print(requests.get('http://polaris:8181/api/management/v1/catalogs/teehr',headers=h,timeout=20).json())" +``` + +Look for: + +- `storageConfigInfo.endpoint = http://minio:9000` +- `storageConfigInfo.pathStyleAccess = true` +- catalog properties containing both: + - `s3.path-style-access = true` + - `table-default.s3.path-style-access = true` + - `table-default.s3.endpoint = http://minio:9000` + +## 5. Durable fix in manifests + +The bootstrap job now writes both catalog-level and table-default S3 properties during catalog create/update. + +- `s3.endpoint` and `table-default.s3.endpoint` +- `s3.path-style-access` and `table-default.s3.path-style-access` +- `s3.region` and `table-default.s3.region` +- `s3.remote-signing-enabled` and `table-default.s3.remote-signing-enabled` + +After pulling these changes, rerun bootstrap: + +```bash +kubectl -n teehr-hub delete job polaris-bootstrap --ignore-not-found=true +kubectl -n teehr-hub apply -f ./polaris-bootstrap/manifests/bootstrap-job.yaml +kubectl -n teehr-hub wait --for=condition=Complete job/polaris-bootstrap --timeout=600s +``` + +## 6. Files for this simple flow + +- `examples/developer/polaris_spark_namespace_table_example.py` +- `scripts/run_jupyter_polaris_spark_example.sh` +- `examples/developer/setup_utils.py` +- `polaris-bootstrap/manifests/bootstrap-job.yaml` +- `polaris-bootstrap/manifests/acl-config.yaml.tpl` diff --git a/examples/developer/01_setup_minio_warehouse.ipynb b/examples/developer/01_setup_minio_warehouse.ipynb index 460c89f..4b56bcc 100644 --- a/examples/developer/01_setup_minio_warehouse.ipynb +++ b/examples/developer/01_setup_minio_warehouse.ipynb @@ -8,13 +8,23 @@ "outputs": [], "source": [ "from pathlib import Path\n", + "import base64\n", + "import json\n", + "import os\n", + "\n", "import pandas as pd\n", + "import requests\n", "\n", "import teehr\n", "from teehr import RemoteReadWriteEvaluation\n", "from teehr.utilities.apply_migrations import evolve_catalog_schema\n", "\n", - "from setup_utils import create_minio_spark_session, DEV_LOCATION_ID_LIST" + "from setup_utils import (\n", + " DEV_LOCATION_ID_LIST,\n", + " apply_polaris_token_to_spark,\n", + " create_minio_spark_session,\n", + " ensure_fresh_polaris_user_token,\n", + ")" ] }, { @@ -25,6 +35,19 @@ "#### Start the spark session" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "6d8287a8-fbf3-492e-9af7-53126b63a5d8", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "print(\"POLARIS_CLIENT_ID:\", os.getenv(\"POLARIS_CLIENT_ID\"))\n", + "print(\"POLARIS_CLIENT_SECRET exists:\", bool(os.getenv(\"POLARIS_CLIENT_SECRET\")))\n", + "print(\"POLARIS_REFRESH_TOKEN exists:\", bool(os.getenv(\"POLARIS_REFRESH_TOKEN\")))" + ] + }, { "cell_type": "code", "execution_count": null, @@ -32,7 +55,151 @@ "metadata": {}, "outputs": [], "source": [ - "spark = create_minio_spark_session()" + "polaris_user_token = os.environ.get(\"POLARIS_USER_TOKEN\")\n", + "polaris_refresh_token = os.getenv(\"POLARIS_REFRESH_TOKEN\", \"\")\n", + "polaris_username = os.getenv(\"POLARIS_USERNAME\", os.getenv(\"JUPYTERHUB_USER\", \"admin\"))\n", + "polaris_password = os.getenv(\"POLARIS_PASSWORD\", \"\")\n", + "polaris_client_id = os.getenv(\"POLARIS_CLIENT_ID\", \"jupyterhub\")\n", + "polaris_client_secret = os.getenv(\"POLARIS_CLIENT_SECRET\")\n", + "\n", + "if not polaris_user_token and not polaris_refresh_token and not polaris_password:\n", + " raise RuntimeError(\n", + " \"No usable Polaris credentials found. Set POLARIS_USER_TOKEN or POLARIS_REFRESH_TOKEN in the session.\"\n", + " )\n", + "\n", + "polaris_user_token, polaris_refresh_token, token_renewed = ensure_fresh_polaris_user_token(\n", + " current_token=polaris_user_token,\n", + " username=polaris_username,\n", + " password=polaris_password,\n", + " client_id=polaris_client_id,\n", + " client_secret=polaris_client_secret,\n", + " refresh_token=polaris_refresh_token,\n", + " refresh_window_seconds=120,\n", + ")\n", + "os.environ[\"POLARIS_USER_TOKEN\"] = polaris_user_token\n", + "if polaris_refresh_token:\n", + " os.environ[\"POLARIS_REFRESH_TOKEN\"] = polaris_refresh_token\n", + "\n", + "payload = polaris_user_token.split(\".\")[1]\n", + "payload += \"=\" * (-len(payload) % 4)\n", + "claims = json.loads(base64.urlsafe_b64decode(payload.encode()))\n", + "roles = sorted(claims.get(\"realm_access\", {}).get(\"roles\", []))\n", + "\n", + "print(\"Token status:\", \"renewed\" if token_renewed else \"reused\")\n", + "print(\"Refresh token available:\", bool(polaris_refresh_token))\n", + "print(\"Token user:\", claims.get(\"preferred_username\"))\n", + "print(\"Token roles:\", \", \".join(roles))\n", + "if \"iceberg-catalog-admin\" not in roles:\n", + " print(\"WARNING: token is missing iceberg-catalog-admin; Polaris may reject writes\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8c468ed5", + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "\n", + "realm = os.getenv(\"POLARIS_DEFAULT_REALM\", \"teehr\")\n", + "remote_warehouse = os.getenv(\"REMOTE_WAREHOUSE_S3_PATH\", \"s3://warehouse/\")\n", + "polaris_api_base = os.getenv(\"POLARIS_CATALOG_URI\", \"http://polaris:8181/api/catalog\").rstrip(\"/\")\n", + "catalog_config_url = f\"{polaris_api_base}/v1/config\"\n", + "mgmt_catalog_url = f\"http://polaris:8181/api/management/v1/catalogs/{realm}\"\n", + "\n", + "# Refresh access token before probing Polaris or recreating Spark.\n", + "polaris_user_token, polaris_refresh_token, refreshed = ensure_fresh_polaris_user_token(\n", + " current_token=polaris_user_token,\n", + " username=polaris_username,\n", + " password=polaris_password,\n", + " client_id=polaris_client_id,\n", + " client_secret=polaris_client_secret,\n", + " refresh_token=polaris_refresh_token,\n", + " refresh_window_seconds=120,\n", + ")\n", + "os.environ[\"POLARIS_USER_TOKEN\"] = polaris_user_token\n", + "if polaris_refresh_token:\n", + " os.environ[\"POLARIS_REFRESH_TOKEN\"] = polaris_refresh_token\n", + "print(\"Cell 4 token status:\", \"renewed\" if refreshed else \"still fresh\")\n", + "\n", + "# Polaris REST config endpoint expects catalog identifier (realm), not S3 URI.\n", + "warehouse_for_config = remote_warehouse\n", + "if polaris_api_base.endswith(\"/api/catalog\"):\n", + " warehouse_for_config = realm\n", + "\n", + "headers = {\n", + " \"Authorization\": f\"Bearer {polaris_user_token}\",\n", + " \"X-Polaris-Realm\": realm,\n", + "}\n", + "\n", + "# Catalog config endpoint requires a warehouse query parameter.\n", + "catalog_resp = requests.get(\n", + " catalog_config_url,\n", + " headers=headers,\n", + " params={\"warehouse\": warehouse_for_config},\n", + " timeout=20,\n", + " )\n", + "print(\"Catalog config API status:\", catalog_resp.status_code)\n", + "print(\"Catalog config warehouse query:\", warehouse_for_config)\n", + "if catalog_resp.status_code >= 400:\n", + " print(\"Catalog config API body:\", catalog_resp.text[:500])\n", + "catalog_resp.raise_for_status()\n", + "\n", + "if \"iceberg-catalog-admin\" in roles:\n", + " mgmt_resp = requests.get(mgmt_catalog_url, headers=headers, timeout=20)\n", + " print(\"Management API status:\", mgmt_resp.status_code)\n", + " if mgmt_resp.status_code >= 400:\n", + " print(\"Management API body:\", mgmt_resp.text[:500])\n", + " mgmt_resp.raise_for_status()\n", + "else:\n", + " print(\n", + " \"Skipping Management API check: token does not include iceberg-catalog-admin. \"\n", + " \"Catalog access is sufficient for non-admin Spark paths.\"\n", + " )\n", + "\n", + "print(\"Polaris access check passed.\")\n", + "\n", + "spark = create_minio_spark_session(\n", + " polaris_token=polaris_user_token,\n", + " force_recreate_session=True,\n", + ")\n", + "\n", + "\n", + "def refresh_polaris_token_for_spark(refresh_window_seconds: int = 120) -> bool:\n", + " global polaris_user_token, polaris_refresh_token\n", + "\n", + " polaris_user_token, polaris_refresh_token, refreshed = ensure_fresh_polaris_user_token(\n", + " current_token=polaris_user_token,\n", + " username=polaris_username,\n", + " password=polaris_password,\n", + " client_id=polaris_client_id,\n", + " client_secret=polaris_client_secret,\n", + " refresh_token=polaris_refresh_token,\n", + " refresh_window_seconds=refresh_window_seconds,\n", + " )\n", + " os.environ[\"POLARIS_USER_TOKEN\"] = polaris_user_token\n", + " if polaris_refresh_token:\n", + " os.environ[\"POLARIS_REFRESH_TOKEN\"] = polaris_refresh_token\n", + " apply_polaris_token_to_spark(spark, polaris_user_token, catalog_name=\"iceberg\", realm=realm)\n", + "\n", + " print(\"Spark token status:\", \"renewed\" if refreshed else \"still fresh\")\n", + " return refreshed\n", + "\n", + "\n", + "apply_polaris_token_to_spark(spark, polaris_user_token, catalog_name=\"iceberg\", realm=realm)\n", + "print(\"Spark catalog namespace probe:\")\n", + "spark.sql(\"SHOW NAMESPACES IN iceberg\").show(truncate=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "07e97944-1dd5-4631-9799-0124e671e472", + "metadata": {}, + "outputs": [], + "source": [ + "os.environ[\"POLARIS_REFRESH_TOKEN\"]" ] }, { @@ -76,7 +243,7 @@ "metadata": {}, "source": [ "#### Now you can pull in a subset of data from the TEEHR-Cloud warehouse via the API\n", - "Note: If you encounter a time out error, try reducing the page_size arg" + "Note: If you encounter a timeout or auth error, run refresh_polaris_token_for_spark() and retry. Prefer setting POLARIS_REFRESH_TOKEN in the session so renewals do not depend on username/password." ] }, { @@ -241,7 +408,7 @@ "ev.download.locations(\n", " ids=df.location_id.unique().tolist(),\n", " load=True\n", - "# )\n", + ")\n", "print(\"Loading forcing pixel weights table complete\")" ] }, @@ -305,6 +472,16 @@ "- `update-forecast-metrics-table`" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "ed4e7c40-c973-486f-b426-d83006c52451", + "metadata": {}, + "outputs": [], + "source": [ + "ev.configurations.to_sdf().show()" + ] + }, { "cell_type": "code", "execution_count": null, @@ -318,7 +495,7 @@ ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -332,7 +509,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.3" + "version": "3.12.11" } }, "nbformat": 4, diff --git a/examples/developer/polaris_spark_namespace_table_example.py b/examples/developer/polaris_spark_namespace_table_example.py new file mode 100644 index 0000000..919f23e --- /dev/null +++ b/examples/developer/polaris_spark_namespace_table_example.py @@ -0,0 +1,311 @@ +#!/usr/bin/env python3 +"""Minimal Polaris + Spark example for local KinD + MinIO development. + +This script is designed to run inside a Jupyter single-user pod. +It verifies Spark catalog connectivity, creates a namespace, and then +attempts to create and read a simple Iceberg table. +""" + +from __future__ import annotations + +import argparse +import base64 +import json +import os +import sys +import time +import traceback + +import requests + + +def _ensure_setup_utils_importable() -> None: + preferred = "/tmp/polaris-spark-example" + if preferred not in sys.path: + sys.path.insert(0, preferred) + + candidates = [ + ".", + "/workspace", + "/workspace/examples/developer", + "/home/jovyan", + "/home/jovyan/examples/developer", + ] + for path in candidates: + if path not in sys.path: + sys.path.append(path) + + +_ensure_setup_utils_importable() + +from setup_utils import ( + apply_polaris_token_to_spark, + create_minio_spark_session, + ensure_fresh_polaris_user_token, +) + + +def _decode_token_claims(token: str) -> dict: + payload = token.split(".")[1] + payload += "=" * (-len(payload) % 4) + return json.loads(base64.urlsafe_b64decode(payload.encode())) + + +def _print_token_claims(token: str) -> None: + claims = _decode_token_claims(token) + roles = sorted(claims.get("realm_access", {}).get("roles", [])) + print(f"[example] Token user: {claims.get('preferred_username')}") + print(f"[example] Token roles: {', '.join(roles)}") + + +def _print_spark_catalog_config(spark) -> None: + keys = [ + "spark.sql.catalog.iceberg.type", + "spark.sql.catalog.iceberg.uri", + "spark.sql.catalog.iceberg.warehouse", + "spark.sql.catalog.iceberg.token", + "spark.sql.catalog.iceberg.header.X-Polaris-Realm", + "spark.sql.catalog.iceberg.rest.auth.type", + "spark.sql.catalog.iceberg.rest.auth.oauth2.token", + "spark.sql.catalog.iceberg.rest.transport.header.X-Polaris-Realm", + "spark.sql.catalog.iceberg.io-impl", + "spark.sql.catalog.iceberg.s3.endpoint", + "spark.sql.catalog.iceberg.s3.path-style-access", + "spark.sql.catalog.iceberg.s3.region", + "spark.hadoop.fs.s3a.endpoint", + "spark.hadoop.fs.s3a.path.style.access", + ] + print("[example] Effective Spark catalog config:") + for key in keys: + value = spark.conf.get(key, "") + if "token" in key and value not in ("", ""): + value = f"{value[:12]}...{value[-8:]}" + print(f" {key}={value}") + + +def _apply_direct_catalog_overrides(spark, token: str) -> None: + realm = os.getenv("POLARIS_DEFAULT_REALM", "teehr") + spark.conf.set("spark.sql.catalog.iceberg.warehouse", realm) + apply_polaris_token_to_spark(spark, token, catalog_name="iceberg", realm=realm) + + +def _is_auth_failure(exc: Exception) -> bool: + msg = str(exc).lower() + markers = [ + "notauthorizedexception", + "unauthorized", + "http error 401", + "401", + "invalid token", + "token expired", + ] + return any(marker in msg for marker in markers) + + +def _sql_with_token_retry(spark, sql_text: str, token_ctx: dict, show: bool = False): + try: + df = spark.sql(sql_text) + if show: + df.show(truncate=False) + return df + except Exception as exc: + if not _is_auth_failure(exc): + raise + + print("[example] SQL failed due to auth; renewing token and retrying once...") + refreshed, refreshed_refresh_token, renewed = ensure_fresh_polaris_user_token( + current_token=token_ctx["token"], + username=token_ctx["username"], + password=token_ctx["password"], + client_id=token_ctx["client_id"], + client_secret=token_ctx["client_secret"], + refresh_token=token_ctx.get("refresh_token"), + refresh_window_seconds=300, + ) + token_ctx["token"] = refreshed + token_ctx["refresh_token"] = refreshed_refresh_token + if refreshed_refresh_token: + os.environ["POLARIS_REFRESH_TOKEN"] = refreshed_refresh_token + apply_polaris_token_to_spark(spark, refreshed, catalog_name="iceberg") + if renewed: + print("[example] Token refreshed; retrying SQL.") + + df = spark.sql(sql_text) + if show: + df.show(truncate=False) + return df + + +def _set_local_polaris_env_defaults() -> None: + realm = os.getenv("POLARIS_DEFAULT_REALM", "teehr") + os.environ["REMOTE_CATALOG_REST_URI"] = os.getenv( + "REMOTE_CATALOG_REST_URI", "http://polaris:8181/api/catalog" + ) + os.environ["REMOTE_WAREHOUSE_S3_PATH"] = realm + os.environ["REMOTE_CATALOG_S3_ENDPOINT"] = os.getenv( + "REMOTE_CATALOG_S3_ENDPOINT", "http://minio:9000" + ) + os.environ["REMOTE_CATALOG_S3_PATH_STYLE_ACCESS"] = os.getenv( + "REMOTE_CATALOG_S3_PATH_STYLE_ACCESS", "true" + ) + os.environ["POLARIS_DEFAULT_REALM"] = realm + + +def _validate_direct_polaris_access(token: str) -> None: + realm = os.getenv("POLARIS_DEFAULT_REALM", "teehr") + headers = { + "Authorization": f"Bearer {token}", + "X-Polaris-Realm": realm, + } + resp = requests.get( + "http://polaris:8181/api/catalog/v1/config", + headers=headers, + params={"warehouse": realm}, + timeout=20, + ) + print(f"[example] Direct Polaris config status: {resp.status_code}") + if resp.status_code >= 400: + print(f"[example] Direct Polaris config body: {resp.text[:500]}") + resp.raise_for_status() + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--username", default=os.getenv("POLARIS_TEST_USERNAME", "admin")) + parser.add_argument("--password", default=os.getenv("POLARIS_TEST_PASSWORD", "admin")) + parser.add_argument( + "--oauth-client-id", + default=os.getenv("POLARIS_OAUTH_CLIENT_ID", "jupyterhub"), + ) + parser.add_argument( + "--oauth-client-secret", + default=os.getenv("POLARIS_OAUTH_CLIENT_SECRET"), + ) + parser.add_argument( + "--refresh-token", + default=os.getenv("POLARIS_REFRESH_TOKEN"), + help="Refresh token from Jupyter login context, if available", + ) + parser.add_argument( + "--namespace", + default=f"spark_demo_{int(time.time())}", + help="Namespace to create (default: spark_demo_)", + ) + parser.add_argument( + "--table", + default="hello_table", + help="Table name to create inside the namespace", + ) + parser.add_argument( + "--keep", + action="store_true", + help="Keep created namespace/table instead of dropping at end", + ) + args = parser.parse_args() + + spark = None + created_namespace = False + created_table = False + + try: + token = os.getenv("POLARIS_USER_TOKEN") + if token: + print("[example] Using POLARIS_USER_TOKEN from Jupyter environment...") + print(f"[example] Ensuring a fresh token for user '{args.username}'...") + token, refresh_token, renewed = ensure_fresh_polaris_user_token( + current_token=token, + username=args.username, + password=args.password, + client_id=args.oauth_client_id, + client_secret=args.oauth_client_secret, + refresh_token=args.refresh_token, + refresh_window_seconds=300, + ) + if refresh_token: + os.environ["POLARIS_REFRESH_TOKEN"] = refresh_token + if renewed: + print("[example] Minted or renewed access token.") + + token_ctx = { + "token": token, + "username": args.username, + "password": args.password, + "client_id": args.oauth_client_id, + "client_secret": args.oauth_client_secret, + "refresh_token": refresh_token, + } + + _print_token_claims(token) + + _set_local_polaris_env_defaults() + _validate_direct_polaris_access(token) + + print("[example] Creating Spark session...") + spark = create_minio_spark_session( + polaris_token=token, + force_recreate_session=True, + ) + + _apply_direct_catalog_overrides(spark, token) + + _print_spark_catalog_config(spark) + + print("[example] Existing namespaces:") + _sql_with_token_retry(spark, "SHOW NAMESPACES IN iceberg", token_ctx, show=True) + + fq_namespace = f"iceberg.{args.namespace}" + fq_table = f"{fq_namespace}.{args.table}" + + print(f"[example] Creating namespace: {fq_namespace}") + _sql_with_token_retry(spark, f"CREATE NAMESPACE IF NOT EXISTS {fq_namespace}", token_ctx) + created_namespace = True + + print(f"[example] Creating table: {fq_table}") + _sql_with_token_retry(spark, f"CREATE TABLE {fq_table} (id int, name string) USING iceberg", token_ctx) + created_table = True + + print(f"[example] Inserting sample rows into: {fq_table}") + _sql_with_token_retry( + spark, + f"INSERT INTO {fq_table} VALUES (1, 'alpha'), (2, 'beta'), (3, 'gamma')", + token_ctx, + ) + + print(f"[example] Reading back rows from: {fq_table}") + _sql_with_token_retry(spark, f"SELECT * FROM {fq_table} ORDER BY id", token_ctx, show=True) + + print("[example] SUCCESS") + return 0 + + except Exception as exc: + msg = str(exc) + print("[example] FAILED") + traceback.print_exc() + + if "warehouse.minio" in msg or "UnknownHostException" in msg: + print("[example] Diagnostic: Polaris server is attempting virtual-host S3 DNS") + print("[example] Diagnostic: expected local path-style access for MinIO") + print("[example] Suggested checks:") + print(" 1) Re-run Polaris bootstrap to refresh catalog properties") + print(" 2) Verify catalog has s3.path-style-access=true and endpoint=http://minio:9000") + print(" 3) Check Polaris logs for the exact hostname in UnknownHostException") + return 1 + + finally: + if spark is not None: + if created_table and created_namespace and not args.keep: + fq_namespace = f"iceberg.{args.namespace}" + fq_table = f"{fq_namespace}.{args.table}" + try: + print(f"[example] Cleaning up table: {fq_table}") + spark.sql(f"DROP TABLE IF EXISTS {fq_table}") + print(f"[example] Cleaning up namespace: {fq_namespace}") + spark.sql(f"DROP NAMESPACE IF EXISTS {fq_namespace}") + except Exception: + print("[example] Cleanup skipped due to error") + print("[example] Stopping Spark session...") + spark.stop() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/developer/setup_utils.py b/examples/developer/setup_utils.py index cc2235f..91ff363 100644 --- a/examples/developer/setup_utils.py +++ b/examples/developer/setup_utils.py @@ -1,3 +1,10 @@ +import base64 +import json +import os +import time +from typing import Dict, Optional, Tuple + +import requests from teehr.evaluation.spark_session_utils import create_spark_session @@ -30,13 +37,250 @@ "usgs-50011128" ] -def create_minio_spark_session(): - """Start a Spark session with MinIO credentials and custom configuration.""" - return create_spark_session( - aws_access_key_id="minioadmin", - aws_secret_access_key="minioadmin123", - update_configs={ - "spark.hadoop.fs.s3a.aws.credentials.provider": - "org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider" + +def _decode_jwt_claims(token: str) -> Dict[str, object]: + payload = token.split(".")[1] + payload += "=" * (-len(payload) % 4) + return json.loads(base64.urlsafe_b64decode(payload.encode())) + + +def _token_expires_soon(token: str, refresh_window_seconds: int = 120) -> bool: + try: + claims = _decode_jwt_claims(token) + except Exception: + return True + exp = int(claims.get("exp", 0)) + now = int(time.time()) + return exp <= now + max(refresh_window_seconds, 1) + + +def _request_oauth_tokens( + data: Dict[str, str], + token_endpoint: Optional[str] = None, + timeout_seconds: int = 20, +) -> Tuple[str, Optional[str]]: + endpoint = token_endpoint or os.getenv("POLARIS_OAUTH2_SERVER_URI") + print(endpoint) + if not endpoint: + raise RuntimeError("POLARIS_OAUTH2_SERVER_URI is required to mint or refresh a user token") + + resp = requests.post(endpoint, data=data, timeout=timeout_seconds) + print(resp) + resp.raise_for_status() + payload = resp.json() + + access_token = payload.get("access_token") + print(access_token) + if not access_token: + raise RuntimeError("Token endpoint did not return access_token") + + return access_token, payload.get("refresh_token") + + +def mint_polaris_user_token( + username: Optional[str], + password: Optional[str], + client_id: str, + client_secret: Optional[str] = None, + token_endpoint: Optional[str] = None, +) -> str: + if not username or not password: + raise RuntimeError("username and password are required for password grant token minting") + + data = { + "grant_type": "password", + "client_id": client_id, + "username": username, + "password": password, + "scope": "openid profile email", + } + if client_secret: + data["client_secret"] = client_secret + + access_token, _ = _request_oauth_tokens(data=data, token_endpoint=token_endpoint) + return access_token + + +def refresh_polaris_user_token( + refresh_token: str, + client_id: str, + client_secret: Optional[str] = None, + token_endpoint: Optional[str] = None, +) -> Tuple[str, Optional[str]]: + if not refresh_token: + raise RuntimeError("refresh_token is required for refresh grant") + + data = { + "grant_type": "refresh_token", + "client_id": client_id, + "refresh_token": refresh_token, } -) \ No newline at end of file + if client_secret: + data["client_secret"] = client_secret + + print(data) + print(token_endpoint) + + return _request_oauth_tokens(data=data, token_endpoint=token_endpoint) + + +def ensure_fresh_polaris_user_token( + current_token: Optional[str], + username: Optional[str], + password: Optional[str], + client_id: str, + client_secret: Optional[str] = None, + refresh_token: Optional[str] = None, + allow_password_fallback: bool = True, + refresh_window_seconds: int = 120, + token_endpoint: Optional[str] = None, +) -> Tuple[str, Optional[str], bool]: + if current_token and not _token_expires_soon(current_token, refresh_window_seconds): + return current_token, refresh_token, False + + if refresh_token: + try: + print("trying to refresh token") + print(refresh_token) + refreshed_access, refreshed_refresh = refresh_polaris_user_token( + refresh_token=refresh_token, + client_id=client_id, + client_secret=client_secret, + token_endpoint=token_endpoint, + ) + return refreshed_access, (refreshed_refresh or refresh_token), True + except requests.RequestException: + if not allow_password_fallback: + print("Passowrd fall back not enabled") + raise + + if allow_password_fallback and username and password: + minted = mint_polaris_user_token( + username=username, + password=password, + client_id=client_id, + client_secret=client_secret, + token_endpoint=token_endpoint, + ) + return minted, refresh_token, True + + raise RuntimeError( + "Unable to obtain a fresh Polaris user token. " + "Provide POLARIS_REFRESH_TOKEN or enable password-grant fallback credentials." + ) + + +def apply_polaris_token_to_spark( + spark, + token: str, + catalog_name: str = "iceberg", + realm: Optional[str] = None, +) -> None: + active_realm = realm or os.getenv("POLARIS_DEFAULT_REALM", "teehr") + base = f"spark.sql.catalog.{catalog_name}" + spark.conf.set(f"{base}.rest.auth.type", "oauth2") + spark.conf.set(f"{base}.token", token) + spark.conf.set(f"{base}.rest.auth.oauth2.token", token) + spark.conf.set(f"{base}.header.X-Polaris-Realm", active_realm) + spark.conf.set(f"{base}.rest.transport.header.X-Polaris-Realm", active_realm) + +def _as_bool_str(value: str, default: str = "true") -> str: + normalized = (value or default).strip().lower() + return "true" if normalized in ("1", "true", "t", "yes", "y", "on") else "false" + + +def _apply_runtime_spark_configs(spark, configs: Dict[str, str]) -> None: + for key, value in configs.items(): + if not key.startswith("spark."): + continue + spark.conf.set(key, value) + + +def create_minio_spark_session( + polaris_token: Optional[str] = None, + force_recreate_session: bool = False, + update_configs: Optional[Dict[str, str]] = None, +): + """Start a Spark session configured for the local Polaris REST catalog. + + If ``polaris_token`` is provided, Spark uses user-token OAuth2 auth. + Otherwise it falls back to client-credential OAuth2 using env vars. + """ + aws_access_key_id = os.getenv("AWS_ACCESS_KEY_ID", "minioadmin") + aws_secret_access_key = os.getenv("AWS_SECRET_ACCESS_KEY", "minioadmin123") + + remote_catalog_uri = os.getenv("REMOTE_CATALOG_REST_URI", "http://polaris:8181/api/catalog") + remote_warehouse_dir = os.getenv("REMOTE_WAREHOUSE_S3_PATH", "s3://warehouse/") + polaris_realm = os.getenv("POLARIS_DEFAULT_REALM", "teehr") + + if remote_catalog_uri.rstrip("/").endswith("/api/catalog"): + # Polaris REST expects the catalog identifier here, not the backing S3 URI. + remote_warehouse_dir = polaris_realm + + s3_endpoint = os.getenv("REMOTE_CATALOG_S3_ENDPOINT", "http://minio:9000") + s3_path_style = _as_bool_str(os.getenv("REMOTE_CATALOG_S3_PATH_STYLE_ACCESS", "true")) + s3_region = os.getenv("AWS_REGION", "us-east-2") + + merged_configs: Dict[str, str] = { + "spark.sql.catalog.iceberg.warehouse": remote_warehouse_dir, + "spark.sql.catalog.iceberg.header.X-Polaris-Realm": polaris_realm, + "spark.sql.catalog.iceberg.rest.transport.header.X-Polaris-Realm": polaris_realm, + "spark.sql.catalog.iceberg.s3.endpoint": s3_endpoint, + "spark.sql.catalog.iceberg.s3.path-style-access": s3_path_style, + "spark.sql.catalog.iceberg.s3.region": s3_region, + "spark.hadoop.fs.s3a.endpoint": s3_endpoint, + "spark.hadoop.fs.s3a.path.style.access": s3_path_style, + "spark.hadoop.fs.s3a.connection.ssl.enabled": "false", + } + + if polaris_token: + merged_configs["spark.sql.catalog.iceberg.rest.auth.type"] = "oauth2" + merged_configs["spark.sql.catalog.iceberg.token"] = polaris_token + merged_configs["spark.sql.catalog.iceberg.rest.auth.oauth2.token"] = polaris_token + else: + oauth_server_uri = os.getenv("POLARIS_OAUTH2_SERVER_URI") + spark_polaris_client_secret = os.getenv("SPARK_POLARIS_CLIENT_SECRET") + + merged_configs["spark.sql.catalog.iceberg.rest.auth.type"] = "oauth2" + merged_configs["spark.sql.catalog.iceberg.scope"] = "openid" + merged_configs["spark.sql.catalog.iceberg.rest.auth.oauth2.scope"] = "openid" + if oauth_server_uri: + merged_configs["spark.sql.catalog.iceberg.oauth2-server-uri"] = oauth_server_uri + merged_configs["spark.sql.catalog.iceberg.rest.auth.oauth2.server-uri"] = oauth_server_uri + if spark_polaris_client_secret: + merged_configs["spark.sql.catalog.iceberg.credential"] = ( + f"spark-polaris:{spark_polaris_client_secret}" + ) + merged_configs["spark.sql.catalog.iceberg.rest.auth.oauth2.credential"] = ( + f"spark-polaris:{spark_polaris_client_secret}" + ) + + if update_configs: + merged_configs.update(update_configs) + + call_kwargs = { + "remote_catalog_uri": remote_catalog_uri, + "remote_warehouse_dir": remote_warehouse_dir, + "aws_access_key_id": aws_access_key_id, + "aws_secret_access_key": aws_secret_access_key, + "force_recreate_session": force_recreate_session, + "update_configs": merged_configs, + } + if polaris_token: + # Preferred path for newer teehr versions that support direct token auth. + call_kwargs["oauth2_token"] = polaris_token + + while True: + try: + spark = create_spark_session(**call_kwargs) + _apply_runtime_spark_configs(spark, merged_configs) + return spark + except TypeError as exc: + msg = str(exc) + if "oauth2_token" in msg and "oauth2_token" in call_kwargs: + call_kwargs.pop("oauth2_token", None) + continue + if "force_recreate_session" in msg and "force_recreate_session" in call_kwargs: + call_kwargs.pop("force_recreate_session", None) + continue + raise \ No newline at end of file diff --git a/iceberg-pg/garden.yaml b/iceberg-pg/garden.yaml index 2618b41..de4c912 100644 --- a/iceberg-pg/garden.yaml +++ b/iceberg-pg/garden.yaml @@ -2,6 +2,7 @@ kind: Deploy type: kubernetes name: iceberg-database description: K8s Deploy Iceberg Catalog Database +disabled: true dependencies: - deploy.secrets environments: diff --git a/iceberg-rest/garden.yaml b/iceberg-rest/garden.yaml index e0e39c3..3d31481 100644 --- a/iceberg-rest/garden.yaml +++ b/iceberg-rest/garden.yaml @@ -2,6 +2,7 @@ kind: Deploy type: kubernetes name: iceberg-rest-deploy description: K8s Deploy Iceberg REST +disabled: true spec: manifestTemplates: - ./manifests/iceberg-rest.yaml.tpl diff --git a/jupyterhub/garden.yaml b/jupyterhub/garden.yaml index 53f52ed..a08934f 100644 --- a/jupyterhub/garden.yaml +++ b/jupyterhub/garden.yaml @@ -54,6 +54,7 @@ spec: - openid - profile - email + - offline_access claim_groups_key: groups allowed_groups: - jupyter-user @@ -128,8 +129,14 @@ spec: auth-state: | c.Authenticator.enable_auth_state = True async def auth_state_hook(spawner, auth_state): - if auth_state and "access_token" in auth_state: - spawner.environment["POLARIS_USER_TOKEN"] = auth_state["access_token"] + spawner.environment["POLARIS_CLIENT_ID"] = os.environ.get("OAUTH_CLIENT_ID", "jupyterhub") + client_secret = os.environ.get("OAUTH_CLIENT_SECRET") + if client_secret: + spawner.environment["POLARIS_CLIENT_SECRET"] = client_secret + if auth_state and "access_token" in auth_state: + spawner.environment["POLARIS_USER_TOKEN"] = auth_state["access_token"] + if auth_state and "refresh_token" in auth_state: + spawner.environment["POLARIS_REFRESH_TOKEN"] = auth_state["refresh_token"] c.Spawner.auth_state_hook = auth_state_hook cull-kernels: | c.MappingKernelManager.cull_idle_timeout = 3600 @@ -163,7 +170,7 @@ spec: key: api-key REMOTE_CATALOG_TYPE: ${var.polaris.catalogType} REMOTE_CATALOG_REST_URI: ${var.polaris.catalogUri} - REMOTE_WAREHOUSE_S3_PATH: ${var.polaris.defaultRealm} + REMOTE_WAREHOUSE_S3_PATH: ${var.polaris.catalogWarehouse} REMOTE_CATALOG_S3_ENDPOINT: ${var.polaris.catalogS3Endpoint} REMOTE_CATALOG_S3_PATH_STYLE_ACCESS: ${var.polaris.catalogS3PathStyleAccess} IN_CLUSTER: ${var.polaris.inCluster} @@ -259,6 +266,7 @@ spec: - openid - profile - email + - offline_access claim_groups_key: groups allowed_groups: - jupyter-user @@ -332,8 +340,14 @@ spec: auth-state: | c.Authenticator.enable_auth_state = True async def auth_state_hook(spawner, auth_state): - if auth_state and "access_token" in auth_state: - spawner.environment["POLARIS_USER_TOKEN"] = auth_state["access_token"] + spawner.environment["POLARIS_CLIENT_ID"] = os.environ.get("OAUTH_CLIENT_ID", "jupyterhub") + client_secret = os.environ.get("OAUTH_CLIENT_SECRET") + if client_secret: + spawner.environment["POLARIS_CLIENT_SECRET"] = client_secret + if auth_state and "access_token" in auth_state: + spawner.environment["POLARIS_USER_TOKEN"] = auth_state["access_token"] + if auth_state and "refresh_token" in auth_state: + spawner.environment["POLARIS_REFRESH_TOKEN"] = auth_state["refresh_token"] c.Spawner.auth_state_hook = auth_state_hook cull-kernels: | c.MappingKernelManager.cull_idle_timeout = 3600 diff --git a/keycloak-bootstrap/manifests/local-users-configmap.yaml.tpl b/keycloak-bootstrap/manifests/local-users-configmap.yaml.tpl index eb7f634..3c1dd0e 100644 --- a/keycloak-bootstrap/manifests/local-users-configmap.yaml.tpl +++ b/keycloak-bootstrap/manifests/local-users-configmap.yaml.tpl @@ -26,6 +26,7 @@ data: "groups": [ "/basic-user", "/iceberg-user", + "/iceberg-catalog-admins", "/jupyter-admin", "/key-management-admin", "/prefect-admin", diff --git a/keycloak-bootstrap/manifests/realm-configmap.yaml.tpl b/keycloak-bootstrap/manifests/realm-configmap.yaml.tpl index 9f8b605..c457ccb 100644 --- a/keycloak-bootstrap/manifests/realm-configmap.yaml.tpl +++ b/keycloak-bootstrap/manifests/realm-configmap.yaml.tpl @@ -9,6 +9,7 @@ data: { "realm": "teehr", "enabled": true, + "accessTokenLifespan": 1800, "loginTheme": "teehr", "registrationAllowed": true, "loginWithEmailAllowed": true, diff --git a/polaris-bootstrap/manifests/acl-config.yaml.tpl b/polaris-bootstrap/manifests/acl-config.yaml.tpl index f67851b..2767648 100644 --- a/polaris-bootstrap/manifests/acl-config.yaml.tpl +++ b/polaris-bootstrap/manifests/acl-config.yaml.tpl @@ -11,6 +11,10 @@ data: "catalog": "${var.polaris.defaultRealm}", "warehouse": "${var.polaris.catalogWarehouse}", "storage_type": "S3", + "s3_endpoint": "${var.polaris.catalogS3Endpoint}", + "path_style_access": "${var.polaris.catalogS3PathStyleAccess}", + "s3_region": "${var.polaris.catalogS3Region}", + "sts_unavailable": ${var.polaris.storageStsUnavailable}, "allowed_locations": [ "${var.polaris.catalogWarehouse}" ], diff --git a/polaris-bootstrap/manifests/bootstrap-job.yaml b/polaris-bootstrap/manifests/bootstrap-job.yaml index 2172e3e..2e3cf11 100644 --- a/polaris-bootstrap/manifests/bootstrap-job.yaml +++ b/polaris-bootstrap/manifests/bootstrap-job.yaml @@ -52,6 +52,13 @@ spec: SUCCESS_CODES = (200, 201, 204, 409) + def parse_bool(value): + if isinstance(value, bool): + return value + if value is None: + return None + return str(value).strip().lower() in ("1", "true", "t", "yes", "y", "on") + def is_duplicate_error(resp): return resp.status_code == 500 and "duplicate key" in resp.text.lower() @@ -101,6 +108,67 @@ spec: return resp.raise_for_status() + def ensure_catalog(realm, token, catalog_name, properties, storage_config_info): + get_resp = requests.get( + f"{MGMT_URL}/api/management/v1/catalogs/{catalog_name}", + headers=realm_headers(realm, token=token, content_type=None), + ) + + if get_resp.status_code == 404: + mgmt( + "post", + realm, + token, + "/catalogs", + json={ + "name": catalog_name, + "type": "INTERNAL", + "properties": properties, + "storageConfigInfo": storage_config_info, + }, + ) + return + + get_resp.raise_for_status() + current = get_resp.json() + current_entity_version = current.get("entityVersion") + if current_entity_version is None: + raise ValueError(f"Catalog '{catalog_name}' missing entityVersion") + + update_payload = { + "currentEntityVersion": current_entity_version, + "properties": properties, + "storageConfigInfo": storage_config_info, + } + + update_resp = requests.put( + f"{MGMT_URL}/api/management/v1/catalogs/{catalog_name}", + headers=realm_headers(realm, token=token), + json=update_payload, + ) + + if update_resp.status_code in SUCCESS_CODES: + return + + # Retry once on optimistic-concurrency conflict. + if update_resp.status_code == 409: + refreshed = requests.get( + f"{MGMT_URL}/api/management/v1/catalogs/{catalog_name}", + headers=realm_headers(realm, token=token, content_type=None), + ) + refreshed.raise_for_status() + update_payload["currentEntityVersion"] = refreshed.json()["entityVersion"] + update_resp = requests.put( + f"{MGMT_URL}/api/management/v1/catalogs/{catalog_name}", + headers=realm_headers(realm, token=token), + json=update_payload, + ) + if update_resp.status_code in SUCCESS_CODES: + return + + print(f"ERROR {update_resp.status_code}: {update_resp.text}", file=sys.stderr) + update_resp.raise_for_status() + def grant_to_catalog_role(realm, token, catalog_name, catalog_role, grant, fallback_namespace=None): payload = { "type": grant["type"], @@ -171,25 +239,50 @@ spec: catalog_name = realm_cfg["catalog"] warehouse = realm_cfg["warehouse"] storage_type = realm_cfg.get("storage_type", "S3") + s3_endpoint = realm_cfg.get("s3_endpoint") + path_style_access = realm_cfg.get("path_style_access") + s3_region = realm_cfg.get("s3_region") + sts_unavailable = realm_cfg.get("sts_unavailable") allowed_locations = realm_cfg.get("allowed_locations") or [warehouse] print(f"[polaris-bootstrap] realm={realm} catalog={catalog_name}") token = request_token(realm) - mgmt( - "post", + storage_config_info = { + "storageType": storage_type, + "allowedLocations": allowed_locations, + } + catalog_properties = { + "default-base-location": warehouse, + } + if s3_endpoint: + storage_config_info["endpoint"] = s3_endpoint + catalog_properties["s3.endpoint"] = s3_endpoint + catalog_properties["table-default.s3.endpoint"] = s3_endpoint + if path_style_access is not None: + path_style_access_bool = parse_bool(path_style_access) + storage_config_info["pathStyleAccess"] = path_style_access_bool + catalog_properties["s3.path-style-access"] = str( + path_style_access_bool + ).lower() + catalog_properties["table-default.s3.path-style-access"] = str( + path_style_access_bool + ).lower() + if s3_region: + storage_config_info["region"] = s3_region + catalog_properties["s3.region"] = s3_region + catalog_properties["table-default.s3.region"] = s3_region + if sts_unavailable is not None: + storage_config_info["stsUnavailable"] = parse_bool(sts_unavailable) + catalog_properties["s3.remote-signing-enabled"] = "false" + catalog_properties["table-default.s3.remote-signing-enabled"] = "false" + + ensure_catalog( realm, token, - "/catalogs", - json={ - "name": catalog_name, - "type": "INTERNAL", - "properties": {"default-base-location": warehouse}, - "storageConfigInfo": { - "storageType": storage_type, - "allowedLocations": allowed_locations, - }, - }, + catalog_name, + catalog_properties, + storage_config_info, ) admin_cfg = realm_cfg.get("admin") diff --git a/polaris-bootstrap/manifests/polaris-principal-sync-cronjob.yaml.tpl b/polaris-bootstrap/manifests/polaris-principal-sync-cronjob.yaml.tpl index 8008358..32451b9 100644 --- a/polaris-bootstrap/manifests/polaris-principal-sync-cronjob.yaml.tpl +++ b/polaris-bootstrap/manifests/polaris-principal-sync-cronjob.yaml.tpl @@ -44,7 +44,7 @@ spec: - name: sync-script mountPath: /scripts readOnly: true - volumes: - - name: sync-script - configMap: - name: polaris-sync-principals-script + volumes: + - name: sync-script + configMap: + name: polaris-sync-principals-script diff --git a/polaris-bootstrap/manifests/polaris-sync-principals-script.yaml b/polaris-bootstrap/manifests/polaris-sync-principals-script.yaml index 01b686f..0314f35 100644 --- a/polaris-bootstrap/manifests/polaris-sync-principals-script.yaml +++ b/polaris-bootstrap/manifests/polaris-sync-principals-script.yaml @@ -5,9 +5,9 @@ metadata: data: sync_principals.py: | """ - Syncs all enabled Keycloak realm users into Polaris as principal entities. - Role assignment is intentionally omitted: the Polaris OIDC role mapper maps - realm_access.roles claims to PRINCIPAL_ROLE: at authentication time. + Syncs all enabled Keycloak realm users into Polaris as principal entities + and grants matching Polaris principal roles for Keycloak realm roles with + the iceberg-* prefix. Required environment variables: POLARIS_MANAGEMENT_URL e.g. http://polaris:8181 @@ -78,6 +78,15 @@ data: first += max_per_page + def get_keycloak_realm_roles(kc_token, user_id): + resp = requests.get( + f"{KC_URL}/admin/realms/{REALM}/users/{user_id}/role-mappings/realm/composite", + headers={"Authorization": f"Bearer {kc_token}"}, + ) + resp.raise_for_status() + return [role.get("name") for role in resp.json() if role.get("name")] + + def ensure_principal(polaris_token, principal_name): resp = requests.post( f"{MGMT_URL}/api/management/v1/principals", @@ -94,14 +103,73 @@ data: resp.raise_for_status() + def ensure_principal_role(polaris_token, principal_role): + resp = requests.post( + f"{MGMT_URL}/api/management/v1/principal-roles", + headers={ + "X-Polaris-Realm": REALM, + "Authorization": f"Bearer {polaris_token}", + "Content-Type": "application/json", + }, + json={"name": principal_role}, + ) + if resp.status_code in (200, 201, 204, 409): + return + print(f"ERROR creating principal role '{principal_role}': {resp.status_code} {resp.text}", file=sys.stderr) + resp.raise_for_status() + + + def _is_duplicate_grant_error(resp): + if resp.status_code != 500: + return False + body = (resp.text or "").lower() + return ( + "duplicate key value violates unique constraint" in body + or "grant_records_pkey" in body + or "sql-state '23505'" in body + or "already exists" in body + ) + + + def ensure_principal_role_binding(polaris_token, principal_name, principal_role): + ensure_principal_role(polaris_token, principal_role) + resp = requests.put( + f"{MGMT_URL}/api/management/v1/principals/{principal_name}/principal-roles", + headers={ + "X-Polaris-Realm": REALM, + "Authorization": f"Bearer {polaris_token}", + "Content-Type": "application/json", + }, + json={"name": principal_role}, + ) + if resp.status_code in (200, 201, 204, 409) or _is_duplicate_grant_error(resp): + return + print( + f"ERROR binding principal role '{principal_role}' to principal '{principal_name}': " + f"{resp.status_code} {resp.text}", + file=sys.stderr, + ) + resp.raise_for_status() + + polaris_token = get_polaris_token() kc_token = get_keycloak_token() synced = 0 + granted = 0 for user in iter_keycloak_users(kc_token): name = user.get("username") + user_id = user.get("id") if name: ensure_principal(polaris_token, name) + if user_id: + for role_name in get_keycloak_realm_roles(kc_token, user_id): + if role_name.startswith("iceberg-"): + ensure_principal_role_binding(polaris_token, name, role_name) + granted += 1 synced += 1 - print(f"[polaris-principal-sync] Synced {synced} principals from Keycloak realm '{REALM}'") + print( + f"[polaris-principal-sync] Synced {synced} principals and ensured {granted} " + f"principal-role grants from Keycloak realm '{REALM}'" + ) diff --git a/polaris/manifests/polaris-config.yaml.tpl b/polaris/manifests/polaris-config.yaml.tpl index f0b160e..05730ea 100644 --- a/polaris/manifests/polaris-config.yaml.tpl +++ b/polaris/manifests/polaris-config.yaml.tpl @@ -32,6 +32,10 @@ data: quarkus.oidc.tenant-enabled=true quarkus.oidc.application-type=service quarkus.oidc.client-id=jupyterhub + # Keycloak advertises external hostnames in discovery metadata; disable discovery + # so Polaris uses the configured auth-server-url and internal JWKS path directly. + quarkus.oidc.discovery-enabled=false + quarkus.oidc.jwks-path=/protocol/openid-connect/certs quarkus.tls.trust-all=true quarkus.oidc.connection-delay=PT10S quarkus.oidc.connection-retry-count=5 @@ -45,4 +49,5 @@ data: # Storage Properties Integration polaris.features."SUPPORTED_CATALOG_STORAGE_TYPES"=["S3","GCS","AZURE","FILE"] polaris.features."ALLOW_INSECURE_STORAGE_TYPES"=true + polaris.features."SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION"=${var.polaris.skipCredentialSubscopingIndirection} polaris.readiness.ignore-severe-issues=true diff --git a/polaris/manifests/polaris.yaml.tpl b/polaris/manifests/polaris.yaml.tpl index 3baf78e..2abb7b7 100644 --- a/polaris/manifests/polaris.yaml.tpl +++ b/polaris/manifests/polaris.yaml.tpl @@ -21,12 +21,6 @@ spec: app: polaris spec: serviceAccountName: polaris - ${if environment.name == "local"} - hostAliases: - - ip: "${var.polaris.keycloakIngressIp}" - hostnames: - - "auth.${var.hostname}" - ${endif} initContainers: - name: schema-bootstrap image: apache/polaris-admin-tool:1.5.0 diff --git a/project.garden.yml.template b/project.garden.yml.template index 41bbee6..767c92f 100644 --- a/project.garden.yml.template +++ b/project.garden.yml.template @@ -13,9 +13,6 @@ variables: ssl: "false" aws: region: us-east-2 - # remoteCluster: - # contextArn: "" - # ecrRegistry: "" environments: - name: local @@ -25,13 +22,27 @@ environments: certificateIssuerName: letsencrypt-prod devTeehrVersion: 267f8a75034132aefe84749af10ab1562a6ac169 stableTeehrVersion: place-holder # Needed for sync to work for some reason. - iceberg: + # iceberg: + # inCluster: "true" + # catalogS3PathStyleAccess: "true" + # catalogS3Endpoint: "http://minio:9000" + # catalogType: rest + # catalogUri: http://iceberg-rest:8181 + # catalogWarehouse: s3://warehouse/ + polaris: inCluster: "true" catalogS3PathStyleAccess: "true" catalogS3Endpoint: "http://minio:9000" + catalogS3Region: ${var.aws.region} catalogType: rest - catalogUri: http://iceberg-rest:8181 + catalogUri: http://polaris:8181/api/catalog catalogWarehouse: s3://warehouse/ + skipCredentialSubscopingIndirection: "true" + storageStsUnavailable: "true" + defaultRealm: teehr + realmsCsv: teehr + oauthServerUri: http://keycloak-service:8080/realms/teehr/protocol/openid-connect/token + oidcIssuerUri: https://auth.${var.hostname}/realms/teehr trino: host: trino port: "8080" diff --git a/scripts/run_jupyter_polaris_spark_example.sh b/scripts/run_jupyter_polaris_spark_example.sh new file mode 100755 index 0000000..4bf345d --- /dev/null +++ b/scripts/run_jupyter_polaris_spark_example.sh @@ -0,0 +1,134 @@ +#!/usr/bin/env bash +set -euo pipefail + +NAMESPACE="${NAMESPACE:-teehr-hub}" +POD_NAME="${POD_NAME:-}" +CONTAINER_NAME="${CONTAINER_NAME:-notebook}" +SCRIPT_PATH="${SCRIPT_PATH:-examples/developer/polaris_spark_namespace_table_example.py}" +SETUP_UTILS_PATH="${SETUP_UTILS_PATH:-examples/developer/setup_utils.py}" +EXTRA_ARGS="${EXTRA_ARGS:-}" +POLARIS_TEST_USERNAME="${POLARIS_TEST_USERNAME:-admin}" +POLARIS_TEST_PASSWORD="${POLARIS_TEST_PASSWORD:-admin}" +POLARIS_OAUTH_CLIENT_ID="${POLARIS_OAUTH_CLIENT_ID:-jupyterhub}" +DEBUG_POLARIS="${DEBUG_POLARIS:-0}" +POLARIS_DOCTOR="${POLARIS_DOCTOR:-0}" + +if [[ -z "$POD_NAME" ]]; then + POD_NAME="$(kubectl -n "$NAMESPACE" get pods -o name | sed 's#^pod/##' | grep -E '^jupyter-' | head -n1 || true)" +fi + +if [[ -z "$POD_NAME" ]]; then + echo "No Jupyter single-user pod found in namespace '$NAMESPACE'." >&2 + echo "Start a notebook server first, or set POD_NAME explicitly." >&2 + exit 1 +fi + +if [[ -z "${POLARIS_OAUTH_CLIENT_SECRET:-}" ]]; then + POLARIS_OAUTH_CLIENT_SECRET="$(kubectl -n "$NAMESPACE" get secret jupyterhub -o jsonpath='{.data.OAUTH_CLIENT_SECRET}' | base64 --decode)" +fi + +if [[ ! -f "$SCRIPT_PATH" ]]; then + echo "Script not found: $SCRIPT_PATH" >&2 + exit 1 +fi + +if [[ ! -f "$SETUP_UTILS_PATH" ]]; then + echo "Setup utils not found: $SETUP_UTILS_PATH" >&2 + exit 1 +fi + +echo "Running $SCRIPT_PATH in pod $POD_NAME (namespace: $NAMESPACE, container: $CONTAINER_NAME)" + +start_epoch="$(date +%s)" + +diagnostics_dir="" +run_log="" +if [[ "$POLARIS_DOCTOR" == "1" ]]; then + diagnostics_dir="${TMPDIR:-/tmp}/polaris-doctor-$(date +%Y%m%d-%H%M%S)-$$" + mkdir -p "$diagnostics_dir" + run_log="$diagnostics_dir/run.log" + echo "[doctor] diagnostics dir: $diagnostics_dir" +fi + +tmpdir="${TMPDIR:-/tmp}/polaris-spark-example.$$" +mkdir -p "$tmpdir" +cp "$SCRIPT_PATH" "$tmpdir/polaris_spark_namespace_table_example.py" +cp "$SETUP_UTILS_PATH" "$tmpdir/setup_utils.py" + +set +e +if [[ "$POLARIS_DOCTOR" == "1" ]]; then + ( + tar -C "$tmpdir" -cf - polaris_spark_namespace_table_example.py setup_utils.py | \ + kubectl -n "$NAMESPACE" exec -i "$POD_NAME" -c "$CONTAINER_NAME" -- sh -lc \ + "mkdir -p /tmp/polaris-spark-example && cd /tmp/polaris-spark-example && tar -xf - && export PYTHONPATH=/tmp/polaris-spark-example:\$PYTHONPATH POLARIS_TEST_USERNAME='$POLARIS_TEST_USERNAME' POLARIS_TEST_PASSWORD='$POLARIS_TEST_PASSWORD' POLARIS_OAUTH_CLIENT_ID='$POLARIS_OAUTH_CLIENT_ID' POLARIS_OAUTH_CLIENT_SECRET='$POLARIS_OAUTH_CLIENT_SECRET'; python /tmp/polaris-spark-example/polaris_spark_namespace_table_example.py $EXTRA_ARGS" + ) 2>&1 | tee "$run_log" + cmd_exit_code=${PIPESTATUS[0]} +else + tar -C "$tmpdir" -cf - polaris_spark_namespace_table_example.py setup_utils.py | \ + kubectl -n "$NAMESPACE" exec -i "$POD_NAME" -c "$CONTAINER_NAME" -- sh -lc \ + "mkdir -p /tmp/polaris-spark-example && cd /tmp/polaris-spark-example && tar -xf - && export PYTHONPATH=/tmp/polaris-spark-example:\$PYTHONPATH POLARIS_TEST_USERNAME='$POLARIS_TEST_USERNAME' POLARIS_TEST_PASSWORD='$POLARIS_TEST_PASSWORD' POLARIS_OAUTH_CLIENT_ID='$POLARIS_OAUTH_CLIENT_ID' POLARIS_OAUTH_CLIENT_SECRET='$POLARIS_OAUTH_CLIENT_SECRET'; python /tmp/polaris-spark-example/polaris_spark_namespace_table_example.py $EXTRA_ARGS" + cmd_exit_code=$? +fi +set -e + +if [[ "$POLARIS_DOCTOR" == "1" ]]; then + { + echo "exit_code=$cmd_exit_code" + echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + echo "namespace=$NAMESPACE" + echo "pod_name=$POD_NAME" + echo "container_name=$CONTAINER_NAME" + echo "script_path=$SCRIPT_PATH" + } > "$diagnostics_dir/context.txt" +fi + +if [[ "$DEBUG_POLARIS" == "1" ]]; then + now_epoch="$(date +%s)" + since_seconds="$((now_epoch - start_epoch + 15))" + if (( since_seconds < 30 )); then + since_seconds=30 + fi + echo + echo "[debug] Polaris logs for the last ${since_seconds}s" + kubectl -n "$NAMESPACE" logs deploy/polaris --since="${since_seconds}s" | \ + grep -E "(POST /api/catalog/v1/oauth/tokens|GET /api/catalog/v1/config|HTTP/1.1\" 401|HTTP/1.1\" 200|principal=|roles=|Some principal roles were not found)" || true + + if [[ "$POLARIS_DOCTOR" == "1" ]]; then + kubectl -n "$NAMESPACE" logs deploy/polaris --since="${since_seconds}s" > "$diagnostics_dir/polaris.log" || true + grep -E "(POST /api/catalog/v1/oauth/tokens|/api/catalog/v1/config|/api/catalog/v1/.*/namespaces|HTTP/1.1\" 401|HTTP/1.1\" 403|HTTP/1.1\" 500|Some principal roles were not found|UnknownHostException|warehouse.minio)" "$diagnostics_dir/polaris.log" > "$diagnostics_dir/polaris-summary.log" || true + fi +fi + +if [[ "$POLARIS_DOCTOR" == "1" ]]; then + diagnosis="unknown" + if [[ "$cmd_exit_code" -eq 0 ]]; then + diagnosis="success" + elif grep -qi "NotAuthorizedException\|HTTP Error 401\|401 Unauthorized" "$run_log" 2>/dev/null; then + diagnosis="authz_or_realm_mismatch" + elif grep -qi "UnknownHostException\|warehouse\.minio\|NoSuchBucket\|AccessDenied" "$run_log" 2>/dev/null; then + diagnosis="storage_or_warehouse_misconfig" + elif grep -qi "Failed to write to grant records\|grant_records_pkey\|duplicate key value" "$run_log" 2>/dev/null; then + diagnosis="principal_role_grant_idempotency" + fi + + { + echo "diagnosis=$diagnosis" + if [[ "$diagnosis" == "authz_or_realm_mismatch" ]]; then + echo "hint=Verify token issuer/realm and Polaris principal-role grants for this principal" + elif [[ "$diagnosis" == "storage_or_warehouse_misconfig" ]]; then + echo "hint=Verify Polaris catalog warehouse and MinIO endpoint/path-style settings" + elif [[ "$diagnosis" == "principal_role_grant_idempotency" ]]; then + echo "hint=Principal sync is re-granting existing role; ensure duplicate grant is treated as success" + fi + } > "$diagnostics_dir/diagnosis.txt" + + echo + echo "[doctor] diagnosis: $diagnosis" + echo "[doctor] bundle: $diagnostics_dir" + if [[ -f "$diagnostics_dir/polaris-summary.log" ]]; then + echo "[doctor] summary:" + tail -n 60 "$diagnostics_dir/polaris-summary.log" || true + fi +fi + +exit "$cmd_exit_code" diff --git a/trino/garden.yaml b/trino/garden.yaml index db58019..f5d68a8 100644 --- a/trino/garden.yaml +++ b/trino/garden.yaml @@ -36,8 +36,8 @@ spec: iceberg.rest-catalog.oauth2.scope=openid # S3 Configuration fs.native-s3.enabled=true - s3.path-style-access=${var.iceberg.catalogS3PathStyleAccess} - s3.endpoint=${var.iceberg.catalogS3Endpoint} + s3.path-style-access=${var.polaris.catalogS3PathStyleAccess} + s3.endpoint=${var.polaris.catalogS3Endpoint} s3.region=${var.aws.region} accessControl: type: configmap From 30229c06e1121925cb311b8a86b0a5adf3e9c0ee Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Thu, 23 Jul 2026 17:07:54 -0400 Subject: [PATCH 13/62] this is a test notebook for human testing in jupyter --- examples/developer/test_token_renewal.ipynb | 615 ++++++++++++++++++++ 1 file changed, 615 insertions(+) create mode 100644 examples/developer/test_token_renewal.ipynb diff --git a/examples/developer/test_token_renewal.ipynb b/examples/developer/test_token_renewal.ipynb new file mode 100644 index 0000000..bbebf6c --- /dev/null +++ b/examples/developer/test_token_renewal.ipynb @@ -0,0 +1,615 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "b02f2f71-c50b-42ff-acd1-4c71cc8b851a", + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "import base64\n", + "import json\n", + "import os\n", + "\n", + "import pandas as pd\n", + "import requests\n", + "\n", + "import teehr\n", + "from teehr import RemoteReadWriteEvaluation\n", + "from teehr.utilities.apply_migrations import evolve_catalog_schema\n", + "\n", + "from setup_utils import (\n", + " DEV_LOCATION_ID_LIST,\n", + " apply_polaris_token_to_spark,\n", + " create_minio_spark_session,\n", + " ensure_fresh_polaris_user_token,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "5319cb94-0510-4695-8127-68f4c71c6776", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "POLARIS_CLIENT_ID: jupyterhub\n", + "POLARIS_CLIENT_SECRET exists: True\n", + "POLARIS_REFRESH_TOKEN exists: True\n" + ] + } + ], + "source": [ + "import os\n", + "print(\"POLARIS_CLIENT_ID:\", os.getenv(\"POLARIS_CLIENT_ID\"))\n", + "print(\"POLARIS_CLIENT_SECRET exists:\", bool(os.getenv(\"POLARIS_CLIENT_SECRET\")))\n", + "print(\"POLARIS_REFRESH_TOKEN exists:\", bool(os.getenv(\"POLARIS_REFRESH_TOKEN\")))" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "60ca0cbf-f004-4e0a-bcdc-9b9319993a93", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "POLARIS_USER_TOKEN claims:\n", + "{\n", + " \"acr\": \"1\",\n", + " \"allowed-origins\": [\n", + " \"https://hub.teehr.local.app.garden\"\n", + " ],\n", + " \"aud\": [\n", + " \"realm-management\",\n", + " \"account\"\n", + " ],\n", + " \"auth_time\": 1784820452,\n", + " \"auth_time_utc\": \"2026-07-23T15:27:32+00:00\",\n", + " \"azp\": \"jupyterhub\",\n", + " \"email\": \"admin@example.local\",\n", + " \"email_verified\": true,\n", + " \"exp\": 1784820752,\n", + " \"exp_utc\": \"2026-07-23T15:32:32+00:00\",\n", + " \"family_name\": \"Admin\",\n", + " \"given_name\": \"Local\",\n", + " \"groups\": [\n", + " \"basic-user\",\n", + " \"iceberg-catalog-admins\",\n", + " \"iceberg-user\",\n", + " \"jupyter-admin\",\n", + " \"key-management-admin\",\n", + " \"prefect-admin\",\n", + " \"webapi-admin\"\n", + " ],\n", + " \"iat\": 1784820452,\n", + " \"iat_utc\": \"2026-07-23T15:27:32+00:00\",\n", + " \"iss\": \"https://auth.teehr.local.app.garden/realms/teehr\",\n", + " \"jti\": \"ofrtac:3719a618-6e89-7a46-38fb-c9a93cddf030\",\n", + " \"name\": \"Local Admin\",\n", + " \"preferred_username\": \"admin\",\n", + " \"realm_access\": {\n", + " \"roles\": [\n", + " \"iceberg-catalog-admin\",\n", + " \"offline_access\",\n", + " \"admin\",\n", + " \"uma_authorization\",\n", + " \"basic-user\",\n", + " \"default-roles-teehr\",\n", + " \"jupyter-user\",\n", + " \"iceberg-user\"\n", + " ]\n", + " },\n", + " \"resource_access\": {\n", + " \"account\": {\n", + " \"roles\": [\n", + " \"manage-account\",\n", + " \"manage-account-links\",\n", + " \"view-profile\"\n", + " ]\n", + " },\n", + " \"realm-management\": {\n", + " \"roles\": [\n", + " \"view-realm\",\n", + " \"manage-users\",\n", + " \"view-users\",\n", + " \"query-groups\",\n", + " \"query-users\"\n", + " ]\n", + " }\n", + " },\n", + " \"scope\": \"openid offline_access profile email\",\n", + " \"sid\": \"DxxUtwd0xu5xkhSEDcsIebzd\",\n", + " \"sub\": \"8f4441da-7756-49e3-9142-64c299ed6c39\",\n", + " \"typ\": \"Bearer\"\n", + "}\n" + ] + } + ], + "source": [ + "import base64\n", + "import json\n", + "import os\n", + "from datetime import datetime, timezone\n", + "\n", + "def decode_jwt(token: str):\n", + " if not token:\n", + " return None\n", + " parts = token.split(\".\")\n", + " if len(parts) != 3:\n", + " return {\"error\": \"Not a JWT (expected 3 dot-separated parts)\"}\n", + " payload = parts[1] + \"=\" * (-len(parts[1]) % 4)\n", + " claims = json.loads(base64.urlsafe_b64decode(payload.encode()))\n", + " return claims\n", + "\n", + "def pretty_claims(name: str, token: str):\n", + " claims = decode_jwt(token)\n", + " print(f\"\\n{name}:\")\n", + " if claims is None:\n", + " print(\" \")\n", + " return\n", + " if \"error\" in claims:\n", + " print(\" \", claims[\"error\"])\n", + " return\n", + " for ts_key in [\"iat\", \"exp\", \"auth_time\"]:\n", + " if ts_key in claims:\n", + " claims[f\"{ts_key}_utc\"] = datetime.fromtimestamp(\n", + " claims[ts_key], tz=timezone.utc\n", + " ).isoformat()\n", + " print(json.dumps(claims, indent=2, sort_keys=True))\n", + "\n", + "access_token = os.getenv(\"POLARIS_USER_TOKEN\", \"\")\n", + "refresh_token = os.getenv(\"POLARIS_REFRESH_TOKEN\", \"\")\n", + "\n", + "pretty_claims(\"POLARIS_USER_TOKEN claims\", access_token)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "c3a50694-a1ec-4923-90ca-efcf145c14b5", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "sub: 8f4441da-7756-49e3-9142-64c299ed6c39\n", + "preferred_username: admin\n", + "groups: ['basic-user', 'iceberg-catalog-admins', 'iceberg-user', 'jupyter-admin', 'key-management-admin', 'prefect-admin', 'webapi-admin']\n", + "realm roles: ['iceberg-catalog-admin', 'offline_access', 'admin', 'uma_authorization', 'basic-user', 'default-roles-teehr', 'jupyter-user', 'iceberg-user']\n", + "exp: 1784820752\n" + ] + } + ], + "source": [ + "claims = decode_jwt(os.getenv(\"POLARIS_USER_TOKEN\", \"\"))\n", + "print(\"sub:\", claims.get(\"sub\"))\n", + "print(\"preferred_username:\", claims.get(\"preferred_username\"))\n", + "print(\"groups:\", claims.get(\"groups\"))\n", + "print(\"realm roles:\", claims.get(\"realm_access\", {}).get(\"roles\", []))\n", + "print(\"exp:\", claims.get(\"exp\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "fe4b02e0-cf82-45db-bb48-3b242e11ee24", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "now_epoch: 1784820789\n", + "now_utc: 2026-07-23T15:33:09+00:00\n" + ] + } + ], + "source": [ + "import time\n", + "from datetime import datetime, timezone\n", + "\n", + "now = int(time.time())\n", + "print(\"now_epoch:\", now)\n", + "print(\"now_utc:\", datetime.fromtimestamp(now, tz=timezone.utc).isoformat())" + ] + }, + { + "cell_type": "code", + "execution_count": 48, + "id": "016380fe-06db-4a3f-87b5-854ad1d3458e", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "exp: 1784825530\n", + "now: 1784825236\n", + "seconds_remaining: 294\n" + ] + } + ], + "source": [ + "claims = decode_jwt(os.getenv(\"POLARIS_USER_TOKEN\", \"\")) # from previous helper\n", + "exp = int(claims.get(\"exp\", 0))\n", + "now = int(time.time())\n", + "print(\"exp:\", exp)\n", + "print(\"now:\", now)\n", + "print(\"seconds_remaining:\", exp - now)" + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "id": "2f8ff9fe-0d4c-43a1-96e3-1eee7cc9f6e0", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "trying to refresh token\n", + "eyJhbGciOiJIUzUxMiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJkMTJhN2FiYy04MzVkLTRiNzAtODgxMi02NjcxMjMzYjI1ODUifQ.eyJpYXQiOjE3ODQ4MjI1MzQsImp0aSI6IjI2ZjI5MmJiLWVkNGMtOWY4NS04MzRmLWEzNjNmOTQ3ZDYwZiIsImlzcyI6Imh0dHBzOi8vYXV0aC50ZWVoci5sb2NhbC5hcHAuZ2FyZGVuL3JlYWxtcy90ZWVociIsImF1ZCI6Imh0dHBzOi8vYXV0aC50ZWVoci5sb2NhbC5hcHAuZ2FyZGVuL3JlYWxtcy90ZWVociIsInN1YiI6IjhmNDQ0MWRhLTc3NTYtNDllMy05MTQyLTY0YzI5OWVkNmMzOSIsInR5cCI6Ik9mZmxpbmUiLCJhenAiOiJqdXB5dGVyaHViIiwic2lkIjoiRHh4VXR3ZDB4dTV4a2hTRURjc0llYnpkIiwic2NvcGUiOiJvcGVuaWQgYWNyIG9mZmxpbmVfYWNjZXNzIHJvbGVzIGJhc2ljIHByb2ZpbGUgZW1haWwgd2ViLW9yaWdpbnMiLCJhdWRfeCI6WyJyZWFsbS1tYW5hZ2VtZW50IiwiYWNjb3VudCJdfQ.VUSlhU7bpO6pRxlDE_RMNNpDv-HcLKZ0bThk-5ewvOoHGtae9QLxBhdqFCnZg7K_-bK8Tbpt9eKteInSImNwKA\n", + "{'grant_type': 'refresh_token', 'client_id': 'jupyterhub', 'refresh_token': 'eyJhbGciOiJIUzUxMiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJkMTJhN2FiYy04MzVkLTRiNzAtODgxMi02NjcxMjMzYjI1ODUifQ.eyJpYXQiOjE3ODQ4MjI1MzQsImp0aSI6IjI2ZjI5MmJiLWVkNGMtOWY4NS04MzRmLWEzNjNmOTQ3ZDYwZiIsImlzcyI6Imh0dHBzOi8vYXV0aC50ZWVoci5sb2NhbC5hcHAuZ2FyZGVuL3JlYWxtcy90ZWVociIsImF1ZCI6Imh0dHBzOi8vYXV0aC50ZWVoci5sb2NhbC5hcHAuZ2FyZGVuL3JlYWxtcy90ZWVociIsInN1YiI6IjhmNDQ0MWRhLTc3NTYtNDllMy05MTQyLTY0YzI5OWVkNmMzOSIsInR5cCI6Ik9mZmxpbmUiLCJhenAiOiJqdXB5dGVyaHViIiwic2lkIjoiRHh4VXR3ZDB4dTV4a2hTRURjc0llYnpkIiwic2NvcGUiOiJvcGVuaWQgYWNyIG9mZmxpbmVfYWNjZXNzIHJvbGVzIGJhc2ljIHByb2ZpbGUgZW1haWwgd2ViLW9yaWdpbnMiLCJhdWRfeCI6WyJyZWFsbS1tYW5hZ2VtZW50IiwiYWNjb3VudCJdfQ.VUSlhU7bpO6pRxlDE_RMNNpDv-HcLKZ0bThk-5ewvOoHGtae9QLxBhdqFCnZg7K_-bK8Tbpt9eKteInSImNwKA', 'client_secret': 'local-jupyterhub-client-secret'}\n", + "None\n", + "http://keycloak-service:8080/realms/teehr/protocol/openid-connect/token\n", + "\n", + "eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJ6UFFnS004WjdiaUdIZnFPWGdrV3JZbjM1NjNTdHFUMFdpcm5XdHBqZXNjIn0.eyJleHAiOjE3ODQ4MjU1MzAsImlhdCI6MTc4NDgyNTIzMCwiYXV0aF90aW1lIjoxNzg0ODIwNDUyLCJqdGkiOiJvZnJ0cnQ6MWJjNTJlNjUtZTc5Ni0zOTlmLWY2NWQtOGEzNThlNWJiNDJmIiwiaXNzIjoiaHR0cHM6Ly9hdXRoLnRlZWhyLmxvY2FsLmFwcC5nYXJkZW4vcmVhbG1zL3RlZWhyIiwiYXVkIjpbInJlYWxtLW1hbmFnZW1lbnQiLCJhY2NvdW50Il0sInN1YiI6IjhmNDQ0MWRhLTc3NTYtNDllMy05MTQyLTY0YzI5OWVkNmMzOSIsInR5cCI6IkJlYXJlciIsImF6cCI6Imp1cHl0ZXJodWIiLCJzaWQiOiJEeHhVdHdkMHh1NXhraFNFRGNzSWViemQiLCJhY3IiOiIxIiwiYWxsb3dlZC1vcmlnaW5zIjpbImh0dHBzOi8vaHViLnRlZWhyLmxvY2FsLmFwcC5nYXJkZW4iXSwicmVhbG1fYWNjZXNzIjp7InJvbGVzIjpbImljZWJlcmctY2F0YWxvZy1hZG1pbiIsIm9mZmxpbmVfYWNjZXNzIiwiYWRtaW4iLCJ1bWFfYXV0aG9yaXphdGlvbiIsImJhc2ljLXVzZXIiLCJkZWZhdWx0LXJvbGVzLXRlZWhyIiwianVweXRlci11c2VyIiwiaWNlYmVyZy11c2VyIl19LCJyZXNvdXJjZV9hY2Nlc3MiOnsicmVhbG0tbWFuYWdlbWVudCI6eyJyb2xlcyI6WyJ2aWV3LXJlYWxtIiwibWFuYWdlLXVzZXJzIiwidmlldy11c2VycyIsInF1ZXJ5LWdyb3VwcyIsInF1ZXJ5LXVzZXJzIl19LCJhY2NvdW50Ijp7InJvbGVzIjpbIm1hbmFnZS1hY2NvdW50IiwibWFuYWdlLWFjY291bnQtbGlua3MiLCJ2aWV3LXByb2ZpbGUiXX19LCJzY29wZSI6Im9wZW5pZCBvZmZsaW5lX2FjY2VzcyBwcm9maWxlIGVtYWlsIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsIm5hbWUiOiJMb2NhbCBBZG1pbiIsImdyb3VwcyI6WyJiYXNpYy11c2VyIiwiaWNlYmVyZy1jYXRhbG9nLWFkbWlucyIsImljZWJlcmctdXNlciIsImp1cHl0ZXItYWRtaW4iLCJrZXktbWFuYWdlbWVudC1hZG1pbiIsInByZWZlY3QtYWRtaW4iLCJ3ZWJhcGktYWRtaW4iXSwicHJlZmVycmVkX3VzZXJuYW1lIjoiYWRtaW4iLCJnaXZlbl9uYW1lIjoiTG9jYWwiLCJmYW1pbHlfbmFtZSI6IkFkbWluIiwiZW1haWwiOiJhZG1pbkBleGFtcGxlLmxvY2FsIn0.V3YgkyyVl9eUu-We4GDV36GS2tlYJzGyw6twBOVLtgWkbu6DzGOKiTOpkGiuabeGtJaQXGFoM9h454zSicXDQUJFa4qUt7FF36C87X81_MAaLiOJ88oFQXV5JxV6VVPTR6C0faO_-j5uw7Rk3P2RNK9yHiVxVcnbTfakWO9TdgZhxk0kSg7w7mzZbPgA2Wg_83pO3kqvUZF0wUyG30QxUQifsnJfiVFfBakFj7ffPizeiw1l67Lusef0qvDkQA5VWA_UVXzp-UoKpcQ4xlK0V1Kn367fP0lX1Ea_IB9GMFjwIARHR-t3ESzowcX4bRtybOyrVbQhOOjrzzs1aX4ZHw\n", + "Token status: renewed\n", + "Refresh token available: True\n", + "Token user: admin\n", + "Token roles: admin, basic-user, default-roles-teehr, iceberg-catalog-admin, iceberg-user, jupyter-user, offline_access, uma_authorization\n" + ] + } + ], + "source": [ + "polaris_user_token = os.environ.get(\"POLARIS_USER_TOKEN\")\n", + "polaris_refresh_token = os.getenv(\"POLARIS_REFRESH_TOKEN\", \"\")\n", + "polaris_username = os.getenv(\"POLARIS_USERNAME\", os.getenv(\"JUPYTERHUB_USER\", \"admin\"))\n", + "polaris_password = os.getenv(\"POLARIS_PASSWORD\", \"\")\n", + "polaris_client_id = os.getenv(\"POLARIS_CLIENT_ID\", \"jupyterhub\")\n", + "polaris_client_secret = os.getenv(\"POLARIS_CLIENT_SECRET\")\n", + "\n", + "if not polaris_user_token and not polaris_refresh_token and not polaris_password:\n", + " raise RuntimeError(\n", + " \"No usable Polaris credentials found. Set POLARIS_USER_TOKEN or POLARIS_REFRESH_TOKEN in the session.\"\n", + " )\n", + "\n", + "polaris_user_token, polaris_refresh_token, token_renewed = ensure_fresh_polaris_user_token(\n", + " current_token=polaris_user_token,\n", + " username=polaris_username,\n", + " password=polaris_password,\n", + " client_id=polaris_client_id,\n", + " client_secret=polaris_client_secret,\n", + " refresh_token=polaris_refresh_token,\n", + " refresh_window_seconds=120,\n", + ")\n", + "os.environ[\"POLARIS_USER_TOKEN\"] = polaris_user_token\n", + "if polaris_refresh_token:\n", + " os.environ[\"POLARIS_REFRESH_TOKEN\"] = polaris_refresh_token\n", + "\n", + "payload = polaris_user_token.split(\".\")[1]\n", + "payload += \"=\" * (-len(payload) % 4)\n", + "claims = json.loads(base64.urlsafe_b64decode(payload.encode()))\n", + "roles = sorted(claims.get(\"realm_access\", {}).get(\"roles\", []))\n", + "\n", + "print(\"Token status:\", \"renewed\" if token_renewed else \"reused\")\n", + "print(\"Refresh token available:\", bool(polaris_refresh_token))\n", + "print(\"Token user:\", claims.get(\"preferred_username\"))\n", + "print(\"Token roles:\", \", \".join(roles))\n", + "if \"iceberg-catalog-admin\" not in roles:\n", + " print(\"WARNING: token is missing iceberg-catalog-admin; Polaris may reject writes\")" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "id": "7d7592da-2753-4557-8d93-8ac1e5c27364", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:teehr.evaluation.spark_session_utils:🚀 Creating Spark session: TEEHR Evaluation\n", + "INFO:teehr.evaluation.spark_session_utils:✅ Spark local configuration successful!\n", + "INFO:teehr.evaluation.spark_session_utils:Setting Hadoop's default AWS credentials provider and AWS region\n", + "INFO:teehr.evaluation.spark_session_utils:🔑 Using user-provided AWS credentials\n", + "INFO:teehr.evaluation.spark_session_utils:Configuring Iceberg catalogs...\n", + "INFO:teehr.evaluation.spark_session_utils:⚙️ All settings applied. Creating Spark session...\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "trying to refresh token\n", + "eyJhbGciOiJIUzUxMiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJkMTJhN2FiYy04MzVkLTRiNzAtODgxMi02NjcxMjMzYjI1ODUifQ.eyJpYXQiOjE3ODQ4MjIzMjQsImp0aSI6IjJhNmQwZjQ3LWQ0YzItZjRiNS0xOTNjLTRhZGYzNjFiN2RiOSIsImlzcyI6Imh0dHBzOi8vYXV0aC50ZWVoci5sb2NhbC5hcHAuZ2FyZGVuL3JlYWxtcy90ZWVociIsImF1ZCI6Imh0dHBzOi8vYXV0aC50ZWVoci5sb2NhbC5hcHAuZ2FyZGVuL3JlYWxtcy90ZWVociIsInN1YiI6IjhmNDQ0MWRhLTc3NTYtNDllMy05MTQyLTY0YzI5OWVkNmMzOSIsInR5cCI6Ik9mZmxpbmUiLCJhenAiOiJqdXB5dGVyaHViIiwic2lkIjoiRHh4VXR3ZDB4dTV4a2hTRURjc0llYnpkIiwic2NvcGUiOiJvcGVuaWQgYWNyIG9mZmxpbmVfYWNjZXNzIHJvbGVzIGJhc2ljIHByb2ZpbGUgZW1haWwgd2ViLW9yaWdpbnMiLCJhdWRfeCI6WyJyZWFsbS1tYW5hZ2VtZW50IiwiYWNjb3VudCJdfQ.GTaneyP2O50XcRm0-BQc8gRuwoI_yJskKOGRFepFrtwlBu5JHz7EVi1sef5GzlyaMVr3VOC02VTGnMk3p1fpnQ\n", + "{'grant_type': 'refresh_token', 'client_id': 'jupyterhub', 'refresh_token': 'eyJhbGciOiJIUzUxMiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJkMTJhN2FiYy04MzVkLTRiNzAtODgxMi02NjcxMjMzYjI1ODUifQ.eyJpYXQiOjE3ODQ4MjIzMjQsImp0aSI6IjJhNmQwZjQ3LWQ0YzItZjRiNS0xOTNjLTRhZGYzNjFiN2RiOSIsImlzcyI6Imh0dHBzOi8vYXV0aC50ZWVoci5sb2NhbC5hcHAuZ2FyZGVuL3JlYWxtcy90ZWVociIsImF1ZCI6Imh0dHBzOi8vYXV0aC50ZWVoci5sb2NhbC5hcHAuZ2FyZGVuL3JlYWxtcy90ZWVociIsInN1YiI6IjhmNDQ0MWRhLTc3NTYtNDllMy05MTQyLTY0YzI5OWVkNmMzOSIsInR5cCI6Ik9mZmxpbmUiLCJhenAiOiJqdXB5dGVyaHViIiwic2lkIjoiRHh4VXR3ZDB4dTV4a2hTRURjc0llYnpkIiwic2NvcGUiOiJvcGVuaWQgYWNyIG9mZmxpbmVfYWNjZXNzIHJvbGVzIGJhc2ljIHByb2ZpbGUgZW1haWwgd2ViLW9yaWdpbnMiLCJhdWRfeCI6WyJyZWFsbS1tYW5hZ2VtZW50IiwiYWNjb3VudCJdfQ.GTaneyP2O50XcRm0-BQc8gRuwoI_yJskKOGRFepFrtwlBu5JHz7EVi1sef5GzlyaMVr3VOC02VTGnMk3p1fpnQ', 'client_secret': 'local-jupyterhub-client-secret'}\n", + "None\n", + "http://keycloak-service:8080/realms/teehr/protocol/openid-connect/token\n", + "\n", + "eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJ6UFFnS004WjdiaUdIZnFPWGdrV3JZbjM1NjNTdHFUMFdpcm5XdHBqZXNjIn0.eyJleHAiOjE3ODQ4MjI4MzQsImlhdCI6MTc4NDgyMjUzNCwiYXV0aF90aW1lIjoxNzg0ODIwNDUyLCJqdGkiOiJvZnJ0cnQ6YjE5MjY0ZjgtNGQ0OC0yM2MzLTYzYjUtZGZkMjEwNjA4MGUyIiwiaXNzIjoiaHR0cHM6Ly9hdXRoLnRlZWhyLmxvY2FsLmFwcC5nYXJkZW4vcmVhbG1zL3RlZWhyIiwiYXVkIjpbInJlYWxtLW1hbmFnZW1lbnQiLCJhY2NvdW50Il0sInN1YiI6IjhmNDQ0MWRhLTc3NTYtNDllMy05MTQyLTY0YzI5OWVkNmMzOSIsInR5cCI6IkJlYXJlciIsImF6cCI6Imp1cHl0ZXJodWIiLCJzaWQiOiJEeHhVdHdkMHh1NXhraFNFRGNzSWViemQiLCJhY3IiOiIxIiwiYWxsb3dlZC1vcmlnaW5zIjpbImh0dHBzOi8vaHViLnRlZWhyLmxvY2FsLmFwcC5nYXJkZW4iXSwicmVhbG1fYWNjZXNzIjp7InJvbGVzIjpbImljZWJlcmctY2F0YWxvZy1hZG1pbiIsIm9mZmxpbmVfYWNjZXNzIiwiYWRtaW4iLCJ1bWFfYXV0aG9yaXphdGlvbiIsImJhc2ljLXVzZXIiLCJkZWZhdWx0LXJvbGVzLXRlZWhyIiwianVweXRlci11c2VyIiwiaWNlYmVyZy11c2VyIl19LCJyZXNvdXJjZV9hY2Nlc3MiOnsicmVhbG0tbWFuYWdlbWVudCI6eyJyb2xlcyI6WyJ2aWV3LXJlYWxtIiwibWFuYWdlLXVzZXJzIiwidmlldy11c2VycyIsInF1ZXJ5LWdyb3VwcyIsInF1ZXJ5LXVzZXJzIl19LCJhY2NvdW50Ijp7InJvbGVzIjpbIm1hbmFnZS1hY2NvdW50IiwibWFuYWdlLWFjY291bnQtbGlua3MiLCJ2aWV3LXByb2ZpbGUiXX19LCJzY29wZSI6Im9wZW5pZCBvZmZsaW5lX2FjY2VzcyBwcm9maWxlIGVtYWlsIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsIm5hbWUiOiJMb2NhbCBBZG1pbiIsImdyb3VwcyI6WyJiYXNpYy11c2VyIiwiaWNlYmVyZy1jYXRhbG9nLWFkbWlucyIsImljZWJlcmctdXNlciIsImp1cHl0ZXItYWRtaW4iLCJrZXktbWFuYWdlbWVudC1hZG1pbiIsInByZWZlY3QtYWRtaW4iLCJ3ZWJhcGktYWRtaW4iXSwicHJlZmVycmVkX3VzZXJuYW1lIjoiYWRtaW4iLCJnaXZlbl9uYW1lIjoiTG9jYWwiLCJmYW1pbHlfbmFtZSI6IkFkbWluIiwiZW1haWwiOiJhZG1pbkBleGFtcGxlLmxvY2FsIn0.maYl1K9wKSbIAOg9q-HjX4AomKObiQ4TNXZtjxnW9DjJG3hiHUA1zdEQXfZuduuK95vBqvuy_cP8kvzuwTa3WNePVVwlp5JZYOk2-mmWutGortefXPwraRSvtP9u28zF4uM4xjcbHB-kCY2ievOCIWRuYy-W-MyhonH-t-bhv76AEj0ysdWZ8caVQOig2Ko1HT6dLZaehegqT2ZPwJwGzHteHchOQfRq9O1NwSrfkyc18qQNi_ba4JGXkS1A3pMIkFjat4HZ0B5bPhYuk5J1HB9ZlN60n1Z_Irjz36aM4vr7oCmD2QtRSewEf8F6bwkCM75f0aYFps5KC7ZLC5pokQ\n", + "Cell 4 token status: renewed\n", + "Catalog config API status: 200\n", + "Catalog config warehouse query: teehr\n", + "Management API status: 200\n", + "Polaris access check passed.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:teehr.evaluation.spark_session_utils:🎉 Spark session created successfully!\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Spark catalog namespace probe:\n", + "+----------------+\n", + "|namespace |\n", + "+----------------+\n", + "|teehr |\n", + "|schema_evolution|\n", + "|restricted |\n", + "|public |\n", + "+----------------+\n", + "\n" + ] + } + ], + "source": [ + "import requests\n", + "\n", + "realm = os.getenv(\"POLARIS_DEFAULT_REALM\", \"teehr\")\n", + "remote_warehouse = os.getenv(\"REMOTE_WAREHOUSE_S3_PATH\", \"s3://warehouse/\")\n", + "polaris_api_base = os.getenv(\"POLARIS_CATALOG_URI\", \"http://polaris:8181/api/catalog\").rstrip(\"/\")\n", + "catalog_config_url = f\"{polaris_api_base}/v1/config\"\n", + "mgmt_catalog_url = f\"http://polaris:8181/api/management/v1/catalogs/{realm}\"\n", + "\n", + "# Refresh access token before probing Polaris or recreating Spark.\n", + "polaris_user_token, polaris_refresh_token, refreshed = ensure_fresh_polaris_user_token(\n", + " current_token=polaris_user_token,\n", + " username=polaris_username,\n", + " password=polaris_password,\n", + " client_id=polaris_client_id,\n", + " client_secret=polaris_client_secret,\n", + " refresh_token=polaris_refresh_token,\n", + " refresh_window_seconds=120,\n", + ")\n", + "os.environ[\"POLARIS_USER_TOKEN\"] = polaris_user_token\n", + "if polaris_refresh_token:\n", + " os.environ[\"POLARIS_REFRESH_TOKEN\"] = polaris_refresh_token\n", + "print(\"Cell 4 token status:\", \"renewed\" if refreshed else \"still fresh\")\n", + "\n", + "# Polaris REST config endpoint expects catalog identifier (realm), not S3 URI.\n", + "warehouse_for_config = remote_warehouse\n", + "if polaris_api_base.endswith(\"/api/catalog\"):\n", + " warehouse_for_config = realm\n", + "\n", + "headers = {\n", + " \"Authorization\": f\"Bearer {polaris_user_token}\",\n", + " \"X-Polaris-Realm\": realm,\n", + "}\n", + "\n", + "# Catalog config endpoint requires a warehouse query parameter.\n", + "catalog_resp = requests.get(\n", + " catalog_config_url,\n", + " headers=headers,\n", + " params={\"warehouse\": warehouse_for_config},\n", + " timeout=20,\n", + " )\n", + "print(\"Catalog config API status:\", catalog_resp.status_code)\n", + "print(\"Catalog config warehouse query:\", warehouse_for_config)\n", + "if catalog_resp.status_code >= 400:\n", + " print(\"Catalog config API body:\", catalog_resp.text[:500])\n", + "catalog_resp.raise_for_status()\n", + "\n", + "if \"iceberg-catalog-admin\" in roles:\n", + " mgmt_resp = requests.get(mgmt_catalog_url, headers=headers, timeout=20)\n", + " print(\"Management API status:\", mgmt_resp.status_code)\n", + " if mgmt_resp.status_code >= 400:\n", + " print(\"Management API body:\", mgmt_resp.text[:500])\n", + " mgmt_resp.raise_for_status()\n", + "else:\n", + " print(\n", + " \"Skipping Management API check: token does not include iceberg-catalog-admin. \"\n", + " \"Catalog access is sufficient for non-admin Spark paths.\"\n", + " )\n", + "\n", + "print(\"Polaris access check passed.\")\n", + "\n", + "spark = create_minio_spark_session(\n", + " polaris_token=polaris_user_token,\n", + " force_recreate_session=True,\n", + ")\n", + "\n", + "\n", + "def refresh_polaris_token_for_spark(refresh_window_seconds: int = 120) -> bool:\n", + " global polaris_user_token, polaris_refresh_token\n", + "\n", + " polaris_user_token, polaris_refresh_token, refreshed = ensure_fresh_polaris_user_token(\n", + " current_token=polaris_user_token,\n", + " username=polaris_username,\n", + " password=polaris_password,\n", + " client_id=polaris_client_id,\n", + " client_secret=polaris_client_secret,\n", + " refresh_token=polaris_refresh_token,\n", + " refresh_window_seconds=refresh_window_seconds,\n", + " )\n", + " os.environ[\"POLARIS_USER_TOKEN\"] = polaris_user_token\n", + " if polaris_refresh_token:\n", + " os.environ[\"POLARIS_REFRESH_TOKEN\"] = polaris_refresh_token\n", + " apply_polaris_token_to_spark(spark, polaris_user_token, catalog_name=\"iceberg\", realm=realm)\n", + "\n", + " print(\"Spark token status:\", \"renewed\" if refreshed else \"still fresh\")\n", + " return refreshed\n", + "\n", + "\n", + "apply_polaris_token_to_spark(spark, polaris_user_token, catalog_name=\"iceberg\", realm=realm)\n", + "print(\"Spark catalog namespace probe:\")\n", + "spark.sql(\"SHOW NAMESPACES IN iceberg\").show(truncate=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 51, + "id": "a0558850-b6a5-4994-a981-23ef845babe2", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:teehr.evaluation.spark_session_utils:🚀 Creating Spark session: TEEHR Evaluation\n", + "INFO:teehr.evaluation.spark_session_utils:✅ Spark local configuration successful!\n", + "INFO:teehr.evaluation.spark_session_utils:Setting Hadoop's default AWS credentials provider and AWS region\n", + "INFO:teehr.evaluation.spark_session_utils:🔑 Using user-provided AWS credentials\n", + "INFO:teehr.evaluation.spark_session_utils:Configuring Iceberg catalogs...\n", + "INFO:teehr.evaluation.spark_session_utils:⚙️ All settings applied. Creating Spark session...\n", + "INFO:teehr.evaluation.spark_session_utils:🎉 Spark session created successfully!\n" + ] + } + ], + "source": [ + "spark.stop()\n", + "spark = create_minio_spark_session(\n", + " polaris_token=polaris_user_token,\n", + " force_recreate_session=True,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 53, + "id": "4c74e5de-c29e-46bb-b5e3-686faec7cbf9", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:teehr.evaluation.evaluation:Using provided Spark session.\n", + "INFO:teehr.evaluation.evaluation:Active catalog set to iceberg.\n" + ] + } + ], + "source": [ + "ev = RemoteReadWriteEvaluation(spark=spark, enable_spark_proxy=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "id": "ff0d1a63-f78a-4516-994d-48b964f18d4b", + "metadata": {}, + "outputs": [], + "source": [ + "spark.conf.set(\"spark.sql.catalog.iceberg.token\", polaris_user_token)\n", + "spark.conf.set(\"spark.sql.catalog.iceberg.rest.auth.oauth2.token\", polaris_user_token)" + ] + }, + { + "cell_type": "code", + "execution_count": 54, + "id": "6c171370-baf5-4a15-84f5-4d0d3a5356bb", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:teehr.evaluation.tables.base_table:Initializing Table for table: configurations.\n", + "INFO:teehr.evaluation.tables.base_table:Loading files from iceberg.teehr.configurations.\n", + "INFO:teehr.evaluation.read:Reading files from iceberg.teehr.configurations.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+--------------------+---------------+--------------------+--------------------+--------------------+----------+\n", + "| name|timeseries_type| description| created_at| updated_at|properties|\n", + "+--------------------+---------------+--------------------+--------------------+--------------------+----------+\n", + "|nrds_v22_cfenom_m...| secondary|NRDS DataStream m...|2026-07-21 19:49:...|2026-07-21 19:49:...| NULL|\n", + "|nrds_v22_cfenom_s...| secondary|POC version of Da...|2026-07-21 19:49:...|2026-07-21 19:49:...| NULL|\n", + "|nrds_v22_lstm0_me...| secondary|NRDS DataStream m...|2026-07-21 19:49:...|2026-07-21 19:49:...| NULL|\n", + "|nrds_v22_lstm0_sh...| secondary|NRDS DataStream s...|2026-07-21 19:49:...|2026-07-21 19:49:...| NULL|\n", + "|nwm30_analysis_as...| secondary|Alaska NWM standa...|2026-07-21 19:49:...|2026-07-21 19:49:...| NULL|\n", + "|nwm30_analysis_as...| secondary|Alaska NWM extend...|2026-07-21 19:49:...|2026-07-21 19:49:...| NULL|\n", + "|nwm30_analysis_as...| secondary|CONUS NWM extende...|2026-07-21 19:49:...|2026-07-21 19:49:...| NULL|\n", + "|nwm30_analysis_as...| secondary|Hawaii NWM standa...|2026-07-21 19:49:...|2026-07-21 19:49:...| NULL|\n", + "|nwm30_analysis_as...| secondary|CONUS NWM standar...|2026-07-21 19:49:...|2026-07-21 19:49:...| NULL|\n", + "|nwm30_analysis_as...| secondary|PRVI NWM standard...|2026-07-21 19:49:...|2026-07-21 19:49:...| NULL|\n", + "|nwm30_forcing_ana...| primary|CONUS MRMS mean a...|2026-07-21 19:49:...|2026-07-21 19:49:...| NULL|\n", + "|nwm30_forcing_ana...| primary|Alaska MRMS mean ...|2026-07-21 19:49:...|2026-07-21 19:49:...| NULL|\n", + "|nwm30_forcing_ana...| primary|CONUS STAGEIV mea...|2026-07-21 19:49:...|2026-07-21 19:49:...| NULL|\n", + "|nwm30_forcing_ana...| primary|Alaska StageIV me...|2026-07-21 19:49:...|2026-07-21 19:49:...| NULL|\n", + "|nwm30_forcing_ana...| primary|Hawaii MRMS mean ...|2026-07-21 19:49:...|2026-07-21 19:49:...| NULL|\n", + "|nwm30_forcing_ana...| primary|PRVI MRMS mean ar...|2026-07-21 19:49:...|2026-07-21 19:49:...| NULL|\n", + "|nwm30_forcing_med...| secondary|CONUS GFS mean ar...|2026-07-21 19:49:...|2026-07-21 19:49:...| NULL|\n", + "|nwm30_forcing_med...| secondary|Alaska GFS mean a...|2026-07-21 19:49:...|2026-07-21 19:49:...| NULL|\n", + "|nwm30_forcing_med...| secondary|CONUS NBM mean ar...|2026-07-21 19:49:...|2026-07-21 19:49:...| NULL|\n", + "|nwm30_forcing_med...| secondary|Alaska NBM mean a...|2026-07-21 19:49:...|2026-07-21 19:49:...| NULL|\n", + "+--------------------+---------------+--------------------+--------------------+--------------------+----------+\n", + "only showing top 20 rows\n" + ] + } + ], + "source": [ + "ev.configurations.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c87d9147-d860-41b0-8dcf-dfad60beaaa4", + "metadata": {}, + "outputs": [], + "source": [ + "# spark.stop()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "85a4d3ab-bfea-4c91-9f05-8ffcc73b0d4f", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.11" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From c3fe966ddd9418e5454a09170283fd7770e2205d Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Thu, 23 Jul 2026 17:08:14 -0400 Subject: [PATCH 14/62] Add broker contract and Spark AuthManager prototype scaffold --- docs/polaris-broker-api-contract.md | 122 ++++++++++ docs/polaris-identity-propagation-plan.md | 230 ++++++++++++++++++ spark/authmanager-prototype/README.md | 35 +++ spark/authmanager-prototype/pom.xml | 53 ++++ .../iceberg/auth/BrokerBackedAuthSession.java | 96 ++++++++ .../org/teehr/iceberg/auth/BrokerToken.java | 10 + .../teehr/iceberg/auth/BrokerTokenClient.java | 65 +++++ .../iceberg/auth/TeehrAuthProperties.java | 21 ++ .../iceberg/auth/TeehrBrokerAuthManager.java | 101 ++++++++ 9 files changed, 733 insertions(+) create mode 100644 docs/polaris-broker-api-contract.md create mode 100644 spark/authmanager-prototype/README.md create mode 100644 spark/authmanager-prototype/pom.xml create mode 100644 spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerBackedAuthSession.java create mode 100644 spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerToken.java create mode 100644 spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerTokenClient.java create mode 100644 spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/TeehrAuthProperties.java create mode 100644 spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/TeehrBrokerAuthManager.java diff --git a/docs/polaris-broker-api-contract.md b/docs/polaris-broker-api-contract.md new file mode 100644 index 0000000..dd8e896 --- /dev/null +++ b/docs/polaris-broker-api-contract.md @@ -0,0 +1,122 @@ +# Polaris Broker API Contract (Prototype v0) + +Last updated: 2026-07-23 + +## Purpose + +Define a minimal broker contract that allows Spark-side Iceberg AuthManager code to acquire and rotate short-lived Polaris bearer tokens without exposing refresh tokens in notebooks. + +## Scope + +This contract is intentionally narrow: + +- one token mint endpoint for interactive notebook Spark sessions +- strict caller/session binding +- short-lived access tokens only +- no refresh token returned to Spark + +## Endpoint + +- Method: `POST` +- Path: `/v1/polaris/token` +- Authn: broker validates caller using cluster-local identity and request signature/session binding +- Content-Type: `application/json` + +## Request Body + +```json +{ + "user_id": "user@example.local", + "session_id": "jupyter-6f9f99b5f9-l9z6k", + "realm": "teehr", + "catalog": "iceberg", + "groups": ["iceberg-user", "hydrology-team"], + "requested_ttl_seconds": 600, + "audience": "polaris" +} +``` + +## Request Field Notes + +- `user_id`: stable user identifier. Prefer immutable subject (`sub`) if available; username is acceptable for prototype. +- `session_id`: Jupyter notebook server identity (pod UID or equivalent) for caller binding and audit. +- `realm`: Polaris realm to target. +- `catalog`: optional catalog hint for policy/audit context. +- `groups`: optional hint for diagnostics only. Broker should derive trusted groups from validated identity when possible. +- `requested_ttl_seconds`: bounded by broker policy (for example 120 to 900). +- `audience`: should be constrained to Polaris data-plane usage. + +## Success Response + +- Status: `200 OK` + +```json +{ + "access_token": "", + "token_type": "Bearer", + "expires_at_epoch_seconds": 1785169492, + "expires_in_seconds": 600, + "issued_for": { + "user_id": "user@example.local", + "session_id": "jupyter-6f9f99b5f9-l9z6k", + "realm": "teehr" + }, + "trace_id": "f8af1e8b3d21469a9adf99c9185d2d10" +} +``` + +## Error Responses + +- `400 Bad Request`: validation failures +- `401 Unauthorized`: caller identity not valid +- `403 Forbidden`: caller/session mismatch or policy denies issuance +- `429 Too Many Requests`: per-user/session throttle exceeded +- `500/502/503`: transient issuer/broker failure + +Error payload shape: + +```json +{ + "error": "forbidden", + "message": "session is not authorized for requested user_id", + "trace_id": "f8af1e8b3d21469a9adf99c9185d2d10" +} +``` + +## Security Requirements + +- Broker must not trust notebook-supplied `groups` blindly. +- Broker must validate caller identity from trusted transport identity (mTLS, workload identity, signed upstream token, or equivalent). +- Broker must bind `user_id` to `session_id` and reject mismatches. +- Broker-issued tokens must be short-lived and audience-restricted. +- Broker credentials used for exchange/mint operations must never be exposed to notebook runtimes. + +## Audit Requirements + +At minimum, log: + +- `trace_id` +- broker caller identity +- `user_id` +- `session_id` +- `realm` +- requested and granted TTL +- outcome (`issued`, `denied`, `error`) +- downstream issuer status/latency + +## Spark AuthManager Expectations + +The Spark-side AuthManager should: + +- request new token on first use +- cache token in-memory +- proactively rotate before expiry (for example 60 seconds early) +- retry once on auth failure when token is near-expiry +- never persist tokens to logs or Spark history + +## Non-goals (v0) + +- multi-tenant broker routing policies +- user-delegated arbitrary scope requests +- long-lived refresh token distribution to Spark +- batch token mint APIs diff --git a/docs/polaris-identity-propagation-plan.md b/docs/polaris-identity-propagation-plan.md index 0c9baf2..3096e4c 100644 --- a/docs/polaris-identity-propagation-plan.md +++ b/docs/polaris-identity-propagation-plan.md @@ -477,6 +477,97 @@ Do not own: ## Migration Phases +## Token Lifecycle Strategy for Jupyter and Spark + +### Problem statement + +Current behavior relies on a short-lived user access token being injected into a +long-lived notebook runtime. Once the token expires, Spark and direct catalog +operations fail until the user logs out and logs back in. + +### Constraints and observations + +- Interactive notebook UX requires sessions that may outlive an access token. +- We should preserve end-user identity at Polaris for authorization and audit. +- We should avoid exposing long-lived refresh tokens directly in notebook + environments unless there is no practical alternative. +- Spark itself does not provide a complete Keycloak session lifecycle model. + Token refresh behavior must be designed at the client/configuration layer. + +### Approved direction + +#### Short-term (now): Notebook-side token renewal helper + +Implement an explicit token renewal path for Jupyter and Spark sessions that: + +1. detects token expiration proactively using token `exp` metadata +2. acquires a fresh user access token before expiry +3. updates Spark catalog auth configuration in-session +4. retries failed catalog operations once after token renewal + +Short-term implementation notes: + +- Keep renewal logic in a small shared helper used by notebooks and example + scripts. +- Do not require a full logout/login roundtrip for normal token expiry. +- Prefer renewal from a controlled server-side endpoint where possible. +- If direct refresh is used temporarily, limit scope and lifetime and avoid + persisting refresh credentials in notebook files or outputs. + +#### Long-term (target): Token broker / exchange service + +Introduce a dedicated token broker that mints short-lived Polaris-compatible +tokens on behalf of the authenticated Jupyter user. + +Target flow: + +1. user authenticates to JupyterHub with Keycloak +2. notebook runtime requests a short-lived data-plane token from broker +3. broker validates caller identity and exchanges/mints token using secure + server-side credentials +4. notebook and Spark use only short-lived access tokens +5. broker refreshes/exchanges as needed without requiring user relogin + +Broker requirements: + +- preserve individual user identity and effective groups in resulting token +- enforce strict caller binding and audience/scope constraints +- issue short-lived tokens only +- never expose long-lived broker credentials to notebook runtimes +- provide auditable logs that correlate user, request, and issued token metadata + +### Option assessment + +1. Increase Keycloak token TTL only + - Pros: fast operational relief + - Cons: weakens security posture and does not solve lifecycle architecture +2. Notebook helper renewal (approved short-term) + - Pros: immediate UX improvement, compatible with identity goals + - Cons: still transitional without centralized brokering +3. Service principal for interactive Spark + - Pros: simple operational model + - Cons: breaks per-user authorization and audit goals for interactive usage +4. Broker/exchange service (approved target) + - Pros: strongest alignment with security + UX + per-user authorization + - Cons: requires new service and integration work + +### Definition of done + +Short-term done when: + +- notebooks continue through normal token expiry without manual relogin +- Spark catalog operations recover automatically after token rotation +- user identity at Polaris remains user-specific (not collapsed to shared + principal) + +Long-term done when: + +- notebook runtimes no longer require direct refresh-token handling +- broker-issued short-lived tokens are the standard interactive data-plane + credential +- audit logs can correlate Keycloak identity, notebook request, broker issuance, + and Polaris authorization decision + ### Phase 0: Discovery and capability validation Validate product capabilities and constraints before committing implementation details: @@ -536,6 +627,12 @@ Deliverables: - audit-context validation notes - project namespace proof of concept +Token-lifecycle deliverables for this phase: + +- notebook token-renewal helper design +- Spark in-session token update mechanism +- relogin-free expiry recovery validation for representative notebooks + ### Phase 3: Spark + Iceberg + Polaris user-context path Implement Spark access through Polaris under end-user identity: @@ -609,6 +706,126 @@ Deliverables: - migration completion checklist - project namespace user guidance +### Phase 7: Token broker and exchange hardening + +Move from notebook-managed renewal to broker-managed short-lived credentials: + +- implement broker service with strict identity and audience validation +- integrate JupyterHub/notebook clients with broker endpoint +- remove notebook dependence on direct refresh-token handling +- add issuance and exchange audit events with trace correlation +- define fallback behavior when broker is unavailable + +Deliverables: + +- broker service design and implementation +- end-to-end token exchange flow validation for Jupyter + Spark +- security review of broker scopes, TTLs, and credential storage +- migration plan to retire temporary notebook-side renewal paths + +## AuthManager Prototype Path + +Iceberg 1.9+ includes the AuthManager API, which provides a more viable medium-term +path than notebook-managed token refresh for long-lived Spark sessions. + +### Why AuthManager matters here + +- Spark currently relies on Iceberg REST catalog auth behavior that does not recover + reliably with Keycloak token refresh in long-lived sessions. +- Updating notebook variables or Spark conf after catalog initialization is not a + dependable fix once the REST catalog client is already live. +- A custom AuthManager moves token acquisition/rotation into the JVM-side catalog + auth layer where Spark is actually making catalog requests. + +### What is available today + +- Iceberg AuthManager API enablement landed in Iceberg 1.9.0. +- Shared/external AuthManager injection into RESTCatalog was discussed but did not + land in core. +- Therefore, the realistic implementation path is a custom AuthManager class loaded + on the Spark classpath and configured through Iceberg auth properties. + +### Recommended prototype shape + +Build a custom `teehr` AuthManager implementation in Java or Scala that: + +1. accepts a stable user/session identifier from the Spark session context or catalog + properties +2. obtains short-lived Polaris-compatible access tokens from a broker endpoint +3. caches tokens in-memory with proactive refresh ahead of `exp` +4. returns auth headers to Iceberg REST calls without depending on notebook-side + Python token mutation +5. supports recovery by reacquiring tokens independently of the notebook kernel state + +### Strongly preferred token source + +Use a broker or local sidecar endpoint, not direct refresh-token handling inside +Spark catalog properties. + +Preferred flow: + +1. user authenticates to JupyterHub with Keycloak +2. notebook runtime receives stable user/session context +3. Spark AuthManager calls broker with that context +4. broker validates caller/session and returns a short-lived Polaris access token +5. AuthManager refreshes from broker as needed for the life of the Spark session + +### Why broker-backed is preferred + +- avoids storing long-lived refresh credentials in Spark config or executors +- centralizes audit, policy, and failure handling +- preserves per-user authorization semantics at Polaris +- allows independent evolution of Keycloak refresh/exchange logic without pushing + auth complexity into notebooks + +Concrete prototype artifacts now in this repo: + +- broker contract: `docs/polaris-broker-api-contract.md` +- AuthManager scaffold root: `spark/authmanager-prototype/` +- prototype manager class: `spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/TeehrBrokerAuthManager.java` +- prototype session class: `spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerBackedAuthSession.java` + +Prototype property contract used by the manager: + +- `rest.auth.teehr.broker.url` +- `rest.auth.teehr.user-id` +- `rest.auth.teehr.session-id` +- `rest.auth.teehr.realm` +- `rest.auth.teehr.catalog` (default `iceberg`) +- `rest.auth.teehr.requested-ttl-seconds` (default `600`) +- `rest.auth.teehr.request-timeout-ms` (default `5000`) +- `rest.auth.teehr.refresh-skew-seconds` (default `60`) + +### Prototype integration points + +- package custom AuthManager in a jar available to Spark driver and executors +- configure Spark Iceberg catalog to use custom auth manager type/class +- pass only minimal user/session context from notebook to Spark +- keep current notebook refresh helper as fallback for direct REST/API calls, but + do not treat it as the primary fix for Spark session longevity + +### Prototype success criteria + +1. Spark interactive session survives normal Keycloak access-token expiry without + full Spark restart +2. per-user Polaris authorization still reflects current Keycloak-derived entitlements +3. no long-lived refresh token is exposed in notebook code or Spark SQL properties +4. audit logs can correlate notebook user, broker issuance, and Polaris access + +### Near-term recommendation while prototyping + +- keep longer access-token TTL for user experience stability +- keep notebook-side proactive refresh for direct API access +- treat custom AuthManager plus broker as the real fix for Spark session continuity + +Execution checklist for this prototype: + +1. build the prototype jar from `spark/authmanager-prototype/` +2. place jar on Spark driver and executor classpaths +3. configure `spark.sql.catalog..rest.auth.type` to the custom class name +4. provide the `rest.auth.teehr.*` properties via Spark config +5. validate session behavior across at least one access-token expiration window + ## Open Design Questions 1. What auth mechanism does the selected Polaris deployment support for end-user principals? @@ -627,6 +844,19 @@ Deliverables: 14. What governance process should control direct per-user exceptions? 15. What quotas, lifecycle rules, or cleanup policies should apply to project namespaces? +## Token Lifecycle Immediate Execution Plan + +1. Build shared notebook token utility with expiry introspection and proactive + renewal threshold. +2. Add Spark catalog token update function and one-time retry wrapper for + authorization failures attributable to token expiry. +3. Wire utility into developer notebooks and example scripts first. +4. Add observability fields: token issue time, expiry, renewal attempts, + renewal outcome, and Spark operation retry outcome. +5. Define broker API contract and security model in parallel. +6. Implement broker, then cut notebooks over from direct renewal to broker + issuance. + ## Immediate Next Steps 1. Inspect current JupyterHub config and identify where to enable `auth_state` and pre-spawn claim handling. diff --git a/spark/authmanager-prototype/README.md b/spark/authmanager-prototype/README.md new file mode 100644 index 0000000..85aa4c9 --- /dev/null +++ b/spark/authmanager-prototype/README.md @@ -0,0 +1,35 @@ +# Spark AuthManager Prototype + +This directory contains a minimal broker-backed Iceberg REST AuthManager prototype intended for local validation. + +## What this prototype does + +- implements an Iceberg AuthManager class (`org.teehr.iceberg.auth.TeehrBrokerAuthManager`) +- requests short-lived Polaris access tokens from a broker endpoint +- caches tokens in-memory and refreshes before expiry +- injects `Authorization: Bearer ` into REST catalog requests + +## Build + +```bash +mvn -f spark/authmanager-prototype/pom.xml -DskipTests package +``` + +## Spark configuration sketch + +Set the auth manager class as the Iceberg REST auth type: + +```bash +--conf spark.sql.catalog.iceberg.rest.auth.type=org.teehr.iceberg.auth.TeehrBrokerAuthManager \ +--conf spark.sql.catalog.iceberg.rest.auth.teehr.broker.url=http://teehr-token-broker.teehr.svc.cluster.local:8080/v1/polaris/token \ +--conf spark.sql.catalog.iceberg.rest.auth.teehr.user-id=${POLARIS_USER_ID} \ +--conf spark.sql.catalog.iceberg.rest.auth.teehr.session-id=${JUPYTERHUB_SERVER_NAME} \ +--conf spark.sql.catalog.iceberg.rest.auth.teehr.realm=teehr \ +--conf spark.sql.catalog.iceberg.rest.auth.teehr.catalog=iceberg +``` + +## Prototype notes + +- Keep this classpath-local to development images until broker authn/authz is production-ready. +- Do not log access tokens. +- Use broker-issued token TTL of 5 to 15 minutes with proactive refresh. diff --git a/spark/authmanager-prototype/pom.xml b/spark/authmanager-prototype/pom.xml new file mode 100644 index 0000000..1006033 --- /dev/null +++ b/spark/authmanager-prototype/pom.xml @@ -0,0 +1,53 @@ + + + 4.0.0 + + org.teehr + teehr-iceberg-authmanager-prototype + 0.0.1-SNAPSHOT + TEEHR Iceberg AuthManager Prototype + Broker-backed AuthManager prototype for Iceberg REST catalog auth. + + + 17 + 17 + UTF-8 + 1.10.0 + 2.18.2 + 2.0.16 + + + + + org.apache.iceberg + iceberg-core + ${iceberg.version} + provided + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + org.slf4j + slf4j-api + ${slf4j.version} + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + + + + + diff --git a/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerBackedAuthSession.java b/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerBackedAuthSession.java new file mode 100644 index 0000000..c738059 --- /dev/null +++ b/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerBackedAuthSession.java @@ -0,0 +1,96 @@ +package org.teehr.iceberg.auth; + +import java.io.IOException; +import java.time.Duration; +import java.time.Instant; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.iceberg.rest.HTTPHeaders; +import org.apache.iceberg.rest.HTTPRequest; +import org.apache.iceberg.rest.auth.AuthSession; +import org.apache.iceberg.rest.auth.DefaultAuthSession; + +final class BrokerBackedAuthSession implements AuthSession { + + private final BrokerTokenClient tokenClient; + private final String brokerUrl; + private final String userId; + private final String sessionId; + private final String realm; + private final String catalog; + private final Duration timeout; + private final long requestedTtlSeconds; + private final long refreshSkewSeconds; + + private final AtomicReference cachedToken = new AtomicReference<>(); + + BrokerBackedAuthSession( + BrokerTokenClient tokenClient, + String brokerUrl, + String userId, + String sessionId, + String realm, + String catalog, + Duration timeout, + long requestedTtlSeconds, + long refreshSkewSeconds) { + this.tokenClient = tokenClient; + this.brokerUrl = brokerUrl; + this.userId = userId; + this.sessionId = sessionId; + this.realm = realm; + this.catalog = catalog; + this.timeout = timeout; + this.requestedTtlSeconds = requestedTtlSeconds; + this.refreshSkewSeconds = refreshSkewSeconds; + } + + @Override + public HTTPRequest authenticate(HTTPRequest request) { + BrokerToken token = currentToken(); + AuthSession delegate = + DefaultAuthSession.of(HTTPHeaders.of(Map.of("Authorization", "Bearer " + token.accessToken()))); + return delegate.authenticate(request); + } + + private BrokerToken currentToken() { + BrokerToken token = cachedToken.get(); + Instant now = Instant.now(); + + if (token != null && !token.expiresWithinSeconds(refreshSkewSeconds, now)) { + return token; + } + + synchronized (this) { + token = cachedToken.get(); + now = Instant.now(); + if (token != null && !token.expiresWithinSeconds(refreshSkewSeconds, now)) { + return token; + } + + try { + BrokerToken refreshed = + tokenClient.mintToken( + brokerUrl, + userId, + sessionId, + realm, + catalog, + requestedTtlSeconds, + timeout); + cachedToken.set(refreshed); + return refreshed; + } catch (IOException | InterruptedException e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + throw new IllegalStateException("Unable to acquire broker-backed Polaris token", e); + } + } + } + + @Override + public void close() { + cachedToken.set(null); + } +} diff --git a/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerToken.java b/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerToken.java new file mode 100644 index 0000000..2ac2ba6 --- /dev/null +++ b/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerToken.java @@ -0,0 +1,10 @@ +package org.teehr.iceberg.auth; + +import java.time.Instant; + +record BrokerToken(String accessToken, Instant expiresAt) { + + boolean expiresWithinSeconds(long seconds, Instant now) { + return !expiresAt.isAfter(now.plusSeconds(seconds)); + } +} diff --git a/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerTokenClient.java b/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerTokenClient.java new file mode 100644 index 0000000..3661cc8 --- /dev/null +++ b/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerTokenClient.java @@ -0,0 +1,65 @@ +package org.teehr.iceberg.auth; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import java.time.Instant; +import java.util.HashMap; +import java.util.Map; + +final class BrokerTokenClient { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private final HttpClient httpClient; + + BrokerTokenClient(HttpClient httpClient) { + this.httpClient = httpClient; + } + + BrokerToken mintToken( + String brokerUrl, + String userId, + String sessionId, + String realm, + String catalog, + long requestedTtlSeconds, + Duration timeout) + throws IOException, InterruptedException { + + Map body = new HashMap<>(); + body.put("user_id", userId); + body.put("session_id", sessionId); + body.put("realm", realm); + body.put("catalog", catalog); + body.put("requested_ttl_seconds", requestedTtlSeconds); + body.put("audience", "polaris"); + + HttpRequest request = + HttpRequest.newBuilder(URI.create(brokerUrl)) + .header("Content-Type", "application/json") + .timeout(timeout) + .POST(HttpRequest.BodyPublishers.ofString(MAPPER.writeValueAsString(body))) + .build(); + + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + if (response.statusCode() != 200) { + throw new IOException("Broker token request failed with status " + response.statusCode()); + } + + JsonNode payload = MAPPER.readTree(response.body()); + String accessToken = payload.path("access_token").asText(null); + long expiresAtEpoch = payload.path("expires_at_epoch_seconds").asLong(0); + + if (accessToken == null || expiresAtEpoch <= 0) { + throw new IOException("Broker response missing access_token or expires_at_epoch_seconds"); + } + + return new BrokerToken(accessToken, Instant.ofEpochSecond(expiresAtEpoch)); + } +} diff --git a/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/TeehrAuthProperties.java b/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/TeehrAuthProperties.java new file mode 100644 index 0000000..8a21822 --- /dev/null +++ b/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/TeehrAuthProperties.java @@ -0,0 +1,21 @@ +package org.teehr.iceberg.auth; + +final class TeehrAuthProperties { + + private TeehrAuthProperties() {} + + static final String PREFIX = "rest.auth.teehr."; + + static final String BROKER_URL = PREFIX + "broker.url"; + static final String USER_ID = PREFIX + "user-id"; + static final String SESSION_ID = PREFIX + "session-id"; + static final String REALM = PREFIX + "realm"; + static final String CATALOG = PREFIX + "catalog"; + static final String REQUESTED_TTL_SECONDS = PREFIX + "requested-ttl-seconds"; + static final String REQUEST_TIMEOUT_MS = PREFIX + "request-timeout-ms"; + static final String REFRESH_SKEW_SECONDS = PREFIX + "refresh-skew-seconds"; + + static final int DEFAULT_REQUEST_TIMEOUT_MS = 5000; + static final long DEFAULT_REFRESH_SKEW_SECONDS = 60L; + static final long DEFAULT_REQUESTED_TTL_SECONDS = 600L; +} diff --git a/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/TeehrBrokerAuthManager.java b/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/TeehrBrokerAuthManager.java new file mode 100644 index 0000000..828f068 --- /dev/null +++ b/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/TeehrBrokerAuthManager.java @@ -0,0 +1,101 @@ +package org.teehr.iceberg.auth; + +import java.net.http.HttpClient; +import java.time.Duration; +import java.util.Map; +import org.apache.iceberg.rest.RESTClient; +import org.apache.iceberg.rest.auth.AuthManager; +import org.apache.iceberg.rest.auth.AuthSession; +import org.apache.iceberg.util.PropertyUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Prototype Iceberg AuthManager that sources short-lived bearer tokens from a broker endpoint. + */ +public class TeehrBrokerAuthManager implements AuthManager { + + private static final Logger LOG = LoggerFactory.getLogger(TeehrBrokerAuthManager.class); + + private final String name; + private volatile AuthSession catalogSession; + + public TeehrBrokerAuthManager(String managerName) { + this.name = managerName; + } + + @Override + public AuthSession catalogSession(RESTClient sharedClient, Map properties) { + AuthSession existing = catalogSession; + if (existing != null) { + return existing; + } + + synchronized (this) { + existing = catalogSession; + if (existing != null) { + return existing; + } + + String brokerUrl = required(properties, TeehrAuthProperties.BROKER_URL); + String userId = required(properties, TeehrAuthProperties.USER_ID); + String sessionId = required(properties, TeehrAuthProperties.SESSION_ID); + String realm = required(properties, TeehrAuthProperties.REALM); + String catalog = properties.getOrDefault(TeehrAuthProperties.CATALOG, "iceberg"); + + int timeoutMs = + PropertyUtil.propertyAsInt( + properties, + TeehrAuthProperties.REQUEST_TIMEOUT_MS, + TeehrAuthProperties.DEFAULT_REQUEST_TIMEOUT_MS); + long refreshSkewSeconds = + PropertyUtil.propertyAsLong( + properties, + TeehrAuthProperties.REFRESH_SKEW_SECONDS, + TeehrAuthProperties.DEFAULT_REFRESH_SKEW_SECONDS); + long requestedTtlSeconds = + PropertyUtil.propertyAsLong( + properties, + TeehrAuthProperties.REQUESTED_TTL_SECONDS, + TeehrAuthProperties.DEFAULT_REQUESTED_TTL_SECONDS); + + BrokerTokenClient tokenClient = new BrokerTokenClient(HttpClient.newHttpClient()); + + catalogSession = + new BrokerBackedAuthSession( + tokenClient, + brokerUrl, + userId, + sessionId, + realm, + catalog, + Duration.ofMillis(timeoutMs), + requestedTtlSeconds, + refreshSkewSeconds); + + LOG.info("Initialized TeehrBrokerAuthManager for catalog {}", name); + return catalogSession; + } + } + + @Override + public void close() { + AuthSession session = catalogSession; + this.catalogSession = null; + if (session != null) { + try { + session.close(); + } catch (Exception e) { + LOG.warn("Error closing broker auth session", e); + } + } + } + + private static String required(Map properties, String key) { + String value = properties.get(key); + if (value == null || value.isBlank()) { + throw new IllegalArgumentException("Missing required property: " + key); + } + return value; + } +} From 34c4df9f974a59fe08b3faaf86306e8f6f2617a0 Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Thu, 23 Jul 2026 22:32:07 -0400 Subject: [PATCH 15/62] feat(auth): add broker token exchange path and keycloak wiring --- api/manifests/configmap.yaml.tpl | 11 +- api/manifests/deployment.yaml.tpl | 45 ++++++++ api/src/auth.py | 18 +++ api/src/broker.py | 109 ++++++++++++++++++ api/src/config.py | 21 +++- api/src/main.py | 12 +- api/src/routes/auth.py | 75 +++++++++++- docs/polaris-broker-api-contract.md | 42 ++++++- .../developer/01_setup_minio_warehouse.ipynb | 26 +++++ examples/developer/setup_utils.py | 75 +++++++++++- .../manifests/realm-configmap.yaml.tpl | 14 +++ scripts/test_jupyter_broker_token.sh | 97 ++++++++++++++++ 12 files changed, 539 insertions(+), 6 deletions(-) create mode 100644 api/src/broker.py create mode 100755 scripts/test_jupyter_broker_token.sh diff --git a/api/manifests/configmap.yaml.tpl b/api/manifests/configmap.yaml.tpl index b0e5750..655f5a8 100644 --- a/api/manifests/configmap.yaml.tpl +++ b/api/manifests/configmap.yaml.tpl @@ -15,7 +15,16 @@ data: KEYCLOAK_ISSUER_URL: "https://auth.${var.hostname}/realms/teehr" KEYCLOAK_JWKS_URL: "http://keycloak-service:8080/realms/teehr/protocol/openid-connect/certs" KEYCLOAK_AUDIENCE: "teehr-api" - KEYCLOAK_ALLOWED_AUDIENCES: "teehr-api,teehr-frontend" + KEYCLOAK_ALLOWED_AUDIENCES: "teehr-api,teehr-frontend,jupyterhub" + BROKER_TOKEN_EXCHANGE_ENABLED: "true" + BROKER_TOKEN_ENDPOINT: "http://keycloak-service:8080/realms/teehr/protocol/openid-connect/token" + BROKER_OAUTH_CLIENT_ID: "teehr-api" + BROKER_OAUTH_CLIENT_SECRET: "" + BROKER_TARGET_AUDIENCE: "account" + BROKER_DEFAULT_SCOPE: "openid profile email" + BROKER_MIN_TTL_SECONDS: "120" + BROKER_MAX_TTL_SECONDS: "900" + BROKER_REQUEST_TIMEOUT_SECONDS: "10" ANON_RATE_LIMIT_RPM: "20" AUTH_RATE_LIMIT_RPM: "120" ROW_LIMIT_ANON: "200" diff --git a/api/manifests/deployment.yaml.tpl b/api/manifests/deployment.yaml.tpl index 49993db..2a050f7 100644 --- a/api/manifests/deployment.yaml.tpl +++ b/api/manifests/deployment.yaml.tpl @@ -76,6 +76,51 @@ spec: configMapKeyRef: name: teehr-api-config key: KEYCLOAK_ALLOWED_AUDIENCES + - name: BROKER_TOKEN_EXCHANGE_ENABLED + valueFrom: + configMapKeyRef: + name: teehr-api-config + key: BROKER_TOKEN_EXCHANGE_ENABLED + - name: BROKER_TOKEN_ENDPOINT + valueFrom: + configMapKeyRef: + name: teehr-api-config + key: BROKER_TOKEN_ENDPOINT + - name: BROKER_OAUTH_CLIENT_ID + valueFrom: + configMapKeyRef: + name: teehr-api-config + key: BROKER_OAUTH_CLIENT_ID + - name: BROKER_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: teehr-api-secrets + key: client-secret + - name: BROKER_TARGET_AUDIENCE + valueFrom: + configMapKeyRef: + name: teehr-api-config + key: BROKER_TARGET_AUDIENCE + - name: BROKER_DEFAULT_SCOPE + valueFrom: + configMapKeyRef: + name: teehr-api-config + key: BROKER_DEFAULT_SCOPE + - name: BROKER_MIN_TTL_SECONDS + valueFrom: + configMapKeyRef: + name: teehr-api-config + key: BROKER_MIN_TTL_SECONDS + - name: BROKER_MAX_TTL_SECONDS + valueFrom: + configMapKeyRef: + name: teehr-api-config + key: BROKER_MAX_TTL_SECONDS + - name: BROKER_REQUEST_TIMEOUT_SECONDS + valueFrom: + configMapKeyRef: + name: teehr-api-config + key: BROKER_REQUEST_TIMEOUT_SECONDS - name: API_KEYS_DB_HOST value: keycloak-pg - name: API_KEYS_DB_PORT diff --git a/api/src/auth.py b/api/src/auth.py index bf2293e..c821a39 100644 --- a/api/src/auth.py +++ b/api/src/auth.py @@ -35,6 +35,8 @@ class AuthIdentity: auth_type: str roles: list[str] = field(default_factory=list) scopes: list[str] = field(default_factory=list) + preferred_username: str | None = None + groups: list[str] = field(default_factory=list) @property def is_authenticated(self) -> bool: @@ -129,11 +131,17 @@ async def validate(self, token: str) -> AuthIdentity: if not subject: raise HTTPException(status_code=401, detail="JWT missing subject") + groups = claims.get("groups", []) + if not isinstance(groups, list): + groups = [] + return AuthIdentity( subject=subject, auth_type="jwt", roles=roles, scopes=scopes, + preferred_username=claims.get("preferred_username"), + groups=[str(group) for group in groups], ) except httpx.HTTPError as exc: logger.error("Keycloak connectivity error during token validation: %s", str(exc)) @@ -172,6 +180,16 @@ async def get_request_identity(request: Request) -> AuthIdentity: return identity +def extract_bearer_token_from_request(request: Request) -> str: + auth_header = request.headers.get("authorization", "") + if not auth_header.lower().startswith("bearer "): + raise HTTPException(status_code=401, detail="Bearer token required") + token = auth_header.split(" ", 1)[1].strip() + if not token: + raise HTTPException(status_code=401, detail="Bearer token required") + return token + + async def get_authenticated_identity( identity: AuthIdentity = Depends(get_request_identity), ) -> AuthIdentity: diff --git a/api/src/broker.py b/api/src/broker.py new file mode 100644 index 0000000..bf6e5ed --- /dev/null +++ b/api/src/broker.py @@ -0,0 +1,109 @@ +import time +import uuid + +import httpx +from fastapi import HTTPException +from jose import JWTError, jwt + +from .config import config + + +TOKEN_EXCHANGE_GRANT = "urn:ietf:params:oauth:grant-type:token-exchange" +ACCESS_TOKEN_TYPE = "urn:ietf:params:oauth:token-type:access_token" + + +def clamp_requested_ttl(requested_ttl_seconds: int | None) -> int: + requested = requested_ttl_seconds or config.BROKER_MAX_TTL_SECONDS + return max(config.BROKER_MIN_TTL_SECONDS, min(requested, config.BROKER_MAX_TTL_SECONDS)) + + +async def exchange_token_for_polaris( + *, + subject_token: str, + audience: str, + requested_ttl_seconds: int | None, +) -> dict: + if not config.BROKER_TOKEN_EXCHANGE_ENABLED: + raise HTTPException(status_code=503, detail="Token broker is disabled") + + trace_id = uuid.uuid4().hex + clamped_ttl = clamp_requested_ttl(requested_ttl_seconds) + + payload = { + "grant_type": TOKEN_EXCHANGE_GRANT, + "client_id": config.BROKER_OAUTH_CLIENT_ID, + "subject_token": subject_token, + "subject_token_type": ACCESS_TOKEN_TYPE, + "requested_token_type": ACCESS_TOKEN_TYPE, + "audience": audience, + "scope": config.BROKER_DEFAULT_SCOPE, + } + if config.BROKER_OAUTH_CLIENT_SECRET: + payload["client_secret"] = config.BROKER_OAUTH_CLIENT_SECRET + + try: + async with httpx.AsyncClient(timeout=config.BROKER_REQUEST_TIMEOUT_SECONDS) as client: + response = await client.post(config.BROKER_TOKEN_ENDPOINT, data=payload) + + if response.status_code >= 400: + upstream_error = None + upstream_error_description = None + try: + upstream_payload = response.json() + upstream_error = upstream_payload.get("error") + upstream_error_description = upstream_payload.get("error_description") + except ValueError: + upstream_error_description = response.text[:500] + + raise HTTPException( + status_code=502, + detail={ + "error": "token_exchange_failed", + "message": "Broker failed to exchange token with identity provider", + "trace_id": trace_id, + "upstream_status": response.status_code, + "upstream_error": upstream_error, + "upstream_error_description": upstream_error_description, + }, + ) + + token_payload = response.json() + access_token = token_payload.get("access_token") + if not access_token: + raise HTTPException( + status_code=502, + detail={ + "error": "invalid_upstream_response", + "message": "Identity provider did not return access_token", + "trace_id": trace_id, + }, + ) + + expires_in = int(token_payload.get("expires_in", clamped_ttl)) + expires_at_epoch_seconds = int(time.time()) + max(expires_in, 1) + + # Prefer JWT exp claim when present. + try: + claims = jwt.get_unverified_claims(access_token) + exp = int(claims.get("exp", 0)) + if exp > 0: + expires_at_epoch_seconds = exp + except (JWTError, ValueError, TypeError): + pass + + return { + "access_token": access_token, + "token_type": token_payload.get("token_type", "Bearer"), + "expires_in_seconds": max(expires_at_epoch_seconds - int(time.time()), 1), + "expires_at_epoch_seconds": expires_at_epoch_seconds, + "trace_id": trace_id, + } + except httpx.HTTPError as exc: + raise HTTPException( + status_code=503, + detail={ + "error": "broker_connectivity_error", + "message": "Unable to contact identity provider for token exchange", + "trace_id": trace_id, + }, + ) from exc diff --git a/api/src/config.py b/api/src/config.py index 4092da6..9d1db5e 100644 --- a/api/src/config.py +++ b/api/src/config.py @@ -40,7 +40,7 @@ class Config: KEYCLOAK_AUDIENCE = os.environ.get("KEYCLOAK_AUDIENCE", "teehr-api") KEYCLOAK_ALLOWED_AUDIENCES = os.environ.get( "KEYCLOAK_ALLOWED_AUDIENCES", - "teehr-api,teehr-frontend", + "teehr-api,teehr-frontend,jupyterhub", ) KEYCLOAK_AUTH_URL = os.environ.get( "KEYCLOAK_AUTH_URL", @@ -70,6 +70,25 @@ class Config: ANON_RATE_LIMIT_RPM = int(os.environ.get("ANON_RATE_LIMIT_RPM", "20")) AUTH_RATE_LIMIT_RPM = int(os.environ.get("AUTH_RATE_LIMIT_RPM", "120")) + # Polaris token broker settings + BROKER_TOKEN_EXCHANGE_ENABLED = ( + os.environ.get("BROKER_TOKEN_EXCHANGE_ENABLED", "true").strip().lower() + in {"1", "true", "t", "yes", "y", "on"} + ) + BROKER_TOKEN_ENDPOINT = os.environ.get( + "BROKER_TOKEN_ENDPOINT", + KEYCLOAK_TOKEN_URL, + ) + BROKER_OAUTH_CLIENT_ID = os.environ.get("BROKER_OAUTH_CLIENT_ID", "teehr-api") + BROKER_OAUTH_CLIENT_SECRET = os.environ.get("BROKER_OAUTH_CLIENT_SECRET", "") + BROKER_TARGET_AUDIENCE = os.environ.get("BROKER_TARGET_AUDIENCE", "account") + BROKER_DEFAULT_SCOPE = os.environ.get("BROKER_DEFAULT_SCOPE", "openid profile email") + BROKER_MIN_TTL_SECONDS = int(os.environ.get("BROKER_MIN_TTL_SECONDS", "120")) + BROKER_MAX_TTL_SECONDS = int(os.environ.get("BROKER_MAX_TTL_SECONDS", "900")) + BROKER_REQUEST_TIMEOUT_SECONDS = int( + os.environ.get("BROKER_REQUEST_TIMEOUT_SECONDS", "10") + ) + # Role-based record/page limits ROW_LIMIT_ANON = int(os.environ.get("ROW_LIMIT_ANON", "200")) ROW_LIMIT_API_KEY = int(os.environ.get("ROW_LIMIT_API_KEY", "50000")) diff --git a/api/src/main.py b/api/src/main.py index 186c9cd..4422649 100644 --- a/api/src/main.py +++ b/api/src/main.py @@ -132,6 +132,13 @@ def custom_openapi(): {"BearerAuth": []}, ] + auth_polaris_token = openapi_schema.get("paths", {}).get("/auth/polaris-token", {}).get("post") + if auth_polaris_token: + auth_polaris_token["security"] = [ + {"OAuth2Keycloak": ["openid", "profile", "email"]}, + {"BearerAuth": []}, + ] + app.openapi_schema = openapi_schema return app.openapi_schema @@ -195,7 +202,10 @@ async def auth_context_middleware(request: Request, call_next): ) # Keep auth optional while attaching identity for routes that need it. - request.state.identity = await resolve_identity(request) + try: + request.state.identity = await resolve_identity(request) + except HTTPException as exc: + return JSONResponse(status_code=exc.status_code, content={"detail": exc.detail}) # Require authentication for all non-diagnostic API paths. if not exempt_paths and not request.state.identity.is_authenticated: diff --git a/api/src/routes/auth.py b/api/src/routes/auth.py index cb42cad..1898ecd 100644 --- a/api/src/routes/auth.py +++ b/api/src/routes/auth.py @@ -1,7 +1,15 @@ from fastapi import APIRouter, Depends, HTTPException, Request from pydantic import BaseModel, Field -from ..auth import AuthIdentity, get_admin_identity, get_request_identity +from ..auth import ( + AuthIdentity, + extract_bearer_token_from_request, + get_admin_identity, + get_authenticated_identity, + get_request_identity, +) +from ..broker import exchange_token_for_polaris +from ..config import config router = APIRouter(prefix="/auth", tags=["Auth"]) @@ -11,6 +19,31 @@ class ApiKeyCreateRequest(BaseModel): scopes: list[str] = Field(default_factory=list) +class PolarisTokenRequest(BaseModel): + user_id: str = Field(min_length=1) + session_id: str = Field(min_length=1) + realm: str = Field(min_length=1) + catalog: str = Field(default="iceberg", min_length=1) + groups: list[str] = Field(default_factory=list) + requested_ttl_seconds: int = Field(default=600, ge=1) + audience: str = Field(default_factory=lambda: config.BROKER_TARGET_AUDIENCE) + + +class PolarisTokenIssuedFor(BaseModel): + user_id: str + session_id: str + realm: str + + +class PolarisTokenResponse(BaseModel): + access_token: str + token_type: str + expires_at_epoch_seconds: int + expires_in_seconds: int + issued_for: PolarisTokenIssuedFor + trace_id: str + + @router.get("/me") async def me(identity: AuthIdentity = Depends(get_request_identity)): return { @@ -54,3 +87,43 @@ async def revoke_api_key( revoked = await request.app.state.api_key_store.revoke_key(identity.subject, key_id) if not revoked: raise HTTPException(status_code=404, detail="API key not found") + + +@router.post("/polaris-token", response_model=PolarisTokenResponse) +async def polaris_token( + request: Request, + payload: PolarisTokenRequest, + identity: AuthIdentity = Depends(get_authenticated_identity), +): + if identity.auth_type != "jwt": + raise HTTPException(status_code=403, detail="JWT identity required") + + allowed_user_ids = {identity.subject} + if identity.preferred_username: + allowed_user_ids.add(identity.preferred_username) + + if payload.user_id not in allowed_user_ids: + raise HTTPException( + status_code=403, + detail="Requested user_id does not match authenticated identity", + ) + + bearer_token = extract_bearer_token_from_request(request) + exchanged = await exchange_token_for_polaris( + subject_token=bearer_token, + audience=payload.audience, + requested_ttl_seconds=payload.requested_ttl_seconds, + ) + + return PolarisTokenResponse( + access_token=exchanged["access_token"], + token_type=exchanged["token_type"], + expires_at_epoch_seconds=exchanged["expires_at_epoch_seconds"], + expires_in_seconds=exchanged["expires_in_seconds"], + issued_for=PolarisTokenIssuedFor( + user_id=payload.user_id, + session_id=payload.session_id, + realm=payload.realm, + ), + trace_id=exchanged["trace_id"], + ) diff --git a/docs/polaris-broker-api-contract.md b/docs/polaris-broker-api-contract.md index dd8e896..1ffe2db 100644 --- a/docs/polaris-broker-api-contract.md +++ b/docs/polaris-broker-api-contract.md @@ -22,6 +22,13 @@ This contract is intentionally narrow: - Authn: broker validates caller using cluster-local identity and request signature/session binding - Content-Type: `application/json` +Current prototype implementation in this repo: + +- Method: `POST` +- Path: `/auth/polaris-token` +- Service: `teehr-api` +- Caller auth: Keycloak bearer token validated by `teehr-api` + ## Request Body ```json @@ -32,7 +39,7 @@ This contract is intentionally narrow: "catalog": "iceberg", "groups": ["iceberg-user", "hydrology-team"], "requested_ttl_seconds": 600, - "audience": "polaris" + "audience": "account" } ``` @@ -120,3 +127,36 @@ The Spark-side AuthManager should: - user-delegated arbitrary scope requests - long-lived refresh token distribution to Spark - batch token mint APIs + +## Prototype Test (Current Repo) + +From a Jupyter single-user pod shell: + +```bash +python - <<'PY' +import json +import os +import requests + +token = os.environ["POLARIS_USER_TOKEN"] +realm = os.getenv("POLARIS_DEFAULT_REALM", "teehr") +user_id = os.getenv("JUPYTERHUB_USER", "admin") +session_id = os.getenv("JUPYTERHUB_SERVER_NAME", user_id) + +resp = requests.post( + "http://teehr-api:8000/auth/polaris-token", + headers={"Authorization": f"Bearer {token}"}, + json={ + "user_id": user_id, + "session_id": session_id, + "realm": realm, + "catalog": "iceberg", + "requested_ttl_seconds": 600, + "audience": "account", + }, + timeout=20, +) +print(resp.status_code) +print(json.dumps(resp.json(), indent=2)[:1200]) +PY +``` diff --git a/examples/developer/01_setup_minio_warehouse.ipynb b/examples/developer/01_setup_minio_warehouse.ipynb index 4b56bcc..08a23b4 100644 --- a/examples/developer/01_setup_minio_warehouse.ipynb +++ b/examples/developer/01_setup_minio_warehouse.ipynb @@ -93,6 +93,32 @@ " print(\"WARNING: token is missing iceberg-catalog-admin; Polaris may reject writes\")" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "4f288bea", + "metadata": {}, + "outputs": [], + "source": [ + "use_broker = os.getenv(\"POLARIS_USE_BROKER\", \"false\").strip().lower() in {\"1\", \"true\", \"t\", \"yes\", \"y\", \"on\"}\n", + "if use_broker:\n", + " broker_user_id = claims.get(\"preferred_username\") or claims.get(\"sub\") or polaris_username\n", + " broker_session_id = os.getenv(\"JUPYTERHUB_SERVER_NAME\", os.getenv(\"JUPYTERHUB_USER\", \"notebook\"))\n", + "\n", + " polaris_user_token, broker_renewed = ensure_fresh_polaris_token_via_broker(\n", + " current_token=polaris_user_token,\n", + " user_id=broker_user_id,\n", + " session_id=broker_session_id,\n", + " bearer_token=polaris_user_token,\n", + " realm=os.getenv(\"POLARIS_DEFAULT_REALM\", \"teehr\"),\n", + " catalog=\"iceberg\",\n", + " refresh_window_seconds=120,\n", + " requested_ttl_seconds=600,\n", + " )\n", + " os.environ[\"POLARIS_USER_TOKEN\"] = polaris_user_token\n", + " print(\"Broker token status:\", \"renewed\" if broker_renewed else \"still fresh\")" + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/developer/setup_utils.py b/examples/developer/setup_utils.py index 91ff363..97cea5e 100644 --- a/examples/developer/setup_utils.py +++ b/examples/developer/setup_utils.py @@ -283,4 +283,77 @@ def create_minio_spark_session( if "force_recreate_session" in msg and "force_recreate_session" in call_kwargs: call_kwargs.pop("force_recreate_session", None) continue - raise \ No newline at end of file + raise + + +def request_broker_polaris_token( + *, + user_id: str, + session_id: str, + bearer_token: str, + realm: Optional[str] = None, + catalog: str = "iceberg", + requested_ttl_seconds: int = 600, + audience: Optional[str] = None, + broker_url: Optional[str] = None, + timeout_seconds: int = 20, +) -> Tuple[str, int, str]: + endpoint = broker_url or os.getenv("POLARIS_BROKER_URL", "http://teehr-api:8000/auth/polaris-token") + active_realm = realm or os.getenv("POLARIS_DEFAULT_REALM", "teehr") + active_audience = audience or os.getenv("POLARIS_BROKER_AUDIENCE", "polaris") + + if not bearer_token: + raise RuntimeError("bearer_token is required to request a broker token") + + resp = requests.post( + endpoint, + headers={"Authorization": f"Bearer {bearer_token}"}, + json={ + "user_id": user_id, + "session_id": session_id, + "realm": active_realm, + "catalog": catalog, + "requested_ttl_seconds": requested_ttl_seconds, + "audience": active_audience, + }, + timeout=timeout_seconds, + ) + resp.raise_for_status() + payload = resp.json() + access_token = payload.get("access_token") + expires_at = int(payload.get("expires_at_epoch_seconds", 0)) + trace_id = payload.get("trace_id", "") + + if not access_token or expires_at <= 0: + raise RuntimeError("Broker response missing access_token or expires_at_epoch_seconds") + + return access_token, expires_at, trace_id + + +def ensure_fresh_polaris_token_via_broker( + *, + current_token: Optional[str], + user_id: str, + session_id: str, + bearer_token: str, + realm: Optional[str] = None, + catalog: str = "iceberg", + refresh_window_seconds: int = 120, + requested_ttl_seconds: int = 600, + audience: Optional[str] = None, + broker_url: Optional[str] = None, +) -> Tuple[str, bool]: + if current_token and not _token_expires_soon(current_token, refresh_window_seconds): + return current_token, False + + refreshed_token, _, _ = request_broker_polaris_token( + user_id=user_id, + session_id=session_id, + bearer_token=bearer_token, + realm=realm, + catalog=catalog, + requested_ttl_seconds=requested_ttl_seconds, + audience=audience, + broker_url=broker_url, + ) + return refreshed_token, True \ No newline at end of file diff --git a/keycloak-bootstrap/manifests/realm-configmap.yaml.tpl b/keycloak-bootstrap/manifests/realm-configmap.yaml.tpl index c457ccb..4fdcc59 100644 --- a/keycloak-bootstrap/manifests/realm-configmap.yaml.tpl +++ b/keycloak-bootstrap/manifests/realm-configmap.yaml.tpl @@ -128,6 +128,9 @@ data: "protocol": "openid-connect", "publicClient": false, "serviceAccountsEnabled": true, + "attributes": { + "standard.token.exchange.enabled": "true" + }, "secret": "$(env:TEEHR_API_CLIENT_SECRET)" }, { @@ -166,6 +169,17 @@ data: "claim.name": "realm_access.roles", "jsonType.label": "String" } + }, + { + "name": "audience-teehr-api", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-mapper", + "consentRequired": false, + "config": { + "included.client.audience": "teehr-api", + "id.token.claim": "false", + "access.token.claim": "true" + } } ], "redirectUris": [ diff --git a/scripts/test_jupyter_broker_token.sh b/scripts/test_jupyter_broker_token.sh new file mode 100755 index 0000000..8670b96 --- /dev/null +++ b/scripts/test_jupyter_broker_token.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +set -euo pipefail + +NAMESPACE="${NAMESPACE:-teehr-hub}" +POD_NAME="${POD_NAME:-}" +CONTAINER_NAME="${CONTAINER_NAME:-notebook}" +BROKER_URL="${BROKER_URL:-http://teehr-api:8000/auth/polaris-token}" +REQUESTED_TTL_SECONDS="${REQUESTED_TTL_SECONDS:-600}" +AUDIENCE="${AUDIENCE:-account}" + +if [[ -z "$POD_NAME" ]]; then + POD_NAME="$(kubectl -n "$NAMESPACE" get pods -o name | sed 's#^pod/##' | grep -E '^jupyter-' | head -n1 || true)" +fi + +if [[ -z "$POD_NAME" ]]; then + echo "No Jupyter single-user pod found in namespace '$NAMESPACE'." >&2 + exit 1 +fi + +echo "Testing broker token endpoint from pod $POD_NAME" + +kubectl -n "$NAMESPACE" exec -i "$POD_NAME" -c "$CONTAINER_NAME" -- env \ + BROKER_URL="$BROKER_URL" \ + REQUESTED_TTL_SECONDS="$REQUESTED_TTL_SECONDS" \ + AUDIENCE="$AUDIENCE" \ + python - <<'PY' +import json +import os +import requests + +def get_fresh_subject_token() -> str: + token_endpoint = os.getenv( + "POLARIS_OAUTH2_SERVER_URI", + "http://keycloak-service:8080/realms/teehr/protocol/openid-connect/token", + ) + client_id = os.getenv("POLARIS_CLIENT_ID", "jupyterhub") + client_secret = os.getenv("POLARIS_CLIENT_SECRET", "") + + refresh_token = os.getenv("POLARIS_REFRESH_TOKEN", "") + if refresh_token: + payload = { + "grant_type": "refresh_token", + "client_id": client_id, + "refresh_token": refresh_token, + } + if client_secret: + payload["client_secret"] = client_secret + resp = requests.post(token_endpoint, data=payload, timeout=20) + if resp.status_code < 400: + token = resp.json().get("access_token") + if token: + return token + + token = os.getenv("POLARIS_USER_TOKEN", "") + if token: + return token + + raise RuntimeError("Unable to obtain a usable subject token from pod environment") + + +token = get_fresh_subject_token() + +realm = os.getenv("POLARIS_DEFAULT_REALM", "teehr") +user_id = os.getenv("JUPYTERHUB_USER", "admin") +session_id = (os.getenv("JUPYTERHUB_SERVER_NAME") or "").strip() or user_id + +response = requests.post( + os.environ["BROKER_URL"], + headers={"Authorization": f"Bearer {token}"}, + json={ + "user_id": user_id, + "session_id": session_id, + "realm": realm, + "catalog": "iceberg", + "requested_ttl_seconds": int(os.environ["REQUESTED_TTL_SECONDS"]), + "audience": os.environ["AUDIENCE"], + }, + timeout=20, +) + +print("status:", response.status_code) +try: + payload = response.json() +except ValueError: + print(response.text[:1500]) + raise + +if response.status_code >= 400: + print(json.dumps(payload, indent=2)[:2000]) + raise RuntimeError("broker token call failed") + +print("trace_id:", payload.get("trace_id")) +print("token_type:", payload.get("token_type")) +print("expires_in_seconds:", payload.get("expires_in_seconds")) +print("issued_for:", payload.get("issued_for")) +print("access_token_prefix:", str(payload.get("access_token", ""))[:24]) +PY From 377fdda5840364eff720d8660be775eb03a5961d Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Thu, 23 Jul 2026 22:40:07 -0400 Subject: [PATCH 16/62] feat(authmanager): wire broker subject token into spark auth flow --- spark/authmanager-prototype/README.md | 6 +++- .../iceberg/auth/BrokerBackedAuthSession.java | 14 +++++++++ .../teehr/iceberg/auth/BrokerTokenClient.java | 15 ++++++++-- .../iceberg/auth/TeehrAuthProperties.java | 5 ++++ .../iceberg/auth/TeehrBrokerAuthManager.java | 29 +++++++++++++++++++ 5 files changed, 66 insertions(+), 3 deletions(-) diff --git a/spark/authmanager-prototype/README.md b/spark/authmanager-prototype/README.md index 85aa4c9..488cb64 100644 --- a/spark/authmanager-prototype/README.md +++ b/spark/authmanager-prototype/README.md @@ -25,7 +25,9 @@ Set the auth manager class as the Iceberg REST auth type: --conf spark.sql.catalog.iceberg.rest.auth.teehr.user-id=${POLARIS_USER_ID} \ --conf spark.sql.catalog.iceberg.rest.auth.teehr.session-id=${JUPYTERHUB_SERVER_NAME} \ --conf spark.sql.catalog.iceberg.rest.auth.teehr.realm=teehr \ ---conf spark.sql.catalog.iceberg.rest.auth.teehr.catalog=iceberg +--conf spark.sql.catalog.iceberg.rest.auth.teehr.catalog=iceberg \ +--conf spark.sql.catalog.iceberg.rest.auth.teehr.audience=account \ +--conf spark.sql.catalog.iceberg.rest.auth.teehr.subject-token-env=POLARIS_USER_TOKEN ``` ## Prototype notes @@ -33,3 +35,5 @@ Set the auth manager class as the Iceberg REST auth type: - Keep this classpath-local to development images until broker authn/authz is production-ready. - Do not log access tokens. - Use broker-issued token TTL of 5 to 15 minutes with proactive refresh. +- The manager sends the subject access token to the broker in the Authorization header and requests an exchanged Polaris token. +- Provide a subject token via `rest.auth.teehr.subject-token` (testing only) or `rest.auth.teehr.subject-token-env`. diff --git a/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerBackedAuthSession.java b/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerBackedAuthSession.java index c738059..51ee1ff 100644 --- a/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerBackedAuthSession.java +++ b/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerBackedAuthSession.java @@ -5,6 +5,7 @@ import java.time.Instant; import java.util.Map; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Supplier; import org.apache.iceberg.rest.HTTPHeaders; import org.apache.iceberg.rest.HTTPRequest; import org.apache.iceberg.rest.auth.AuthSession; @@ -18,6 +19,8 @@ final class BrokerBackedAuthSession implements AuthSession { private final String sessionId; private final String realm; private final String catalog; + private final String audience; + private final Supplier subjectTokenSupplier; private final Duration timeout; private final long requestedTtlSeconds; private final long refreshSkewSeconds; @@ -31,6 +34,8 @@ final class BrokerBackedAuthSession implements AuthSession { String sessionId, String realm, String catalog, + String audience, + Supplier subjectTokenSupplier, Duration timeout, long requestedTtlSeconds, long refreshSkewSeconds) { @@ -40,6 +45,8 @@ final class BrokerBackedAuthSession implements AuthSession { this.sessionId = sessionId; this.realm = realm; this.catalog = catalog; + this.audience = audience; + this.subjectTokenSupplier = subjectTokenSupplier; this.timeout = timeout; this.requestedTtlSeconds = requestedTtlSeconds; this.refreshSkewSeconds = refreshSkewSeconds; @@ -69,6 +76,11 @@ private BrokerToken currentToken() { } try { + String subjectToken = subjectTokenSupplier.get(); + if (subjectToken == null || subjectToken.isBlank()) { + throw new IllegalStateException("Subject token supplier returned an empty token"); + } + BrokerToken refreshed = tokenClient.mintToken( brokerUrl, @@ -76,6 +88,8 @@ private BrokerToken currentToken() { sessionId, realm, catalog, + audience, + subjectToken, requestedTtlSeconds, timeout); cachedToken.set(refreshed); diff --git a/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerTokenClient.java b/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerTokenClient.java index 3661cc8..28ddf5c 100644 --- a/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerTokenClient.java +++ b/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerTokenClient.java @@ -28,6 +28,8 @@ BrokerToken mintToken( String sessionId, String realm, String catalog, + String audience, + String subjectToken, long requestedTtlSeconds, Duration timeout) throws IOException, InterruptedException { @@ -38,18 +40,27 @@ BrokerToken mintToken( body.put("realm", realm); body.put("catalog", catalog); body.put("requested_ttl_seconds", requestedTtlSeconds); - body.put("audience", "polaris"); + body.put("audience", audience); HttpRequest request = HttpRequest.newBuilder(URI.create(brokerUrl)) .header("Content-Type", "application/json") + .header("Authorization", "Bearer " + subjectToken) .timeout(timeout) .POST(HttpRequest.BodyPublishers.ofString(MAPPER.writeValueAsString(body))) .build(); HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); if (response.statusCode() != 200) { - throw new IOException("Broker token request failed with status " + response.statusCode()); + String bodySnippet = response.body() == null ? "" : response.body(); + if (bodySnippet.length() > 500) { + bodySnippet = bodySnippet.substring(0, 500); + } + throw new IOException( + "Broker token request failed with status " + + response.statusCode() + + ": " + + bodySnippet); } JsonNode payload = MAPPER.readTree(response.body()); diff --git a/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/TeehrAuthProperties.java b/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/TeehrAuthProperties.java index 8a21822..c5b8953 100644 --- a/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/TeehrAuthProperties.java +++ b/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/TeehrAuthProperties.java @@ -11,6 +11,9 @@ private TeehrAuthProperties() {} static final String SESSION_ID = PREFIX + "session-id"; static final String REALM = PREFIX + "realm"; static final String CATALOG = PREFIX + "catalog"; + static final String AUDIENCE = PREFIX + "audience"; + static final String SUBJECT_TOKEN = PREFIX + "subject-token"; + static final String SUBJECT_TOKEN_ENV = PREFIX + "subject-token-env"; static final String REQUESTED_TTL_SECONDS = PREFIX + "requested-ttl-seconds"; static final String REQUEST_TIMEOUT_MS = PREFIX + "request-timeout-ms"; static final String REFRESH_SKEW_SECONDS = PREFIX + "refresh-skew-seconds"; @@ -18,4 +21,6 @@ private TeehrAuthProperties() {} static final int DEFAULT_REQUEST_TIMEOUT_MS = 5000; static final long DEFAULT_REFRESH_SKEW_SECONDS = 60L; static final long DEFAULT_REQUESTED_TTL_SECONDS = 600L; + static final String DEFAULT_AUDIENCE = "account"; + static final String DEFAULT_SUBJECT_TOKEN_ENV = "POLARIS_USER_TOKEN"; } diff --git a/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/TeehrBrokerAuthManager.java b/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/TeehrBrokerAuthManager.java index 828f068..44c087e 100644 --- a/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/TeehrBrokerAuthManager.java +++ b/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/TeehrBrokerAuthManager.java @@ -3,6 +3,7 @@ import java.net.http.HttpClient; import java.time.Duration; import java.util.Map; +import java.util.function.Supplier; import org.apache.iceberg.rest.RESTClient; import org.apache.iceberg.rest.auth.AuthManager; import org.apache.iceberg.rest.auth.AuthSession; @@ -42,6 +43,9 @@ public AuthSession catalogSession(RESTClient sharedClient, Map p String sessionId = required(properties, TeehrAuthProperties.SESSION_ID); String realm = required(properties, TeehrAuthProperties.REALM); String catalog = properties.getOrDefault(TeehrAuthProperties.CATALOG, "iceberg"); + String audience = + properties.getOrDefault(TeehrAuthProperties.AUDIENCE, TeehrAuthProperties.DEFAULT_AUDIENCE); + Supplier subjectTokenSupplier = subjectTokenSupplier(properties); int timeoutMs = PropertyUtil.propertyAsInt( @@ -69,6 +73,8 @@ public AuthSession catalogSession(RESTClient sharedClient, Map p sessionId, realm, catalog, + audience, + subjectTokenSupplier, Duration.ofMillis(timeoutMs), requestedTtlSeconds, refreshSkewSeconds); @@ -98,4 +104,27 @@ private static String required(Map properties, String key) { } return value; } + + private static Supplier subjectTokenSupplier(Map properties) { + String explicitToken = properties.get(TeehrAuthProperties.SUBJECT_TOKEN); + if (explicitToken != null && !explicitToken.isBlank()) { + return () -> explicitToken; + } + + String tokenEnv = + properties.getOrDefault( + TeehrAuthProperties.SUBJECT_TOKEN_ENV, TeehrAuthProperties.DEFAULT_SUBJECT_TOKEN_ENV); + + return () -> { + String token = System.getenv(tokenEnv); + if (token == null || token.isBlank()) { + throw new IllegalStateException( + "Missing subject token: set " + + TeehrAuthProperties.SUBJECT_TOKEN + + " or export env var " + + tokenEnv); + } + return token; + }; + } } From e60112584d713a47b60f4b061524e1bd98c3b974 Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Thu, 23 Jul 2026 22:51:35 -0400 Subject: [PATCH 17/62] build(garden): automate authmanager jar staging for spark and jupyter images --- .gitignore | 4 ++- jupyterhub/docker/Dockerfile.jupyter-driver | 3 +++ jupyterhub/docker/garden.yaml | 6 +++++ scripts/stage_authmanager_jar.sh | 27 +++++++++++++++++++++ spark/authmanager-prototype/garden.yaml | 14 +++++++++++ spark/docker/Dockerfile.spark-executor | 3 +++ spark/docker/garden.yaml | 2 ++ 7 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 scripts/stage_authmanager_jar.sh create mode 100644 spark/authmanager-prototype/garden.yaml diff --git a/.gitignore b/.gitignore index a1418bc..6c15a71 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,6 @@ secrets.remote.yaml:Zone.Identifier __pycache__/ .python-version *.egg-info -project.garden.yml \ No newline at end of file +project.garden.yml +jupyterhub/docker/teehr-authmanager.jar +spark/docker/teehr-authmanager.jar \ No newline at end of file diff --git a/jupyterhub/docker/Dockerfile.jupyter-driver b/jupyterhub/docker/Dockerfile.jupyter-driver index 8a1fbd2..a1e9b02 100644 --- a/jupyterhub/docker/Dockerfile.jupyter-driver +++ b/jupyterhub/docker/Dockerfile.jupyter-driver @@ -78,6 +78,9 @@ RUN mkdir -p /opt/teehr && chown -R ${NB_USER}:${NB_USER} /opt/teehr COPY executor-pod-template.yaml /opt/teehr/executor-pod-template.yaml +# Keep driver and executor classpaths aligned for the custom AuthManager. +COPY teehr-authmanager.jar /opt/spark/jars/teehr-authmanager.jar + # Configure IPython system-wide COPY ipython_kernel_config.py /etc/ipython/ipython_kernel_config.py # RUN fix-permissions /etc/ipython/ diff --git a/jupyterhub/docker/garden.yaml b/jupyterhub/docker/garden.yaml index 86ebd91..60fa067 100644 --- a/jupyterhub/docker/garden.yaml +++ b/jupyterhub/docker/garden.yaml @@ -2,6 +2,8 @@ kind: Build type: container name: teehr-jupyter-driver-image-edge description: Build TEEHR Jupyter Driver Image +dependencies: + - deploy.stage-authmanager-jar spec: dockerfile: Dockerfile.jupyter-driver localId: jupyter-driver @@ -16,6 +18,8 @@ kind: Build type: container name: teehr-jupyter-driver-image-stable description: Build TEEHR Jupyter Driver Image +dependencies: + - deploy.stage-authmanager-jar spec: dockerfile: Dockerfile.jupyter-driver localId: jupyter-driver @@ -31,6 +35,8 @@ kind: Build type: container name: teehr-jupyter-driver-image-previous description: Build TEEHR Jupyter Driver Image +dependencies: + - deploy.stage-authmanager-jar spec: dockerfile: Dockerfile.jupyter-driver localId: jupyter-driver diff --git a/scripts/stage_authmanager_jar.sh b/scripts/stage_authmanager_jar.sh new file mode 100644 index 0000000..2134ccb --- /dev/null +++ b/scripts/stage_authmanager_jar.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +AUTHMANAGER_DIR="$ROOT_DIR/spark/authmanager-prototype" +SPARK_DOCKER_DIR="$ROOT_DIR/spark/docker" +JUPYTER_DOCKER_DIR="$ROOT_DIR/jupyterhub/docker" + +cd "$ROOT_DIR" + +echo "[authmanager-stage] Building AuthManager jar..." +mvn -f "$AUTHMANAGER_DIR/pom.xml" -DskipTests package >/dev/null + +JAR_PATH="$(find "$AUTHMANAGER_DIR/target" -maxdepth 1 -type f -name 'teehr-iceberg-authmanager-prototype-*.jar' ! -name '*-sources.jar' ! -name '*-javadoc.jar' | head -n1)" + +if [[ -z "${JAR_PATH:-}" || ! -f "$JAR_PATH" ]]; then + echo "[authmanager-stage] Failed to find built jar under $AUTHMANAGER_DIR/target" >&2 + exit 1 +fi + +echo "[authmanager-stage] Staging jar from $JAR_PATH" +cp "$JAR_PATH" "$SPARK_DOCKER_DIR/teehr-authmanager.jar" +cp "$JAR_PATH" "$JUPYTER_DOCKER_DIR/teehr-authmanager.jar" + +echo "[authmanager-stage] Staged jar to:" +echo " - $SPARK_DOCKER_DIR/teehr-authmanager.jar" +echo " - $JUPYTER_DOCKER_DIR/teehr-authmanager.jar" diff --git a/spark/authmanager-prototype/garden.yaml b/spark/authmanager-prototype/garden.yaml new file mode 100644 index 0000000..5a4ebec --- /dev/null +++ b/spark/authmanager-prototype/garden.yaml @@ -0,0 +1,14 @@ +kind: Deploy +type: exec +name: stage-authmanager-jar +description: Build and stage AuthManager jar into Spark and Jupyter Docker contexts +environments: + - local + - remote +spec: + deployCommand: + - bash + - -c + - >- + cd "${project.root}" && + bash ./scripts/stage_authmanager_jar.sh diff --git a/spark/docker/Dockerfile.spark-executor b/spark/docker/Dockerfile.spark-executor index 2eaf5b2..9b65861 100644 --- a/spark/docker/Dockerfile.spark-executor +++ b/spark/docker/Dockerfile.spark-executor @@ -72,5 +72,8 @@ RUN usermod -u 1000 spark && groupmod -g 1000 spark # Ensure work directory is owned by spark user RUN chown -R spark:spark /opt/spark/work-dir +# Stage custom Iceberg AuthManager extension into Spark classpath. +COPY teehr-authmanager.jar /opt/spark/jars/teehr-authmanager.jar + # Switch back to spark user USER 1000 \ No newline at end of file diff --git a/spark/docker/garden.yaml b/spark/docker/garden.yaml index 0cb4a32..010b441 100644 --- a/spark/docker/garden.yaml +++ b/spark/docker/garden.yaml @@ -2,6 +2,8 @@ kind: Build type: container name: teehr-spark-executor-image description: Build TEEHR Spark Executor Image +dependencies: + - deploy.stage-authmanager-jar environments: - local - remote From dc3cf5f5e1ee6ebe767ac18615187f9fb9902082 Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Thu, 23 Jul 2026 22:52:42 -0400 Subject: [PATCH 18/62] feat(notebook): add authmanager toggle for spark broker flow --- examples/developer/setup_utils.py | 33 +++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/examples/developer/setup_utils.py b/examples/developer/setup_utils.py index 97cea5e..c95139d 100644 --- a/examples/developer/setup_utils.py +++ b/examples/developer/setup_utils.py @@ -200,6 +200,7 @@ def create_minio_spark_session( polaris_token: Optional[str] = None, force_recreate_session: bool = False, update_configs: Optional[Dict[str, str]] = None, + use_authmanager: Optional[bool] = None, ): """Start a Spark session configured for the local Polaris REST catalog. @@ -233,7 +234,35 @@ def create_minio_spark_session( "spark.hadoop.fs.s3a.connection.ssl.enabled": "false", } - if polaris_token: + resolved_use_authmanager = ( + use_authmanager + if use_authmanager is not None + else _as_bool_str(os.getenv("POLARIS_USE_AUTHMANAGER", "false")) == "true" + ) + + if resolved_use_authmanager: + broker_url = os.getenv("POLARIS_BROKER_URL", "http://teehr-api:8000/auth/polaris-token") + authmanager_user_id = os.getenv("JUPYTERHUB_USER", "admin") + authmanager_session_id = ( + os.getenv("JUPYTERHUB_SERVER_NAME", "").strip() or authmanager_user_id + ) + broker_audience = os.getenv("POLARIS_BROKER_AUDIENCE", "account") + + merged_configs["spark.sql.catalog.iceberg.rest.auth.type"] = ( + "org.teehr.iceberg.auth.TeehrBrokerAuthManager" + ) + merged_configs["spark.sql.catalog.iceberg.rest.auth.teehr.broker.url"] = broker_url + merged_configs["spark.sql.catalog.iceberg.rest.auth.teehr.user-id"] = authmanager_user_id + merged_configs["spark.sql.catalog.iceberg.rest.auth.teehr.session-id"] = ( + authmanager_session_id + ) + merged_configs["spark.sql.catalog.iceberg.rest.auth.teehr.realm"] = polaris_realm + merged_configs["spark.sql.catalog.iceberg.rest.auth.teehr.catalog"] = "iceberg" + merged_configs["spark.sql.catalog.iceberg.rest.auth.teehr.audience"] = broker_audience + merged_configs["spark.sql.catalog.iceberg.rest.auth.teehr.subject-token-env"] = ( + "POLARIS_USER_TOKEN" + ) + elif polaris_token: merged_configs["spark.sql.catalog.iceberg.rest.auth.type"] = "oauth2" merged_configs["spark.sql.catalog.iceberg.token"] = polaris_token merged_configs["spark.sql.catalog.iceberg.rest.auth.oauth2.token"] = polaris_token @@ -300,7 +329,7 @@ def request_broker_polaris_token( ) -> Tuple[str, int, str]: endpoint = broker_url or os.getenv("POLARIS_BROKER_URL", "http://teehr-api:8000/auth/polaris-token") active_realm = realm or os.getenv("POLARIS_DEFAULT_REALM", "teehr") - active_audience = audience or os.getenv("POLARIS_BROKER_AUDIENCE", "polaris") + active_audience = audience or os.getenv("POLARIS_BROKER_AUDIENCE", "account") if not bearer_token: raise RuntimeError("bearer_token is required to request a broker token") From cb3e399e6571d7d66ce72a08b8902adbfa5d09f6 Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Mon, 27 Jul 2026 09:23:45 -0400 Subject: [PATCH 19/62] fix duplicate keys --- jupyterhub/docker/garden.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/jupyterhub/docker/garden.yaml b/jupyterhub/docker/garden.yaml index 60fa067..57a3d0f 100644 --- a/jupyterhub/docker/garden.yaml +++ b/jupyterhub/docker/garden.yaml @@ -18,8 +18,6 @@ kind: Build type: container name: teehr-jupyter-driver-image-stable description: Build TEEHR Jupyter Driver Image -dependencies: - - deploy.stage-authmanager-jar spec: dockerfile: Dockerfile.jupyter-driver localId: jupyter-driver @@ -29,14 +27,13 @@ spec: environments: - remote dependencies: + - deploy.stage-authmanager-jar - build.teehr-jupyter-driver-image-edge --- kind: Build type: container name: teehr-jupyter-driver-image-previous description: Build TEEHR Jupyter Driver Image -dependencies: - - deploy.stage-authmanager-jar spec: dockerfile: Dockerfile.jupyter-driver localId: jupyter-driver @@ -46,4 +43,5 @@ spec: environments: - remote dependencies: + - deploy.stage-authmanager-jar - build.teehr-jupyter-driver-image-stable \ No newline at end of file From 7b402d86181d736b8784cdc69dad925d05d9e7a7 Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Mon, 27 Jul 2026 09:24:13 -0400 Subject: [PATCH 20/62] change token lifespan --- keycloak-bootstrap/manifests/realm-configmap.yaml.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/keycloak-bootstrap/manifests/realm-configmap.yaml.tpl b/keycloak-bootstrap/manifests/realm-configmap.yaml.tpl index 4fdcc59..6b0dae5 100644 --- a/keycloak-bootstrap/manifests/realm-configmap.yaml.tpl +++ b/keycloak-bootstrap/manifests/realm-configmap.yaml.tpl @@ -9,7 +9,7 @@ data: { "realm": "teehr", "enabled": true, - "accessTokenLifespan": 1800, + "accessTokenLifespan": 300, "loginTheme": "teehr", "registrationAllowed": true, "loginWithEmailAllowed": true, From 35008ef7109cfd4745f5a64c09070447e42fa924 Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Mon, 27 Jul 2026 09:24:29 -0400 Subject: [PATCH 21/62] fix bash script --- spark/authmanager-prototype/garden.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/spark/authmanager-prototype/garden.yaml b/spark/authmanager-prototype/garden.yaml index 5a4ebec..3bb735f 100644 --- a/spark/authmanager-prototype/garden.yaml +++ b/spark/authmanager-prototype/garden.yaml @@ -10,5 +10,4 @@ spec: - bash - -c - >- - cd "${project.root}" && - bash ./scripts/stage_authmanager_jar.sh + bash ../../scripts/stage_authmanager_jar.sh From 97b05c29ee198fdcbb45f234db6d4d52be7f191e Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Tue, 28 Jul 2026 07:43:07 -0400 Subject: [PATCH 22/62] working jupyter polaris auth with refresh --- api/manifests/configmap.yaml.tpl | 2 + api/manifests/deployment.yaml.tpl | 20 + api/src/broker.py | 158 +++ api/src/config.py | 9 + api/src/delegated_session_store.py | 165 +++ api/src/main.py | 13 + api/src/routes/auth.py | 77 + examples/developer/setup_utils.py | 389 +---- examples/developer/spark_session_utils.py | 1262 +++++++++++++++++ spark/authmanager-prototype/README.md | 6 +- .../iceberg/auth/BrokerBackedAuthSession.java | 21 +- .../teehr/iceberg/auth/BrokerTokenClient.java | 64 +- .../iceberg/auth/TeehrAuthProperties.java | 3 + .../iceberg/auth/TeehrBrokerAuthManager.java | 30 +- .../auth/BrokerBackedAuthSession.class | Bin 0 -> 5275 bytes .../org/teehr/iceberg/auth/BrokerToken.class | Bin 0 -> 1937 bytes .../iceberg/auth/BrokerTokenClient.class | Bin 0 -> 6356 bytes .../iceberg/auth/TeehrAuthProperties.class | Bin 0 -> 1677 bytes .../iceberg/auth/TeehrBrokerAuthManager.class | Bin 0 -> 7223 bytes .../target/maven-archiver/pom.properties | 3 + .../compile/default-compile/createdFiles.lst | 5 + .../compile/default-compile/inputFiles.lst | 5 + ...g-authmanager-prototype-0.0.1-SNAPSHOT.jar | Bin 0 -> 12971 bytes 23 files changed, 1864 insertions(+), 368 deletions(-) create mode 100644 api/src/delegated_session_store.py create mode 100644 examples/developer/spark_session_utils.py create mode 100644 spark/authmanager-prototype/target/classes/org/teehr/iceberg/auth/BrokerBackedAuthSession.class create mode 100644 spark/authmanager-prototype/target/classes/org/teehr/iceberg/auth/BrokerToken.class create mode 100644 spark/authmanager-prototype/target/classes/org/teehr/iceberg/auth/BrokerTokenClient.class create mode 100644 spark/authmanager-prototype/target/classes/org/teehr/iceberg/auth/TeehrAuthProperties.class create mode 100644 spark/authmanager-prototype/target/classes/org/teehr/iceberg/auth/TeehrBrokerAuthManager.class create mode 100644 spark/authmanager-prototype/target/maven-archiver/pom.properties create mode 100644 spark/authmanager-prototype/target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst create mode 100644 spark/authmanager-prototype/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst create mode 100644 spark/authmanager-prototype/target/teehr-iceberg-authmanager-prototype-0.0.1-SNAPSHOT.jar diff --git a/api/manifests/configmap.yaml.tpl b/api/manifests/configmap.yaml.tpl index 655f5a8..edddbff 100644 --- a/api/manifests/configmap.yaml.tpl +++ b/api/manifests/configmap.yaml.tpl @@ -25,6 +25,8 @@ data: BROKER_MIN_TTL_SECONDS: "120" BROKER_MAX_TTL_SECONDS: "900" BROKER_REQUEST_TIMEOUT_SECONDS: "10" + BROKER_SUBJECT_CLIENT_ID: "jupyterhub" + BROKER_DELEGATED_SESSION_TTL_SECONDS: "43200" ANON_RATE_LIMIT_RPM: "20" AUTH_RATE_LIMIT_RPM: "120" ROW_LIMIT_ANON: "200" diff --git a/api/manifests/deployment.yaml.tpl b/api/manifests/deployment.yaml.tpl index 2a050f7..812406f 100644 --- a/api/manifests/deployment.yaml.tpl +++ b/api/manifests/deployment.yaml.tpl @@ -121,6 +121,26 @@ spec: configMapKeyRef: name: teehr-api-config key: BROKER_REQUEST_TIMEOUT_SECONDS + - name: BROKER_SUBJECT_CLIENT_ID + valueFrom: + configMapKeyRef: + name: teehr-api-config + key: BROKER_SUBJECT_CLIENT_ID + - name: BROKER_DELEGATED_SESSION_TTL_SECONDS + valueFrom: + configMapKeyRef: + name: teehr-api-config + key: BROKER_DELEGATED_SESSION_TTL_SECONDS + - name: BROKER_SUBJECT_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: jupyterhub + key: OAUTH_CLIENT_SECRET + - name: BROKER_SESSION_SIGNING_SECRET + valueFrom: + secretKeyRef: + name: teehr-api-secrets + key: client-secret - name: API_KEYS_DB_HOST value: keycloak-pg - name: API_KEYS_DB_PORT diff --git a/api/src/broker.py b/api/src/broker.py index bf6e5ed..f60932f 100644 --- a/api/src/broker.py +++ b/api/src/broker.py @@ -1,17 +1,36 @@ import time import uuid +from asyncio import Lock import httpx from fastapi import HTTPException from jose import JWTError, jwt from .config import config +from .delegated_session_store import DelegatedSessionStore TOKEN_EXCHANGE_GRANT = "urn:ietf:params:oauth:grant-type:token-exchange" ACCESS_TOKEN_TYPE = "urn:ietf:params:oauth:token-type:access_token" +_DELEGATED_SESSION_STORE: DelegatedSessionStore | None = None +_DELEGATED_SESSION_STORE_LOCK = Lock() + + +async def set_delegated_session_store(store: DelegatedSessionStore): + async with _DELEGATED_SESSION_STORE_LOCK: + global _DELEGATED_SESSION_STORE + _DELEGATED_SESSION_STORE = store + + +async def _get_delegated_session_store() -> DelegatedSessionStore: + async with _DELEGATED_SESSION_STORE_LOCK: + if _DELEGATED_SESSION_STORE is None: + raise HTTPException(status_code=503, detail="Delegated session store unavailable") + return _DELEGATED_SESSION_STORE + + def clamp_requested_ttl(requested_ttl_seconds: int | None) -> int: requested = requested_ttl_seconds or config.BROKER_MAX_TTL_SECONDS return max(config.BROKER_MIN_TTL_SECONDS, min(requested, config.BROKER_MAX_TTL_SECONDS)) @@ -107,3 +126,142 @@ async def exchange_token_for_polaris( "trace_id": trace_id, }, ) from exc + + +async def _refresh_subject_access_token(refresh_token: str) -> tuple[str, str | None]: + payload = { + "grant_type": "refresh_token", + "client_id": config.BROKER_SUBJECT_CLIENT_ID, + "refresh_token": refresh_token, + } + if config.BROKER_SUBJECT_CLIENT_SECRET: + payload["client_secret"] = config.BROKER_SUBJECT_CLIENT_SECRET + + async with httpx.AsyncClient(timeout=config.BROKER_REQUEST_TIMEOUT_SECONDS) as client: + response = await client.post(config.BROKER_TOKEN_ENDPOINT, data=payload) + + if response.status_code >= 400: + try: + upstream_payload = response.json() + detail = upstream_payload.get("error_description") or upstream_payload.get("error") + except ValueError: + detail = response.text[:500] + raise HTTPException( + status_code=401, + detail={ + "error": "subject_refresh_failed", + "message": "Unable to refresh delegated subject token", + "upstream_status": response.status_code, + "upstream_detail": detail, + }, + ) + + token_payload = response.json() + access_token = token_payload.get("access_token") + if not access_token: + raise HTTPException( + status_code=502, + detail={ + "error": "invalid_upstream_response", + "message": "Identity provider did not return refreshed access_token", + }, + ) + + return access_token, token_payload.get("refresh_token") + + +async def create_delegated_broker_session( + *, + subject: str, + user_id: str, + session_id: str, + realm: str, + catalog: str, + audience: str, + refresh_token: str, +) -> dict: + if not refresh_token: + raise HTTPException(status_code=400, detail="refresh_token is required") + + now = int(time.time()) + ttl = max(config.BROKER_DELEGATED_SESSION_TTL_SECONDS, 300) + expires_at = now + ttl + delegated_session_id = uuid.uuid4().hex + + claims = { + "sid": delegated_session_id, + "sub": subject, + "uid": user_id, + "exp": expires_at, + "iat": now, + "typ": "teehr-broker-session", + } + + broker_session_token = jwt.encode( + claims, + config.BROKER_SESSION_SIGNING_SECRET, + algorithm="HS256", + ) + + store = await _get_delegated_session_store() + await store.put_session( + sid=delegated_session_id, + subject=subject, + user_id=user_id, + session_id=session_id, + realm=realm, + catalog=catalog, + audience=audience, + refresh_token=refresh_token, + expires_at_epoch_seconds=expires_at, + ) + + return { + "broker_session_token": broker_session_token, + "expires_at_epoch_seconds": expires_at, + } + + +async def exchange_token_for_polaris_via_broker_session( + *, + broker_session_token: str, + requested_ttl_seconds: int | None, +) -> dict: + if not broker_session_token: + raise HTTPException(status_code=401, detail="broker session token required") + + try: + claims = jwt.decode( + broker_session_token, + config.BROKER_SESSION_SIGNING_SECRET, + algorithms=["HS256"], + ) + except JWTError as exc: + raise HTTPException(status_code=401, detail="Invalid broker session token") from exc + + delegated_session_id = claims.get("sid") + if not delegated_session_id: + raise HTTPException(status_code=401, detail="Invalid broker session token") + + store = await _get_delegated_session_store() + record = await store.get_session(delegated_session_id) + + if not record: + raise HTTPException(status_code=401, detail="Delegated broker session not found") + + if int(record.get("expires_at", 0)) <= int(time.time()): + await store.delete_session(delegated_session_id) + raise HTTPException(status_code=401, detail="Delegated broker session expired") + + refreshed_subject_token, maybe_new_refresh_token = await _refresh_subject_access_token( + record["refresh_token"] + ) + + if maybe_new_refresh_token: + await store.update_refresh_token(delegated_session_id, maybe_new_refresh_token) + + return await exchange_token_for_polaris( + subject_token=refreshed_subject_token, + audience=record["audience"], + requested_ttl_seconds=requested_ttl_seconds, + ) diff --git a/api/src/config.py b/api/src/config.py index 9d1db5e..b02b010 100644 --- a/api/src/config.py +++ b/api/src/config.py @@ -88,6 +88,15 @@ class Config: BROKER_REQUEST_TIMEOUT_SECONDS = int( os.environ.get("BROKER_REQUEST_TIMEOUT_SECONDS", "10") ) + BROKER_SUBJECT_CLIENT_ID = os.environ.get("BROKER_SUBJECT_CLIENT_ID", "jupyterhub") + BROKER_SUBJECT_CLIENT_SECRET = os.environ.get("BROKER_SUBJECT_CLIENT_SECRET", "") + BROKER_DELEGATED_SESSION_TTL_SECONDS = int( + os.environ.get("BROKER_DELEGATED_SESSION_TTL_SECONDS", "43200") + ) + BROKER_SESSION_SIGNING_SECRET = os.environ.get( + "BROKER_SESSION_SIGNING_SECRET", + API_KEY_HASH_SALT, + ) # Role-based record/page limits ROW_LIMIT_ANON = int(os.environ.get("ROW_LIMIT_ANON", "200")) diff --git a/api/src/delegated_session_store.py b/api/src/delegated_session_store.py new file mode 100644 index 0000000..37b5772 --- /dev/null +++ b/api/src/delegated_session_store.py @@ -0,0 +1,165 @@ +from datetime import UTC, datetime + +import asyncpg + + +class DelegatedSessionStore: + def __init__(self, dsn: str): + self._dsn = dsn + self._pool: asyncpg.Pool | None = None + + async def startup(self): + self._pool = await asyncpg.create_pool(dsn=self._dsn, min_size=1, max_size=5) + async with self._pool.acquire() as conn: + await conn.execute( + """ + CREATE TABLE IF NOT EXISTS delegated_sessions ( + sid TEXT PRIMARY KEY, + subject TEXT NOT NULL, + user_id TEXT NOT NULL, + session_id TEXT NOT NULL, + realm TEXT NOT NULL, + catalog TEXT NOT NULL, + audience TEXT NOT NULL, + refresh_token TEXT NOT NULL, + expires_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + ); + """ + ) + await conn.execute( + """ + CREATE INDEX IF NOT EXISTS delegated_sessions_expires_at_idx + ON delegated_sessions (expires_at); + """ + ) + + async def shutdown(self): + if self._pool: + await self._pool.close() + + async def put_session( + self, + *, + sid: str, + subject: str, + user_id: str, + session_id: str, + realm: str, + catalog: str, + audience: str, + refresh_token: str, + expires_at_epoch_seconds: int, + ): + if self._pool is None: + raise RuntimeError("Delegated session store is not initialized") + + expires_at = datetime.fromtimestamp(expires_at_epoch_seconds, tz=UTC) + async with self._pool.acquire() as conn: + await conn.execute( + """ + INSERT INTO delegated_sessions ( + sid, + subject, + user_id, + session_id, + realm, + catalog, + audience, + refresh_token, + expires_at, + updated_at + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, NOW()) + ON CONFLICT (sid) + DO UPDATE SET + subject = EXCLUDED.subject, + user_id = EXCLUDED.user_id, + session_id = EXCLUDED.session_id, + realm = EXCLUDED.realm, + catalog = EXCLUDED.catalog, + audience = EXCLUDED.audience, + refresh_token = EXCLUDED.refresh_token, + expires_at = EXCLUDED.expires_at, + updated_at = NOW() + """, + sid, + subject, + user_id, + session_id, + realm, + catalog, + audience, + refresh_token, + expires_at, + ) + + async def get_session(self, sid: str) -> dict | None: + if self._pool is None: + raise RuntimeError("Delegated session store is not initialized") + + async with self._pool.acquire() as conn: + row = await conn.fetchrow( + """ + SELECT + sid, + subject, + user_id, + session_id, + realm, + catalog, + audience, + refresh_token, + expires_at + FROM delegated_sessions + WHERE sid = $1 + """, + sid, + ) + + if row is None: + return None + + expires_at = row["expires_at"] + if expires_at.tzinfo is None: + expires_at = expires_at.replace(tzinfo=UTC) + + return { + "sid": row["sid"], + "subject": row["subject"], + "user_id": row["user_id"], + "session_id": row["session_id"], + "realm": row["realm"], + "catalog": row["catalog"], + "audience": row["audience"], + "refresh_token": row["refresh_token"], + "expires_at": int(expires_at.timestamp()), + } + + async def update_refresh_token(self, sid: str, refresh_token: str): + if self._pool is None: + raise RuntimeError("Delegated session store is not initialized") + + async with self._pool.acquire() as conn: + await conn.execute( + """ + UPDATE delegated_sessions + SET refresh_token = $1, updated_at = NOW() + WHERE sid = $2 + """, + refresh_token, + sid, + ) + + async def delete_session(self, sid: str): + if self._pool is None: + raise RuntimeError("Delegated session store is not initialized") + + async with self._pool.acquire() as conn: + await conn.execute( + """ + DELETE FROM delegated_sessions + WHERE sid = $1 + """, + sid, + ) diff --git a/api/src/main.py b/api/src/main.py index 4422649..aa704d3 100644 --- a/api/src/main.py +++ b/api/src/main.py @@ -10,7 +10,9 @@ from .api_key_store import ApiKeyStore from .auth import KeycloakJWTValidator, resolve_identity +from .broker import set_delegated_session_store from .config import config +from .delegated_session_store import DelegatedSessionStore from .models import HealthResponse from .rate_limit import InMemoryRateLimiter from .routes import router @@ -139,6 +141,10 @@ def custom_openapi(): {"BearerAuth": []}, ] + auth_polaris_token_session = openapi_schema.get("paths", {}).get("/auth/polaris-token/session", {}).get("post") + if auth_polaris_token_session: + auth_polaris_token_session["security"] = [] + app.openapi_schema = openapi_schema return app.openapi_schema @@ -152,10 +158,14 @@ async def startup_event(): app.state.rate_limiter = InMemoryRateLimiter() app.state.api_key_store = ApiKeyStore(config.API_KEYS_DB_DSN) await app.state.api_key_store.startup() + app.state.delegated_session_store = DelegatedSessionStore(config.API_KEYS_DB_DSN) + await app.state.delegated_session_store.startup() + await set_delegated_session_store(app.state.delegated_session_store) @app.on_event("shutdown") async def shutdown_event(): + await app.state.delegated_session_store.shutdown() await app.state.api_key_store.shutdown() # CORS middleware to allow frontend requests - MUST be first middleware @@ -194,6 +204,9 @@ async def auth_context_middleware(request: Request, call_next): return await call_next(request) path = request.url.path + if path == "/auth/polaris-token/session": + return await call_next(request) + exempt_paths = ( path == "/health" or path == "/openapi.json" diff --git a/api/src/routes/auth.py b/api/src/routes/auth.py index 1898ecd..526e59b 100644 --- a/api/src/routes/auth.py +++ b/api/src/routes/auth.py @@ -9,6 +9,10 @@ get_request_identity, ) from ..broker import exchange_token_for_polaris +from ..broker import ( + create_delegated_broker_session, + exchange_token_for_polaris_via_broker_session, +) from ..config import config router = APIRouter(prefix="/auth", tags=["Auth"]) @@ -44,6 +48,20 @@ class PolarisTokenResponse(BaseModel): trace_id: str +class PolarisSessionRequest(BaseModel): + user_id: str = Field(min_length=1) + session_id: str = Field(min_length=1) + realm: str = Field(min_length=1) + catalog: str = Field(default="iceberg", min_length=1) + audience: str = Field(default_factory=lambda: config.BROKER_TARGET_AUDIENCE) + refresh_token: str = Field(min_length=1) + + +class PolarisSessionResponse(BaseModel): + broker_session_token: str + expires_at_epoch_seconds: int + + @router.get("/me") async def me(identity: AuthIdentity = Depends(get_request_identity)): return { @@ -127,3 +145,62 @@ async def polaris_token( ), trace_id=exchanged["trace_id"], ) + + +@router.post("/polaris-session", response_model=PolarisSessionResponse) +async def polaris_session( + payload: PolarisSessionRequest, + identity: AuthIdentity = Depends(get_authenticated_identity), +): + if identity.auth_type != "jwt": + raise HTTPException(status_code=403, detail="JWT identity required") + + allowed_user_ids = {identity.subject} + if identity.preferred_username: + allowed_user_ids.add(identity.preferred_username) + + if payload.user_id not in allowed_user_ids: + raise HTTPException( + status_code=403, + detail="Requested user_id does not match authenticated identity", + ) + + created = await create_delegated_broker_session( + subject=identity.subject, + user_id=payload.user_id, + session_id=payload.session_id, + realm=payload.realm, + catalog=payload.catalog, + audience=payload.audience, + refresh_token=payload.refresh_token, + ) + + return PolarisSessionResponse( + broker_session_token=created["broker_session_token"], + expires_at_epoch_seconds=created["expires_at_epoch_seconds"], + ) + + +@router.post("/polaris-token/session", response_model=PolarisTokenResponse) +async def polaris_token_via_session( + request: Request, + payload: PolarisTokenRequest, +): + broker_session_token = request.headers.get("x-broker-session-token", "").strip() + exchanged = await exchange_token_for_polaris_via_broker_session( + broker_session_token=broker_session_token, + requested_ttl_seconds=payload.requested_ttl_seconds, + ) + + return PolarisTokenResponse( + access_token=exchanged["access_token"], + token_type=exchanged["token_type"], + expires_at_epoch_seconds=exchanged["expires_at_epoch_seconds"], + expires_in_seconds=exchanged["expires_in_seconds"], + issued_for=PolarisTokenIssuedFor( + user_id=payload.user_id, + session_id=payload.session_id, + realm=payload.realm, + ), + trace_id=exchanged["trace_id"], + ) diff --git a/examples/developer/setup_utils.py b/examples/developer/setup_utils.py index c95139d..c35dee8 100644 --- a/examples/developer/setup_utils.py +++ b/examples/developer/setup_utils.py @@ -1,14 +1,27 @@ -import base64 -import json -import os -import time -from typing import Dict, Optional, Tuple +"""Compatibility layer for developer Spark/Polaris helpers. -import requests -from teehr.evaluation.spark_session_utils import create_spark_session +Canonical implementations now live in spark_session_utils to keep one shared +utility surface for local and remote Polaris support. +""" + +import sys +from pathlib import Path + +_CURRENT_DIR = Path(__file__).resolve().parent +if str(_CURRENT_DIR) not in sys.path: + sys.path.insert(0, str(_CURRENT_DIR)) + +from spark_session_utils import apply_polaris_token_to_spark +from spark_session_utils import create_minio_spark_session +from spark_session_utils import create_spark_session +from spark_session_utils import ensure_broker_session_token +from spark_session_utils import ensure_fresh_polaris_token_via_broker +from spark_session_utils import ensure_fresh_polaris_user_token +from spark_session_utils import mint_polaris_user_token +from spark_session_utils import refresh_polaris_user_token +from spark_session_utils import request_broker_polaris_token -# Handpicked sites that seemed interesting DEV_LOCATION_ID_LIST = [ # CONUS "usgs-02424000", @@ -38,351 +51,15 @@ ] -def _decode_jwt_claims(token: str) -> Dict[str, object]: - payload = token.split(".")[1] - payload += "=" * (-len(payload) % 4) - return json.loads(base64.urlsafe_b64decode(payload.encode())) - - -def _token_expires_soon(token: str, refresh_window_seconds: int = 120) -> bool: - try: - claims = _decode_jwt_claims(token) - except Exception: - return True - exp = int(claims.get("exp", 0)) - now = int(time.time()) - return exp <= now + max(refresh_window_seconds, 1) - - -def _request_oauth_tokens( - data: Dict[str, str], - token_endpoint: Optional[str] = None, - timeout_seconds: int = 20, -) -> Tuple[str, Optional[str]]: - endpoint = token_endpoint or os.getenv("POLARIS_OAUTH2_SERVER_URI") - print(endpoint) - if not endpoint: - raise RuntimeError("POLARIS_OAUTH2_SERVER_URI is required to mint or refresh a user token") - - resp = requests.post(endpoint, data=data, timeout=timeout_seconds) - print(resp) - resp.raise_for_status() - payload = resp.json() - - access_token = payload.get("access_token") - print(access_token) - if not access_token: - raise RuntimeError("Token endpoint did not return access_token") - - return access_token, payload.get("refresh_token") - - -def mint_polaris_user_token( - username: Optional[str], - password: Optional[str], - client_id: str, - client_secret: Optional[str] = None, - token_endpoint: Optional[str] = None, -) -> str: - if not username or not password: - raise RuntimeError("username and password are required for password grant token minting") - - data = { - "grant_type": "password", - "client_id": client_id, - "username": username, - "password": password, - "scope": "openid profile email", - } - if client_secret: - data["client_secret"] = client_secret - - access_token, _ = _request_oauth_tokens(data=data, token_endpoint=token_endpoint) - return access_token - - -def refresh_polaris_user_token( - refresh_token: str, - client_id: str, - client_secret: Optional[str] = None, - token_endpoint: Optional[str] = None, -) -> Tuple[str, Optional[str]]: - if not refresh_token: - raise RuntimeError("refresh_token is required for refresh grant") - - data = { - "grant_type": "refresh_token", - "client_id": client_id, - "refresh_token": refresh_token, - } - if client_secret: - data["client_secret"] = client_secret - - print(data) - print(token_endpoint) - - return _request_oauth_tokens(data=data, token_endpoint=token_endpoint) - - -def ensure_fresh_polaris_user_token( - current_token: Optional[str], - username: Optional[str], - password: Optional[str], - client_id: str, - client_secret: Optional[str] = None, - refresh_token: Optional[str] = None, - allow_password_fallback: bool = True, - refresh_window_seconds: int = 120, - token_endpoint: Optional[str] = None, -) -> Tuple[str, Optional[str], bool]: - if current_token and not _token_expires_soon(current_token, refresh_window_seconds): - return current_token, refresh_token, False - - if refresh_token: - try: - print("trying to refresh token") - print(refresh_token) - refreshed_access, refreshed_refresh = refresh_polaris_user_token( - refresh_token=refresh_token, - client_id=client_id, - client_secret=client_secret, - token_endpoint=token_endpoint, - ) - return refreshed_access, (refreshed_refresh or refresh_token), True - except requests.RequestException: - if not allow_password_fallback: - print("Passowrd fall back not enabled") - raise - - if allow_password_fallback and username and password: - minted = mint_polaris_user_token( - username=username, - password=password, - client_id=client_id, - client_secret=client_secret, - token_endpoint=token_endpoint, - ) - return minted, refresh_token, True - - raise RuntimeError( - "Unable to obtain a fresh Polaris user token. " - "Provide POLARIS_REFRESH_TOKEN or enable password-grant fallback credentials." - ) - - -def apply_polaris_token_to_spark( - spark, - token: str, - catalog_name: str = "iceberg", - realm: Optional[str] = None, -) -> None: - active_realm = realm or os.getenv("POLARIS_DEFAULT_REALM", "teehr") - base = f"spark.sql.catalog.{catalog_name}" - spark.conf.set(f"{base}.rest.auth.type", "oauth2") - spark.conf.set(f"{base}.token", token) - spark.conf.set(f"{base}.rest.auth.oauth2.token", token) - spark.conf.set(f"{base}.header.X-Polaris-Realm", active_realm) - spark.conf.set(f"{base}.rest.transport.header.X-Polaris-Realm", active_realm) - -def _as_bool_str(value: str, default: str = "true") -> str: - normalized = (value or default).strip().lower() - return "true" if normalized in ("1", "true", "t", "yes", "y", "on") else "false" - - -def _apply_runtime_spark_configs(spark, configs: Dict[str, str]) -> None: - for key, value in configs.items(): - if not key.startswith("spark."): - continue - spark.conf.set(key, value) - - -def create_minio_spark_session( - polaris_token: Optional[str] = None, - force_recreate_session: bool = False, - update_configs: Optional[Dict[str, str]] = None, - use_authmanager: Optional[bool] = None, -): - """Start a Spark session configured for the local Polaris REST catalog. - - If ``polaris_token`` is provided, Spark uses user-token OAuth2 auth. - Otherwise it falls back to client-credential OAuth2 using env vars. - """ - aws_access_key_id = os.getenv("AWS_ACCESS_KEY_ID", "minioadmin") - aws_secret_access_key = os.getenv("AWS_SECRET_ACCESS_KEY", "minioadmin123") - - remote_catalog_uri = os.getenv("REMOTE_CATALOG_REST_URI", "http://polaris:8181/api/catalog") - remote_warehouse_dir = os.getenv("REMOTE_WAREHOUSE_S3_PATH", "s3://warehouse/") - polaris_realm = os.getenv("POLARIS_DEFAULT_REALM", "teehr") - - if remote_catalog_uri.rstrip("/").endswith("/api/catalog"): - # Polaris REST expects the catalog identifier here, not the backing S3 URI. - remote_warehouse_dir = polaris_realm - - s3_endpoint = os.getenv("REMOTE_CATALOG_S3_ENDPOINT", "http://minio:9000") - s3_path_style = _as_bool_str(os.getenv("REMOTE_CATALOG_S3_PATH_STYLE_ACCESS", "true")) - s3_region = os.getenv("AWS_REGION", "us-east-2") - - merged_configs: Dict[str, str] = { - "spark.sql.catalog.iceberg.warehouse": remote_warehouse_dir, - "spark.sql.catalog.iceberg.header.X-Polaris-Realm": polaris_realm, - "spark.sql.catalog.iceberg.rest.transport.header.X-Polaris-Realm": polaris_realm, - "spark.sql.catalog.iceberg.s3.endpoint": s3_endpoint, - "spark.sql.catalog.iceberg.s3.path-style-access": s3_path_style, - "spark.sql.catalog.iceberg.s3.region": s3_region, - "spark.hadoop.fs.s3a.endpoint": s3_endpoint, - "spark.hadoop.fs.s3a.path.style.access": s3_path_style, - "spark.hadoop.fs.s3a.connection.ssl.enabled": "false", - } - - resolved_use_authmanager = ( - use_authmanager - if use_authmanager is not None - else _as_bool_str(os.getenv("POLARIS_USE_AUTHMANAGER", "false")) == "true" - ) - - if resolved_use_authmanager: - broker_url = os.getenv("POLARIS_BROKER_URL", "http://teehr-api:8000/auth/polaris-token") - authmanager_user_id = os.getenv("JUPYTERHUB_USER", "admin") - authmanager_session_id = ( - os.getenv("JUPYTERHUB_SERVER_NAME", "").strip() or authmanager_user_id - ) - broker_audience = os.getenv("POLARIS_BROKER_AUDIENCE", "account") - - merged_configs["spark.sql.catalog.iceberg.rest.auth.type"] = ( - "org.teehr.iceberg.auth.TeehrBrokerAuthManager" - ) - merged_configs["spark.sql.catalog.iceberg.rest.auth.teehr.broker.url"] = broker_url - merged_configs["spark.sql.catalog.iceberg.rest.auth.teehr.user-id"] = authmanager_user_id - merged_configs["spark.sql.catalog.iceberg.rest.auth.teehr.session-id"] = ( - authmanager_session_id - ) - merged_configs["spark.sql.catalog.iceberg.rest.auth.teehr.realm"] = polaris_realm - merged_configs["spark.sql.catalog.iceberg.rest.auth.teehr.catalog"] = "iceberg" - merged_configs["spark.sql.catalog.iceberg.rest.auth.teehr.audience"] = broker_audience - merged_configs["spark.sql.catalog.iceberg.rest.auth.teehr.subject-token-env"] = ( - "POLARIS_USER_TOKEN" - ) - elif polaris_token: - merged_configs["spark.sql.catalog.iceberg.rest.auth.type"] = "oauth2" - merged_configs["spark.sql.catalog.iceberg.token"] = polaris_token - merged_configs["spark.sql.catalog.iceberg.rest.auth.oauth2.token"] = polaris_token - else: - oauth_server_uri = os.getenv("POLARIS_OAUTH2_SERVER_URI") - spark_polaris_client_secret = os.getenv("SPARK_POLARIS_CLIENT_SECRET") - - merged_configs["spark.sql.catalog.iceberg.rest.auth.type"] = "oauth2" - merged_configs["spark.sql.catalog.iceberg.scope"] = "openid" - merged_configs["spark.sql.catalog.iceberg.rest.auth.oauth2.scope"] = "openid" - if oauth_server_uri: - merged_configs["spark.sql.catalog.iceberg.oauth2-server-uri"] = oauth_server_uri - merged_configs["spark.sql.catalog.iceberg.rest.auth.oauth2.server-uri"] = oauth_server_uri - if spark_polaris_client_secret: - merged_configs["spark.sql.catalog.iceberg.credential"] = ( - f"spark-polaris:{spark_polaris_client_secret}" - ) - merged_configs["spark.sql.catalog.iceberg.rest.auth.oauth2.credential"] = ( - f"spark-polaris:{spark_polaris_client_secret}" - ) - - if update_configs: - merged_configs.update(update_configs) - - call_kwargs = { - "remote_catalog_uri": remote_catalog_uri, - "remote_warehouse_dir": remote_warehouse_dir, - "aws_access_key_id": aws_access_key_id, - "aws_secret_access_key": aws_secret_access_key, - "force_recreate_session": force_recreate_session, - "update_configs": merged_configs, - } - if polaris_token: - # Preferred path for newer teehr versions that support direct token auth. - call_kwargs["oauth2_token"] = polaris_token - - while True: - try: - spark = create_spark_session(**call_kwargs) - _apply_runtime_spark_configs(spark, merged_configs) - return spark - except TypeError as exc: - msg = str(exc) - if "oauth2_token" in msg and "oauth2_token" in call_kwargs: - call_kwargs.pop("oauth2_token", None) - continue - if "force_recreate_session" in msg and "force_recreate_session" in call_kwargs: - call_kwargs.pop("force_recreate_session", None) - continue - raise - - -def request_broker_polaris_token( - *, - user_id: str, - session_id: str, - bearer_token: str, - realm: Optional[str] = None, - catalog: str = "iceberg", - requested_ttl_seconds: int = 600, - audience: Optional[str] = None, - broker_url: Optional[str] = None, - timeout_seconds: int = 20, -) -> Tuple[str, int, str]: - endpoint = broker_url or os.getenv("POLARIS_BROKER_URL", "http://teehr-api:8000/auth/polaris-token") - active_realm = realm or os.getenv("POLARIS_DEFAULT_REALM", "teehr") - active_audience = audience or os.getenv("POLARIS_BROKER_AUDIENCE", "account") - - if not bearer_token: - raise RuntimeError("bearer_token is required to request a broker token") - - resp = requests.post( - endpoint, - headers={"Authorization": f"Bearer {bearer_token}"}, - json={ - "user_id": user_id, - "session_id": session_id, - "realm": active_realm, - "catalog": catalog, - "requested_ttl_seconds": requested_ttl_seconds, - "audience": active_audience, - }, - timeout=timeout_seconds, - ) - resp.raise_for_status() - payload = resp.json() - access_token = payload.get("access_token") - expires_at = int(payload.get("expires_at_epoch_seconds", 0)) - trace_id = payload.get("trace_id", "") - - if not access_token or expires_at <= 0: - raise RuntimeError("Broker response missing access_token or expires_at_epoch_seconds") - - return access_token, expires_at, trace_id - - -def ensure_fresh_polaris_token_via_broker( - *, - current_token: Optional[str], - user_id: str, - session_id: str, - bearer_token: str, - realm: Optional[str] = None, - catalog: str = "iceberg", - refresh_window_seconds: int = 120, - requested_ttl_seconds: int = 600, - audience: Optional[str] = None, - broker_url: Optional[str] = None, -) -> Tuple[str, bool]: - if current_token and not _token_expires_soon(current_token, refresh_window_seconds): - return current_token, False - - refreshed_token, _, _ = request_broker_polaris_token( - user_id=user_id, - session_id=session_id, - bearer_token=bearer_token, - realm=realm, - catalog=catalog, - requested_ttl_seconds=requested_ttl_seconds, - audience=audience, - broker_url=broker_url, - ) - return refreshed_token, True \ No newline at end of file +__all__ = [ + "DEV_LOCATION_ID_LIST", + "apply_polaris_token_to_spark", + "create_minio_spark_session", + "create_spark_session", + "ensure_broker_session_token", + "ensure_fresh_polaris_token_via_broker", + "ensure_fresh_polaris_user_token", + "mint_polaris_user_token", + "refresh_polaris_user_token", + "request_broker_polaris_token", +] \ No newline at end of file diff --git a/examples/developer/spark_session_utils.py b/examples/developer/spark_session_utils.py new file mode 100644 index 0000000..1141c5c --- /dev/null +++ b/examples/developer/spark_session_utils.py @@ -0,0 +1,1262 @@ +"""Module to create and configure Spark sessions and Polaris auth helpers.""" +# flake8: noqa +import base64 +import json +import logging +import os +import socket +import time +from pathlib import Path +from typing import Dict, List, Optional, Tuple, Union +from urllib.parse import urlsplit, urlunsplit + +import psutil +import requests + +from pyspark import SparkConf +from pyspark.sql import SparkSession +from sedona.spark import SedonaContext +import pandas as pd +import botocore.session + +import teehr.const as const + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# Note: Scala version: 2.13 in pyspark 4.0 +SCALA_VERSION = "2.13" +PYSPARK_VERSION = "4.0" +ICEBERG_VERSION = "1.10.1" +SEDONA_VERSION = "1.8.0" + + + +def create_spark_session( + # App name and catalog settings + app_name: str = "TEEHR Evaluation", + local_catalog_name: str = const.LOCAL_CATALOG_NAME, + local_catalog_type: str = const.LOCAL_CATALOG_TYPE, + remote_warehouse_dir: str = const.REMOTE_WAREHOUSE_S3_PATH, + remote_catalog_name: str = const.REMOTE_CATALOG_NAME, + remote_catalog_type: str = const.REMOTE_CATALOG_TYPE, + remote_catalog_uri: str = const.REMOTE_CATALOG_REST_URI, + # Spark K8'specific parameters + start_spark_cluster: bool = False, + force_recreate_session: bool = False, + executor_instances: int = 2, + executor_memory: str = "1g", + executor_cores: int = 1, + executor_image: str = None, + executor_namespace: str = None, + driver_memory: str = None, + driver_max_result_size: str = None, + pod_template_path: Union[str, Path] = const.POD_TEMPLATE_PATH, + # AWS credential parameters + aws_access_key_id: str = None, + aws_secret_access_key: str = None, + aws_session_token: str = None, + aws_region: str = const.AWS_REGION, + aws_profile: str = None, + # GCS credential parameters + enable_gcs: bool = False, + gcs_project_id: str = None, + gcs_service_account_key_file: str = None, + # Simple extensibility parameters + add_jars: List[str] = None, + add_packages: List[str] = None, + update_configs: Dict[str, str] = None, + debug_config: bool = False +) -> SparkSession: + """Create and return a Spark session for evaluation. + + Parameters + ---------- + app_name : str + Name of the Spark application. Default is "TEEHR Evaluation". + local_catalog_name : str + Name of the local Iceberg catalog. Default is "local". + local_catalog_type : str + Type of the local Iceberg catalog. Default is "jdbc". + remote_warehouse_dir : str + Remote warehouse directory for Iceberg catalog. Default is TEEHR + warehouse S3 path. + remote_catalog_name : str + Name of the remote Iceberg catalog. Default is "iceberg". + remote_catalog_type : str + Type of the remote Iceberg catalog. Default is "rest". + remote_catalog_uri : str + URI for the remote Iceberg catalog. Default is TEEHR catalog REST URI. + start_spark_cluster : bool + Whether to start a Spark cluster (Kubernetes mode). + Default is False (local mode). + force_recreate_session : bool + Whether to stop an existing Spark session before creating a new one. + Default is False. + executor_instances : int + Number of executor instances for the Spark cluster. Default is 2. + executor_memory : str + Memory allocation for each executor. Default is "1g". + executor_cores : int + Number of CPU cores for each executor. Default is 1. + executor_image : str + Container image for Spark executors. Default is None. + executor_namespace : str + Kubernetes namespace for Spark executors. Default is None. + driver_memory : str + Memory allocation for the Spark driver. Default is None. + driver_max_result_size : str + Maximum result size for the Spark driver. Default is None. + pod_template_path : Union[str, Path] + Path to the pod template file for Spark executors. + Default is "/opt/teehr/executor-pod-template.yaml". + aws_access_key_id : str + AWS access key ID for S3 access. Default is None. + aws_secret_access_key : str + AWS secret access key for S3 access. Default is None. + aws_session_token : str + AWS session token for temporary credentials. Default is None. + aws_region : str + AWS region name. Default is "us-east-2". + aws_profile : str + AWS profile name to use from ~/.aws/credentials. Only reads credentials + file if this parameter is explicitly provided. Default is None. + enable_gcs : bool + Whether to add GCS (Google Cloud Storage) connector support. + Default is False. + gcs_project_id : str + GCS project ID. Used for billing and quota tracking. When accessing + public buckets without credentials, set to "anonymous" or any + non-empty string. Default is None (will be set to "anonymous" when + enable_gcs is True and no service account key is provided). + gcs_service_account_key_file : str + Path to a GCS service account JSON key file. When provided, + authenticated access is used. When None, unauthenticated + (public-bucket) access is used. Default is None. + add_packages : List[str] + Provided Spark packages will be added if they do not already exist. + Default is None. + >>> add_packages=["com.example:my-package:1.0.0"] + add_jars : List[str] + Provided local jar paths will be added if they do not already exist. + Default is None. + >>> add_jars=["/opt/spark/jars/teehr-authmanager.jar"] + update_configs : Dict[str, str] + Provided Spark configurations will be added if they do not already + exist, or overwritten if they do exist. Default is None. + >>> update_configs={"spark.sql.shuffle.partitions": "100"} + debug_config : bool + Whether to log the final Spark configuration for debugging. + Default is False. + + Returns + ------- + SparkSession + Configured Spark session. + """ + logger.info(f"🚀 Creating Spark session: {app_name}") + + if force_recreate_session: + existing_session = SparkSession.getActiveSession() + if existing_session is not None: + logger.info("♻️ Stopping the active Spark session before recreation") + existing_session.stop() + + # Get the base configuration with common settings + conf = _create_spark_base_session( + conf=SparkConf(), + aws_region=aws_region, + driver_memory=driver_memory, + driver_maxresultsize=driver_max_result_size + ) + + if start_spark_cluster is False: + logger.info("✅ Spark local configuration successful!") + else: + logger.info(f"📦 Configuring Spark cluster with container image: {executor_image}") + _set_spark_cluster_configuration( + conf=conf, + executor_instances=executor_instances, + executor_memory=executor_memory, + executor_cores=executor_cores, + container_image=executor_image, + spark_namespace=executor_namespace, + pod_template_path=pod_template_path + ) + logger.info("✅ Spark cluster configuration successful!") + logger.info(f" - Executor instances: {executor_instances}") + logger.info(f" - Executor memory: {executor_memory}") + logger.info(f" - Executor cores: {executor_cores}") + + # Set AWS credentials if available + _set_aws_credentials_in_spark( + conf=conf, + remote_catalog_name=remote_catalog_name, + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + aws_session_token=aws_session_token, + aws_region=aws_region, + aws_profile=aws_profile, + ) + + # Set GCS configuration if available + if enable_gcs: + _set_gcs_configuration( + conf=conf, + gcs_project_id=gcs_project_id, + gcs_service_account_key_file=gcs_service_account_key_file, + ) + + # Set catalog metadata in Spark configuration + _set_catalog_metadata( + conf=conf, + local_catalog_name=local_catalog_name, + local_catalog_type=local_catalog_type, + remote_catalog_name=remote_catalog_name, + remote_catalog_type=remote_catalog_type, + remote_catalog_uri=remote_catalog_uri, + remote_warehouse_dir=remote_warehouse_dir + ) + + # Apply catalog configurations + _configure_iceberg_catalogs( + conf=conf, + local_catalog_name=local_catalog_name, + local_catalog_type=local_catalog_type, + remote_warehouse_dir=remote_warehouse_dir, + remote_catalog_name=remote_catalog_name, + remote_catalog_type=remote_catalog_type, + remote_catalog_uri=remote_catalog_uri + ) + + # Update configs and packages if provided + _update_configs_and_packages( + conf=conf, + update_configs=update_configs, + add_jars=add_jars, + add_packages=add_packages + ) + + logger.info("⚙️ All settings applied. Creating Spark session...") + spark = SparkSession.builder.appName(app_name).config(conf=conf).getOrCreate() + sedona_spark = SedonaContext.create(spark) + + if debug_config: + log_session_config(sedona_spark) + + logger.info("🎉 Spark session created successfully!") + + return sedona_spark + + +def _create_spark_base_session( + conf: SparkConf, + aws_region: str, + driver_memory: float = None, + driver_maxresultsize: float = None +): + """Create a base Spark builder.""" + conf.setMaster("local[*]") + + # Set base packages + base_packages = [ + f"org.apache.sedona:sedona-spark-shaded-{PYSPARK_VERSION}_{SCALA_VERSION}:{SEDONA_VERSION}", + f"org.apache.iceberg:iceberg-spark-runtime-{PYSPARK_VERSION}_{SCALA_VERSION}:{ICEBERG_VERSION}", + f"org.apache.iceberg:iceberg-core:{ICEBERG_VERSION}", + "org.datasyslab:geotools-wrapper:1.8.0-33.1", + f"org.apache.iceberg:iceberg-spark-extensions-{PYSPARK_VERSION}_{SCALA_VERSION}:{ICEBERG_VERSION}", + "org.apache.hadoop:hadoop-aws:3.4.1", # Note. Need 3.4.1 for compatibility + "com.amazonaws:aws-java-sdk-bundle:1.12.791", + "org.xerial:sqlite-jdbc:3.42.0.0" + ] + conf.set("spark.jars.packages", ",".join(base_packages)) + + # Set configurations + conf.set("spark.driver.extraJavaOptions", f"-Daws.region={aws_region}") + conf.set("spark.executor.extraJavaOptions", f"-Daws.region={aws_region}") + conf.set("spark.sql.session.timeZone", "UTC") + conf.set("spark.sql.extensions", "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions") + conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer") + conf.set("spark.hadoop.fs.s3a.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem") + + # Enable Spark decommissioning for graceful spot instance handling + conf.set("spark.decommission.enabled", "true") + conf.set("spark.storage.decommission.enabled", "true") + conf.set("spark.storage.decommission.rddBlocks.enabled", "true") + conf.set("spark.storage.decommission.shuffleBlocks.enabled", "true") + # Grace period for executors to decommission before being terminated + conf.set("spark.kubernetes.executor.decommission.gracePeriodSeconds", "30") + + # Memory settings + memory_info = psutil.virtual_memory() + driver_memory_int = int(0.75 * memory_info.available / (1024**3)) + if driver_memory is None: + driver_memory = f"{driver_memory_int}g" + if driver_maxresultsize is None: + driver_maxresultsize = f"{int(0.5 * driver_memory_int)}g" + conf.set("spark.driver.memory", f"{driver_memory}") + conf.set("spark.driver.maxResultSize", f"{driver_maxresultsize}") + + # Driver binding configuration for local mode + conf.set("spark.driver.bindAddress", "127.0.0.1") + conf.set("spark.driver.host", "127.0.0.1") + conf.set("spark.driver.port", "0") # Let Spark choose an available port + + # Default shuffle partitions: 2x local CPU cores + local_cores = os.cpu_count() or 1 + default_shuffle_partitions = 2 * local_cores + conf.set("spark.sql.shuffle.partitions", str(default_shuffle_partitions)) + + return conf + + +def _set_spark_cluster_configuration( + conf: SparkConf, + executor_instances: int, + executor_memory: str, + executor_cores: int, + container_image: str, + spark_namespace: str, + pod_template_path: Union[str, Path] +): + """Configure Spark for Kubernetes cluster mode. + + These settings are considered immutable after session creation. + """ + # Default container image - use the same image as the current pod + if container_image is None: + container_image = os.environ["TEEHR_SPARK_IMAGE"] + + # Get Kubernetes API server - use HTTPS port specifically + k8s_host = os.environ.get('KUBERNETES_SERVICE_HOST', 'kubernetes.default.svc.cluster.local') + k8s_port_https = os.environ.get('KUBERNETES_SERVICE_PORT_HTTPS', '443') + k8s_api_server = f"https://{k8s_host}:{k8s_port_https}" + + # First try getting it from environment variable + if spark_namespace is None: + spark_namespace = os.environ.get("TEEHR_NAMESPACE", "") + logger.info(f"🔍 Initial spark namespace from ENV: {spark_namespace}") + + if spark_namespace is None: + # Then get it from here + namespace_file = "/var/run/secrets/kubernetes.io/serviceaccount/namespace" + if os.path.exists(namespace_file): + with open(namespace_file, 'r') as f: + spark_namespace = f.read().strip() + + # Finally get it here if still None + if spark_namespace is None: + spark_namespace = "default" # last resort, will probably fail + + logger.info(f"🔍 Connecting to Kubernetes API: {k8s_api_server}") + logger.info(f"🎯 Executor namespace: {spark_namespace}") + logger.info(f"🔐 Executor service account: spark (in {spark_namespace})") + + # Create Spark configuration + conf.setMaster(f"k8s://{k8s_api_server}") + + # Check project ID to specify appropriate node group name. + teehr_project_id = os.environ.get("TEEHR_PROJECT_ID", "") + if teehr_project_id != "TEEHR" and teehr_project_id != "": + conf.set( + "spark.kubernetes.executor.node.selector.teehr-hub/nodegroup-name", + f"spark-r5-4xlarge-spot-{teehr_project_id.lower()}" + ) + + # Basic Kubernetes settings + conf.set("spark.executor.instances", str(executor_instances)) + conf.set("spark.executor.memory", executor_memory) + conf.set("spark.executor.cores", str(executor_cores)) + conf.set("spark.kubernetes.container.image", container_image) + conf.set("spark.kubernetes.namespace", spark_namespace) + conf.set("spark.kubernetes.authenticate.executor.serviceAccountName", "spark") + conf.set("spark.kubernetes.container.image.pullPolicy", "Always") + + # Enable Spark's dynamic allocation and decommissioning features for better + # handling of spot instances and resource management. + conf.set("spark.decommission.enabled", "true") + conf.set("spark.executor.decommission.signal", "SIGTERM") + conf.set("spark.storage.decommission.enabled", "true") + + # Set pod name prefix for executors in TEEHR-HUB for easy identification in cluster. + # Truncated to 46 chars since K8s pod names have a 63-char limit and Spark appends suffixes. + jupyterhub_user = os.environ.get("JUPYTERHUB_USER", None) + if jupyterhub_user is not None: + conf.set("spark.kubernetes.executor.podNamePrefix", jupyterhub_user[:46]) + + if os.path.exists(pod_template_path): + conf.set("spark.kubernetes.executor.podTemplateFile", pod_template_path) + else: + logger.info(f"⚠️ Executor pod template not found: {pod_template_path}") + logger.info(" You must provide a valid pod template for executors to launch correctly.") + raise FileNotFoundError(f"Executor pod template not found: {pod_template_path}") + + conf.set("spark.kubernetes.executor.deleteOnTermination", "true") + + # Default shuffle partitions: 2x total executor cores + total_executor_cores = executor_cores * executor_instances + default_shuffle_partitions = 2 * total_executor_cores + conf.set("spark.sql.shuffle.partitions", str(default_shuffle_partitions)) + + # Authentication - use service account token if available + token_file = const.SERVICE_ACCOUNT_TOKEN_PATH + ca_file = const.CA_CERTIFICATE_PATH + if os.path.exists(token_file) and os.path.exists(ca_file): + logger.info("🔐 Using in-cluster authentication") + conf.set("spark.kubernetes.authenticate.submission.oauthTokenFile", token_file) + conf.set("spark.kubernetes.authenticate.submission.caCertFile", ca_file) + conf.set("spark.kubernetes.authenticate.driver.oauthTokenFile", token_file) + conf.set("spark.kubernetes.authenticate.executor.oauthTokenFile", token_file) + + # Critical: Set the CA cert file for SSL validation + conf.set("spark.kubernetes.authenticate.caCertFile", ca_file) + else: + logger.info("⚠️ No service account tokens found - may have authentication issues") + logger.info(f" Checked: {token_file}") + logger.info(f" Checked: {ca_file}") + + # Driver binding configuration - use pod IP for Kubernetes + conf.set("spark.driver.bindAddress", "0.0.0.0") + conf.set("spark.driver.port", "0") # Let Spark choose an available port + + # Get pod IP and set as driver host so executors can connect back + pod_ip = os.environ.get('POD_IP') + if not pod_ip: + try: + hostname = socket.gethostname() + pod_ip = socket.gethostbyname(hostname) + except: + pod_ip = None + + if pod_ip: + logger.info(f"🔗 Setting driver host to pod IP: {pod_ip}") + conf.set("spark.driver.host", pod_ip) + else: + logger.info("⚠️ Could not determine pod IP - using default driver host") + + return + + +def _set_aws_credentials_in_spark( + conf: SparkConf, + remote_catalog_name: str, + aws_access_key_id: str, + aws_secret_access_key: str, + aws_session_token: str, + aws_region: str, + aws_profile: str = None, +): + """Set AWS credentials in Spark configuration with multiple options.""" + logger.info("Setting Hadoop's default AWS credentials provider and AWS region") + conf.set( + "spark.hadoop.fs.s3a.aws.credentials.provider", + "com.amazonaws.auth.DefaultAWSCredentialsProviderChain" + ) + conf.set("spark.hadoop.fs.s3a.endpoint.region", aws_region) + + # Priority 1: Explicit credentials provided by user + if aws_access_key_id and aws_secret_access_key: + logger.info("🔑 Using user-provided AWS credentials") + conf.set(f"spark.sql.catalog.{remote_catalog_name}.s3.access-key-id", aws_access_key_id) + conf.set(f"spark.sql.catalog.{remote_catalog_name}.s3.secret-access-key", aws_secret_access_key) + conf.set("spark.hadoop.fs.s3a.access.key", aws_access_key_id) + conf.set("spark.hadoop.fs.s3a.secret.key", aws_secret_access_key) + return + + # Priority 2: Explicit token + if aws_session_token: + logger.info("🔑 Using user-provided AWS session token") + conf.set(f"spark.sql.catalog.{remote_catalog_name}.s3.session-token", aws_session_token) + conf.set("spark.hadoop.fs.s3a.session.token", aws_session_token) + return + + # Priority 3: Check ~/.aws/credentials file only if profile explicitly specified (full access) + if aws_profile: + aws_credentials_file = Path.home() / ".aws" / "credentials" + if aws_credentials_file.exists(): + try: + import configparser + config = configparser.ConfigParser() + config.read(aws_credentials_file) + + if config.has_section(aws_profile): + if config.has_option(aws_profile, "aws_access_key_id") and config.has_option(aws_profile, "aws_secret_access_key"): + creds_access_key = config.get(aws_profile, "aws_access_key_id") + creds_secret_key = config.get(aws_profile, "aws_secret_access_key") + creds_session_token = config.get(aws_profile, "aws_session_token", fallback=None) + + logger.info(f"🔑 Using AWS credentials from ~/.aws/credentials profile '{aws_profile}") + conf.set(f"spark.sql.catalog.{remote_catalog_name}.s3.access-key-id", creds_access_key) + conf.set(f"spark.sql.catalog.{remote_catalog_name}.s3.secret-access-key", creds_secret_key) + conf.set("spark.hadoop.fs.s3a.access.key", creds_access_key) + conf.set("spark.hadoop.fs.s3a.secret.key", creds_secret_key) + + if creds_session_token: + conf.set(f"spark.sql.catalog.{remote_catalog_name}.s3.session-token", creds_session_token) + conf.set("spark.hadoop.fs.s3a.session.token", creds_session_token) + return + except Exception as e: + logger.debug(f"Could not read ~/.aws/credentials: {e}") + + session = botocore.session.Session() + credentials = session.get_credentials() + + # Priority 4: Check boto token + if credentials and credentials.token: + logger.info("🔑 Using AWS session token from boto3") + conf.set(f"spark.sql.catalog.{remote_catalog_name}.s3.session-token", credentials.token) + conf.set("spark.hadoop.fs.s3a.session.token", credentials.token) + return + + # Priority 5: Check boto credentials + if credentials and credentials.access_key and credentials.secret_key: + logger.info("🔑 Using AWS credentials from boto3") + conf.set(f"spark.sql.catalog.{remote_catalog_name}.s3.access-key-id", credentials.access_key) + conf.set(f"spark.sql.catalog.{remote_catalog_name}.s3.secret-access-key", credentials.secret_key) + conf.set("spark.hadoop.fs.s3a.access.key", credentials.access_key) + conf.set("spark.hadoop.fs.s3a.secret.key", credentials.secret_key) + return + + # Priority 6: Fall back to anonymous or default provider + logger.info("🔑 Using anonymous AWS credentials for S3 access") + conf.set( + "spark.hadoop.fs.s3a.aws.credentials.provider", + "org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider" + ) + return + +def _set_gcs_configuration( + conf: SparkConf, + gcs_project_id: str = None, + gcs_service_account_key_file: str = None, +): + """Configure Spark for Google Cloud Storage (GCS) access. + + Parameters + ---------- + conf : SparkConf + The Spark configuration object to update. + gcs_project_id : str + GCS project ID for billing/quota. Defaults to "anonymous" when + no service account key is provided (public bucket access). + gcs_service_account_key_file : str + Path to a GCS service account JSON key file. When None, + unauthenticated access is used (suitable for public buckets). + """ + GCS_CONNECTOR_VERSION = "hadoop3-2.2.32" + gcs_package = f"com.google.cloud.bigdataoss:gcs-connector:{GCS_CONNECTOR_VERSION}" + + # Add the GCS connector package + current_packages = conf.get("spark.jars.packages").split(",") + if gcs_package not in current_packages: + current_packages.append(gcs_package) + conf.set("spark.jars.packages", ",".join(current_packages)) + + # Register GCS filesystem implementations + conf.set( + "spark.hadoop.fs.gs.impl", + "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem" + ) + conf.set( + "spark.hadoop.fs.AbstractFileSystem.gs.impl", + "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS" + ) + + if gcs_service_account_key_file: + key_path = Path(gcs_service_account_key_file) + if not key_path.exists(): + raise FileNotFoundError( + f"GCS service account key file not found: {gcs_service_account_key_file}" + ) + logger.info(f"🔑 Using GCS service account key: {gcs_service_account_key_file}") + conf.set("spark.hadoop.google.cloud.auth.service.account.enable", "true") + conf.set( + "spark.hadoop.google.cloud.auth.service.account.keyfile", + str(key_path) + ) + if gcs_project_id: + conf.set("spark.hadoop.fs.gs.project.id", gcs_project_id) + else: + # Unauthenticated access for public GCS buckets + logger.info("🔑 Using unauthenticated GCS access (public buckets)") + # Set both old-style (fs.gs.*) and new-style (google.cloud.*) auth + # properties so the connector picks up UNAUTHENTICATED regardless of + # which configuration namespace it checks first. + conf.set("spark.hadoop.fs.gs.auth.type", "UNAUTHENTICATED") + conf.set("spark.hadoop.google.cloud.auth.type", "UNAUTHENTICATED") + # Explicitly disable service account auth to prevent the connector from + # attempting to contact the GCE metadata server (http://metadata.google.internal/) + # for credentials. Without this, the connector may hang on machines + # that are not running on GCP. + conf.set("spark.hadoop.google.cloud.auth.service.account.enable", "false") + # Enable null (anonymous) credentials — required for the connector to + # accept a configuration with no active credential source. + conf.set("spark.hadoop.fs.gs.auth.null.enable", "true") + conf.set( + "spark.hadoop.fs.gs.project.id", + gcs_project_id if gcs_project_id else "anonymous" + ) + + +def _set_catalog_metadata( + conf: SparkConf, + local_catalog_name: str, + local_catalog_type: str, + remote_catalog_name: str, + remote_catalog_type: str, + remote_catalog_uri: str, + remote_warehouse_dir: str, +): + """Set catalog metadata in Spark configuration.""" + metadata_configs = { + "local_catalog_name": local_catalog_name, + "local_catalog_type": local_catalog_type, + "remote_warehouse_dir": remote_warehouse_dir, + "remote_catalog_name": remote_catalog_name, + "remote_catalog_type": remote_catalog_type, + "remote_catalog_uri": remote_catalog_uri + } + for key, value in metadata_configs.items(): + conf.set(key, value) + logger.debug(f"Metadata config: {key}: {value}") + + +def _configure_iceberg_catalogs( + conf: SparkConf, + local_catalog_name: str, + local_catalog_type: str, + remote_warehouse_dir: str, + remote_catalog_name: str, + remote_catalog_type: str, + remote_catalog_uri: str +): + """Configure Iceberg catalogs through conf.set().""" + logger.info("Configuring Iceberg catalogs...") + # Local catalog configuration + conf.set(f"spark.sql.catalog.{local_catalog_name}", "org.apache.iceberg.spark.SparkCatalog") + conf.set(f"spark.sql.catalog.{local_catalog_name}.type", local_catalog_type) + conf.set(f"spark.sql.catalog.{local_catalog_name}.jdbc.driver", "org.sqlite.JDBC") + conf.set(f"spark.sql.catalog.{local_catalog_name}.jdbc.initialize", "true") + conf.set(f"spark.sql.catalog.{local_catalog_name}.jdbc.schema-version", "V1") + # conf.set("spark.sql.catalog.local.jdbc.user", "user") + # conf.set("spark.sql.catalog.local.jdbc.password", "password") + + # Remote catalog configuration + conf.set(f"spark.sql.catalog.{remote_catalog_name}", "org.apache.iceberg.spark.SparkCatalog") + conf.set(f"spark.sql.catalog.{remote_catalog_name}.type", remote_catalog_type) + conf.set(f"spark.sql.catalog.{remote_catalog_name}.uri", remote_catalog_uri) + conf.set(f"spark.sql.catalog.{remote_catalog_name}.warehouse", remote_warehouse_dir) + conf.set(f"spark.sql.catalog.{remote_catalog_name}.io-impl", "org.apache.iceberg.aws.s3.S3FileIO") + # S3 end point and path style access + if os.environ.get("REMOTE_CATALOG_S3_PATH_STYLE_ACCESS", "false").lower() == "true": + conf.set(f"spark.sql.catalog.{remote_catalog_name}.s3.endpoint", os.environ.get("REMOTE_CATALOG_S3_ENDPOINT")) + conf.set(f"spark.sql.catalog.{remote_catalog_name}.s3.path-style-access", os.environ.get("REMOTE_CATALOG_S3_PATH_STYLE_ACCESS").lower()) + + +def _update_configs_and_packages( + conf: SparkConf, + update_configs: Dict[str, str], + add_jars: List[str], + add_packages: List[str] +) -> Dict[str, str]: + """Update Spark configurations and packages.""" + # Add specified local jars + if add_jars is not None: + current_jars = conf.get("spark.jars").split(",") if conf.contains("spark.jars") else [] + for jar_path in add_jars: + if jar_path not in current_jars: + current_jars.append(jar_path) + if current_jars: + conf.set("spark.jars", ",".join(current_jars)) + + # Add specified packages + if add_packages is not None: + current_packages = conf.get("spark.jars.packages").split(",") + for package in add_packages: + if package not in current_packages: + current_packages.append(package) + conf.set("spark.jars.packages", ",".join(current_packages)) + + # Update or add specified configs + if update_configs is not None: + for key, value in update_configs.items(): + conf.set(key, value) + return + + +def log_session_config(spark: SparkSession): + """Log the current Spark session configuration for debugging. + + Parameters + ---------- + spark : SparkSession + The Spark session whose configuration should be logged. + + Notes + ----- + This function logs all Spark configuration properties to the + logger at INFO level for troubleshooting purposes. + """ + logger.info("Final Spark configuration:") + df = pd.DataFrame(list(spark.conf.getAll.items()), columns=["Key", "Value"]) + gps = df.groupby(by="Key") + for key, group in gps: + value = ",".join(group["Value"].tolist()) + values = value.split(",") + if key.startswith("spark."): + if len(values) > 1: + logger.info(f" {key}: ") + for val in values: + logger.info(f" {val}") + else: + logger.info(f" {key}: {value}") + + +def remove_or_update_configs( + spark: SparkSession, + remove_configs: List[str] = None, + update_configs: Dict[str, str] = None +) -> Dict[str, str]: + """Add, remove, or update Spark configurations. + + Parameters + ---------- + spark : SparkSession + The Spark session whose configuration should be logged. + remove_configs : List[str] + List of configuration keys to remove from the Spark session. + Default is None. + >>> remove_configs=["spark.sql.shuffle.partitions"] + update_configs : Dict[str, str] + Provided Spark configurations will be added if they do not already + exist, or overwritten if they do exist. Default is None. + >>> update_configs={"spark.sql.shuffle.partitions": "100"} + + Notes + ----- + This function allows for dynamic modification of the Spark session's + configuration by removing specified keys and updating or adding new + key-value pairs. + """ + # Remove specified configs + if remove_configs is not None: + for key in remove_configs: + try: + current_value = spark.conf.get(key) + logger.info(f"Removing config: {key} (was: {current_value})") + spark.conf.unset(key) + except Exception as e: + logger.warning(f"Could not remove config {key}: {e}") + + # Update or add specified configs + if update_configs is not None: + for key, value in update_configs.items(): + spark.conf.set(key, value) + return + + +def _decode_jwt_claims(token: str) -> Dict[str, object]: + payload = token.split(".")[1] + payload += "=" * (-len(payload) % 4) + return json.loads(base64.urlsafe_b64decode(payload.encode())) + + +def _token_expires_soon(token: str, refresh_window_seconds: int = 120) -> bool: + try: + claims = _decode_jwt_claims(token) + except Exception: + return True + exp = int(claims.get("exp", 0)) + now = int(time.time()) + return exp <= now + max(refresh_window_seconds, 1) + + +def _request_oauth_tokens( + data: Dict[str, str], + token_endpoint: Optional[str] = None, + timeout_seconds: int = 20, +) -> Tuple[str, Optional[str]]: + endpoint = token_endpoint or os.getenv("POLARIS_OAUTH2_SERVER_URI") + if not endpoint: + raise RuntimeError("POLARIS_OAUTH2_SERVER_URI is required to mint or refresh a user token") + + resp = requests.post(endpoint, data=data, timeout=timeout_seconds) + resp.raise_for_status() + payload = resp.json() + + access_token = payload.get("access_token") + if not access_token: + raise RuntimeError("Token endpoint did not return access_token") + + return access_token, payload.get("refresh_token") + + +def mint_polaris_user_token( + username: Optional[str], + password: Optional[str], + client_id: str, + client_secret: Optional[str] = None, + token_endpoint: Optional[str] = None, +) -> str: + if not username or not password: + raise RuntimeError("username and password are required for password grant token minting") + + data = { + "grant_type": "password", + "client_id": client_id, + "username": username, + "password": password, + "scope": "openid profile email", + } + if client_secret: + data["client_secret"] = client_secret + + access_token, _ = _request_oauth_tokens(data=data, token_endpoint=token_endpoint) + return access_token + + +def refresh_polaris_user_token( + refresh_token: str, + client_id: str, + client_secret: Optional[str] = None, + token_endpoint: Optional[str] = None, +) -> Tuple[str, Optional[str]]: + if not refresh_token: + raise RuntimeError("refresh_token is required for refresh grant") + + data = { + "grant_type": "refresh_token", + "client_id": client_id, + "refresh_token": refresh_token, + } + if client_secret: + data["client_secret"] = client_secret + + return _request_oauth_tokens(data=data, token_endpoint=token_endpoint) + + +def ensure_fresh_polaris_user_token( + current_token: Optional[str], + username: Optional[str], + password: Optional[str], + client_id: str, + client_secret: Optional[str] = None, + refresh_token: Optional[str] = None, + allow_password_fallback: bool = True, + refresh_window_seconds: int = 120, + token_endpoint: Optional[str] = None, +) -> Tuple[str, Optional[str], bool]: + if current_token and not _token_expires_soon(current_token, refresh_window_seconds): + return current_token, refresh_token, False + + if refresh_token: + try: + refreshed_access, refreshed_refresh = refresh_polaris_user_token( + refresh_token=refresh_token, + client_id=client_id, + client_secret=client_secret, + token_endpoint=token_endpoint, + ) + return refreshed_access, (refreshed_refresh or refresh_token), True + except requests.RequestException: + if not allow_password_fallback: + raise + + if allow_password_fallback and username and password: + minted = mint_polaris_user_token( + username=username, + password=password, + client_id=client_id, + client_secret=client_secret, + token_endpoint=token_endpoint, + ) + return minted, refresh_token, True + + raise RuntimeError( + "Unable to obtain a fresh Polaris user token. " + "Provide POLARIS_REFRESH_TOKEN or enable password-grant fallback credentials." + ) + + +def apply_polaris_token_to_spark( + spark, + token: str, + catalog_name: str = "iceberg", + realm: Optional[str] = None, +) -> None: + active_realm = realm or os.getenv("POLARIS_DEFAULT_REALM", "teehr") + base = f"spark.sql.catalog.{catalog_name}" + spark.conf.set(f"{base}.rest.auth.type", "oauth2") + spark.conf.set(f"{base}.token", token) + spark.conf.set(f"{base}.rest.auth.oauth2.token", token) + spark.conf.set(f"{base}.header.X-Polaris-Realm", active_realm) + spark.conf.set(f"{base}.rest.transport.header.X-Polaris-Realm", active_realm) + + +def _as_bool_str(value: str, default: str = "true") -> str: + normalized = (value or default).strip().lower() + return "true" if normalized in ("1", "true", "t", "yes", "y", "on") else "false" + + +def _apply_runtime_spark_configs(spark, configs: Dict[str, str]) -> None: + immutable_after_start = { + "spark.jars", + "spark.driver.extraClassPath", + "spark.executor.extraClassPath", + } + for key, value in configs.items(): + if not key.startswith("spark."): + continue + if key in immutable_after_start: + continue + spark.conf.set(key, value) + + +def _is_http_error_with_status(exc: Exception, status_code: int) -> bool: + if not isinstance(exc, requests.HTTPError): + return False + response = getattr(exc, "response", None) + return bool(response is not None and response.status_code == status_code) + + +def _normalize_internal_broker_url(url: str) -> str: + parsed = urlsplit(url) + if parsed.scheme != "https": + return url + if parsed.hostname != "teehr-api": + return url + + host = parsed.hostname + port = parsed.port or 8000 + netloc = f"{host}:{port}" + return urlunsplit(("http", netloc, parsed.path, parsed.query, parsed.fragment)) + + +def _broker_session_endpoint_from_token_endpoint(token_endpoint: str) -> str: + normalized = _normalize_internal_broker_url(token_endpoint) + parsed = urlsplit(normalized) + path = parsed.path or "" + + if path.endswith("/auth/polaris-token/session"): + return normalized + + if path.endswith("/auth/polaris-token"): + session_path = path[:-len("/auth/polaris-token")] + "/auth/polaris-token/session" + return urlunsplit((parsed.scheme, parsed.netloc, session_path, parsed.query, parsed.fragment)) + + raise RuntimeError( + "POLARIS_BROKER_URL must end with /auth/polaris-token " + "(or /auth/polaris-token/session if already session-scoped)" + ) + + +def ensure_broker_session_token( + *, + user_id: str, + session_id: str, + realm: str, + refresh_token: str, + bearer_token: Optional[str] = None, + catalog: str = "iceberg", + audience: Optional[str] = None, + broker_url: Optional[str] = None, + timeout_seconds: int = 20, +) -> str: + endpoint = broker_url or os.getenv("POLARIS_BROKER_URL", "http://teehr-api:8000/auth/polaris-token") + endpoint = _normalize_internal_broker_url(endpoint) + session_endpoint = _broker_session_endpoint_from_token_endpoint(endpoint).replace( + "/auth/polaris-token/session", + "/auth/polaris-session", + ) + active_audience = audience or os.getenv("POLARIS_BROKER_AUDIENCE", "account") + subject_token = bearer_token or os.getenv("POLARIS_USER_TOKEN", "") + + if not subject_token: + raise RuntimeError("A valid bearer subject token is required to create a broker session") + if not refresh_token: + raise RuntimeError("POLARIS_REFRESH_TOKEN is required to create a broker session") + + resp = requests.post( + session_endpoint, + headers={"Authorization": f"Bearer {subject_token}"}, + json={ + "user_id": user_id, + "session_id": session_id, + "realm": realm, + "catalog": catalog, + "audience": active_audience, + "refresh_token": refresh_token, + }, + timeout=timeout_seconds, + ) + resp.raise_for_status() + payload = resp.json() + broker_session_token = payload.get("broker_session_token") + if not broker_session_token: + raise RuntimeError("Broker session endpoint did not return broker_session_token") + os.environ["POLARIS_BROKER_SESSION_TOKEN"] = broker_session_token + return broker_session_token + + +def create_minio_spark_session( + polaris_token: Optional[str] = None, + force_recreate_session: bool = False, + update_configs: Optional[Dict[str, str]] = None, + use_authmanager: Optional[bool] = None, +): + """Start a Spark session configured for the local Polaris REST catalog. + + If ``polaris_token`` is provided, Spark uses user-token OAuth2 auth. + Otherwise it falls back to client-credential OAuth2 using env vars. + """ + aws_access_key_id = os.getenv("AWS_ACCESS_KEY_ID", "minioadmin") + aws_secret_access_key = os.getenv("AWS_SECRET_ACCESS_KEY", "minioadmin123") + + remote_catalog_uri = os.getenv("REMOTE_CATALOG_REST_URI", "http://polaris:8181/api/catalog") + remote_warehouse_dir = os.getenv("REMOTE_WAREHOUSE_S3_PATH", "s3://warehouse/") + polaris_realm = os.getenv("POLARIS_DEFAULT_REALM", "teehr") + + if remote_catalog_uri.rstrip("/").endswith("/api/catalog"): + # Polaris REST expects the catalog identifier here, not the backing S3 URI. + remote_warehouse_dir = polaris_realm + + s3_endpoint = os.getenv("REMOTE_CATALOG_S3_ENDPOINT", "http://minio:9000") + s3_path_style = _as_bool_str(os.getenv("REMOTE_CATALOG_S3_PATH_STYLE_ACCESS", "true")) + s3_region = os.getenv("AWS_REGION", "us-east-2") + + merged_configs: Dict[str, str] = { + "spark.sql.catalog.iceberg.warehouse": remote_warehouse_dir, + "spark.sql.catalog.iceberg.header.X-Polaris-Realm": polaris_realm, + "spark.sql.catalog.iceberg.rest.transport.header.X-Polaris-Realm": polaris_realm, + "spark.sql.catalog.iceberg.s3.endpoint": s3_endpoint, + "spark.sql.catalog.iceberg.s3.path-style-access": s3_path_style, + "spark.sql.catalog.iceberg.s3.region": s3_region, + "spark.hadoop.fs.s3a.endpoint": s3_endpoint, + "spark.hadoop.fs.s3a.path.style.access": s3_path_style, + "spark.hadoop.fs.s3a.connection.ssl.enabled": "false", + } + + resolved_use_authmanager = ( + use_authmanager + if use_authmanager is not None + else _as_bool_str(os.getenv("POLARIS_USE_AUTHMANAGER", "false")) == "true" + ) + + if resolved_use_authmanager: + broker_url = os.getenv("POLARIS_BROKER_URL", "http://teehr-api:8000/auth/polaris-token") + broker_url = _normalize_internal_broker_url(broker_url) + broker_session_url = _broker_session_endpoint_from_token_endpoint(broker_url) + authmanager_user_id = os.getenv("JUPYTERHUB_USER", "admin") + authmanager_session_id = ( + os.getenv("JUPYTERHUB_SERVER_NAME", "").strip() or authmanager_user_id + ) + broker_audience = os.getenv("POLARIS_BROKER_AUDIENCE", "account") + refresh_token = os.getenv("POLARIS_REFRESH_TOKEN", "") + current_user_token = polaris_token or os.getenv("POLARIS_USER_TOKEN", "") + broker_session_token = os.getenv("POLARIS_BROKER_SESSION_TOKEN", "") + if broker_session_token and _token_expires_soon(broker_session_token, 300): + broker_session_token = "" + authmanager_jar = os.getenv( + "POLARIS_AUTHMANAGER_JAR", + "/opt/spark/jars/teehr-authmanager.jar", + ) + + current_user_token, refresh_token, _ = ensure_fresh_polaris_user_token( + current_token=current_user_token, + username=os.getenv("POLARIS_USERNAME"), + password=os.getenv("POLARIS_PASSWORD"), + client_id=os.getenv("POLARIS_CLIENT_ID", "spark-polaris"), + client_secret=os.getenv("POLARIS_CLIENT_SECRET"), + refresh_token=refresh_token, + allow_password_fallback=False, + refresh_window_seconds=300, + token_endpoint=os.getenv("POLARIS_OAUTH2_TOKEN_ENDPOINT"), + ) + os.environ["POLARIS_USER_TOKEN"] = current_user_token + if refresh_token: + os.environ["POLARIS_REFRESH_TOKEN"] = refresh_token + + if not broker_session_token: + try: + broker_session_token = ensure_broker_session_token( + user_id=authmanager_user_id, + session_id=authmanager_session_id, + realm=polaris_realm, + refresh_token=refresh_token, + bearer_token=current_user_token, + catalog="iceberg", + audience=broker_audience, + broker_url=broker_url, + ) + except requests.HTTPError as exc: + if not _is_http_error_with_status(exc, 401): + raise + + # The bearer token may have been invalidated server-side; force refresh and retry once. + current_user_token, refresh_token, _ = ensure_fresh_polaris_user_token( + current_token=None, + username=os.getenv("POLARIS_USERNAME"), + password=os.getenv("POLARIS_PASSWORD"), + client_id=os.getenv("POLARIS_CLIENT_ID", "spark-polaris"), + client_secret=os.getenv("POLARIS_CLIENT_SECRET"), + refresh_token=refresh_token, + allow_password_fallback=True, + refresh_window_seconds=300, + token_endpoint=os.getenv("POLARIS_OAUTH2_TOKEN_ENDPOINT"), + ) + os.environ["POLARIS_USER_TOKEN"] = current_user_token + if refresh_token: + os.environ["POLARIS_REFRESH_TOKEN"] = refresh_token + + broker_session_token = ensure_broker_session_token( + user_id=authmanager_user_id, + session_id=authmanager_session_id, + realm=polaris_realm, + refresh_token=refresh_token, + bearer_token=current_user_token, + catalog="iceberg", + audience=broker_audience, + broker_url=broker_url, + ) + + os.environ["POLARIS_BROKER_SESSION_TOKEN"] = broker_session_token + + merged_configs["spark.sql.catalog.iceberg.rest.auth.type"] = ( + "org.teehr.iceberg.auth.TeehrBrokerAuthManager" + ) + merged_configs["spark.sql.catalog.iceberg.rest.auth.teehr.broker.url"] = broker_session_url + merged_configs["spark.sql.catalog.iceberg.rest.auth.teehr.user-id"] = authmanager_user_id + merged_configs["spark.sql.catalog.iceberg.rest.auth.teehr.session-id"] = ( + authmanager_session_id + ) + merged_configs["spark.sql.catalog.iceberg.rest.auth.teehr.realm"] = polaris_realm + merged_configs["spark.sql.catalog.iceberg.rest.auth.teehr.catalog"] = "iceberg" + merged_configs["spark.sql.catalog.iceberg.rest.auth.teehr.audience"] = broker_audience + merged_configs["spark.sql.catalog.iceberg.rest.auth.teehr.broker-session-token-env"] = ( + "POLARIS_BROKER_SESSION_TOKEN" + ) + merged_configs["spark.jars"] = authmanager_jar + elif polaris_token: + merged_configs["spark.sql.catalog.iceberg.rest.auth.type"] = "oauth2" + merged_configs["spark.sql.catalog.iceberg.token"] = polaris_token + merged_configs["spark.sql.catalog.iceberg.rest.auth.oauth2.token"] = polaris_token + else: + oauth_server_uri = os.getenv("POLARIS_OAUTH2_SERVER_URI") + spark_polaris_client_secret = os.getenv("SPARK_POLARIS_CLIENT_SECRET") + + merged_configs["spark.sql.catalog.iceberg.rest.auth.type"] = "oauth2" + merged_configs["spark.sql.catalog.iceberg.scope"] = "openid" + merged_configs["spark.sql.catalog.iceberg.rest.auth.oauth2.scope"] = "openid" + if oauth_server_uri: + merged_configs["spark.sql.catalog.iceberg.oauth2-server-uri"] = oauth_server_uri + merged_configs["spark.sql.catalog.iceberg.rest.auth.oauth2.server-uri"] = oauth_server_uri + if spark_polaris_client_secret: + merged_configs["spark.sql.catalog.iceberg.credential"] = ( + f"spark-polaris:{spark_polaris_client_secret}" + ) + merged_configs["spark.sql.catalog.iceberg.rest.auth.oauth2.credential"] = ( + f"spark-polaris:{spark_polaris_client_secret}" + ) + + if update_configs: + merged_configs.update(update_configs) + + call_kwargs = { + "remote_catalog_uri": remote_catalog_uri, + "remote_warehouse_dir": remote_warehouse_dir, + "aws_access_key_id": aws_access_key_id, + "aws_secret_access_key": aws_secret_access_key, + "force_recreate_session": force_recreate_session or resolved_use_authmanager, + "update_configs": merged_configs, + } + if polaris_token: + # Preferred path for newer teehr versions that support direct token auth. + call_kwargs["oauth2_token"] = polaris_token + + while True: + try: + spark = create_spark_session(**call_kwargs) + _apply_runtime_spark_configs(spark, merged_configs) + return spark + except TypeError as exc: + msg = str(exc) + if "oauth2_token" in msg and "oauth2_token" in call_kwargs: + call_kwargs.pop("oauth2_token", None) + continue + if "force_recreate_session" in msg and "force_recreate_session" in call_kwargs: + call_kwargs.pop("force_recreate_session", None) + continue + raise + + +def request_broker_polaris_token( + *, + user_id: str, + session_id: str, + bearer_token: str, + realm: Optional[str] = None, + catalog: str = "iceberg", + requested_ttl_seconds: int = 600, + audience: Optional[str] = None, + broker_url: Optional[str] = None, + timeout_seconds: int = 20, +) -> Tuple[str, int, str]: + endpoint = broker_url or os.getenv("POLARIS_BROKER_URL", "http://teehr-api:8000/auth/polaris-token") + active_realm = realm or os.getenv("POLARIS_DEFAULT_REALM", "teehr") + active_audience = audience or os.getenv("POLARIS_BROKER_AUDIENCE", "account") + + if not bearer_token: + raise RuntimeError("bearer_token is required to request a broker token") + + resp = requests.post( + endpoint, + headers={"Authorization": f"Bearer {bearer_token}"}, + json={ + "user_id": user_id, + "session_id": session_id, + "realm": active_realm, + "catalog": catalog, + "requested_ttl_seconds": requested_ttl_seconds, + "audience": active_audience, + }, + timeout=timeout_seconds, + ) + resp.raise_for_status() + payload = resp.json() + access_token = payload.get("access_token") + expires_at = int(payload.get("expires_at_epoch_seconds", 0)) + trace_id = payload.get("trace_id", "") + + if not access_token or expires_at <= 0: + raise RuntimeError("Broker response missing access_token or expires_at_epoch_seconds") + + return access_token, expires_at, trace_id + + +def ensure_fresh_polaris_token_via_broker( + *, + current_token: Optional[str], + user_id: str, + session_id: str, + bearer_token: str, + realm: Optional[str] = None, + catalog: str = "iceberg", + refresh_window_seconds: int = 120, + requested_ttl_seconds: int = 600, + audience: Optional[str] = None, + broker_url: Optional[str] = None, +) -> Tuple[str, bool]: + if current_token and not _token_expires_soon(current_token, refresh_window_seconds): + return current_token, False + + refreshed_token, _, _ = request_broker_polaris_token( + user_id=user_id, + session_id=session_id, + bearer_token=bearer_token, + realm=realm, + catalog=catalog, + requested_ttl_seconds=requested_ttl_seconds, + audience=audience, + broker_url=broker_url, + ) + return refreshed_token, True \ No newline at end of file diff --git a/spark/authmanager-prototype/README.md b/spark/authmanager-prototype/README.md index 488cb64..39c851c 100644 --- a/spark/authmanager-prototype/README.md +++ b/spark/authmanager-prototype/README.md @@ -27,7 +27,7 @@ Set the auth manager class as the Iceberg REST auth type: --conf spark.sql.catalog.iceberg.rest.auth.teehr.realm=teehr \ --conf spark.sql.catalog.iceberg.rest.auth.teehr.catalog=iceberg \ --conf spark.sql.catalog.iceberg.rest.auth.teehr.audience=account \ ---conf spark.sql.catalog.iceberg.rest.auth.teehr.subject-token-env=POLARIS_USER_TOKEN +--conf spark.sql.catalog.iceberg.rest.auth.teehr.broker-session-token-env=POLARIS_BROKER_SESSION_TOKEN ``` ## Prototype notes @@ -35,5 +35,5 @@ Set the auth manager class as the Iceberg REST auth type: - Keep this classpath-local to development images until broker authn/authz is production-ready. - Do not log access tokens. - Use broker-issued token TTL of 5 to 15 minutes with proactive refresh. -- The manager sends the subject access token to the broker in the Authorization header and requests an exchanged Polaris token. -- Provide a subject token via `rest.auth.teehr.subject-token` (testing only) or `rest.auth.teehr.subject-token-env`. +- Preferred: provide a broker delegated session token via `rest.auth.teehr.broker-session-token` or `rest.auth.teehr.broker-session-token-env`. +- Fallback without delegated session token: `rest.auth.teehr.subject-token` or `rest.auth.teehr.subject-token-env`. diff --git a/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerBackedAuthSession.java b/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerBackedAuthSession.java index 51ee1ff..399a840 100644 --- a/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerBackedAuthSession.java +++ b/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerBackedAuthSession.java @@ -20,6 +20,7 @@ final class BrokerBackedAuthSession implements AuthSession { private final String realm; private final String catalog; private final String audience; + private final Supplier brokerSessionTokenSupplier; private final Supplier subjectTokenSupplier; private final Duration timeout; private final long requestedTtlSeconds; @@ -35,6 +36,7 @@ final class BrokerBackedAuthSession implements AuthSession { String realm, String catalog, String audience, + Supplier brokerSessionTokenSupplier, Supplier subjectTokenSupplier, Duration timeout, long requestedTtlSeconds, @@ -46,6 +48,7 @@ final class BrokerBackedAuthSession implements AuthSession { this.realm = realm; this.catalog = catalog; this.audience = audience; + this.brokerSessionTokenSupplier = brokerSessionTokenSupplier; this.subjectTokenSupplier = subjectTokenSupplier; this.timeout = timeout; this.requestedTtlSeconds = requestedTtlSeconds; @@ -76,8 +79,21 @@ private BrokerToken currentToken() { } try { - String subjectToken = subjectTokenSupplier.get(); - if (subjectToken == null || subjectToken.isBlank()) { + String brokerSessionToken = + brokerSessionTokenSupplier == null ? null : brokerSessionTokenSupplier.get(); + String subjectToken = null; + if (subjectTokenSupplier != null) { + try { + subjectToken = subjectTokenSupplier.get(); + } catch (Exception e) { + if (brokerSessionToken == null || brokerSessionToken.isBlank()) { + throw e; + } + } + } + + if ((brokerSessionToken == null || brokerSessionToken.isBlank()) + && (subjectToken == null || subjectToken.isBlank())) { throw new IllegalStateException("Subject token supplier returned an empty token"); } @@ -89,6 +105,7 @@ private BrokerToken currentToken() { realm, catalog, audience, + brokerSessionToken, subjectToken, requestedTtlSeconds, timeout); diff --git a/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerTokenClient.java b/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerTokenClient.java index 28ddf5c..41342df 100644 --- a/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerTokenClient.java +++ b/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerTokenClient.java @@ -29,6 +29,7 @@ BrokerToken mintToken( String realm, String catalog, String audience, + String brokerSessionToken, String subjectToken, long requestedTtlSeconds, Duration timeout) @@ -42,23 +43,51 @@ BrokerToken mintToken( body.put("requested_ttl_seconds", requestedTtlSeconds); body.put("audience", audience); - HttpRequest request = + HttpRequest.Builder requestBuilder = HttpRequest.newBuilder(URI.create(brokerUrl)) .header("Content-Type", "application/json") - .header("Authorization", "Bearer " + subjectToken) .timeout(timeout) - .POST(HttpRequest.BodyPublishers.ofString(MAPPER.writeValueAsString(body))) - .build(); + .POST(HttpRequest.BodyPublishers.ofString(MAPPER.writeValueAsString(body))); + + if (brokerSessionToken != null && !brokerSessionToken.isBlank()) { + requestBuilder.header("X-Broker-Session-Token", brokerSessionToken); + } else { + requestBuilder.header("Authorization", "Bearer " + subjectToken); + } + + HttpRequest request = requestBuilder.build(); HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + + if (shouldFallbackToSubjectExchange(response, brokerUrl, brokerSessionToken, subjectToken)) { + String directBrokerUrl = brokerUrl.replace("/auth/polaris-token/session", "/auth/polaris-token"); + HttpRequest directRequest = + HttpRequest.newBuilder(URI.create(directBrokerUrl)) + .header("Content-Type", "application/json") + .header("Authorization", "Bearer " + subjectToken) + .timeout(timeout) + .POST(HttpRequest.BodyPublishers.ofString(MAPPER.writeValueAsString(body))) + .build(); + + response = httpClient.send(directRequest, HttpResponse.BodyHandlers.ofString()); + } + if (response.statusCode() != 200) { String bodySnippet = response.body() == null ? "" : response.body(); if (bodySnippet.length() > 500) { bodySnippet = bodySnippet.substring(0, 500); } + String authMode = + brokerSessionToken != null && !brokerSessionToken.isBlank() + ? "broker-session-header" + : "subject-authorization-header"; throw new IOException( "Broker token request failed with status " + response.statusCode() + + " against " + + brokerUrl + + " using " + + authMode + ": " + bodySnippet); } @@ -73,4 +102,31 @@ BrokerToken mintToken( return new BrokerToken(accessToken, Instant.ofEpochSecond(expiresAtEpoch)); } + + private static boolean shouldFallbackToSubjectExchange( + HttpResponse response, + String brokerUrl, + String brokerSessionToken, + String subjectToken) { + if (response.statusCode() != 401) { + return false; + } + if (brokerSessionToken == null || brokerSessionToken.isBlank()) { + return false; + } + if (subjectToken == null || subjectToken.isBlank()) { + return false; + } + if (!brokerUrl.endsWith("/auth/polaris-token/session")) { + return false; + } + + String body = response.body(); + if (body == null || body.isBlank()) { + return false; + } + + return body.contains("Delegated broker session not found") + || body.contains("Delegated broker session expired"); + } } diff --git a/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/TeehrAuthProperties.java b/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/TeehrAuthProperties.java index c5b8953..ea8bc83 100644 --- a/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/TeehrAuthProperties.java +++ b/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/TeehrAuthProperties.java @@ -7,6 +7,8 @@ private TeehrAuthProperties() {} static final String PREFIX = "rest.auth.teehr."; static final String BROKER_URL = PREFIX + "broker.url"; + static final String BROKER_SESSION_TOKEN = PREFIX + "broker-session-token"; + static final String BROKER_SESSION_TOKEN_ENV = PREFIX + "broker-session-token-env"; static final String USER_ID = PREFIX + "user-id"; static final String SESSION_ID = PREFIX + "session-id"; static final String REALM = PREFIX + "realm"; @@ -23,4 +25,5 @@ private TeehrAuthProperties() {} static final long DEFAULT_REQUESTED_TTL_SECONDS = 600L; static final String DEFAULT_AUDIENCE = "account"; static final String DEFAULT_SUBJECT_TOKEN_ENV = "POLARIS_USER_TOKEN"; + static final String DEFAULT_BROKER_SESSION_TOKEN_ENV = "POLARIS_BROKER_SESSION_TOKEN"; } diff --git a/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/TeehrBrokerAuthManager.java b/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/TeehrBrokerAuthManager.java index 44c087e..c8f4b3e 100644 --- a/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/TeehrBrokerAuthManager.java +++ b/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/TeehrBrokerAuthManager.java @@ -1,6 +1,7 @@ package org.teehr.iceberg.auth; import java.net.http.HttpClient; +import java.net.http.HttpClient.Version; import java.time.Duration; import java.util.Map; import java.util.function.Supplier; @@ -43,9 +44,10 @@ public AuthSession catalogSession(RESTClient sharedClient, Map p String sessionId = required(properties, TeehrAuthProperties.SESSION_ID); String realm = required(properties, TeehrAuthProperties.REALM); String catalog = properties.getOrDefault(TeehrAuthProperties.CATALOG, "iceberg"); - String audience = + String audience = properties.getOrDefault(TeehrAuthProperties.AUDIENCE, TeehrAuthProperties.DEFAULT_AUDIENCE); - Supplier subjectTokenSupplier = subjectTokenSupplier(properties); + Supplier brokerSessionTokenSupplier = brokerSessionTokenSupplier(properties); + Supplier subjectTokenSupplier = subjectTokenSupplier(properties); int timeoutMs = PropertyUtil.propertyAsInt( @@ -63,7 +65,8 @@ public AuthSession catalogSession(RESTClient sharedClient, Map p TeehrAuthProperties.REQUESTED_TTL_SECONDS, TeehrAuthProperties.DEFAULT_REQUESTED_TTL_SECONDS); - BrokerTokenClient tokenClient = new BrokerTokenClient(HttpClient.newHttpClient()); + BrokerTokenClient tokenClient = + new BrokerTokenClient(HttpClient.newBuilder().version(Version.HTTP_1_1).build()); catalogSession = new BrokerBackedAuthSession( @@ -74,6 +77,7 @@ public AuthSession catalogSession(RESTClient sharedClient, Map p realm, catalog, audience, + brokerSessionTokenSupplier, subjectTokenSupplier, Duration.ofMillis(timeoutMs), requestedTtlSeconds, @@ -127,4 +131,24 @@ private static Supplier subjectTokenSupplier(Map propert return token; }; } + + private static Supplier brokerSessionTokenSupplier(Map properties) { + String explicitToken = properties.get(TeehrAuthProperties.BROKER_SESSION_TOKEN); + if (explicitToken != null && !explicitToken.isBlank()) { + return () -> explicitToken; + } + + String tokenEnv = + properties.getOrDefault( + TeehrAuthProperties.BROKER_SESSION_TOKEN_ENV, + TeehrAuthProperties.DEFAULT_BROKER_SESSION_TOKEN_ENV); + + return () -> { + String token = System.getenv(tokenEnv); + if (token == null || token.isBlank()) { + return null; + } + return token; + }; + } } diff --git a/spark/authmanager-prototype/target/classes/org/teehr/iceberg/auth/BrokerBackedAuthSession.class b/spark/authmanager-prototype/target/classes/org/teehr/iceberg/auth/BrokerBackedAuthSession.class new file mode 100644 index 0000000000000000000000000000000000000000..f6046b288a6903cd13c28644d59b8948e0c1978c GIT binary patch literal 5275 zcmdT|`*Rc575=U*Yi(&Az!->l8Unr>oOG@8qNYnR&zDv@`yxKrQOJ>^tpugnTw$n_1>`W&q{qEhBERBt!Go8+K#_N0c z?z!ijd(L;xJx71|*OfN_Y{A85+ zR%_*xR6QZpVzY@BLiD+OO2_lrEYIi&EGskBNIso#Y0g;jvsQJTi8jTiF_+i= zu3(_##7u0!160VhCuAm1yES@D)q8A5-mw%@&FeuEAHYKb^Bs9CFLN$E=BDD37)j;? z#4g2dvxyHXng&OXI5IaDAD1V5k`O<)o9IBNfa#-3t0SG+~X)ZU%zo9?mXIulUS$XmW2y z7<=(y1N#)`xWIkt^j1~{Qz`Q31UJ?*IJmb*T1n|p>rK9>5*N?*ci2Z+LtuV5pky5s zh}}+>4)q;!#LB1KiYPzWtbKeZIk(s5Hk_&1VLWEwaT6cGM+KT^)vQ7#Q?X>Su<2VV z&^Nn=`#jip++IUv=OUHLj7LmZ7-ni6l`60*kmkEIge0Va5fh^r6PTyLp<}Pt!x?v{ zGbaQV6nB?@>`?eWYGNGBR*mvx)~3G|O6_zhFc&rNDvM~JYe&oBFw!_;AS1A33X#*Y zh=F4j*M=7$1_-42Q(^PS$%M?RL=i$3c@xJ$jSV~3#nd^jua?6GP6@1-+R~d!$x$m6 zcWL_4_F8<5_eb2zRZXr+=S?|3-8DJV%{ytCY_if#aw6-V@(4mWiKo^5c3K&!ENJ{# zZrLBg$MFdRpEU6)JnIAGw)c}x3&lI6Zo*Exexmr}47{gH`(Lh<`?Zv*?<$ijaDK+b zb9kQ4$tiGFSgx3E81J7e`q|FJdiRtZW#9#YVIF-JV$IanvpX~EJt z-TPZR1mb0T?pRTE-px&@xcgu0r&Vt%IDL01K3 zw>tL;ZdgzT=Tb)Ew}$EUG|SQQ>X`BF7q5~jYJS=Ii+3jz7q6|vI7={%I0LU2v!ren zO4)-aPNXt9suPN5@=ikTv=wNVcFo)gtylS!iLxt`adWO?W&0&d-DIv7S7?kbX*tqq z5_5w1O$fim8wRea67@RmIhBa*^l{#rF)zx!s&%K8a5D~X-oq)3I67BO4+;vNt$)!=JN2CdUGWy+AADPxB9OqsJDKj3_!FRO)aDDHfPJ3ZI@kUFs;;C~vJ6={{$ zVpVh!AL?(|vThO&U%?~5CG=jxo=X@UXluQULshsy>MCum2EC(`I&UHWt!(-q))t6* zoJtm0g&*NO<X^@8y84&|JyLcdCXDM%a{ zv|e)_3)+K`z)gfBfl0XM5R3$`mA@~J)Toa&`eXf7oIKR<C-F>skdsg(IEhb3f>g3S5DC1AGv`ou37?Kln2Vp|7u4Uv3;3l*dXn1zN+bOp{vLLLcnEU; zVok?55OaOpghh=;3(FcI*!{gqth~n4WKsVrN9v6#k|cwv?@fLQ%Y-2+|gz-fusD_x1I64@*-|%<5Sm)M8 z@tH7er^?VVX|4B@)LSEjOWVnsTpwNX-_$8@C8&Cn@6G)zah(~_ar7nSCgD3&c(a!Zzy$e>q4 zpN=g08G1ahumss_)SK=Kd2dx+hW^|_{y#V;bPV7mgJ!zZucdI5a2wJ5iwwo;w2m`4 z%aD+^?>58jT;*{7rNFuu(}1wTJE!A3h6u05-P){O6%3;|snrx|Vn|Oa-GSL89^#RPIUSEN&oH>}o71$smhgQH zP^q`p_Y5(73)3{MchssyUvQpV)w*ygXZB{ZQ3_EGsyB2JD6W2Nh=;;89J3*9C#*7C z=dK$*2ryp$3lg^eSJSj2t(PrJI3A>MH9TP$+2?!!2Mvo1XAc&oeE5WMnWj;MIxIqy zhK3yNGHNu@TcanAQM$>|Zc$B6dZ!CNzzSpA&=VS3G~RxJqQAxJG($M0%%ah|*-c9@tVs$}O$1i&MZ?+WyN%_eBEq9vWmf z+Nci#%DKbUM{qMxpCoA&`$S=E7Z?7g-|x$`$R%(Kw+Vqpe~X8Tofm$JnRXD1cmTsF z>~}~-1w5>rI*M`(cRi*hOw#)p`R7-TUpM!>Glm(;VgmOug=v)OEJ@}TP{9%g{s3$c Bv7!I~ literal 0 HcmV?d00001 diff --git a/spark/authmanager-prototype/target/classes/org/teehr/iceberg/auth/BrokerTokenClient.class b/spark/authmanager-prototype/target/classes/org/teehr/iceberg/auth/BrokerTokenClient.class new file mode 100644 index 0000000000000000000000000000000000000000..bca23d112b001072234bb116d430bb8182474a3a GIT binary patch literal 6356 zcmcgwd3+Sr9sj;$li6%0*Mh+Xk5y4}5Z1Fuf(Qv9i{U66K(sbac86qOv$M|3a)?^A zm%VIltqQHxwo|(2vpBCGKMoxpt7ZP zO$~ymQ4ms53ss=Sv<732t`Au;BdHJQ9BDacDAsM68+EIXKbfAip=Zb@G~_th{9tOY zdrn4oVsbjRr0}gvLA^ke`#I+r>DUs@9$KblL#RV#5KRI~&epAdBUOt@n5^J970s9; zPI7`1Wr`wL6>UW%UzpW?DOv(=76r8HyG!@fvxqjHfu z#~WcnX{8hws+f-j^o?T->!ysLIX-_m5nGtET&atVIw3!qD?KU}ViCn%wqWJTMXLll zdXwgGY`w-{v9=7SV;i*OMp_`2VmJ*LnUtS)4B@P9(ODN^iGsL_rMOt2{dMdW2yeCw zM_(g@Y=Q0BLEwyHb9pJASH{f|-(Dds!*T^HRIJ1snde9Fh?&~DGB=Pm>>=H<8M5a3 z{8<+ipLHybC}!0q-H}kyht&dsl`9f`0?U1dCAi^->-6rD-NzE1A*{tE3f8H(6ql7? zUtR;nu)ArNjV}0I8pP!S&6jj|=+=?&z@fv1xgh$9K??|ere$2^21%F*tEm`3Qeeuk zwozvyFzDAB&JZcv(J~J0RpO9RBNaPDT0>W{9t4i60hyEnC$=0xOU90Y4GK0&7p4Vf zlyjk-r69U%x+PjBm3BSYHUZo5i+M%{rh=@BD?yAne*77E#1Ghdh9YZ6@SqROhdH;* zspYly7E+`j92Gfi5;)l&GIQzFVlAB>V4n4viJV*17HvrmvBVKDSC^N^=QSQjj%B8m zfwDzlnpX<5W?HihyTg&iFyz+Ndi&78($&p!lxAMR&lpHX~r>fqEEA$^wBQdqvA6Xu&On?Pv7DM zaUauK-;y;f-R{?%em!d@hYoc(yYM*`_hXN{!E0**Y-W}Q@u0wb4=AJD>f%gv*kC)A z8H^T-95t=zMEn&z%%0}xJ!S}dF{)rp#XjuMcaLttj(a#uXg1d`lAwO_N+|C7ok!X4^Nj8 zxb@Jwt91<{!b|>!UL&I~&keJ;@6!g-jM;Fnnbgv22x@ZdPX?SJgVMG29^F~~8H9mh zBjdQmQ{ahWwfw9ZpL+k#nWaUpE^7OZu3wixLyGB=biD9iZL#`>$BWLG^p+L+?P=bk z5|8*F$+T9&ZX#o_F=SiHn}IX5%*E##FNEB4s^#S<&O%$`2b844-hzbJulN>MeFXJ~ zldSv|rq4-xNkV^$rNsGihMvonvNL>VS>OI^ZYqe6T7M~4cv%U19=fs@>-<{lmAt;(`;S%OGm5&fPp1~aVP z7Bkf-m*V*%Usix3Jzhr=wq8o#%}u8LcY;vD%vni&u_4i-*-HY~=3eH{k}9y>yqyn{HjRbSo-?ctv2Iw>j#_-ic+zjg#nljhs`_%>>z~*QZ29R2!7| z=#a{}B}Q~cMME`S3*o;E7a>^nMTNlhL-sQ=o5(ii>FKrRi?yU<)|Tikz*H(1F=dHXH1{IVjGm${DlEPB|%CU}`+gsAaMMz?y>n#~NSYK3g8 zFS!8OicltK`durQUY8hnCA)cy9Ov%@im2s9#Wmew;N)dw@dbiuRVYR&r%ei|^ul{6?OYPoWx<0rA{^ z)UDf#hE6r2h8t&&AzX>IB#-YXkW(Xp0)0ZHsz9IATr-MjVR%NQ+N0Y?(K3oz;d8@p z7{mFMh}4c@UIiXNcSIe*;!*UDVO0e>>mtfQOpPcbShEx1NL~0%5yjoGb~h$-_~uA( zAFcqrxk%j@h?Sl7Z4W_-)Q@6l1jC&Tk%sn&5($pN-hoTn!<)lfN3gB4G17RL5T)EE z5Xn2C$vvUz&*QYYZI3iX8u#Hgu>bqW0cj-f2=W}k$0XbECq{6W z7{RAIxiWk&2Q?%3?Cfx)=^#SQ;hLLoN1bFCen4{AkB8PpnwrC-7}<-@-%qCp+(5Yp zZKTg&kj$XtTW~U8V#$V8bm#|O>J`<@}EGA)zIE9Zrv(PKzSSI?gT=eq|CXH3% zN+d)MePSEdh#UFfa3`)1_dyf;*s&j^ho8it8^q5tjGxBEcnaS~BRSS{^aH+~#+fM3 zxN|u~@Iz8fGK}-VPU=UTW3ZF@F)1bvsh^NyqL7+Kis?n_L{e2QHG@>OOZ}9T;!@9& z3cA#D_!+Gxcg)AnnMyU{XiR3nSC0cCieKY5XrN{n;G_4G7kdbuanvV zC9oTT%6oZ~7j>Pq2w+mXEGdf~v+02x<8TvWa0}yXD>c2EF?9`neJx+?uES-xo^M3g zxlwSgC$?)*GGc7_-X$e{C1G(8(pQz_(MrEa?@HTxfH%mUFK{F+ItjJ)A;Ev1{luio z`2v6EhYGw%SO0^S^YHdGmn~4c9~TiDpLh`65^#l^M{*GubCvj~yY(gfi+}664dV}E9jXEHYqgAk3u&@jY6ATV~MpX%wBZZ*=S+7UB60z+G-WqNN246LqI5f95W zFMT91A@{V3=a^RGHGz?=ZMmLqc~!mTGXX}>rNdk=6~s@qQ%Vtt@03diw5(OivH)WQ zX8R>I$3A9G%6D1>5*@KZ6-6o)H5E;Z0!$KE4mT5B=DMbBi5@z!2qeQCXtY=r;3k39 zt2hyvbxI&sQ7~yKC%_DW>3%!kWsYdpad+KJ1La)E?z;H};%J#>1EQ8@0DV3dlnpL}%TqQ8y zuXg?R>32&w{*TF7?H&+V{IA(r(Wc7ueT6D2&1tGCYYNSliaA99Y=f14bBBHLu~}GM z^t=`xqhVWh7dx%vLQ|yzEmc&lfc6gwEL~zRdgckUeNQ|=;{>*AnJRmlazH=y!aoUV zABTNHk4@1%W?y^JXLB^4smQ8!F$)4j8UUF5MqsX|x)=_1`%%1~!DkDr-P4AYrX!5t z6&vdcdqePqq1qSqJM%fjH=gKxH2KzhTf{`ju~)ganNg;coG}()t@DNO5%#`I|j>}f!}1v z)0GO|AJ_y;2kV-EBwT|y(siT=3?tv*cocb@;|b&h$5Y7D94C=yIlhHF&+%>K1&$Yy z?{d71e2?S%$ZH(0BX4jlBBwdtM1I8aW8|ms?0kREkzc^ebAE-q#ql=sTh9I+@(#yY wWXf?KJw<{)0UYm*_q{jMNoJZ~;D__ayyc#I z&v(!H&hqY4|K0yt0B4H%et1x(pj?F)6#~^;v@tD~)KY`7^?h6Pge6e1!blm`N`bN^ z^_wc;L!|=0iU3rB#Q#u9@foOeD2ax+MsR2 zT_D(bL`~c>jnrU^TzH0xnWz?+ko`w$hFH+ctIw%^cG3EhV#;?xp}bT(y4v!ASDDNdnH9B;TD?jfmC z@$Bem(vXa5n(PP{jlHLVY${=qL9v{Ci(L0K70cwhwVA9<@d+#XaJsJV@Hs!V%|20n2z`eM$s*SGs{ z7Bv)O8sMv;#>}w3wK1b7(y9K8mq9ImD?$ntJr`yk{LD`Q@Rx!vaHeATK+k5 z0{)b~wKZ!b`=!h4mbgAHAzJ53F`vZIqhOUq;&+2nZh^greM2@OK};~vUF?Vx-82^YJlcG$>oHd?u(6>OZS@s*nuk)yj8{9WY1v7 z`*eykX_`o;GrEGe3)I;BuvN8fOX#DLAU<3zu&B*6C4Ne>F$g=!KP-!L*e!-~L+?=W zPFzdqTQyU52vbAv88XvbwZ5cIZp#vQor-ti-841poxtM4$+=w?aK(M)M?G#(@gBTa z*3(QYu@Bo_5%a)JOd3Zv)SgW0gIaQxIhY+`p3Cnq$kcp`id(UZ{2$SV_0{PV{dB%z z4e>5xX(@~Tp8&^ogBvP^p7G;G+%5y<4z>k_JIGDhw@e(@ABck62;B& zI_rzw0p_rlN%4=8ag$Y+5_dFzj5#r>xXX`E;!_Gftzx%y zO+gbJ-=4Ab5jMdj3U{L5L4o6n938hbOE2)U7q|Ga2eGH{uw?<1? zX>+?kC6{-C{77}SeZ4xVWisrrCeLJ_``C0M??~spI^ECCduFGR(${52*sAx)c(C(K zLQ8JaOhZ1qFUzeVnWLF_UQK`BA`lpHTK#o)pWo}`?^3y;yDi@1RChOX+X+*4(#~c} z&5=1ER#0)MuhQ}SMw1n%LN?OvtZkJpZREYzZjc$UZa!oU$~ByfjVam9h1`Cxi?$H$ zw3VsbJy)D9SKR5iIMBgl<~E3We~*>Sj|A)x%k?nMN6fTp85{A;kj4*<+}t|b5v6QW zU`Mt|Zl}lZ8tH1Ar+K?_=lV7yV=<`;%;`)QdVTw7&htwPW)-#1E1bc?EVPnNSXMBH z9xNeMdR`UdCNazzHtQ7pXhK!U=WO~RZVaY0D{GRk*SrQ!O%u_pVxAzD*#-N`hd&LX zxCCT%7}goy-Z3qirTm3%hRyk8nz#zI|Nv6Z#rM=Dr1``==%uI4lFL>9m!xOl`DFw}#UFJS+GMt4Wu^n#9H=cNh@P zeO@((MK%7$@5)>cZpVq+Jj|@XKjg>fODcYcUDf#KWI5Nfl>4`oJFMcj*u|&+%BNRU z{6;O}n*_g+e*Hnsf!44>9e-ovXu zvKJ!K@CcMWeDXjZU(VnvKnUers}c)vkWX_QX*`b4@rm>B1XpL3=IT#!jB9$*n1x!7 zy%2dARqGlL!P~eyvbnkZiQKcVF;xEK?r5-P9J8Cf(O^hkE26;#@~T9G$I7cO8aytz zXdg}}qq6$u%4o15*qDFxN5>Is4)FifW;GgIKC9;1ah$QdDjHl_i?hekdisoLa7``N z%G>G#=-51t&Ii%G4;OgYEnY0J7k$ma$N}h^L&1Pna3f>KxGEbQbV)b(h?y*L+-amJTd}{W|ZeiR4Ut1K%X^NP#?2Uck5TZPG-(`wqTKs(A5pe2?;F#0y%c%+>WwUe`0! z<;Xuv`5*AU7G;MKP_W}Lf(kq@;c*2&#E&`Rq2-1AVCKIh_T5}h(zi}}t1}|##fC$e z*RT(l3*3*HUDBslMfc&FGTMZvcwBofuH+m^{o`p3;38tYm^SF;sK)VrO7vkK5;y?^ zwsj@ydBpo1eo6$?=I+-^Z~u&Y6E^A^cwXUDj{+Z|Lnyd1z-4*p%}EmMjV{>55#rnF zk%aGx9)dsmD6YQ)vku_K%?AQzOAUb2vL`)maV1 zc_;kD_S=sfx=D%&c2y;@CUy0;E(tdugiHZ=cUA-MSvrJ#ozI= rd-t#W-G6d-G~CNlT9gTy66bTqW^{>~3VDX!C2G$OiP>VV2qE-8N_OV) literal 0 HcmV?d00001 diff --git a/spark/authmanager-prototype/target/maven-archiver/pom.properties b/spark/authmanager-prototype/target/maven-archiver/pom.properties new file mode 100644 index 0000000..41b0911 --- /dev/null +++ b/spark/authmanager-prototype/target/maven-archiver/pom.properties @@ -0,0 +1,3 @@ +artifactId=teehr-iceberg-authmanager-prototype +groupId=org.teehr +version=0.0.1-SNAPSHOT diff --git a/spark/authmanager-prototype/target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst b/spark/authmanager-prototype/target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst new file mode 100644 index 0000000..8cca880 --- /dev/null +++ b/spark/authmanager-prototype/target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst @@ -0,0 +1,5 @@ +org/teehr/iceberg/auth/BrokerBackedAuthSession.class +org/teehr/iceberg/auth/TeehrAuthProperties.class +org/teehr/iceberg/auth/BrokerTokenClient.class +org/teehr/iceberg/auth/TeehrBrokerAuthManager.class +org/teehr/iceberg/auth/BrokerToken.class diff --git a/spark/authmanager-prototype/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst b/spark/authmanager-prototype/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst new file mode 100644 index 0000000..dd63896 --- /dev/null +++ b/spark/authmanager-prototype/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst @@ -0,0 +1,5 @@ +/Users/mdenno/repos/teehr-cloud-core/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerBackedAuthSession.java +/Users/mdenno/repos/teehr-cloud-core/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerToken.java +/Users/mdenno/repos/teehr-cloud-core/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerTokenClient.java +/Users/mdenno/repos/teehr-cloud-core/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/TeehrAuthProperties.java +/Users/mdenno/repos/teehr-cloud-core/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/TeehrBrokerAuthManager.java diff --git a/spark/authmanager-prototype/target/teehr-iceberg-authmanager-prototype-0.0.1-SNAPSHOT.jar b/spark/authmanager-prototype/target/teehr-iceberg-authmanager-prototype-0.0.1-SNAPSHOT.jar new file mode 100644 index 0000000000000000000000000000000000000000..ccd6947a74fc8fb5de21afcc4d226f8851194321 GIT binary patch literal 12971 zcmb_?by!?W@;2`7?ry=|Ex_RJKDfKPYj7vHLkJ$+-3buf-6cT+;Uo9%Z*w=hcc1;^ z>*qN$ea_TdRo&Cwr>gr^1b{=pfq=k(fXFs{*9G}YfqkvZNvMi2O3O>WQG6A{col>B zLyWFjpHKVsXOq|K_31A$IT3kjNeLBICOOF)x$!Ys03*{3B7l)@a(t>@nfWu@=AILi zj2wfs+>C2QgYu;u17Jes(g{HsW|&rPYFveRfeo_!N`?U-K5I5Pwx^nzkyLP!l30ze z`!3U_*wA!vY+wu$pBS9}+G=poPeR!?s3lK)e4UjQj7xuT5q8y&=Eb5cqGfKfsLt zXdUBEZ*7h2jLd;fj1ErrF7_^-4!}P=`t{Ej@u{%x!GnNs5rBZO{e7ENe|E2^lf5<2 zN#s?l+@D&ROl*vtof|bheN^W$f7mr~WwZ<9^s?qLOLd#FCgGrdhEM_%L8BJVKq+QA z8r!AxG-X9!Z->n@ww_nFX>{!?ZdBK?K8vCUwfVR_yRhtApVhRG+UjD^z&P)BH<2~M ziCg;P_=)9W-F-^nfNxCTVD)a-8oCQ+6Mq?Bd6FQUo}ui0#`NAaP_pZ*`;Hnii^yvc`_rl;qioTx6+JFFfXf; z$9~aR`@J0;#z2FJUN<@(CVG=B`NARvu7U~+Atu-yS!S`sj<1nG0`oQtsK3#ki(SuJLj z>A|1Cm{c}$*=0N48hX0QgcbqCPgWZ9OGfbKW&QSg`2%^-Ln+Z)aoN3extZV-#bw@j z`v!JbMV^$@NI>gd0=G^yXF|~-FouwTWxZ&S0JZE0tH5hV8JIHP*2Wp#$~&tO8a;n) zR|{i)b~!&4wucn3mljQ#qYF`C!*lU;?Yg096V$a(-tEt12_CFw()I6)3P7pnsA%oE ziOx5lhwu@Nu78j)Et-LdMg@I}9jyR1A4XqAT zl(K`L$26XM`U5)2)2GR=G!+VP$~R>9k82Ov9e1y2?CF4(O@f1B={UD+V-WBqWSft_Gg_DV3A`w~fSspz z7V=n%dl!vWRuB@KlU6&Gr7QFvD%MI!P)}lB%6-aL;Zl(vZ0syFQPB|70Dl(Z@DFb< zN6@))gFL8g0m0tiejiwkP)b={>1Xo8vy6t+rQ8FWJ#gFZL3y)X>2j_rfCw&ks@dZDn|CCm+1Hd^Gx$B`Bn zv@6GASAwYusqsa^V)T_C^Ce=VzY-aI<2^Zv%@U;k{+($_FO2X@xnWvVv?J`?wRk7P zm}FJjzTijR((5Ca4Xv=X7TF#?wRXTag!u<7^Ds*aJcUEJbw+0AWXG||2tf+Dt=<%{ zXl+@QZE8x;d*zW0YR`vkV;W4a_^7*WE76Voj5@Jl#E!y^n2YJw!q(VMxWzIL;&la; z`gK@_3FZAd3=S?Uhny(n&EUFRs?1NEhVByM;9Sg>F?JTQl3xWGTbEA(NUWS}kDloi z_R5ZI-&lV@H-h@g@kDw7Ch6HB6NrDC!4CgrdnkGGhd&zh%*98kDnJjn`T8fcTCXQ^p+- z?DEi`)GQyZ>FAs$OuJNyyp#CRP7{3|4c5y@nMv`|LXg_N;YJ_*!stLiz_kAsaYQ81 zZTQCUAR_|n?6a)cWKNodde`{OSC!(8Q_oWAJR4??Rl~any?GVR^DLcS!X4JTbSS0^ zv-e>DPrvAqC>1jNBlU$QWoo3|mM?biJPmVvhVhc^xIHTp7r@`NwqEMOAp}c z_uxvkiJ2N$GJwkkNc5RnwDpoFDvxMM8jL37xrd7$Gg24TqeF2vNnxyact^)qZpHhL z^5jOib&%Nt>nCktv$^)ez{Hd!^nSW)JwbhCtcug`qn>tT;HySs>X$c;YNSc^N8z_7b!cNe72o$eA1eaeYNhRX z7C&t?vZA>P8n^Ua72(b-;p$(}8(^AkI4!QuMvKh_7Uu@^tT%gU4HM zAliuv!n{Cu;G%YsG73V}l0EKNaHS^pG?~F*vh~y?dcY$xD>dGAf~zV+P~-=tT|P`2 znaol#3>W$ojW9Xqcr4PS>P}+0L33QS7Lw7Z#Yv;4f~o>@-~e>ul3k{XZ88rn{KLa1 z9ggq;JVrc3{8V{oxOv9W0d@&l4T;cnmKp`;GBLMX#j1rY^)efW)6?>1{f;MW6*A|G z2_PC)+JdGm*V%$7798+4>UdQUlO@YnN}rD2xTZU3wTNMY)|kPV0lRjOzhB)hxBaY^ z^xLpPqz~s%y(CeE$b|=vG)6M;Sj2p-F z`XyQ`{;)So8x*S^*{c~fBhRok3FaaMY>Fd|g+Ia-J4rnt26}vdRL4rhmmbvQH6KR$ zf=-R2j$1Jxd9+_^px&YzSm6zt2LIv%a)4Q@(=KHw)QT_g zLu|U)OJLpDK2qbVwY^xsxp};DlCH}N!ndB@OkpLyCtT?VNEkoX3_UcnE=+ltYMvw2 zO9vbBoGd0c(UkHYq{pD2VD!P6d4lb*Q4JWpr5iEgCz$If)j-Ph&|>zB$kn}rGAvgZ zL&zY_0VTfUkGRxZVC;GL)Hm1uaK@VQX}NKF7X84YRy1>COb|HACm7AYQRu zC?wttM@TW|4<}COeCHuDhQV8xJ2~vNjHE05f@48 zi?n1uoP%vfU3)JFQzNOXoF?w!qJ)L^z|Q@uIQY{88jX_ zI`9}RFodE#dq7jZaGf*szAQ7iQ)Qv6Gun5k>qOjO?whE(-uPw&y$Nw_!HZ zLV}RD3SXu#zOHg%J(g07b!GP6p*(Vg_%Y@iytUwZ^3T_30YYz!JB29V>ZJTY^^r7o zTU<6$V)!;MV-sp3x+Uq!WwyFD{=jI0=2pyTFu3Q^Zb&}d(iY?d3E7#Bic@>T6L^0I zv=WJY-p9)V$^?Sx@HiX*GIAV)61IKN^H_r}HjafNpn(uVE*@y70GLzd-gUVoeHN&u zH{jlWoL}}ZemD`;q9+;6?RQP>MoD~w!}ZoB_qZ297{Q|%WHwyxAwHe(?|*-<8;GVf zo+1>@TJwbIKU6t+Xpk}EKTFQ)p{5)yy7e)*3`J*#$t-o){_w2en&Cq_nmhoum z+8M-zgf&acWDEy&R@e$3jFYtKBcg{&QanqS~!5$xJ5$I*HTsoaW7u{UY# z_yQD~wv+sK;#)vxlfcB$Q0`n(@(;81dd4ltQsU(~=Ua9*_wx?tgY}KoPdDNIdcKI= zob9-hxTA;=vJj|f>He#{mIKP_UNlYan=P^}VELSUwrwkrqxg8pcx5OSv$CFx@&jlY zE@T(ntY?D^Tx?d<#(b{z21qi>=&T6fdnsi>mU+wl0_oJXEISATy0qar={fJf6Upf> zkcO6X#v@#otNSBl&nZZmoO$??u*zgKBaJdn57^&M2wWIFt2cfFL+8+x5~!J-PN*As zgL0NJANNCAzuGh*azK+zkteSdnVU8+GgsPwKyOGM*>WKwl6L@a^q^=iMYRY>m1=#I z-Ao(bIxF5$07Yn7EQtwhjO2(ireIi<0fu4N=0tvtQa9R5YFbVv<4m^@SIwJb8@fy3 zZ68((esJlf1lag!XSMc-*DI@Jjh%R;(I_IYpl3Mijx#{6!vC1U!ess}4Tmxz}2~ z*>)^5QJkQ2@fF0^UlC)SVbrZ5VdktLb7D+{$`RpBpxy8+xMl+Nz$!0lLYY6J!M4s3 z&pVybD%ETqE!l1>xxg{tHcFa82U>gzw{cJJ$hIzC%Vzbi*kvEBn2r^I@{+wRh{sQl z(fW{rNi&83$FEs6%9mNKLK`bUS@*denV7lOz%mZLN~x{6?wDE5=Dmo6NzU{0ci!TZ zWTfi#S%+|}sXOEd zEs@CEVql2%qiN};bZABvY)#5}=5FwJ-nCuPU4x%g&G^lFdqGrA-nm?eXA;jcC%%J% zl0z`kH0;@3I=Oa`JaXAj?k;dV+8$94ec7Rs?6y&(S1`9-&R4W5rAFFM3_G`kz>ila zjWvN{1p}CmkU_~O>*5oYASzzNKyVc~>lRsMVSZ&61k4h)h zsLCK|mA3#E6(Te?``U%J^=m2?kJ<$fbd)coxGsu-b?Y*VV>4CWgC;OL?78Z9D_VO+ zmh!tj%z{zzxc4oQ3Y)mZB+j3((5mzTNX;-a`RG!7Z&xC?kWt=pD%vE*HHQfj0p)tjBji zY+vu(xfSvA1f`$M zi2>fe{^rx2Bd?eoYxsyVx4U~K3Exh}lchMRn$qf#fZAp>Qi|WQ914&`w zff@k2zR1%aLTcpM{niB@AwYA$;6UO9OXh_*m;fkaAR(}&GxT#218tny z2Ro0I1cPLVl?;NcijWkAtcsG10b7tH$AnzPj#Vt8#e>DJWod1x2$+}|n1{6aBsH+- zk@2$fl}t|8hsPUV5wJu4B?2?Th!HV<_Ezpo45J5#?W!=J+{JDgvd+v@tWVnP)U+o% z#zx;A=$c9#Mr*0AR*Cm$#$IdbD(vw-3R3lFfxXe%b6Rut zqg*F*4!`an#{Sikt-iX~lrmmSKS*u2g9Bb%nCWL(Z3m7Ian?6<>TUaQh&rR*28miIq?+TikFSZ%hgZR0*JbWta0XN3w#3!YG19Q>V{?e!J<(UodJ7W3KXSE+x=!o1Y%2A+A;Ec z79Uo$f&|7usZCdYZ={s8s|OYzHF{q`w3ZO;nr55{F7iQFV_W9PSB!1Shv;Xm7 zfwwya$+3Fq)3QNEF8p zoX2-wY>YqkE|x&& zKSQM0loRziVU!{Bz}X0rW->?-P!Uk+=sWnlRL)T0Wy3`EuDzn8WvFFw=N^qq*QZ?` zze`Xbn)oFB!&xlkGoGbq+aGN8e0*&WH+PHnAYZ_F;J_g`37ZpdX{%eM$BHPUOd*+#s&J!D|d~Vh&@{JAQ~{_A6j1h6qgXl@P?j=*W{I zkVQ$qNKMu~h-r=Nu11Y$?61hHD9B#qDB_2ngHZpTbw%`*f9#_t@dy>l#ep#PwFHtK zx@K1#dhYT8L;)n`S_CfAKFqk=3M7%X7Eo7@Pq9q*u)ekFM3F!#BsqC!lPEC5E(Z{+ zP1skWqAin%%&mEZzQFENxUUN1P%bKoacvnG%%SVjJP9+3)hHL`U4->5KBQq`a+f!7 zAeR;@cFn%j;W|;8%{?B5;bpNZqXAdqUJ*rAZul^hYKy99&qs0(WX!PIy~hBYpa8gj zXWar`Vv;6M)sPCWhwdDtJAF&AEJRi76e@6`<9ZAsXz+FiC$*BSEWFi=!RIu@f1%Vz z_YpU|qTOAx%e7$2JjAAg#M0}60S3hp#qg0YFf^`$i{LnpZ2IZUxT%d_UtSKt9q`ts zfbrDE7Ve~nS&q8)%3`u{<8CPzVgF1#RXM8cIJVWcoxJvQ)2ByOf=#F8l4n(juvvLr zdQ06pn@r?mk3I{@a>+hG4@<1}j6?N1onSG)`Cf55>BX*8w_cN7h#Kj7^+U4tw@EM; z#9yEg0v@772<*CHW+#}vnWKE7w;Xq+!%E+oAWV@=u@Vfu@b|gD!^Z85ZtuTjz4i^$ zNFZi@Lh%gVlG|kr^W3HATNzfs@r!yQ(AT{L>}p&-25pdBu(pRk#eMXTZ=Z6&(=z0a z?g+dj4>%)@qxeYqMUvCMcQZary#K(#871zLXZe^oID|1!S1jcP`d?>KgfV&m&+9S8 zXB-d^roW#_e^c=fWF_tLU9rgo)mv1VnpIm~oaN7+14Q{y*P9q)~_ z>T0!>i>wSAv@ZaE9&SQ4e!mDJJaN{M>>t4mBKs34YoE!VpJdM@`nbYfMOrG%sp)ND^jt zfiaalW*Lz{;n*mG;sBOF4x9QNysV|B-~z-08)7;o%7U}BB&Ct~bOaUT1f5x-gz!YT z{o1m835`gfuGr@8$8)!~CTWRSJrnGyc#^m|%V|C}%;@RC)t3D{BWZ{U_}j6_`|a#AEdg5JI(1%E zhC#HD)Uc(3Y)J^0Z(+>K$%5c(WkpJ( z_B2VSpM*{^BkUUG7B%09EbQMqqHtFWi?9>net=OoP6?W)Ti6s`@Ir}wVABTgFJXM@k%^*VrN1tdnIiO?AUk`?)bR+DEabRrF|y2ea+NOyyXZ+(v_ z$st#|T~y-EG43J{1L;mMyPJ$;pW+O%w%P(UAqsbAeni^sWIq*T77%U@tRAk~rH;M- zQnXoiiegS5DvL_dd)GKOA+VUFa@z^TaY(-Zcs$R<)S^zGBKU^d+luAJLK&wfil7N5 z`>?1=m-myZLg{x+dI5F!NwKFy+ei!(K(DUGmt5gR)AVBDZ!pQWU~Es7%+|%OD=yl) zk%=A-`5}&wT3y{>up{(px}lY@f-A!%Um>b>`LW0eY1htEQFj`d{R00Cm zbS~BPd(gi#z7f=Lca{%NVdp<|ADyt*a1^jJr2C!@A6SkE`C-BxPT1?p4)Y@6Ah|Sl zJRmDk4z34r^^-TS2^Yu+a zMVr6`nmak;;uqKP-M;(m@!i7qTdGrrLED6pwSXa2A0h)oCu!%nqI=kP!glv)zS=fL zv7k*hBkA#1ngLgugH9&T>P$(=GRWoka73lb%csX?VY~b$qIB;n-lIgwl`Y3RDT{to z)h9x~+l)QQknFm*ZXJ*LNXIqYa(C!!IVs*&6_Ts3Fqybx$huo1>*?wu_f3TPJv+07 znXbLe=7_Ybo`Da3-jWGzN_on&cJ70w z&Iev^z_}V{0ZI}Zhdg`Nj-NA<2~N!QMwR?UrawTSYJfu_hN)aDq!V$SVZ+|grY0T0 zLyCf#tOiH3%?C#L#QTj>crjmO!apIeJis5e7%#HknSMR91eDq*nFA!J4|2M9jsskm zb>nfQV0Qv17G*%rod?@m^Jsj6q8&)5_&OrcA(Mw0)z+8@H%WVD5}wv1Sf47pVGG-M zA1dn3bW>@^-Wad9O#~ub*kxr+7*EKFfa}}$^{p~W6$N*pi?>#1=P;*c&IUe`)A`4( zPT94s)#uYOP#*wW_o&RQ9qRjQrl zn!6Cm$nTcEx9S$aSjwQ=WC&w>_q(jaDTX+P~U zP%Y@_I#iC9?uBCD0_7!b%i88^r@SSt#;|KbNe7DDM7*$P_sUm9KlQ@o_d^bE1?ZR& zH+IX`4Pr|#8M~8;)W0>{b9VM^l-1u_CY#WabR* z*orkeHe?C(d|~up^S}W8N2^it98Yo@5MF|AY(=xFlKTaD@Sc&om-i z1E#f$X~UWIRKjZgunV-+ZeugIZNJn)tMK?FVXeoHf%wZ;E<-y3tzGB(6dR6>io%$` zXcx}2TSs9Lh1H{DbG&VqY(_SEYhA7Go&LHw$r&d4i{`kCt{uXDzck}C^>{*}*xj@Z zB9!BPj1wq>3yU`j=68a*s6cn>MQf&JOe%Yq9G50g&7gp%D8I9#GMjz71QD4LehZ-2hDnh}3sQsZq3= zRPw!WEYNxd==2iE>XG`oG_#uYOmCZq^WUC!dpVMO4dk@Qu0rdbnh*I!T=~O+6Ggly zk9pq%#~*u9kcwBy^GI}DW)C}+;>=@l4_eF%yMtP*@<@6RLj4?}97+A0u3Sz%k%w_z zv_qTXIm;lkmwU@VIgpfFi1DrH;0j$*PA2+NB~>}pBP@OxTT_`&98IWiN@-xIkKi>E zRNKjm*h;&{v^rdRxaaT_16-Ie_|r2Mrnj7kno_D$GI9?I|ZWTqaMif^;i)(oYUf>pFp+QxKA5)$;LEKR#(CC?grQpKluOwe~G8%lD0C#O5LqRbQ4Odbc^2esYzt@kZL5 zREyEob;z-a<)#BKBzuU*HH`Cwl_nh(E4?TN+9K*CIb)_+XpgeZgs7o(bDW2<2}^Ck zi~y6vl=FpUeUdQyGgi41!q6G;P5$Nz*DdAA(+RUe7e!zLevOY!1avP4LH*W}JfC6B zq#jrAP-S1?i?gLRb#*EufV&g0Md#7WypR9|2|y@C3Re~cV`ft1ikd-;RtnFfdRr*C zlzCV=Js0Y@JY&&7!AI>|KR|3a5NS&@Fk)-k`A`DsMcv~QT+n_7ih-UE@s!{)@p=*U zVZ^qXG%%#?+T-zpniefeO%o4ecagJk)CI2-BKQU+-p7bP!Q1w)Y!@$Y@9A7!Q=AlZ zy&s~C!X5&?*Qt6DJ z)xiH<)zKK+9J!7dftk^fR(A+d$Uy-RPK0(;z@~zvGK1NgpBt8iI+!6JgWj@NC{qN& z9i*wrBeJ%7wTNQ|o7SthI)pIVkXq6R$*@fu_VNc?@=yLAjOcgWv1IeLfY&W1xYs!; z{yHarqaXfa6#g&NKnHtUCJ$Si6jddK6=oE_iw_YM9fNk-4f&%uEam7@%qV9i9j$R$ zcxZfedT^QFHrziZy$h+Ce^lM@-Pk*Co1#klAVJgtV^XY+y#paPn4qXzstm;5A+EH) zdbm%Fld`-`+GGN2sKef=cHSe{3F?3e)f~B3C$VLHa_|qInue%|DcC=tz0si8S@!FC z`#pnNmO2UIrL|2@G&E>HS!)ObTdy_6`J}!%lkH%I|KZ`#AVMQ6%<$9^X3_+Xm^@To z;s@KIuy5R^zKv==YoYs|oJ>B3lc)`1k6f7y+sMs?|Dt4)Scz}|k6 z;<@u`3<&p2%np6->3MtCyO_+y|OCb-NM2pMed*m|BVipkI6a5**LQhI%T(wY!E zW**~iWwmrV&RY-Ujq^$C$y)ZK%Ffy`x(FFlXA_F~OiagLJTVXhel#7BgKAEbp~huxNGj}5 z&B?lgyYJ;VjFSLw)`vd(_2u{2}fCw0!R=*x^udOFF_IPTFYSl@{A#&Xv+usH>}$q0w{@5OjmPA zs1|rO!w5LorKvF@hFe)-M^uy~O4Pdm1QRdwu|7b$Njz)bvr@mSzV-@iMF1!m2Ke7I zuwPs7T2MgFM1Fq%bGrTs`*U9Zr2PKQ$o?t(vlInVzkdD`_7{HkUk(2!9`zsaKMdEu zR{vyG|8n(r#7|!JpXH~k^RLyPh(8vQR|fIl<9_n2|13Xo&#%>AxL>U6U-;kY%|Ch9 zf0mzkM$rF+|H*6q7bkvK@{@=CONsHTl7DvC|FN?F-h-dK>|fyce+U0sQ~%l#zx(i$ zr~OOK;D1x|zbwtakB{GZ+y7^*D3J8^ZSe1X{zsqvanSHLEbiY8{7&)sWk3VzZw>rc z!pH9xem@=gWx?-t6#Ta){HL>$-?jg~WBg0|Fy7y4|C{~e-*x{U2>;S8NA Date: Tue, 28 Jul 2026 08:15:36 -0400 Subject: [PATCH 23/62] add username/password docs login --- api/src/main.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/api/src/main.py b/api/src/main.py index aa704d3..e8e1f45 100644 --- a/api/src/main.py +++ b/api/src/main.py @@ -74,6 +74,20 @@ def custom_openapi(): } }, } + security_schemes["OAuth2KeycloakPassword"] = { + "type": "oauth2", + "description": "Login with Keycloak username/password using the password grant", + "flows": { + "password": { + "tokenUrl": config.KEYCLOAK_TOKEN_URL, + "scopes": { + "openid": "OpenID Connect scope", + "profile": "User profile", + "email": "User email", + }, + } + }, + } security_schemes["ApiKeyAuth"] = { "type": "apiKey", "in": "header", @@ -98,6 +112,7 @@ def custom_openapi(): operation.setdefault( "security", [ + {"OAuth2KeycloakPassword": ["openid", "profile", "email"]}, {"OAuth2Keycloak": ["openid", "profile", "email"]}, {"BearerAuth": []}, {"ApiKeyAuth": []}, @@ -108,6 +123,7 @@ def custom_openapi(): auth_me = openapi_schema.get("paths", {}).get("/auth/me", {}).get("get") if auth_me: auth_me["security"] = [ + {"OAuth2KeycloakPassword": ["openid", "profile", "email"]}, {"OAuth2Keycloak": ["openid", "profile", "email"]}, {"BearerAuth": []}, {"ApiKeyAuth": []}, @@ -116,6 +132,7 @@ def custom_openapi(): auth_keys_get = openapi_schema.get("paths", {}).get("/auth/api-keys", {}).get("get") if auth_keys_get: auth_keys_get["security"] = [ + {"OAuth2KeycloakPassword": ["openid", "profile", "email"]}, {"OAuth2Keycloak": ["openid", "profile", "email"]}, {"BearerAuth": []}, ] @@ -123,6 +140,7 @@ def custom_openapi(): auth_keys_post = openapi_schema.get("paths", {}).get("/auth/api-keys", {}).get("post") if auth_keys_post: auth_keys_post["security"] = [ + {"OAuth2KeycloakPassword": ["openid", "profile", "email"]}, {"OAuth2Keycloak": ["openid", "profile", "email"]}, {"BearerAuth": []}, ] @@ -130,6 +148,7 @@ def custom_openapi(): auth_keys_delete = openapi_schema.get("paths", {}).get("/auth/api-keys/{key_id}", {}).get("delete") if auth_keys_delete: auth_keys_delete["security"] = [ + {"OAuth2KeycloakPassword": ["openid", "profile", "email"]}, {"OAuth2Keycloak": ["openid", "profile", "email"]}, {"BearerAuth": []}, ] @@ -137,6 +156,7 @@ def custom_openapi(): auth_polaris_token = openapi_schema.get("paths", {}).get("/auth/polaris-token", {}).get("post") if auth_polaris_token: auth_polaris_token["security"] = [ + {"OAuth2KeycloakPassword": ["openid", "profile", "email"]}, {"OAuth2Keycloak": ["openid", "profile", "email"]}, {"BearerAuth": []}, ] From a396035e98df1080b2f422407a394be2f1adf145 Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Wed, 29 Jul 2026 17:34:36 -0400 Subject: [PATCH 24/62] setup initial integration tests for polaris --- .gitignore | 2 +- api/src/broker.py | 26 +- docs/access-control-matrix.md | 14 +- .../Dockerfile.jupyter-driver | 2 + .../executor-pod-template.yaml | 0 .../docker => jupyterhub-docker}/garden.yaml | 0 .../ipython_kernel_config.py | 0 jupyterhub-docker/spark_session_utils.py | 1262 +++++++++++++++++ .../manifests/bootstrap-job.yaml | 8 + .../manifests/local-users-configmap.yaml.tpl | 24 +- .../manifests/realm-configmap.yaml.tpl | 34 +- .../manifests/acl-config.yaml.tpl | 52 +- .../polaris-sync-principals-script.yaml | 27 +- scripts/stage_authmanager_jar.sh | 2 +- tests/Dockerfile | 17 + tests/README.md | 130 ++ tests/garden.yaml | 141 ++ tests/keycloak_users_test.py | 101 ++ tests/polaris_auth_token_test.py | 169 +++ tests/polaris_namespace_test.py | 127 ++ tests/polaris_oidc_test.py | 102 ++ tests/polaris_permissions_config_test.py | 204 +++ tests/polaris_roles_test.py | 134 ++ tests/spark_authmanager_test.py | 247 ++++ tests/spark_iceberg_permissions_test.py | 178 +++ tests/spark_permission_test.py | 227 +++ 26 files changed, 3152 insertions(+), 78 deletions(-) rename {jupyterhub/docker => jupyterhub-docker}/Dockerfile.jupyter-driver (97%) rename {jupyterhub/docker => jupyterhub-docker}/executor-pod-template.yaml (100%) rename {jupyterhub/docker => jupyterhub-docker}/garden.yaml (100%) rename {jupyterhub/docker => jupyterhub-docker}/ipython_kernel_config.py (100%) create mode 100644 jupyterhub-docker/spark_session_utils.py create mode 100644 tests/Dockerfile create mode 100644 tests/README.md create mode 100644 tests/garden.yaml create mode 100644 tests/keycloak_users_test.py create mode 100644 tests/polaris_auth_token_test.py create mode 100644 tests/polaris_namespace_test.py create mode 100644 tests/polaris_oidc_test.py create mode 100644 tests/polaris_permissions_config_test.py create mode 100644 tests/polaris_roles_test.py create mode 100644 tests/spark_authmanager_test.py create mode 100644 tests/spark_iceberg_permissions_test.py create mode 100644 tests/spark_permission_test.py diff --git a/.gitignore b/.gitignore index 6c15a71..2bfb397 100644 --- a/.gitignore +++ b/.gitignore @@ -12,5 +12,5 @@ __pycache__/ .python-version *.egg-info project.garden.yml -jupyterhub/docker/teehr-authmanager.jar +jupyterhub-docker/teehr-authmanager.jar spark/docker/teehr-authmanager.jar \ No newline at end of file diff --git a/api/src/broker.py b/api/src/broker.py index f60932f..9380558 100644 --- a/api/src/broker.py +++ b/api/src/broker.py @@ -260,8 +260,24 @@ async def exchange_token_for_polaris_via_broker_session( if maybe_new_refresh_token: await store.update_refresh_token(delegated_session_id, maybe_new_refresh_token) - return await exchange_token_for_polaris( - subject_token=refreshed_subject_token, - audience=record["audience"], - requested_ttl_seconds=requested_ttl_seconds, - ) + # Return the refreshed subject token directly — it preserves the user's + # group claims and is accepted by Polaris for per-user permission enforcement. + # Token exchange with a different audience would strip group claims and + # prevent Polaris from mapping the user to their correct principal roles. + trace_id = uuid.uuid4().hex + expires_at_epoch_seconds = int(time.time()) + clamp_requested_ttl(requested_ttl_seconds) + try: + token_claims = jwt.get_unverified_claims(refreshed_subject_token) + exp = int(token_claims.get("exp", 0)) + if exp > 0: + expires_at_epoch_seconds = exp + except (JWTError, ValueError, TypeError): + pass + + return { + "access_token": refreshed_subject_token, + "token_type": "Bearer", + "expires_in_seconds": max(expires_at_epoch_seconds - int(time.time()), 1), + "expires_at_epoch_seconds": expires_at_epoch_seconds, + "trace_id": trace_id, + } diff --git a/docs/access-control-matrix.md b/docs/access-control-matrix.md index 52d1852..0c0b4df 100644 --- a/docs/access-control-matrix.md +++ b/docs/access-control-matrix.md @@ -20,7 +20,8 @@ Source: [keycloak-bootstrap/manifests/realm-configmap.yaml.tpl](../keycloak-boot | Group | Realm role(s) granted | Extra client roles | |---|---|---| | basic-user | basic-user | None | -| iceberg-user | iceberg-user | None | +| teehr-read-only | teehr-read-only | None | +| teehr-read-write | teehr-read-write, teehr-read-only | None | | jupyter-user | jupyter-user | None | | jupyter-admin | jupyter-user | None | | key-management-admin | admin | None | @@ -39,23 +40,24 @@ Source: [keycloak-bootstrap/manifests/realm-configmap.yaml.tpl](../keycloak-boot | JupyterHub admin privileges | JupyterHub Authenticator admin_groups | Group: jupyter-admin | Users in jupyter-admin | | Keycloak admin console link in TEEHR admin page | TEEHR frontend admin visibility | Role: admin | Users with admin role (same as TEEHR admin UI) | | Keycloak admin console capabilities | Keycloak permissions | Role and client roles | Intended primary group appears to be webapi-admin due to realm-management client roles | -| Iceberg or Trino end-user auth | Not fully wired in this repo | No clear active user-facing Keycloak gate | Undetermined from current implementation | +| Iceberg or Trino end-user auth | Polaris principal roles via JWT realm roles | Realm roles: teehr-read-only, teehr-read-write, admin | Users default to teehr-read-only; teehr-read-write and admin add broader access | ## Important Notes 1. key-management-admin and prefect-admin both grant the admin realm role, so both can administer API keys in the current API and frontend implementation. 2. Prefect is intentionally stricter: it checks membership in the prefect-admin group directly, not just the admin role. 3. JupyterHub authorization is also group-based, not based on the admin realm role. -4. Iceberg REST auth rollout is deferred per plan and appears not fully enforced for end-user role mapping yet. +4. Polaris access is now centered on a single `iceberg.teehr` namespace with default read-only access for all users. ## Local Test Users -Local environments now seed two Keycloak users automatically via a local-only bootstrap job: +Local environments now seed three Keycloak users automatically via a local-only bootstrap job: | User type | Default username | Default password | Group membership | |---|---|---|---| -| Admin test user | admin | admin |basic-user, iceberg-user, jupyter-admin, key-management-admin, prefect-admin, webapi-admin | -| Regular test user | user | user |basic-user, jupyter-user | +| Admin test user | admin | admin |basic-user, teehr-read-only, teehr-read-write, jupyter-admin, key-management-admin, prefect-admin, webapi-admin | +| PowerUser test user | poweruser | poweruser |basic-user, teehr-read-write, jupyter-user | +| Regular test user | user | user |basic-user, teehr-read-only, jupyter-user | To add more personas as permissions evolve, update user entries and group assignments in `keycloak-bootstrap/manifests/local-users-configmap.yaml.tpl`. diff --git a/jupyterhub/docker/Dockerfile.jupyter-driver b/jupyterhub-docker/Dockerfile.jupyter-driver similarity index 97% rename from jupyterhub/docker/Dockerfile.jupyter-driver rename to jupyterhub-docker/Dockerfile.jupyter-driver index a1e9b02..0dbcdc5 100644 --- a/jupyterhub/docker/Dockerfile.jupyter-driver +++ b/jupyterhub-docker/Dockerfile.jupyter-driver @@ -6,6 +6,7 @@ ARG TEEHR_VERSION USER root ENV DEBIAN_FRONTEND=noninteractive ENV PATH=${NB_PYTHON_PREFIX}/bin:$PATH +ENV PYTHONPATH=/opt/teehr:${PYTHONPATH} # Use HTTPS apt mirrors so builds work when plain HTTP egress is blocked. RUN set -eux; \ @@ -77,6 +78,7 @@ RUN ARCH=$(uname -m) && \ RUN mkdir -p /opt/teehr && chown -R ${NB_USER}:${NB_USER} /opt/teehr COPY executor-pod-template.yaml /opt/teehr/executor-pod-template.yaml +COPY spark_session_utils.py /opt/teehr/spark_session_utils.py # Keep driver and executor classpaths aligned for the custom AuthManager. COPY teehr-authmanager.jar /opt/spark/jars/teehr-authmanager.jar diff --git a/jupyterhub/docker/executor-pod-template.yaml b/jupyterhub-docker/executor-pod-template.yaml similarity index 100% rename from jupyterhub/docker/executor-pod-template.yaml rename to jupyterhub-docker/executor-pod-template.yaml diff --git a/jupyterhub/docker/garden.yaml b/jupyterhub-docker/garden.yaml similarity index 100% rename from jupyterhub/docker/garden.yaml rename to jupyterhub-docker/garden.yaml diff --git a/jupyterhub/docker/ipython_kernel_config.py b/jupyterhub-docker/ipython_kernel_config.py similarity index 100% rename from jupyterhub/docker/ipython_kernel_config.py rename to jupyterhub-docker/ipython_kernel_config.py diff --git a/jupyterhub-docker/spark_session_utils.py b/jupyterhub-docker/spark_session_utils.py new file mode 100644 index 0000000..1141c5c --- /dev/null +++ b/jupyterhub-docker/spark_session_utils.py @@ -0,0 +1,1262 @@ +"""Module to create and configure Spark sessions and Polaris auth helpers.""" +# flake8: noqa +import base64 +import json +import logging +import os +import socket +import time +from pathlib import Path +from typing import Dict, List, Optional, Tuple, Union +from urllib.parse import urlsplit, urlunsplit + +import psutil +import requests + +from pyspark import SparkConf +from pyspark.sql import SparkSession +from sedona.spark import SedonaContext +import pandas as pd +import botocore.session + +import teehr.const as const + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# Note: Scala version: 2.13 in pyspark 4.0 +SCALA_VERSION = "2.13" +PYSPARK_VERSION = "4.0" +ICEBERG_VERSION = "1.10.1" +SEDONA_VERSION = "1.8.0" + + + +def create_spark_session( + # App name and catalog settings + app_name: str = "TEEHR Evaluation", + local_catalog_name: str = const.LOCAL_CATALOG_NAME, + local_catalog_type: str = const.LOCAL_CATALOG_TYPE, + remote_warehouse_dir: str = const.REMOTE_WAREHOUSE_S3_PATH, + remote_catalog_name: str = const.REMOTE_CATALOG_NAME, + remote_catalog_type: str = const.REMOTE_CATALOG_TYPE, + remote_catalog_uri: str = const.REMOTE_CATALOG_REST_URI, + # Spark K8'specific parameters + start_spark_cluster: bool = False, + force_recreate_session: bool = False, + executor_instances: int = 2, + executor_memory: str = "1g", + executor_cores: int = 1, + executor_image: str = None, + executor_namespace: str = None, + driver_memory: str = None, + driver_max_result_size: str = None, + pod_template_path: Union[str, Path] = const.POD_TEMPLATE_PATH, + # AWS credential parameters + aws_access_key_id: str = None, + aws_secret_access_key: str = None, + aws_session_token: str = None, + aws_region: str = const.AWS_REGION, + aws_profile: str = None, + # GCS credential parameters + enable_gcs: bool = False, + gcs_project_id: str = None, + gcs_service_account_key_file: str = None, + # Simple extensibility parameters + add_jars: List[str] = None, + add_packages: List[str] = None, + update_configs: Dict[str, str] = None, + debug_config: bool = False +) -> SparkSession: + """Create and return a Spark session for evaluation. + + Parameters + ---------- + app_name : str + Name of the Spark application. Default is "TEEHR Evaluation". + local_catalog_name : str + Name of the local Iceberg catalog. Default is "local". + local_catalog_type : str + Type of the local Iceberg catalog. Default is "jdbc". + remote_warehouse_dir : str + Remote warehouse directory for Iceberg catalog. Default is TEEHR + warehouse S3 path. + remote_catalog_name : str + Name of the remote Iceberg catalog. Default is "iceberg". + remote_catalog_type : str + Type of the remote Iceberg catalog. Default is "rest". + remote_catalog_uri : str + URI for the remote Iceberg catalog. Default is TEEHR catalog REST URI. + start_spark_cluster : bool + Whether to start a Spark cluster (Kubernetes mode). + Default is False (local mode). + force_recreate_session : bool + Whether to stop an existing Spark session before creating a new one. + Default is False. + executor_instances : int + Number of executor instances for the Spark cluster. Default is 2. + executor_memory : str + Memory allocation for each executor. Default is "1g". + executor_cores : int + Number of CPU cores for each executor. Default is 1. + executor_image : str + Container image for Spark executors. Default is None. + executor_namespace : str + Kubernetes namespace for Spark executors. Default is None. + driver_memory : str + Memory allocation for the Spark driver. Default is None. + driver_max_result_size : str + Maximum result size for the Spark driver. Default is None. + pod_template_path : Union[str, Path] + Path to the pod template file for Spark executors. + Default is "/opt/teehr/executor-pod-template.yaml". + aws_access_key_id : str + AWS access key ID for S3 access. Default is None. + aws_secret_access_key : str + AWS secret access key for S3 access. Default is None. + aws_session_token : str + AWS session token for temporary credentials. Default is None. + aws_region : str + AWS region name. Default is "us-east-2". + aws_profile : str + AWS profile name to use from ~/.aws/credentials. Only reads credentials + file if this parameter is explicitly provided. Default is None. + enable_gcs : bool + Whether to add GCS (Google Cloud Storage) connector support. + Default is False. + gcs_project_id : str + GCS project ID. Used for billing and quota tracking. When accessing + public buckets without credentials, set to "anonymous" or any + non-empty string. Default is None (will be set to "anonymous" when + enable_gcs is True and no service account key is provided). + gcs_service_account_key_file : str + Path to a GCS service account JSON key file. When provided, + authenticated access is used. When None, unauthenticated + (public-bucket) access is used. Default is None. + add_packages : List[str] + Provided Spark packages will be added if they do not already exist. + Default is None. + >>> add_packages=["com.example:my-package:1.0.0"] + add_jars : List[str] + Provided local jar paths will be added if they do not already exist. + Default is None. + >>> add_jars=["/opt/spark/jars/teehr-authmanager.jar"] + update_configs : Dict[str, str] + Provided Spark configurations will be added if they do not already + exist, or overwritten if they do exist. Default is None. + >>> update_configs={"spark.sql.shuffle.partitions": "100"} + debug_config : bool + Whether to log the final Spark configuration for debugging. + Default is False. + + Returns + ------- + SparkSession + Configured Spark session. + """ + logger.info(f"🚀 Creating Spark session: {app_name}") + + if force_recreate_session: + existing_session = SparkSession.getActiveSession() + if existing_session is not None: + logger.info("♻️ Stopping the active Spark session before recreation") + existing_session.stop() + + # Get the base configuration with common settings + conf = _create_spark_base_session( + conf=SparkConf(), + aws_region=aws_region, + driver_memory=driver_memory, + driver_maxresultsize=driver_max_result_size + ) + + if start_spark_cluster is False: + logger.info("✅ Spark local configuration successful!") + else: + logger.info(f"📦 Configuring Spark cluster with container image: {executor_image}") + _set_spark_cluster_configuration( + conf=conf, + executor_instances=executor_instances, + executor_memory=executor_memory, + executor_cores=executor_cores, + container_image=executor_image, + spark_namespace=executor_namespace, + pod_template_path=pod_template_path + ) + logger.info("✅ Spark cluster configuration successful!") + logger.info(f" - Executor instances: {executor_instances}") + logger.info(f" - Executor memory: {executor_memory}") + logger.info(f" - Executor cores: {executor_cores}") + + # Set AWS credentials if available + _set_aws_credentials_in_spark( + conf=conf, + remote_catalog_name=remote_catalog_name, + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + aws_session_token=aws_session_token, + aws_region=aws_region, + aws_profile=aws_profile, + ) + + # Set GCS configuration if available + if enable_gcs: + _set_gcs_configuration( + conf=conf, + gcs_project_id=gcs_project_id, + gcs_service_account_key_file=gcs_service_account_key_file, + ) + + # Set catalog metadata in Spark configuration + _set_catalog_metadata( + conf=conf, + local_catalog_name=local_catalog_name, + local_catalog_type=local_catalog_type, + remote_catalog_name=remote_catalog_name, + remote_catalog_type=remote_catalog_type, + remote_catalog_uri=remote_catalog_uri, + remote_warehouse_dir=remote_warehouse_dir + ) + + # Apply catalog configurations + _configure_iceberg_catalogs( + conf=conf, + local_catalog_name=local_catalog_name, + local_catalog_type=local_catalog_type, + remote_warehouse_dir=remote_warehouse_dir, + remote_catalog_name=remote_catalog_name, + remote_catalog_type=remote_catalog_type, + remote_catalog_uri=remote_catalog_uri + ) + + # Update configs and packages if provided + _update_configs_and_packages( + conf=conf, + update_configs=update_configs, + add_jars=add_jars, + add_packages=add_packages + ) + + logger.info("⚙️ All settings applied. Creating Spark session...") + spark = SparkSession.builder.appName(app_name).config(conf=conf).getOrCreate() + sedona_spark = SedonaContext.create(spark) + + if debug_config: + log_session_config(sedona_spark) + + logger.info("🎉 Spark session created successfully!") + + return sedona_spark + + +def _create_spark_base_session( + conf: SparkConf, + aws_region: str, + driver_memory: float = None, + driver_maxresultsize: float = None +): + """Create a base Spark builder.""" + conf.setMaster("local[*]") + + # Set base packages + base_packages = [ + f"org.apache.sedona:sedona-spark-shaded-{PYSPARK_VERSION}_{SCALA_VERSION}:{SEDONA_VERSION}", + f"org.apache.iceberg:iceberg-spark-runtime-{PYSPARK_VERSION}_{SCALA_VERSION}:{ICEBERG_VERSION}", + f"org.apache.iceberg:iceberg-core:{ICEBERG_VERSION}", + "org.datasyslab:geotools-wrapper:1.8.0-33.1", + f"org.apache.iceberg:iceberg-spark-extensions-{PYSPARK_VERSION}_{SCALA_VERSION}:{ICEBERG_VERSION}", + "org.apache.hadoop:hadoop-aws:3.4.1", # Note. Need 3.4.1 for compatibility + "com.amazonaws:aws-java-sdk-bundle:1.12.791", + "org.xerial:sqlite-jdbc:3.42.0.0" + ] + conf.set("spark.jars.packages", ",".join(base_packages)) + + # Set configurations + conf.set("spark.driver.extraJavaOptions", f"-Daws.region={aws_region}") + conf.set("spark.executor.extraJavaOptions", f"-Daws.region={aws_region}") + conf.set("spark.sql.session.timeZone", "UTC") + conf.set("spark.sql.extensions", "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions") + conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer") + conf.set("spark.hadoop.fs.s3a.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem") + + # Enable Spark decommissioning for graceful spot instance handling + conf.set("spark.decommission.enabled", "true") + conf.set("spark.storage.decommission.enabled", "true") + conf.set("spark.storage.decommission.rddBlocks.enabled", "true") + conf.set("spark.storage.decommission.shuffleBlocks.enabled", "true") + # Grace period for executors to decommission before being terminated + conf.set("spark.kubernetes.executor.decommission.gracePeriodSeconds", "30") + + # Memory settings + memory_info = psutil.virtual_memory() + driver_memory_int = int(0.75 * memory_info.available / (1024**3)) + if driver_memory is None: + driver_memory = f"{driver_memory_int}g" + if driver_maxresultsize is None: + driver_maxresultsize = f"{int(0.5 * driver_memory_int)}g" + conf.set("spark.driver.memory", f"{driver_memory}") + conf.set("spark.driver.maxResultSize", f"{driver_maxresultsize}") + + # Driver binding configuration for local mode + conf.set("spark.driver.bindAddress", "127.0.0.1") + conf.set("spark.driver.host", "127.0.0.1") + conf.set("spark.driver.port", "0") # Let Spark choose an available port + + # Default shuffle partitions: 2x local CPU cores + local_cores = os.cpu_count() or 1 + default_shuffle_partitions = 2 * local_cores + conf.set("spark.sql.shuffle.partitions", str(default_shuffle_partitions)) + + return conf + + +def _set_spark_cluster_configuration( + conf: SparkConf, + executor_instances: int, + executor_memory: str, + executor_cores: int, + container_image: str, + spark_namespace: str, + pod_template_path: Union[str, Path] +): + """Configure Spark for Kubernetes cluster mode. + + These settings are considered immutable after session creation. + """ + # Default container image - use the same image as the current pod + if container_image is None: + container_image = os.environ["TEEHR_SPARK_IMAGE"] + + # Get Kubernetes API server - use HTTPS port specifically + k8s_host = os.environ.get('KUBERNETES_SERVICE_HOST', 'kubernetes.default.svc.cluster.local') + k8s_port_https = os.environ.get('KUBERNETES_SERVICE_PORT_HTTPS', '443') + k8s_api_server = f"https://{k8s_host}:{k8s_port_https}" + + # First try getting it from environment variable + if spark_namespace is None: + spark_namespace = os.environ.get("TEEHR_NAMESPACE", "") + logger.info(f"🔍 Initial spark namespace from ENV: {spark_namespace}") + + if spark_namespace is None: + # Then get it from here + namespace_file = "/var/run/secrets/kubernetes.io/serviceaccount/namespace" + if os.path.exists(namespace_file): + with open(namespace_file, 'r') as f: + spark_namespace = f.read().strip() + + # Finally get it here if still None + if spark_namespace is None: + spark_namespace = "default" # last resort, will probably fail + + logger.info(f"🔍 Connecting to Kubernetes API: {k8s_api_server}") + logger.info(f"🎯 Executor namespace: {spark_namespace}") + logger.info(f"🔐 Executor service account: spark (in {spark_namespace})") + + # Create Spark configuration + conf.setMaster(f"k8s://{k8s_api_server}") + + # Check project ID to specify appropriate node group name. + teehr_project_id = os.environ.get("TEEHR_PROJECT_ID", "") + if teehr_project_id != "TEEHR" and teehr_project_id != "": + conf.set( + "spark.kubernetes.executor.node.selector.teehr-hub/nodegroup-name", + f"spark-r5-4xlarge-spot-{teehr_project_id.lower()}" + ) + + # Basic Kubernetes settings + conf.set("spark.executor.instances", str(executor_instances)) + conf.set("spark.executor.memory", executor_memory) + conf.set("spark.executor.cores", str(executor_cores)) + conf.set("spark.kubernetes.container.image", container_image) + conf.set("spark.kubernetes.namespace", spark_namespace) + conf.set("spark.kubernetes.authenticate.executor.serviceAccountName", "spark") + conf.set("spark.kubernetes.container.image.pullPolicy", "Always") + + # Enable Spark's dynamic allocation and decommissioning features for better + # handling of spot instances and resource management. + conf.set("spark.decommission.enabled", "true") + conf.set("spark.executor.decommission.signal", "SIGTERM") + conf.set("spark.storage.decommission.enabled", "true") + + # Set pod name prefix for executors in TEEHR-HUB for easy identification in cluster. + # Truncated to 46 chars since K8s pod names have a 63-char limit and Spark appends suffixes. + jupyterhub_user = os.environ.get("JUPYTERHUB_USER", None) + if jupyterhub_user is not None: + conf.set("spark.kubernetes.executor.podNamePrefix", jupyterhub_user[:46]) + + if os.path.exists(pod_template_path): + conf.set("spark.kubernetes.executor.podTemplateFile", pod_template_path) + else: + logger.info(f"⚠️ Executor pod template not found: {pod_template_path}") + logger.info(" You must provide a valid pod template for executors to launch correctly.") + raise FileNotFoundError(f"Executor pod template not found: {pod_template_path}") + + conf.set("spark.kubernetes.executor.deleteOnTermination", "true") + + # Default shuffle partitions: 2x total executor cores + total_executor_cores = executor_cores * executor_instances + default_shuffle_partitions = 2 * total_executor_cores + conf.set("spark.sql.shuffle.partitions", str(default_shuffle_partitions)) + + # Authentication - use service account token if available + token_file = const.SERVICE_ACCOUNT_TOKEN_PATH + ca_file = const.CA_CERTIFICATE_PATH + if os.path.exists(token_file) and os.path.exists(ca_file): + logger.info("🔐 Using in-cluster authentication") + conf.set("spark.kubernetes.authenticate.submission.oauthTokenFile", token_file) + conf.set("spark.kubernetes.authenticate.submission.caCertFile", ca_file) + conf.set("spark.kubernetes.authenticate.driver.oauthTokenFile", token_file) + conf.set("spark.kubernetes.authenticate.executor.oauthTokenFile", token_file) + + # Critical: Set the CA cert file for SSL validation + conf.set("spark.kubernetes.authenticate.caCertFile", ca_file) + else: + logger.info("⚠️ No service account tokens found - may have authentication issues") + logger.info(f" Checked: {token_file}") + logger.info(f" Checked: {ca_file}") + + # Driver binding configuration - use pod IP for Kubernetes + conf.set("spark.driver.bindAddress", "0.0.0.0") + conf.set("spark.driver.port", "0") # Let Spark choose an available port + + # Get pod IP and set as driver host so executors can connect back + pod_ip = os.environ.get('POD_IP') + if not pod_ip: + try: + hostname = socket.gethostname() + pod_ip = socket.gethostbyname(hostname) + except: + pod_ip = None + + if pod_ip: + logger.info(f"🔗 Setting driver host to pod IP: {pod_ip}") + conf.set("spark.driver.host", pod_ip) + else: + logger.info("⚠️ Could not determine pod IP - using default driver host") + + return + + +def _set_aws_credentials_in_spark( + conf: SparkConf, + remote_catalog_name: str, + aws_access_key_id: str, + aws_secret_access_key: str, + aws_session_token: str, + aws_region: str, + aws_profile: str = None, +): + """Set AWS credentials in Spark configuration with multiple options.""" + logger.info("Setting Hadoop's default AWS credentials provider and AWS region") + conf.set( + "spark.hadoop.fs.s3a.aws.credentials.provider", + "com.amazonaws.auth.DefaultAWSCredentialsProviderChain" + ) + conf.set("spark.hadoop.fs.s3a.endpoint.region", aws_region) + + # Priority 1: Explicit credentials provided by user + if aws_access_key_id and aws_secret_access_key: + logger.info("🔑 Using user-provided AWS credentials") + conf.set(f"spark.sql.catalog.{remote_catalog_name}.s3.access-key-id", aws_access_key_id) + conf.set(f"spark.sql.catalog.{remote_catalog_name}.s3.secret-access-key", aws_secret_access_key) + conf.set("spark.hadoop.fs.s3a.access.key", aws_access_key_id) + conf.set("spark.hadoop.fs.s3a.secret.key", aws_secret_access_key) + return + + # Priority 2: Explicit token + if aws_session_token: + logger.info("🔑 Using user-provided AWS session token") + conf.set(f"spark.sql.catalog.{remote_catalog_name}.s3.session-token", aws_session_token) + conf.set("spark.hadoop.fs.s3a.session.token", aws_session_token) + return + + # Priority 3: Check ~/.aws/credentials file only if profile explicitly specified (full access) + if aws_profile: + aws_credentials_file = Path.home() / ".aws" / "credentials" + if aws_credentials_file.exists(): + try: + import configparser + config = configparser.ConfigParser() + config.read(aws_credentials_file) + + if config.has_section(aws_profile): + if config.has_option(aws_profile, "aws_access_key_id") and config.has_option(aws_profile, "aws_secret_access_key"): + creds_access_key = config.get(aws_profile, "aws_access_key_id") + creds_secret_key = config.get(aws_profile, "aws_secret_access_key") + creds_session_token = config.get(aws_profile, "aws_session_token", fallback=None) + + logger.info(f"🔑 Using AWS credentials from ~/.aws/credentials profile '{aws_profile}") + conf.set(f"spark.sql.catalog.{remote_catalog_name}.s3.access-key-id", creds_access_key) + conf.set(f"spark.sql.catalog.{remote_catalog_name}.s3.secret-access-key", creds_secret_key) + conf.set("spark.hadoop.fs.s3a.access.key", creds_access_key) + conf.set("spark.hadoop.fs.s3a.secret.key", creds_secret_key) + + if creds_session_token: + conf.set(f"spark.sql.catalog.{remote_catalog_name}.s3.session-token", creds_session_token) + conf.set("spark.hadoop.fs.s3a.session.token", creds_session_token) + return + except Exception as e: + logger.debug(f"Could not read ~/.aws/credentials: {e}") + + session = botocore.session.Session() + credentials = session.get_credentials() + + # Priority 4: Check boto token + if credentials and credentials.token: + logger.info("🔑 Using AWS session token from boto3") + conf.set(f"spark.sql.catalog.{remote_catalog_name}.s3.session-token", credentials.token) + conf.set("spark.hadoop.fs.s3a.session.token", credentials.token) + return + + # Priority 5: Check boto credentials + if credentials and credentials.access_key and credentials.secret_key: + logger.info("🔑 Using AWS credentials from boto3") + conf.set(f"spark.sql.catalog.{remote_catalog_name}.s3.access-key-id", credentials.access_key) + conf.set(f"spark.sql.catalog.{remote_catalog_name}.s3.secret-access-key", credentials.secret_key) + conf.set("spark.hadoop.fs.s3a.access.key", credentials.access_key) + conf.set("spark.hadoop.fs.s3a.secret.key", credentials.secret_key) + return + + # Priority 6: Fall back to anonymous or default provider + logger.info("🔑 Using anonymous AWS credentials for S3 access") + conf.set( + "spark.hadoop.fs.s3a.aws.credentials.provider", + "org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider" + ) + return + +def _set_gcs_configuration( + conf: SparkConf, + gcs_project_id: str = None, + gcs_service_account_key_file: str = None, +): + """Configure Spark for Google Cloud Storage (GCS) access. + + Parameters + ---------- + conf : SparkConf + The Spark configuration object to update. + gcs_project_id : str + GCS project ID for billing/quota. Defaults to "anonymous" when + no service account key is provided (public bucket access). + gcs_service_account_key_file : str + Path to a GCS service account JSON key file. When None, + unauthenticated access is used (suitable for public buckets). + """ + GCS_CONNECTOR_VERSION = "hadoop3-2.2.32" + gcs_package = f"com.google.cloud.bigdataoss:gcs-connector:{GCS_CONNECTOR_VERSION}" + + # Add the GCS connector package + current_packages = conf.get("spark.jars.packages").split(",") + if gcs_package not in current_packages: + current_packages.append(gcs_package) + conf.set("spark.jars.packages", ",".join(current_packages)) + + # Register GCS filesystem implementations + conf.set( + "spark.hadoop.fs.gs.impl", + "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem" + ) + conf.set( + "spark.hadoop.fs.AbstractFileSystem.gs.impl", + "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS" + ) + + if gcs_service_account_key_file: + key_path = Path(gcs_service_account_key_file) + if not key_path.exists(): + raise FileNotFoundError( + f"GCS service account key file not found: {gcs_service_account_key_file}" + ) + logger.info(f"🔑 Using GCS service account key: {gcs_service_account_key_file}") + conf.set("spark.hadoop.google.cloud.auth.service.account.enable", "true") + conf.set( + "spark.hadoop.google.cloud.auth.service.account.keyfile", + str(key_path) + ) + if gcs_project_id: + conf.set("spark.hadoop.fs.gs.project.id", gcs_project_id) + else: + # Unauthenticated access for public GCS buckets + logger.info("🔑 Using unauthenticated GCS access (public buckets)") + # Set both old-style (fs.gs.*) and new-style (google.cloud.*) auth + # properties so the connector picks up UNAUTHENTICATED regardless of + # which configuration namespace it checks first. + conf.set("spark.hadoop.fs.gs.auth.type", "UNAUTHENTICATED") + conf.set("spark.hadoop.google.cloud.auth.type", "UNAUTHENTICATED") + # Explicitly disable service account auth to prevent the connector from + # attempting to contact the GCE metadata server (http://metadata.google.internal/) + # for credentials. Without this, the connector may hang on machines + # that are not running on GCP. + conf.set("spark.hadoop.google.cloud.auth.service.account.enable", "false") + # Enable null (anonymous) credentials — required for the connector to + # accept a configuration with no active credential source. + conf.set("spark.hadoop.fs.gs.auth.null.enable", "true") + conf.set( + "spark.hadoop.fs.gs.project.id", + gcs_project_id if gcs_project_id else "anonymous" + ) + + +def _set_catalog_metadata( + conf: SparkConf, + local_catalog_name: str, + local_catalog_type: str, + remote_catalog_name: str, + remote_catalog_type: str, + remote_catalog_uri: str, + remote_warehouse_dir: str, +): + """Set catalog metadata in Spark configuration.""" + metadata_configs = { + "local_catalog_name": local_catalog_name, + "local_catalog_type": local_catalog_type, + "remote_warehouse_dir": remote_warehouse_dir, + "remote_catalog_name": remote_catalog_name, + "remote_catalog_type": remote_catalog_type, + "remote_catalog_uri": remote_catalog_uri + } + for key, value in metadata_configs.items(): + conf.set(key, value) + logger.debug(f"Metadata config: {key}: {value}") + + +def _configure_iceberg_catalogs( + conf: SparkConf, + local_catalog_name: str, + local_catalog_type: str, + remote_warehouse_dir: str, + remote_catalog_name: str, + remote_catalog_type: str, + remote_catalog_uri: str +): + """Configure Iceberg catalogs through conf.set().""" + logger.info("Configuring Iceberg catalogs...") + # Local catalog configuration + conf.set(f"spark.sql.catalog.{local_catalog_name}", "org.apache.iceberg.spark.SparkCatalog") + conf.set(f"spark.sql.catalog.{local_catalog_name}.type", local_catalog_type) + conf.set(f"spark.sql.catalog.{local_catalog_name}.jdbc.driver", "org.sqlite.JDBC") + conf.set(f"spark.sql.catalog.{local_catalog_name}.jdbc.initialize", "true") + conf.set(f"spark.sql.catalog.{local_catalog_name}.jdbc.schema-version", "V1") + # conf.set("spark.sql.catalog.local.jdbc.user", "user") + # conf.set("spark.sql.catalog.local.jdbc.password", "password") + + # Remote catalog configuration + conf.set(f"spark.sql.catalog.{remote_catalog_name}", "org.apache.iceberg.spark.SparkCatalog") + conf.set(f"spark.sql.catalog.{remote_catalog_name}.type", remote_catalog_type) + conf.set(f"spark.sql.catalog.{remote_catalog_name}.uri", remote_catalog_uri) + conf.set(f"spark.sql.catalog.{remote_catalog_name}.warehouse", remote_warehouse_dir) + conf.set(f"spark.sql.catalog.{remote_catalog_name}.io-impl", "org.apache.iceberg.aws.s3.S3FileIO") + # S3 end point and path style access + if os.environ.get("REMOTE_CATALOG_S3_PATH_STYLE_ACCESS", "false").lower() == "true": + conf.set(f"spark.sql.catalog.{remote_catalog_name}.s3.endpoint", os.environ.get("REMOTE_CATALOG_S3_ENDPOINT")) + conf.set(f"spark.sql.catalog.{remote_catalog_name}.s3.path-style-access", os.environ.get("REMOTE_CATALOG_S3_PATH_STYLE_ACCESS").lower()) + + +def _update_configs_and_packages( + conf: SparkConf, + update_configs: Dict[str, str], + add_jars: List[str], + add_packages: List[str] +) -> Dict[str, str]: + """Update Spark configurations and packages.""" + # Add specified local jars + if add_jars is not None: + current_jars = conf.get("spark.jars").split(",") if conf.contains("spark.jars") else [] + for jar_path in add_jars: + if jar_path not in current_jars: + current_jars.append(jar_path) + if current_jars: + conf.set("spark.jars", ",".join(current_jars)) + + # Add specified packages + if add_packages is not None: + current_packages = conf.get("spark.jars.packages").split(",") + for package in add_packages: + if package not in current_packages: + current_packages.append(package) + conf.set("spark.jars.packages", ",".join(current_packages)) + + # Update or add specified configs + if update_configs is not None: + for key, value in update_configs.items(): + conf.set(key, value) + return + + +def log_session_config(spark: SparkSession): + """Log the current Spark session configuration for debugging. + + Parameters + ---------- + spark : SparkSession + The Spark session whose configuration should be logged. + + Notes + ----- + This function logs all Spark configuration properties to the + logger at INFO level for troubleshooting purposes. + """ + logger.info("Final Spark configuration:") + df = pd.DataFrame(list(spark.conf.getAll.items()), columns=["Key", "Value"]) + gps = df.groupby(by="Key") + for key, group in gps: + value = ",".join(group["Value"].tolist()) + values = value.split(",") + if key.startswith("spark."): + if len(values) > 1: + logger.info(f" {key}: ") + for val in values: + logger.info(f" {val}") + else: + logger.info(f" {key}: {value}") + + +def remove_or_update_configs( + spark: SparkSession, + remove_configs: List[str] = None, + update_configs: Dict[str, str] = None +) -> Dict[str, str]: + """Add, remove, or update Spark configurations. + + Parameters + ---------- + spark : SparkSession + The Spark session whose configuration should be logged. + remove_configs : List[str] + List of configuration keys to remove from the Spark session. + Default is None. + >>> remove_configs=["spark.sql.shuffle.partitions"] + update_configs : Dict[str, str] + Provided Spark configurations will be added if they do not already + exist, or overwritten if they do exist. Default is None. + >>> update_configs={"spark.sql.shuffle.partitions": "100"} + + Notes + ----- + This function allows for dynamic modification of the Spark session's + configuration by removing specified keys and updating or adding new + key-value pairs. + """ + # Remove specified configs + if remove_configs is not None: + for key in remove_configs: + try: + current_value = spark.conf.get(key) + logger.info(f"Removing config: {key} (was: {current_value})") + spark.conf.unset(key) + except Exception as e: + logger.warning(f"Could not remove config {key}: {e}") + + # Update or add specified configs + if update_configs is not None: + for key, value in update_configs.items(): + spark.conf.set(key, value) + return + + +def _decode_jwt_claims(token: str) -> Dict[str, object]: + payload = token.split(".")[1] + payload += "=" * (-len(payload) % 4) + return json.loads(base64.urlsafe_b64decode(payload.encode())) + + +def _token_expires_soon(token: str, refresh_window_seconds: int = 120) -> bool: + try: + claims = _decode_jwt_claims(token) + except Exception: + return True + exp = int(claims.get("exp", 0)) + now = int(time.time()) + return exp <= now + max(refresh_window_seconds, 1) + + +def _request_oauth_tokens( + data: Dict[str, str], + token_endpoint: Optional[str] = None, + timeout_seconds: int = 20, +) -> Tuple[str, Optional[str]]: + endpoint = token_endpoint or os.getenv("POLARIS_OAUTH2_SERVER_URI") + if not endpoint: + raise RuntimeError("POLARIS_OAUTH2_SERVER_URI is required to mint or refresh a user token") + + resp = requests.post(endpoint, data=data, timeout=timeout_seconds) + resp.raise_for_status() + payload = resp.json() + + access_token = payload.get("access_token") + if not access_token: + raise RuntimeError("Token endpoint did not return access_token") + + return access_token, payload.get("refresh_token") + + +def mint_polaris_user_token( + username: Optional[str], + password: Optional[str], + client_id: str, + client_secret: Optional[str] = None, + token_endpoint: Optional[str] = None, +) -> str: + if not username or not password: + raise RuntimeError("username and password are required for password grant token minting") + + data = { + "grant_type": "password", + "client_id": client_id, + "username": username, + "password": password, + "scope": "openid profile email", + } + if client_secret: + data["client_secret"] = client_secret + + access_token, _ = _request_oauth_tokens(data=data, token_endpoint=token_endpoint) + return access_token + + +def refresh_polaris_user_token( + refresh_token: str, + client_id: str, + client_secret: Optional[str] = None, + token_endpoint: Optional[str] = None, +) -> Tuple[str, Optional[str]]: + if not refresh_token: + raise RuntimeError("refresh_token is required for refresh grant") + + data = { + "grant_type": "refresh_token", + "client_id": client_id, + "refresh_token": refresh_token, + } + if client_secret: + data["client_secret"] = client_secret + + return _request_oauth_tokens(data=data, token_endpoint=token_endpoint) + + +def ensure_fresh_polaris_user_token( + current_token: Optional[str], + username: Optional[str], + password: Optional[str], + client_id: str, + client_secret: Optional[str] = None, + refresh_token: Optional[str] = None, + allow_password_fallback: bool = True, + refresh_window_seconds: int = 120, + token_endpoint: Optional[str] = None, +) -> Tuple[str, Optional[str], bool]: + if current_token and not _token_expires_soon(current_token, refresh_window_seconds): + return current_token, refresh_token, False + + if refresh_token: + try: + refreshed_access, refreshed_refresh = refresh_polaris_user_token( + refresh_token=refresh_token, + client_id=client_id, + client_secret=client_secret, + token_endpoint=token_endpoint, + ) + return refreshed_access, (refreshed_refresh or refresh_token), True + except requests.RequestException: + if not allow_password_fallback: + raise + + if allow_password_fallback and username and password: + minted = mint_polaris_user_token( + username=username, + password=password, + client_id=client_id, + client_secret=client_secret, + token_endpoint=token_endpoint, + ) + return minted, refresh_token, True + + raise RuntimeError( + "Unable to obtain a fresh Polaris user token. " + "Provide POLARIS_REFRESH_TOKEN or enable password-grant fallback credentials." + ) + + +def apply_polaris_token_to_spark( + spark, + token: str, + catalog_name: str = "iceberg", + realm: Optional[str] = None, +) -> None: + active_realm = realm or os.getenv("POLARIS_DEFAULT_REALM", "teehr") + base = f"spark.sql.catalog.{catalog_name}" + spark.conf.set(f"{base}.rest.auth.type", "oauth2") + spark.conf.set(f"{base}.token", token) + spark.conf.set(f"{base}.rest.auth.oauth2.token", token) + spark.conf.set(f"{base}.header.X-Polaris-Realm", active_realm) + spark.conf.set(f"{base}.rest.transport.header.X-Polaris-Realm", active_realm) + + +def _as_bool_str(value: str, default: str = "true") -> str: + normalized = (value or default).strip().lower() + return "true" if normalized in ("1", "true", "t", "yes", "y", "on") else "false" + + +def _apply_runtime_spark_configs(spark, configs: Dict[str, str]) -> None: + immutable_after_start = { + "spark.jars", + "spark.driver.extraClassPath", + "spark.executor.extraClassPath", + } + for key, value in configs.items(): + if not key.startswith("spark."): + continue + if key in immutable_after_start: + continue + spark.conf.set(key, value) + + +def _is_http_error_with_status(exc: Exception, status_code: int) -> bool: + if not isinstance(exc, requests.HTTPError): + return False + response = getattr(exc, "response", None) + return bool(response is not None and response.status_code == status_code) + + +def _normalize_internal_broker_url(url: str) -> str: + parsed = urlsplit(url) + if parsed.scheme != "https": + return url + if parsed.hostname != "teehr-api": + return url + + host = parsed.hostname + port = parsed.port or 8000 + netloc = f"{host}:{port}" + return urlunsplit(("http", netloc, parsed.path, parsed.query, parsed.fragment)) + + +def _broker_session_endpoint_from_token_endpoint(token_endpoint: str) -> str: + normalized = _normalize_internal_broker_url(token_endpoint) + parsed = urlsplit(normalized) + path = parsed.path or "" + + if path.endswith("/auth/polaris-token/session"): + return normalized + + if path.endswith("/auth/polaris-token"): + session_path = path[:-len("/auth/polaris-token")] + "/auth/polaris-token/session" + return urlunsplit((parsed.scheme, parsed.netloc, session_path, parsed.query, parsed.fragment)) + + raise RuntimeError( + "POLARIS_BROKER_URL must end with /auth/polaris-token " + "(or /auth/polaris-token/session if already session-scoped)" + ) + + +def ensure_broker_session_token( + *, + user_id: str, + session_id: str, + realm: str, + refresh_token: str, + bearer_token: Optional[str] = None, + catalog: str = "iceberg", + audience: Optional[str] = None, + broker_url: Optional[str] = None, + timeout_seconds: int = 20, +) -> str: + endpoint = broker_url or os.getenv("POLARIS_BROKER_URL", "http://teehr-api:8000/auth/polaris-token") + endpoint = _normalize_internal_broker_url(endpoint) + session_endpoint = _broker_session_endpoint_from_token_endpoint(endpoint).replace( + "/auth/polaris-token/session", + "/auth/polaris-session", + ) + active_audience = audience or os.getenv("POLARIS_BROKER_AUDIENCE", "account") + subject_token = bearer_token or os.getenv("POLARIS_USER_TOKEN", "") + + if not subject_token: + raise RuntimeError("A valid bearer subject token is required to create a broker session") + if not refresh_token: + raise RuntimeError("POLARIS_REFRESH_TOKEN is required to create a broker session") + + resp = requests.post( + session_endpoint, + headers={"Authorization": f"Bearer {subject_token}"}, + json={ + "user_id": user_id, + "session_id": session_id, + "realm": realm, + "catalog": catalog, + "audience": active_audience, + "refresh_token": refresh_token, + }, + timeout=timeout_seconds, + ) + resp.raise_for_status() + payload = resp.json() + broker_session_token = payload.get("broker_session_token") + if not broker_session_token: + raise RuntimeError("Broker session endpoint did not return broker_session_token") + os.environ["POLARIS_BROKER_SESSION_TOKEN"] = broker_session_token + return broker_session_token + + +def create_minio_spark_session( + polaris_token: Optional[str] = None, + force_recreate_session: bool = False, + update_configs: Optional[Dict[str, str]] = None, + use_authmanager: Optional[bool] = None, +): + """Start a Spark session configured for the local Polaris REST catalog. + + If ``polaris_token`` is provided, Spark uses user-token OAuth2 auth. + Otherwise it falls back to client-credential OAuth2 using env vars. + """ + aws_access_key_id = os.getenv("AWS_ACCESS_KEY_ID", "minioadmin") + aws_secret_access_key = os.getenv("AWS_SECRET_ACCESS_KEY", "minioadmin123") + + remote_catalog_uri = os.getenv("REMOTE_CATALOG_REST_URI", "http://polaris:8181/api/catalog") + remote_warehouse_dir = os.getenv("REMOTE_WAREHOUSE_S3_PATH", "s3://warehouse/") + polaris_realm = os.getenv("POLARIS_DEFAULT_REALM", "teehr") + + if remote_catalog_uri.rstrip("/").endswith("/api/catalog"): + # Polaris REST expects the catalog identifier here, not the backing S3 URI. + remote_warehouse_dir = polaris_realm + + s3_endpoint = os.getenv("REMOTE_CATALOG_S3_ENDPOINT", "http://minio:9000") + s3_path_style = _as_bool_str(os.getenv("REMOTE_CATALOG_S3_PATH_STYLE_ACCESS", "true")) + s3_region = os.getenv("AWS_REGION", "us-east-2") + + merged_configs: Dict[str, str] = { + "spark.sql.catalog.iceberg.warehouse": remote_warehouse_dir, + "spark.sql.catalog.iceberg.header.X-Polaris-Realm": polaris_realm, + "spark.sql.catalog.iceberg.rest.transport.header.X-Polaris-Realm": polaris_realm, + "spark.sql.catalog.iceberg.s3.endpoint": s3_endpoint, + "spark.sql.catalog.iceberg.s3.path-style-access": s3_path_style, + "spark.sql.catalog.iceberg.s3.region": s3_region, + "spark.hadoop.fs.s3a.endpoint": s3_endpoint, + "spark.hadoop.fs.s3a.path.style.access": s3_path_style, + "spark.hadoop.fs.s3a.connection.ssl.enabled": "false", + } + + resolved_use_authmanager = ( + use_authmanager + if use_authmanager is not None + else _as_bool_str(os.getenv("POLARIS_USE_AUTHMANAGER", "false")) == "true" + ) + + if resolved_use_authmanager: + broker_url = os.getenv("POLARIS_BROKER_URL", "http://teehr-api:8000/auth/polaris-token") + broker_url = _normalize_internal_broker_url(broker_url) + broker_session_url = _broker_session_endpoint_from_token_endpoint(broker_url) + authmanager_user_id = os.getenv("JUPYTERHUB_USER", "admin") + authmanager_session_id = ( + os.getenv("JUPYTERHUB_SERVER_NAME", "").strip() or authmanager_user_id + ) + broker_audience = os.getenv("POLARIS_BROKER_AUDIENCE", "account") + refresh_token = os.getenv("POLARIS_REFRESH_TOKEN", "") + current_user_token = polaris_token or os.getenv("POLARIS_USER_TOKEN", "") + broker_session_token = os.getenv("POLARIS_BROKER_SESSION_TOKEN", "") + if broker_session_token and _token_expires_soon(broker_session_token, 300): + broker_session_token = "" + authmanager_jar = os.getenv( + "POLARIS_AUTHMANAGER_JAR", + "/opt/spark/jars/teehr-authmanager.jar", + ) + + current_user_token, refresh_token, _ = ensure_fresh_polaris_user_token( + current_token=current_user_token, + username=os.getenv("POLARIS_USERNAME"), + password=os.getenv("POLARIS_PASSWORD"), + client_id=os.getenv("POLARIS_CLIENT_ID", "spark-polaris"), + client_secret=os.getenv("POLARIS_CLIENT_SECRET"), + refresh_token=refresh_token, + allow_password_fallback=False, + refresh_window_seconds=300, + token_endpoint=os.getenv("POLARIS_OAUTH2_TOKEN_ENDPOINT"), + ) + os.environ["POLARIS_USER_TOKEN"] = current_user_token + if refresh_token: + os.environ["POLARIS_REFRESH_TOKEN"] = refresh_token + + if not broker_session_token: + try: + broker_session_token = ensure_broker_session_token( + user_id=authmanager_user_id, + session_id=authmanager_session_id, + realm=polaris_realm, + refresh_token=refresh_token, + bearer_token=current_user_token, + catalog="iceberg", + audience=broker_audience, + broker_url=broker_url, + ) + except requests.HTTPError as exc: + if not _is_http_error_with_status(exc, 401): + raise + + # The bearer token may have been invalidated server-side; force refresh and retry once. + current_user_token, refresh_token, _ = ensure_fresh_polaris_user_token( + current_token=None, + username=os.getenv("POLARIS_USERNAME"), + password=os.getenv("POLARIS_PASSWORD"), + client_id=os.getenv("POLARIS_CLIENT_ID", "spark-polaris"), + client_secret=os.getenv("POLARIS_CLIENT_SECRET"), + refresh_token=refresh_token, + allow_password_fallback=True, + refresh_window_seconds=300, + token_endpoint=os.getenv("POLARIS_OAUTH2_TOKEN_ENDPOINT"), + ) + os.environ["POLARIS_USER_TOKEN"] = current_user_token + if refresh_token: + os.environ["POLARIS_REFRESH_TOKEN"] = refresh_token + + broker_session_token = ensure_broker_session_token( + user_id=authmanager_user_id, + session_id=authmanager_session_id, + realm=polaris_realm, + refresh_token=refresh_token, + bearer_token=current_user_token, + catalog="iceberg", + audience=broker_audience, + broker_url=broker_url, + ) + + os.environ["POLARIS_BROKER_SESSION_TOKEN"] = broker_session_token + + merged_configs["spark.sql.catalog.iceberg.rest.auth.type"] = ( + "org.teehr.iceberg.auth.TeehrBrokerAuthManager" + ) + merged_configs["spark.sql.catalog.iceberg.rest.auth.teehr.broker.url"] = broker_session_url + merged_configs["spark.sql.catalog.iceberg.rest.auth.teehr.user-id"] = authmanager_user_id + merged_configs["spark.sql.catalog.iceberg.rest.auth.teehr.session-id"] = ( + authmanager_session_id + ) + merged_configs["spark.sql.catalog.iceberg.rest.auth.teehr.realm"] = polaris_realm + merged_configs["spark.sql.catalog.iceberg.rest.auth.teehr.catalog"] = "iceberg" + merged_configs["spark.sql.catalog.iceberg.rest.auth.teehr.audience"] = broker_audience + merged_configs["spark.sql.catalog.iceberg.rest.auth.teehr.broker-session-token-env"] = ( + "POLARIS_BROKER_SESSION_TOKEN" + ) + merged_configs["spark.jars"] = authmanager_jar + elif polaris_token: + merged_configs["spark.sql.catalog.iceberg.rest.auth.type"] = "oauth2" + merged_configs["spark.sql.catalog.iceberg.token"] = polaris_token + merged_configs["spark.sql.catalog.iceberg.rest.auth.oauth2.token"] = polaris_token + else: + oauth_server_uri = os.getenv("POLARIS_OAUTH2_SERVER_URI") + spark_polaris_client_secret = os.getenv("SPARK_POLARIS_CLIENT_SECRET") + + merged_configs["spark.sql.catalog.iceberg.rest.auth.type"] = "oauth2" + merged_configs["spark.sql.catalog.iceberg.scope"] = "openid" + merged_configs["spark.sql.catalog.iceberg.rest.auth.oauth2.scope"] = "openid" + if oauth_server_uri: + merged_configs["spark.sql.catalog.iceberg.oauth2-server-uri"] = oauth_server_uri + merged_configs["spark.sql.catalog.iceberg.rest.auth.oauth2.server-uri"] = oauth_server_uri + if spark_polaris_client_secret: + merged_configs["spark.sql.catalog.iceberg.credential"] = ( + f"spark-polaris:{spark_polaris_client_secret}" + ) + merged_configs["spark.sql.catalog.iceberg.rest.auth.oauth2.credential"] = ( + f"spark-polaris:{spark_polaris_client_secret}" + ) + + if update_configs: + merged_configs.update(update_configs) + + call_kwargs = { + "remote_catalog_uri": remote_catalog_uri, + "remote_warehouse_dir": remote_warehouse_dir, + "aws_access_key_id": aws_access_key_id, + "aws_secret_access_key": aws_secret_access_key, + "force_recreate_session": force_recreate_session or resolved_use_authmanager, + "update_configs": merged_configs, + } + if polaris_token: + # Preferred path for newer teehr versions that support direct token auth. + call_kwargs["oauth2_token"] = polaris_token + + while True: + try: + spark = create_spark_session(**call_kwargs) + _apply_runtime_spark_configs(spark, merged_configs) + return spark + except TypeError as exc: + msg = str(exc) + if "oauth2_token" in msg and "oauth2_token" in call_kwargs: + call_kwargs.pop("oauth2_token", None) + continue + if "force_recreate_session" in msg and "force_recreate_session" in call_kwargs: + call_kwargs.pop("force_recreate_session", None) + continue + raise + + +def request_broker_polaris_token( + *, + user_id: str, + session_id: str, + bearer_token: str, + realm: Optional[str] = None, + catalog: str = "iceberg", + requested_ttl_seconds: int = 600, + audience: Optional[str] = None, + broker_url: Optional[str] = None, + timeout_seconds: int = 20, +) -> Tuple[str, int, str]: + endpoint = broker_url or os.getenv("POLARIS_BROKER_URL", "http://teehr-api:8000/auth/polaris-token") + active_realm = realm or os.getenv("POLARIS_DEFAULT_REALM", "teehr") + active_audience = audience or os.getenv("POLARIS_BROKER_AUDIENCE", "account") + + if not bearer_token: + raise RuntimeError("bearer_token is required to request a broker token") + + resp = requests.post( + endpoint, + headers={"Authorization": f"Bearer {bearer_token}"}, + json={ + "user_id": user_id, + "session_id": session_id, + "realm": active_realm, + "catalog": catalog, + "requested_ttl_seconds": requested_ttl_seconds, + "audience": active_audience, + }, + timeout=timeout_seconds, + ) + resp.raise_for_status() + payload = resp.json() + access_token = payload.get("access_token") + expires_at = int(payload.get("expires_at_epoch_seconds", 0)) + trace_id = payload.get("trace_id", "") + + if not access_token or expires_at <= 0: + raise RuntimeError("Broker response missing access_token or expires_at_epoch_seconds") + + return access_token, expires_at, trace_id + + +def ensure_fresh_polaris_token_via_broker( + *, + current_token: Optional[str], + user_id: str, + session_id: str, + bearer_token: str, + realm: Optional[str] = None, + catalog: str = "iceberg", + refresh_window_seconds: int = 120, + requested_ttl_seconds: int = 600, + audience: Optional[str] = None, + broker_url: Optional[str] = None, +) -> Tuple[str, bool]: + if current_token and not _token_expires_soon(current_token, refresh_window_seconds): + return current_token, False + + refreshed_token, _, _ = request_broker_polaris_token( + user_id=user_id, + session_id=session_id, + bearer_token=bearer_token, + realm=realm, + catalog=catalog, + requested_ttl_seconds=requested_ttl_seconds, + audience=audience, + broker_url=broker_url, + ) + return refreshed_token, True \ No newline at end of file diff --git a/keycloak-bootstrap/manifests/bootstrap-job.yaml b/keycloak-bootstrap/manifests/bootstrap-job.yaml index a9731fd..e0cc5e8 100644 --- a/keycloak-bootstrap/manifests/bootstrap-job.yaml +++ b/keycloak-bootstrap/manifests/bootstrap-job.yaml @@ -118,6 +118,8 @@ spec: volumeMounts: - name: keycloak-bootstrap-config mountPath: /config + - name: keycloak-local-users-config + mountPath: /config/users volumes: - name: keycloak-bootstrap-config configMap: @@ -125,3 +127,9 @@ spec: items: - key: teehr-realm.json path: teehr-realm.json + - name: keycloak-local-users-config + configMap: + name: keycloak-local-users-bootstrap + items: + - key: teehr-local-users.json + path: teehr-local-users.json diff --git a/keycloak-bootstrap/manifests/local-users-configmap.yaml.tpl b/keycloak-bootstrap/manifests/local-users-configmap.yaml.tpl index 3c1dd0e..7a7b735 100644 --- a/keycloak-bootstrap/manifests/local-users-configmap.yaml.tpl +++ b/keycloak-bootstrap/manifests/local-users-configmap.yaml.tpl @@ -25,7 +25,8 @@ data: ], "groups": [ "/basic-user", - "/iceberg-user", + "/teehr-read-only", + "/teehr-read-write", "/iceberg-catalog-admins", "/jupyter-admin", "/key-management-admin", @@ -49,6 +50,27 @@ data: ], "groups": [ "/basic-user", + "/teehr-read-only", + "/jupyter-user" + ] + }, + { + "username": "poweruser", + "enabled": true, + "email": "poweruser@example.local", + "emailVerified": true, + "firstName": "Local", + "lastName": "PowerUser", + "credentials": [ + { + "type": "password", + "value": "poweruser", + "temporary": false + } + ], + "groups": [ + "/basic-user", + "/teehr-read-write", "/jupyter-user" ] } diff --git a/keycloak-bootstrap/manifests/realm-configmap.yaml.tpl b/keycloak-bootstrap/manifests/realm-configmap.yaml.tpl index 6b0dae5..77e2169 100644 --- a/keycloak-bootstrap/manifests/realm-configmap.yaml.tpl +++ b/keycloak-bootstrap/manifests/realm-configmap.yaml.tpl @@ -34,12 +34,9 @@ data: { "name": "admin" }, { "name": "basic-user" }, { "name": "jupyter-user" }, - { "name": "iceberg-user" }, + { "name": "teehr-read-only" }, + { "name": "teehr-read-write" }, { "name": "iceberg-catalog-admin" }, - { "name": "iceberg-namespace-public-read" }, - { "name": "iceberg-namespace-public-write" }, - { "name": "iceberg-namespace-restricted-read" }, - { "name": "iceberg-namespace-restricted-write" } ] }, "groups": [ @@ -56,8 +53,12 @@ data: "realmRoles": ["jupyter-user"] }, { - "name": "iceberg-user", - "realmRoles": ["iceberg-user"] + "name": "teehr-read-only", + "realmRoles": ["teehr-read-only"] + }, + { + "name": "teehr-read-write", + "realmRoles": ["teehr-read-write", "teehr-read-only"] }, { "name": "key-management-admin", @@ -80,29 +81,14 @@ data: ] } }, - { - "name": "iceberg-public-readers", - "realmRoles": ["iceberg-namespace-public-read"] - }, - { - "name": "iceberg-public-writers", - "realmRoles": ["iceberg-namespace-public-write", "iceberg-namespace-public-read"] - }, - { - "name": "iceberg-restricted-readers", - "realmRoles": ["iceberg-namespace-restricted-read"] - }, - { - "name": "iceberg-restricted-writers", - "realmRoles": ["iceberg-namespace-restricted-write", "iceberg-namespace-restricted-read"] - }, { "name": "iceberg-catalog-admins", "realmRoles": ["iceberg-catalog-admin"] } ], "defaultGroups": [ - "/basic-user" + "/basic-user", + "/teehr-read-only" ], "clients": [ { diff --git a/polaris-bootstrap/manifests/acl-config.yaml.tpl b/polaris-bootstrap/manifests/acl-config.yaml.tpl index 2767648..8ddb048 100644 --- a/polaris-bootstrap/manifests/acl-config.yaml.tpl +++ b/polaris-bootstrap/manifests/acl-config.yaml.tpl @@ -28,55 +28,26 @@ data: }, "namespace_policies": [ { - "namespace": "public", + "namespace": "teehr", "roles": [ { - "principal_role": "iceberg-namespace-public-read", - "catalog_role": "public_read_role", - "grants": [ - { - "type": "namespace", - "privileges": [ - "NAMESPACE_READ_PROPERTIES" - ] - } - ] - }, - { - "principal_role": "iceberg-namespace-public-write", - "catalog_role": "public_write_role", + "principal_role": "teehr-read-only", + "catalog_role": "teehr_read_only_role", "grants": [ { "type": "namespace", "privileges": [ "NAMESPACE_READ_PROPERTIES", - "NAMESPACE_WRITE_PROPERTIES", - "TABLE_CREATE", - "TABLE_DROP" - ] - } - ] - } - ] - }, - { - "namespace": "restricted", - "roles": [ - { - "principal_role": "iceberg-namespace-restricted-read", - "catalog_role": "restricted_read_role", - "grants": [ - { - "type": "namespace", - "privileges": [ - "NAMESPACE_READ_PROPERTIES" + "TABLE_LIST", + "TABLE_READ_PROPERTIES", + "TABLE_READ_DATA" ] } ] }, { - "principal_role": "iceberg-namespace-restricted-write", - "catalog_role": "restricted_write_role", + "principal_role": "teehr-read-write", + "catalog_role": "teehr_read_write_role", "grants": [ { "type": "namespace", @@ -84,7 +55,12 @@ data: "NAMESPACE_READ_PROPERTIES", "NAMESPACE_WRITE_PROPERTIES", "TABLE_CREATE", - "TABLE_DROP" + "TABLE_DROP", + "TABLE_LIST", + "TABLE_READ_PROPERTIES", + "TABLE_WRITE_PROPERTIES", + "TABLE_READ_DATA", + "TABLE_WRITE_DATA" ] } ] diff --git a/polaris-bootstrap/manifests/polaris-sync-principals-script.yaml b/polaris-bootstrap/manifests/polaris-sync-principals-script.yaml index 0314f35..75a0388 100644 --- a/polaris-bootstrap/manifests/polaris-sync-principals-script.yaml +++ b/polaris-bootstrap/manifests/polaris-sync-principals-script.yaml @@ -7,7 +7,7 @@ data: """ Syncs all enabled Keycloak realm users into Polaris as principal entities and grants matching Polaris principal roles for Keycloak realm roles with - the iceberg-* prefix. + the teehr-* prefix. Required environment variables: POLARIS_MANAGEMENT_URL e.g. http://polaris:8181 @@ -87,6 +87,22 @@ data: return [role.get("name") for role in resp.json() if role.get("name")] + def get_keycloak_user_groups(kc_token, user_id): + """Get user's groups, including group paths like /teehr-read-only""" + resp = requests.get( + f"{KC_URL}/admin/realms/{REALM}/users/{user_id}/groups", + headers={"Authorization": f"Bearer {kc_token}"}, + ) + resp.raise_for_status() + groups = [] + for group in resp.json(): + # Extract the group name from the path (e.g., "/teehr-read-only" -> "teehr-read-only") + path = group.get("path", "").strip("/") + if path: + groups.append(path) + return groups + + def ensure_principal(polaris_token, principal_name): resp = requests.post( f"{MGMT_URL}/api/management/v1/principals", @@ -163,10 +179,17 @@ data: if name: ensure_principal(polaris_token, name) if user_id: + # Check realm roles for role_name in get_keycloak_realm_roles(kc_token, user_id): - if role_name.startswith("iceberg-"): + if role_name.startswith("teehr-"): ensure_principal_role_binding(polaris_token, name, role_name) granted += 1 + + # Check group memberships and map to principal roles + for group_name in get_keycloak_user_groups(kc_token, user_id): + if group_name.startswith("teehr-"): + ensure_principal_role_binding(polaris_token, name, group_name) + granted += 1 synced += 1 print( diff --git a/scripts/stage_authmanager_jar.sh b/scripts/stage_authmanager_jar.sh index 2134ccb..649482b 100644 --- a/scripts/stage_authmanager_jar.sh +++ b/scripts/stage_authmanager_jar.sh @@ -4,7 +4,7 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" AUTHMANAGER_DIR="$ROOT_DIR/spark/authmanager-prototype" SPARK_DOCKER_DIR="$ROOT_DIR/spark/docker" -JUPYTER_DOCKER_DIR="$ROOT_DIR/jupyterhub/docker" +JUPYTER_DOCKER_DIR="$ROOT_DIR/jupyterhub-docker" cd "$ROOT_DIR" diff --git a/tests/Dockerfile b/tests/Dockerfile new file mode 100644 index 0000000..dc87003 --- /dev/null +++ b/tests/Dockerfile @@ -0,0 +1,17 @@ +ARG BASE_IMAGE=python:3.11-slim +FROM ${BASE_IMAGE} + +WORKDIR /app + +# Create tests directory +RUN mkdir -p /app/tests + +# Copy test scripts +COPY keycloak_users_test.py /app/tests/ +COPY polaris_oidc_test.py /app/tests/ +COPY polaris_namespace_test.py /app/tests/ +COPY polaris_roles_test.py /app/tests/ +COPY polaris_auth_token_test.py /app/tests/ +COPY polaris_permissions_config_test.py /app/tests/ +COPY spark_permission_test.py /app/tests/ +COPY spark_authmanager_test.py /app/tests/ diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..b7526dc --- /dev/null +++ b/tests/README.md @@ -0,0 +1,130 @@ +# Polaris Integration Tests + +This directory contains **Python-based integration tests** for the Polaris + Keycloak + Spark ecosystem running in KinD. + +## Test Files + +- `keycloak_users_test.py` — Validates user provisioning in Keycloak +- `polaris_oidc_test.py` — Validates Polaris OIDC token acceptance +- `polaris_namespace_test.py` — Validates namespace provisioning +- `polaris_roles_test.py` — Validates role-based access control setup +- `spark_session_auth_test.py` — Validates Spark session creation and write permissions + +## What These Tests Do + +These are **end-to-end integration tests** that validate: + +1. **Keycloak User Provisioning** (`polaris-keycloak-users`) + - Admin, user, and poweruser accounts exist + - Users have correct group membership + - Users can authenticate to Keycloak + +2. **Polaris OIDC Integration** (`polaris-oidc-token-validation`) + - Polaris accepts Keycloak-issued tokens + - Token validation succeeds with correct realm header + - JWT claims are properly mapped + +3. **Namespace Provisioning** (`polaris-namespace-list`) + - The `iceberg` catalog exists + - The `teehr` namespace exists in the catalog + - Root credentials can list namespaces + +4. **Role-Based Access Control** (`polaris-role-acl-validation`) + - Principal roles (`teehr-read-only`, `teehr-read-write`, `iceberg-catalog-admin`) exist + - Catalog roles (`teehr_read_only_role`, `teehr_read_write_role`, `catalog_admin_role`) exist + - Roles are properly bound to namespaces and privileges + +5. **Spark Session Authentication** (`spark-session-auth`) + - Poweruser can create Spark sessions with Keycloak credentials + - Read-only user can create Spark sessions with Keycloak credentials + - Poweruser can list tables in `iceberg.teehr` namespace + - Read-only user can list tables in `iceberg.teehr` namespace + - Poweruser **can create tables** in `iceberg.teehr` namespace + - Read-only user **cannot create tables** (write denied) + +## Running the Tests + +### Run all tests through Garden: +```bash +garden test +``` + +### Run a specific test through Garden: +```bash +garden test polaris-keycloak-users +garden test polaris-oidc-token-validation +garden test spark-session-auth +``` + +### Run tests locally (for development): +```bash +# Each test can be run directly from the command line +python3 tests/keycloak_users_test.py +python3 tests/polaris_oidc_test.py +python3 tests/polaris_namespace_test.py +python3 tests/polaris_roles_test.py +python3 tests/spark_session_auth_test.py +``` + +### Run tests with verbose output: +```bash +garden test --verbose +``` + +## Test Execution Order + +Garden automatically resolves dependencies. The test order is: + +1. `polaris-keycloak-users` (runs after `deploy.keycloak-bootstrap`) +2. `polaris-oidc-token-validation` (runs after `deploy.polaris-bootstrap`) +3. `polaris-namespace-list` (runs after `deploy.polaris-bootstrap`) +4. `polaris-role-acl-validation` (runs after `deploy.polaris-bootstrap`) +5. `spark-session-auth` (runs after `deploy.polaris-bootstrap` and `deploy.spark`) + +## Future Tests + +### Spark Integration Tests (to implement) +- Spark session creation with Keycloak credentials +- Table creation in iceberg.teehr namespace as read-write user +- Read-only access validation (write attempt should fail) +- Namespace isolation across roles + +### Trino Integration Tests (to implement) +- Trino catalog configuration validation +- Query access with different user roles +- Namespace-level permission enforcement + +### End-to-End Flow (to implement) +- Full user journey: Keycloak login → JupyterHub → Spark session → Polaris access +- Data pipeline execution with role-based filtering + +## Extending These Tests + +To add a new test, add a new `kind: Test` block to `garden.yaml`: + +```yaml +--- +kind: Test +name: my-new-test +dependencies: + - deploy.some-service +timeout: 60 +spec: + image: some-container-image + command: + - /bin/sh + - -c + - | + # your test logic here + exit 0 # success + exit 1 # failure +``` + +### Tips for Writing Tests + +- Use `set -e` to fail fast on errors +- Print progress with `echo "[test] ..."` for clarity +- Use `grep` and pipe to `/dev/null` for silent checks +- Use `curl -w "\n%{http_code}"` to capture HTTP status separately +- Resolve service names via internal DNS (e.g., `keycloak-service:8080`) +- Use environment secrets from Garden where available diff --git a/tests/garden.yaml b/tests/garden.yaml new file mode 100644 index 0000000..b1366c0 --- /dev/null +++ b/tests/garden.yaml @@ -0,0 +1,141 @@ +kind: Build +type: container +name: test-scripts +dependencies: + - build.teehr-jupyter-driver-image-edge +environments: + - local +spec: + dockerfile: Dockerfile + localId: test-scripts + buildArgs: + BASE_IMAGE: ${actions.build.teehr-jupyter-driver-image-edge.outputs.deploymentImageName}:${actions.build.teehr-jupyter-driver-image-edge.version} + +--- +kind: Test +name: polaris-keycloak-users +dependencies: + - deploy.keycloak-local-users-bootstrap + - build.test-scripts +timeout: 180 +type: container +spec: + image: ${actions.build.test-scripts.outputs.deploymentImageName}:${actions.build.test-scripts.version} + command: + - python3 + - /app/tests/keycloak_users_test.py + +--- +kind: Test +name: polaris-oidc-token-validation +dependencies: + - deploy.polaris-bootstrap + - build.test-scripts +timeout: 180 +type: container +spec: + image: ${actions.build.test-scripts.outputs.deploymentImageName}:${actions.build.test-scripts.version} + command: + - python3 + - /app/tests/polaris_oidc_test.py + +--- +kind: Test +name: polaris-namespace-list +dependencies: + - deploy.polaris-bootstrap + - build.test-scripts +timeout: 180 +type: container +spec: + image: ${actions.build.test-scripts.outputs.deploymentImageName}:${actions.build.test-scripts.version} + command: + - python3 + - /app/tests/polaris_namespace_test.py + +--- +kind: Test +name: polaris-role-acl-validation +dependencies: + - deploy.polaris-bootstrap + - build.test-scripts +timeout: 180 +type: container +spec: + image: ${actions.build.test-scripts.outputs.deploymentImageName}:${actions.build.test-scripts.version} + command: + - python3 + - /app/tests/polaris_roles_test.py + +--- +kind: Test +name: polaris-auth-token +dependencies: + - deploy.polaris-bootstrap + - deploy.keycloak-local-users-bootstrap + - build.test-scripts +timeout: 240 +type: container +spec: + image: ${actions.build.test-scripts.outputs.deploymentImageName}:${actions.build.test-scripts.version} + command: + - python3 + - /app/tests/polaris_auth_token_test.py + +--- +kind: Test +name: polaris-permissions-config +dependencies: + - deploy.polaris-bootstrap + - deploy.keycloak-local-users-bootstrap + - build.test-scripts +timeout: 180 +type: container +spec: + image: ${actions.build.test-scripts.outputs.deploymentImageName}:${actions.build.test-scripts.version} + command: + - python3 + - /app/tests/polaris_permissions_config_test.py + +--- +kind: Test +name: spark-permission-enforcement +dependencies: + - deploy.polaris-bootstrap + - deploy.keycloak-local-users-bootstrap + - build.test-scripts +timeout: 300 +type: container +spec: + image: ${actions.build.test-scripts.outputs.deploymentImageName}:${actions.build.test-scripts.version} + command: + - python3 + - /app/tests/spark_permission_test.py + memory: + min: 2048 + max: 3072 + cpu: + min: 1000 + max: 2000 + +--- +kind: Test +name: spark-authmanager-enforcement +dependencies: + - deploy.polaris-bootstrap + - deploy.keycloak-local-users-bootstrap + - deploy.teehr-api + - build.test-scripts +timeout: 600 +type: container +spec: + image: ${actions.build.test-scripts.outputs.deploymentImageName}:${actions.build.test-scripts.version} + command: + - python3 + - /app/tests/spark_authmanager_test.py + memory: + min: 2048 + max: 3072 + cpu: + min: 1000 + max: 2000 diff --git a/tests/keycloak_users_test.py b/tests/keycloak_users_test.py new file mode 100644 index 0000000..2aa5e32 --- /dev/null +++ b/tests/keycloak_users_test.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +""" +Integration test: Keycloak user provisioning + +Validates: +- Admin, user, and poweruser accounts exist in Keycloak +- Users have correct group membership +""" + +import sys +import requests +import time + +KEYCLOAK_URL = "http://keycloak-service:8080" +REALM = "teehr" +ADMIN_USERNAME = "admin" +ADMIN_PASSWORD = "admin123" + +EXPECTED_USERS = ["admin", "user", "poweruser"] + +# Retry configuration +MAX_RETRIES = 10 +RETRY_DELAY = 2 # seconds + + +def get_admin_token(): + """Get admin token to query Keycloak""" + for attempt in range(MAX_RETRIES): + try: + token_url = f"{KEYCLOAK_URL}/realms/master/protocol/openid-connect/token" + payload = { + "grant_type": "password", + "client_id": "admin-cli", + "username": ADMIN_USERNAME, + "password": ADMIN_PASSWORD + } + + response = requests.post(token_url, data=payload, timeout=5) + if response.status_code == 200: + return response.json()["access_token"] + elif attempt < MAX_RETRIES - 1: + print(f" Attempt {attempt + 1}/{MAX_RETRIES}: Retrying Keycloak connection...") + time.sleep(RETRY_DELAY) + else: + raise Exception(f"Failed to get admin token: {response.text}") + except requests.exceptions.RequestException as e: + if attempt < MAX_RETRIES - 1: + print(f" Attempt {attempt + 1}/{MAX_RETRIES}: Keycloak not ready, retrying...") + time.sleep(RETRY_DELAY) + else: + raise Exception(f"Failed to connect to Keycloak: {e}") + + +def check_user_exists(admin_token, username): + """Check if a user exists in Keycloak""" + url = f"{KEYCLOAK_URL}/admin/realms/{REALM}/users?username={username}&exact=true" + headers = {"Authorization": f"Bearer {admin_token}"} + + response = requests.get(url, headers=headers) + if response.status_code != 200: + raise Exception(f"Failed to query users: {response.text}") + + users = response.json() + if len(users) == 0: + print(f" DEBUG: No users found for '{username}'. Response: {users}") + return False + + if users[0]["username"] != username: + print(f" DEBUG: Expected '{username}' but got '{users[0]['username']}'") + return False + + return True + + +def main(): + """Run user provisioning validation""" + print("[test] Validating Keycloak users...") + + try: + admin_token = get_admin_token() + print(" ✓ Got admin token") + + for username in EXPECTED_USERS: + print(f" Checking user: {username}") + + if check_user_exists(admin_token, username): + print(f" ✓ User {username} exists") + else: + print(f" ✗ ERROR: User {username} not found") + return 1 + + print("[test] All Keycloak users validated successfully") + return 0 + + except Exception as e: + print(f"[test] ERROR: {e}") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/polaris_auth_token_test.py b/tests/polaris_auth_token_test.py new file mode 100644 index 0000000..4e13c91 --- /dev/null +++ b/tests/polaris_auth_token_test.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 +""" +Integration test: Spark session authentication with Polaris + Keycloak + +Validates: +- Keycloak token acquisition for test users (poweruser, user) +- Polaris accepts Keycloak JWT tokens with correct realm and warehouse +- Different users can obtain tokens and authenticate with Polaris +""" + +import sys +import json +import requests +import time +import base64 + +# Configuration +KEYCLOAK_URL = "http://keycloak-service:8080" +POLARIS_URL = "http://polaris:8181" +REALM = "teehr" +CATALOG = "teehr" +NAMESPACE = "teehr" + +TEST_USERS = { + "poweruser": {"password": "poweruser", "should_write": True}, + "user": {"password": "user", "should_write": False} +} + +# Retry configuration +MAX_RETRIES = 10 +RETRY_DELAY = 2 # seconds + + +def get_keycloak_token(username, password): + """Get a JWT token from Keycloak for a user""" + for attempt in range(MAX_RETRIES): + try: + token_url = f"{KEYCLOAK_URL}/realms/{REALM}/protocol/openid-connect/token" + payload = { + "grant_type": "password", + "client_id": "jupyterhub", + "client_secret": "local-jupyterhub-client-secret", + "username": username, + "password": password, + "scope": "openid" + } + + response = requests.post(token_url, data=payload, timeout=5) + if response.status_code == 200: + return response.json()["access_token"] + elif response.status_code == 401 and attempt < MAX_RETRIES - 1: + print(f" Attempt {attempt + 1}/{MAX_RETRIES}: Keycloak not ready, retrying...") + time.sleep(RETRY_DELAY) + else: + raise Exception(f"Failed to get token for {username}: {response.text}") + except requests.exceptions.RequestException as e: + if attempt < MAX_RETRIES - 1: + print(f" Attempt {attempt + 1}/{MAX_RETRIES}: Connection failed, retrying...") + time.sleep(RETRY_DELAY) + else: + raise Exception(f"Failed to get token for {username}: {e}") + + +def validate_token_with_polaris(token): + """Validate that Polaris accepts the token""" + for attempt in range(MAX_RETRIES): + try: + headers = { + "Authorization": f"Bearer {token}", + "X-Polaris-Realm": REALM + } + response = requests.get( + f"{POLARIS_URL}/api/catalog/v1/config", + headers=headers, + params={"warehouse": CATALOG}, + timeout=5 + ) + if response.status_code == 200: + return True + elif response.status_code >= 500 and attempt < MAX_RETRIES - 1: + print(f" Attempt {attempt + 1}/{MAX_RETRIES}: Polaris error, retrying...") + time.sleep(RETRY_DELAY) + else: + print(f" DEBUG: HTTP {response.status_code}: {response.text[:300]}") + return False + except requests.exceptions.RequestException as e: + if attempt < MAX_RETRIES - 1: + print(f" Attempt {attempt + 1}/{MAX_RETRIES}: Connection failed, retrying...") + time.sleep(RETRY_DELAY) + else: + return False + return False + + +def check_token_roles(token, username): + """Verify expected roles in JWT token""" + try: + parts = token.split(".") + # Add padding if needed + payload_part = parts[1] + padding = 4 - len(payload_part) % 4 + if padding != 4: + payload_part += "=" * padding + + payload = json.loads(base64.urlsafe_b64decode(payload_part)) + roles = payload.get("realm_access", {}).get("roles", []) + + # Check if user has expected roles + if username == "poweruser": + if "teehr-read-write" in roles: + print(f" ✓ User has teehr-read-write role") + return True + else: + print(f" WARNING: User missing teehr-read-write role (has: {roles})") + return False + elif username == "user": + if "teehr-read-only" in roles: + print(f" ✓ User has teehr-read-only role") + return True + else: + print(f" WARNING: User missing teehr-read-only role (has: {roles})") + return False + except Exception as e: + print(f" WARNING: Could not decode token roles: {e}") + return False + + +def main(): + """Run Spark session authentication tests""" + print("[test] Starting Spark session authentication tests...") + print() + + all_passed = True + + for username, config in TEST_USERS.items(): + print(f"Testing user: {username}") + + try: + # Step 1: Get Keycloak token + print(" Getting Keycloak token...") + token = get_keycloak_token(username, config["password"]) + print(" ✓ Token obtained") + + # Step 2: Validate token with Polaris + print(" Validating token with Polaris...") + if validate_token_with_polaris(token): + print(" ✓ Polaris accepted token") + else: + print(" ✗ ERROR: Polaris rejected token") + all_passed = False + continue + + # Step 3: Verify role in token + print(" Checking token roles...") + check_token_roles(token, username) + + except Exception as e: + print(f" ✗ ERROR: {e}") + all_passed = False + + print() + + print("[test] Spark session auth tests", "PASSED" if all_passed else "FAILED") + + return 0 if all_passed else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/polaris_namespace_test.py b/tests/polaris_namespace_test.py new file mode 100644 index 0000000..c6c3da5 --- /dev/null +++ b/tests/polaris_namespace_test.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +""" +Integration test: Polaris namespace provisioning + +Validates: +- The iceberg catalog exists in Polaris +- The teehr namespace exists in the iceberg catalog +- Root credentials can be used to query Polaris +""" + +import sys +import requests +import time + +POLARIS_URL = "http://polaris:8181" +REALM = "teehr" +ROOT_CLIENT_ID = "root" +ROOT_CLIENT_SECRET = "secret123" + +# Retry configuration +MAX_RETRIES = 10 +RETRY_DELAY = 2 # seconds + + +def get_root_token(): + """Get root token using client credentials""" + for attempt in range(MAX_RETRIES): + try: + token_url = f"{POLARIS_URL}/api/catalog/v1/oauth/tokens" + payload = { + "grant_type": "client_credentials", + "client_id": ROOT_CLIENT_ID, + "client_secret": ROOT_CLIENT_SECRET, + "scope": "PRINCIPAL_ROLE:ALL" + } + headers = { + "X-Polaris-Realm": REALM, + "Content-Type": "application/x-www-form-urlencoded" + } + + response = requests.post(token_url, data=payload, headers=headers, timeout=5) + if response.status_code == 200: + return response.json()["access_token"] + elif attempt < MAX_RETRIES - 1: + print(f" Attempt {attempt + 1}/{MAX_RETRIES}: Polaris not ready, retrying...") + time.sleep(RETRY_DELAY) + else: + raise Exception(f"Failed to get root token: {response.text}") + except requests.exceptions.RequestException as e: + if attempt < MAX_RETRIES - 1: + print(f" Attempt {attempt + 1}/{MAX_RETRIES}: Polaris not reachable, retrying...") + time.sleep(RETRY_DELAY) + else: + raise Exception(f"Failed to get root token: {e}") + + +def list_namespaces(root_token): + """List namespaces in the teehr catalog""" + headers = { + "Authorization": f"Bearer {root_token}", + "X-Polaris-Realm": REALM + } + + response = requests.get( + f"{POLARIS_URL}/api/catalog/v1/teehr/namespaces", + headers=headers + ) + + if response.status_code != 200: + raise Exception(f"Failed to list namespaces: {response.text}") + + return response.json() + + +def main(): + """Run namespace provisioning validation""" + print("[test] Validating teehr namespace exists...") + + try: + # Step 1: Get root token + print(" Getting root credentials token...") + root_token = get_root_token() + print(" ✓ Root token obtained") + + # Step 2: List namespaces + print(" Listing namespaces in iceberg catalog...") + namespaces_response = list_namespaces(root_token) + print(" ✓ Namespace list retrieved") + + # Step 3: Check if teehr namespace exists + # Response is a list of namespace objects directly + if isinstance(namespaces_response, list): + namespaces = namespaces_response + else: + namespaces = namespaces_response.get("namespaces", []) + + found_teehr = False + namespace_names = [] + for ns in namespaces: + # namespace can be a list or a dict + namespace_path = ns if isinstance(ns, (list, tuple)) else ns.get("namespace", []) + if namespace_path: + namespace_names.append(".".join(namespace_path) if isinstance(namespace_path, (list, tuple)) else str(namespace_path)) + if isinstance(namespace_path, (list, tuple)) and len(namespace_path) > 0 and namespace_path[0] == "teehr": + found_teehr = True + elif isinstance(namespace_path, str) and namespace_path == "teehr": + found_teehr = True + + if found_teehr: + print(" ✓ teehr namespace found") + else: + print(f" WARNING: teehr namespace not found") + if namespace_names: + print(f" Available namespaces: {namespace_names}") + else: + print(f" Response type: {type(namespaces_response)}, content: {str(namespaces_response)[:200]}") + + print("[test] Namespace validation successful") + return 0 + + except Exception as e: + print(f"[test] ERROR: {e}") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/polaris_oidc_test.py b/tests/polaris_oidc_test.py new file mode 100644 index 0000000..d9d1e68 --- /dev/null +++ b/tests/polaris_oidc_test.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +""" +Integration test: Polaris OIDC token validation + +Validates: +- Keycloak can issue tokens to users +- Polaris accepts and validates Keycloak-issued JWT tokens +- JWT claims are properly mapped +""" + +import sys +import requests +import time + +KEYCLOAK_URL = "http://keycloak-service:8080" +POLARIS_URL = "http://polaris:8181" +REALM = "teehr" +TEST_USERNAME = "user" +TEST_PASSWORD = "user" + +# Retry configuration +MAX_RETRIES = 10 +RETRY_DELAY = 2 # seconds + + +def get_user_token(username, password): + """Get a JWT token from Keycloak for a user""" + for attempt in range(MAX_RETRIES): + try: + token_url = f"{KEYCLOAK_URL}/realms/{REALM}/protocol/openid-connect/token" + payload = { + "grant_type": "password", + "client_id": "jupyterhub", + "client_secret": "local-jupyterhub-client-secret", + "username": username, + "password": password, + "scope": "openid" + } + + response = requests.post(token_url, data=payload, timeout=5) + if response.status_code == 200: + return response.json()["access_token"] + elif response.status_code == 401 and attempt < MAX_RETRIES - 1: + print(f" Attempt {attempt + 1}/{MAX_RETRIES}: Keycloak not ready, retrying...") + time.sleep(RETRY_DELAY) + else: + raise Exception(f"Failed to get token for {username}: {response.text}") + except requests.exceptions.RequestException as e: + if attempt < MAX_RETRIES - 1: + print(f" Attempt {attempt + 1}/{MAX_RETRIES}: Connection failed, retrying...") + time.sleep(RETRY_DELAY) + else: + raise Exception(f"Failed to get token for {username}: {e}") + + +def validate_token_with_polaris(token): + """Test that Polaris accepts and validates the token""" + headers = { + "Authorization": f"Bearer {token}", + "X-Polaris-Realm": REALM + } + + response = requests.get( + f"{POLARIS_URL}/api/catalog/v1/config", + headers=headers, + params={"warehouse": REALM} + ) + + if response.status_code != 200: + print(f" DEBUG: HTTP {response.status_code}: {response.text[:300]}") + + return response.status_code == 200 + + +def main(): + """Run Polaris OIDC validation tests""" + print("[test] Validating Polaris OIDC token acceptance...") + + try: + # Step 1: Get user token from Keycloak + print(" Getting user token from Keycloak...") + token = get_user_token(TEST_USERNAME, TEST_PASSWORD) + print(" ✓ Token obtained") + + # Step 2: Test Polaris accepts the token + print(" Testing Polaris accepts the token...") + if validate_token_with_polaris(token): + print(" ✓ Polaris accepted token") + else: + print(" ✗ ERROR: Polaris rejected token") + return 1 + + print("[test] Polaris OIDC validation successful") + return 0 + + except Exception as e: + print(f"[test] ERROR: {e}") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/polaris_permissions_config_test.py b/tests/polaris_permissions_config_test.py new file mode 100644 index 0000000..87847f0 --- /dev/null +++ b/tests/polaris_permissions_config_test.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +""" +Integration test: Polaris permission configuration verification + +Validates: +- Catalog roles and principal roles are correctly created +- Principal roles have the expected permissions granted on the teehr namespace +- Permissions are configured for read-only and read-write access patterns +""" + +import sys +import json +import requests +import time + +# Configuration +POLARIS_URL = "http://polaris:8181" +REALM = "teehr" +CATALOG = "teehr" +NAMESPACE = "teehr" + +# Root credentials for admin access +ROOT_CLIENT_ID = "root" +ROOT_CLIENT_SECRET = "secret123" + +# Retry configuration +MAX_RETRIES = 10 +RETRY_DELAY = 2 # seconds + + +def get_root_token(): + """Get root token using client credentials""" + for attempt in range(MAX_RETRIES): + try: + token_url = f"{POLARIS_URL}/api/catalog/v1/oauth/tokens" + payload = { + "grant_type": "client_credentials", + "client_id": ROOT_CLIENT_ID, + "client_secret": ROOT_CLIENT_SECRET, + "scope": "PRINCIPAL_ROLE:ALL" + } + headers = { + "X-Polaris-Realm": REALM, + "Content-Type": "application/x-www-form-urlencoded" + } + + response = requests.post(token_url, data=payload, headers=headers, timeout=5) + if response.status_code == 200: + return response.json()["access_token"] + elif attempt < MAX_RETRIES - 1: + print(f" Attempt {attempt + 1}/{MAX_RETRIES}: Polaris not ready, retrying...") + time.sleep(RETRY_DELAY) + else: + raise Exception(f"Failed to get root token: {response.text}") + except requests.exceptions.RequestException as e: + if attempt < MAX_RETRIES - 1: + print(f" Attempt {attempt + 1}/{MAX_RETRIES}: Polaris not reachable, retrying...") + time.sleep(RETRY_DELAY) + else: + raise Exception(f"Failed to get root token: {e}") + + +def get_principal_role_grants(root_token, principal_role): + """Get all grants for a principal role""" + headers = { + "Authorization": f"Bearer {root_token}", + "X-Polaris-Realm": REALM + } + + response = requests.get( + f"{POLARIS_URL}/api/management/v1/principal-roles/{principal_role}", + headers=headers + ) + + status = response.status_code + if status != 200: + return None + + return response.json() + + +def get_catalog_role_grants(root_token, catalog_role): + """Get all grants for a catalog role""" + headers = { + "Authorization": f"Bearer {root_token}", + "X-Polaris-Realm": REALM + } + + response = requests.get( + f"{POLARIS_URL}/api/management/v1/catalogs/{CATALOG}/catalog-roles/{catalog_role}/grants", + headers=headers + ) + + status = response.status_code + if status != 200: + return None + + return response.json() + + +def main(): + """Verify Polaris permission configuration""" + print("[test] Verifying Polaris permission configuration...\n") + + all_passed = True + + try: + # Step 1: Get root token + print(" Getting root credentials token...") + root_token = get_root_token() + print(" ✓ Root token obtained\n") + + # Step 2: Verify teehr-read-only role has correct permissions + print(" Checking teehr-read-only role permissions...") + read_only_role_info = get_principal_role_grants(root_token, "teehr-read-only") + if read_only_role_info: + print(" ✓ teehr-read-only principal role exists") + else: + print(" ✗ ERROR: teehr-read-only principal role not found") + all_passed = False + + # Step 3: Verify teehr-read-write role has correct permissions + print(" Checking teehr-read-write role permissions...") + read_write_role_info = get_principal_role_grants(root_token, "teehr-read-write") + if read_write_role_info: + print(" ✓ teehr-read-write principal role exists") + else: + print(" ✗ ERROR: teehr-read-write principal role not found") + all_passed = False + + # Step 4: Verify catalog roles have the expected grants + print(" Checking teehr_read_only_role grants...") + read_only_catalog_grants = get_catalog_role_grants(root_token, "teehr_read_only_role") + if read_only_catalog_grants: + print(" ✓ teehr_read_only_role catalog role exists") + grants = read_only_catalog_grants.get("grants", []) + privileges = [g.get("privilege") for g in grants] + has_read_props = any("NAMESPACE_READ_PROPERTIES" in p for p in privileges) + has_table_list = any("TABLE_LIST" in p for p in privileges) + has_table_read = any("TABLE_READ_DATA" in p for p in privileges) + if has_read_props: + print(" ✓ NAMESPACE_READ_PROPERTIES permission is granted") + else: + print(" ✗ NAMESPACE_READ_PROPERTIES missing") + all_passed = False + if has_table_list: + print(" ✓ TABLE_LIST permission is granted") + else: + print(" ✗ TABLE_LIST missing") + all_passed = False + if has_table_read: + print(" ✓ TABLE_READ_DATA permission is granted") + else: + print(" ✗ TABLE_READ_DATA missing") + all_passed = False + else: + print(" ✗ ERROR: teehr_read_only_role not found") + all_passed = False + + print(" Checking teehr_read_write_role grants...") + read_write_catalog_grants = get_catalog_role_grants(root_token, "teehr_read_write_role") + if read_write_catalog_grants: + print(" ✓ teehr_read_write_role catalog role exists") + grants = read_write_catalog_grants.get("grants", []) + privileges = [g.get("privilege") for g in grants] + has_create = any("TABLE_CREATE" in p for p in privileges) + has_write = any("TABLE_WRITE_DATA" in p for p in privileges) + has_read = any("TABLE_READ_DATA" in p for p in privileges) + if has_create: + print(" ✓ TABLE_CREATE permission is granted") + else: + print(" ✗ TABLE_CREATE missing") + all_passed = False + if has_write: + print(" ✓ TABLE_WRITE_DATA permission is granted") + else: + print(" ✗ TABLE_WRITE_DATA missing") + all_passed = False + if has_read: + print(" ✓ TABLE_READ_DATA permission is granted") + else: + print(" ✗ TABLE_READ_DATA missing") + all_passed = False + else: + print(" ✗ ERROR: teehr_read_write_role not found") + all_passed = False + + print() + if all_passed: + print("[test] Polaris permission configuration verification PASSED") + return 0 + else: + print("[test] Polaris permission configuration verification FAILED") + return 1 + + except Exception as e: + print(f"[test] ERROR: {e}") + import traceback + traceback.print_exc() + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/polaris_roles_test.py b/tests/polaris_roles_test.py new file mode 100644 index 0000000..f41b254 --- /dev/null +++ b/tests/polaris_roles_test.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +""" +Integration test: Polaris role-based ACL validation + +Validates: +- Principal roles (teehr-read-only, teehr-read-write, iceberg-catalog-admin) exist +- Catalog roles (teehr_read_only_role, teehr_read_write_role, catalog_admin_role) exist +- Roles are properly bound to permissions +""" + +import sys +import requests +import time + +POLARIS_URL = "http://polaris:8181" +REALM = "teehr" +ROOT_CLIENT_ID = "root" +ROOT_CLIENT_SECRET = "secret123" + +EXPECTED_CATALOG_ROLES = [ + "teehr_read_only_role", + "teehr_read_write_role", + "catalog_admin_role" +] + +EXPECTED_PRINCIPAL_ROLES = [ + "teehr-read-only", + "teehr-read-write", + "iceberg-catalog-admin" +] + +# Retry configuration +MAX_RETRIES = 10 +RETRY_DELAY = 2 # seconds + + +def get_root_token(): + """Get root token using client credentials""" + for attempt in range(MAX_RETRIES): + try: + token_url = f"{POLARIS_URL}/api/catalog/v1/oauth/tokens" + payload = { + "grant_type": "client_credentials", + "client_id": ROOT_CLIENT_ID, + "client_secret": ROOT_CLIENT_SECRET, + "scope": "PRINCIPAL_ROLE:ALL" + } + headers = { + "X-Polaris-Realm": REALM, + "Content-Type": "application/x-www-form-urlencoded" + } + + response = requests.post(token_url, data=payload, headers=headers, timeout=5) + if response.status_code == 200: + return response.json()["access_token"] + elif attempt < MAX_RETRIES - 1: + print(f" Attempt {attempt + 1}/{MAX_RETRIES}: Polaris not ready, retrying...") + time.sleep(RETRY_DELAY) + else: + raise Exception(f"Failed to get root token: {response.text}") + except requests.exceptions.RequestException as e: + if attempt < MAX_RETRIES - 1: + print(f" Attempt {attempt + 1}/{MAX_RETRIES}: Polaris not reachable, retrying...") + time.sleep(RETRY_DELAY) + else: + raise Exception(f"Failed to get root token: {e}") + + +def check_catalog_role(root_token, role_name): + """Check if a catalog role exists""" + headers = { + "Authorization": f"Bearer {root_token}", + "X-Polaris-Realm": REALM + } + + response = requests.get( + f"{POLARIS_URL}/api/management/v1/catalogs/teehr/catalog-roles/{role_name}", + headers=headers + ) + + return response.status_code == 200 + + +def check_principal_role(root_token, role_name): + """Check if a principal role exists""" + headers = { + "Authorization": f"Bearer {root_token}", + "X-Polaris-Realm": REALM + } + + response = requests.get( + f"{POLARIS_URL}/api/management/v1/principal-roles/{role_name}", + headers=headers + ) + + return response.status_code == 200 + + +def main(): + """Run role-based ACL validation""" + print("[test] Validating Polaris role-based ACLs...") + + try: + root_token = get_root_token() + + # Check catalog roles + print(" Checking catalog roles...") + all_passed = True + for role in EXPECTED_CATALOG_ROLES: + if check_catalog_role(root_token, role): + print(f" ✓ Catalog role {role} exists") + else: + print(f" WARNING: Catalog role {role} not found") + all_passed = False + + # Check principal roles + print(" Checking principal roles...") + for role in EXPECTED_PRINCIPAL_ROLES: + if check_principal_role(root_token, role): + print(f" ✓ Principal role {role} exists") + else: + print(f" WARNING: Principal role {role} not found") + all_passed = False + + print("[test] Role-based ACL validation complete") + return 0 if all_passed else 1 + + except Exception as e: + print(f"[test] ERROR: {e}") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/spark_authmanager_test.py b/tests/spark_authmanager_test.py new file mode 100644 index 0000000..0643dc2 --- /dev/null +++ b/tests/spark_authmanager_test.py @@ -0,0 +1,247 @@ +#!/usr/bin/env python3 +""" +Integration test: Spark session permission enforcement via AuthManager + +When run without TEST_USERNAME set, orchestrates per-user subprocesses so each +user gets a fresh JVM (matching JupyterHub's per-user-pod model and avoiding +static state leakage in TeehrBrokerAuthManager across spark.stop() calls). + +Single-user mode (invoked by orchestrator or directly): + TEST_USERNAME - Keycloak username (default: admin) + TEST_PASSWORD - Keycloak password (default: admin) + TEST_EXPECTED_READ - "true"/"false" (default: true) + TEST_EXPECTED_WRITE - "true"/"false" (default: true) +""" + +import sys +import os +import subprocess +import requests +import time +import gc + +# --- Orchestrator mode: spawn one subprocess per user for clean JVM isolation --- +TEST_USERS = { + "admin": {"password": "admin", "expected_read": True, "expected_write": True}, + "poweruser": {"password": "poweruser", "expected_read": True, "expected_write": True}, + "user": {"password": "user", "expected_read": True, "expected_write": False}, +} + +if "TEST_USERNAME" not in os.environ: + all_passed = True + for username, config in TEST_USERS.items(): + print(f"\n{'='*60}") + print(f"Running AuthManager test for: {username}") + print('='*60) + env = os.environ.copy() + env["TEST_USERNAME"] = username + env["TEST_PASSWORD"] = config["password"] + env["TEST_EXPECTED_READ"] = str(config["expected_read"]).lower() + env["TEST_EXPECTED_WRITE"] = str(config["expected_write"]).lower() + result = subprocess.run( + [sys.executable, __file__], + env=env, + timeout=300, + ) + if result.returncode != 0: + all_passed = False + + print(f"\n[test] Spark AuthManager enforcement tests {'PASSED' if all_passed else 'FAILED'}") + sys.exit(0 if all_passed else 1) + +# --- Single-user mode: actual test logic, called by orchestrator subprocess --- + +# Set up environment before importing PySpark +os.environ.setdefault("POLARIS_DEFAULT_REALM", "teehr") +os.environ.setdefault("REMOTE_CATALOG_REST_URI", "http://polaris:8181/api/catalog") +os.environ.setdefault("REMOTE_WAREHOUSE_S3_PATH", "s3://warehouse/") +os.environ.setdefault("REMOTE_CATALOG_S3_ENDPOINT", "http://minio:9000") +os.environ.setdefault("REMOTE_CATALOG_S3_PATH_STYLE_ACCESS", "true") +os.environ.setdefault("AWS_ACCESS_KEY_ID", "minioadmin") +os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "minioadmin123") +os.environ.setdefault("AWS_REGION", "us-east-2") +# Broker and OAuth endpoints +os.environ.setdefault("POLARIS_BROKER_URL", "http://teehr-api:8000/auth/polaris-token") +os.environ.setdefault( + "POLARIS_OAUTH2_TOKEN_ENDPOINT", + "http://keycloak-service:8080/realms/teehr/protocol/openid-connect/token", +) +# Use jupyterhub client (same as the password grant below) +os.environ.setdefault("POLARIS_CLIENT_ID", "jupyterhub") +os.environ.setdefault("POLARIS_CLIENT_SECRET", "local-jupyterhub-client-secret") +# JVM heap +os.environ["JAVA_TOOL_OPTIONS"] = "-Xmx1g" +os.environ.setdefault("SPARK_LOCAL_IP", "127.0.0.1") + +sys.path.insert(0, "/opt/teehr") + +# Configuration +KEYCLOAK_URL = "http://keycloak-service:8080" +REALM = "teehr" +CATALOG = "iceberg" +NAMESPACE = "teehr" + +MAX_RETRIES = 10 +RETRY_DELAY = 2 + +# Single-user mode — configured via env vars to match JupyterHub's per-user-pod model +USERNAME = os.getenv("TEST_USERNAME", "admin") +PASSWORD = os.getenv("TEST_PASSWORD", "admin") +EXPECTED_READ = os.getenv("TEST_EXPECTED_READ", "true").lower() == "true" +EXPECTED_WRITE = os.getenv("TEST_EXPECTED_WRITE", "true").lower() == "true" + + +def get_keycloak_tokens(username: str, password: str) -> tuple: + """Get access_token and refresh_token from Keycloak via password grant.""" + for attempt in range(MAX_RETRIES): + try: + token_url = f"{KEYCLOAK_URL}/realms/{REALM}/protocol/openid-connect/token" + payload = { + "grant_type": "password", + "client_id": "jupyterhub", + "client_secret": "local-jupyterhub-client-secret", + "username": username, + "password": password, + "scope": "openid", + } + response = requests.post(token_url, data=payload, timeout=10) + if response.status_code == 200: + data = response.json() + return data["access_token"], data.get("refresh_token", "") + elif attempt < MAX_RETRIES - 1: + print(f" Attempt {attempt + 1}/{MAX_RETRIES}: Keycloak not ready, retrying...") + time.sleep(RETRY_DELAY) + else: + raise Exception(f"Failed to get token for {username}: {response.text}") + except requests.exceptions.RequestException as e: + if attempt < MAX_RETRIES - 1: + print(f" Attempt {attempt + 1}/{MAX_RETRIES}: Connection failed, retrying...") + time.sleep(RETRY_DELAY) + else: + raise Exception(f"Failed to get token for {username}: {e}") + + raise Exception(f"Failed to get token for {username} after {MAX_RETRIES} attempts") + + +def test_spark_read_access(spark, username: str) -> bool: + """Test read access by listing tables in namespace.""" + try: + print(f" Testing READ access...") + tables = spark.sql(f"SHOW TABLES IN {CATALOG}.{NAMESPACE}").collect() + print(f" ✓ {username} can READ from {NAMESPACE} namespace (found {len(tables)} tables)") + return True + except Exception as e: + error_msg = str(e).lower() + if any(w in error_msg for w in ("permission", "forbidden", "denied", "403")): + print(f" ✓ {username} correctly denied READ access") + else: + print(f" ✗ ERROR during READ test: {type(e).__name__}: {str(e)[:100]}") + return False + + +def test_spark_write_access(spark, username: str, should_write: bool = True) -> bool: + """Test write access via CREATE TABLE + INSERT. Returns True if write succeeded.""" + table_name = f"test_authmanager_{username}_{int(time.time() * 1000)}" + full_table_name = f"{CATALOG}.{NAMESPACE}.{table_name}" + + try: + print(f" Testing WRITE access (CREATE TABLE + INSERT)...") + spark.sql(f""" + CREATE TABLE {full_table_name} ( + id INT, + name STRING + ) + USING iceberg + """) + spark.sql(f"INSERT INTO {full_table_name} VALUES (1, 'test')") + + try: + spark.sql(f"DROP TABLE {full_table_name}") + except Exception: + pass + + if should_write: + print(f" ✓ {username} successfully created table and inserted data") + else: + print(f" ✗ ERROR: {username} should NOT be able to write but succeeded!") + return True + + except Exception as e: + try: + spark.sql(f"DROP TABLE IF EXISTS {full_table_name}") + except Exception: + pass + + error_msg = str(e).lower() + is_permission_error = any(w in error_msg for w in ( + "permission", "forbidden", "denied", "403", "not authorized", "unauthorized", "access" + )) + + if should_write: + print(f" ✗ ERROR: {username} should be able to write but got: {type(e).__name__}") + print(f" {str(e)[:150]}") + elif is_permission_error: + print(f" ✓ {username} correctly denied WRITE access (permission error)") + else: + print(f" ✓ {username} denied WRITE access: {type(e).__name__}") + return False + + +def main(): + print(f"[test] Starting Spark AuthManager test for user: {USERNAME}") + print(f" Expected: read={EXPECTED_READ}, write={EXPECTED_WRITE}\n") + + all_passed = True + spark = None + try: + print(" Getting Keycloak tokens...") + access_token, refresh_token = get_keycloak_tokens(USERNAME, PASSWORD) + if not refresh_token: + print(" ✗ ERROR: No refresh_token returned - required for AuthManager broker session") + return 1 + print(" ✓ Tokens obtained (access + refresh)") + + os.environ["POLARIS_USER_TOKEN"] = access_token + os.environ["POLARIS_REFRESH_TOKEN"] = refresh_token + os.environ["JUPYTERHUB_USER"] = USERNAME + + print(" Creating Spark session via AuthManager...") + try: + from spark_session_utils import create_minio_spark_session + spark = create_minio_spark_session( + use_authmanager=True, + force_recreate_session=True, + ) + print(" ✓ Spark session created via AuthManager") + except ImportError: + print(" ✗ ERROR: spark_session_utils not available") + return 1 + + can_read = test_spark_read_access(spark, USERNAME) + if can_read != EXPECTED_READ: + print(f" ✗ ERROR: Expected read={EXPECTED_READ} but got {can_read}") + all_passed = False + + can_write = test_spark_write_access(spark, USERNAME, should_write=EXPECTED_WRITE) + if can_write != EXPECTED_WRITE: + print(f" ✗ ERROR: Expected write={EXPECTED_WRITE} but got {can_write}") + all_passed = False + + except Exception as e: + print(f" ✗ ERROR: {type(e).__name__}: {str(e)[:200]}") + all_passed = False + finally: + if spark: + try: + spark.stop() + except Exception: + pass + gc.collect() + + result = "PASSED" if all_passed else "FAILED" + print(f"\n[test] Spark AuthManager test for {USERNAME}: {result}") + return 0 if all_passed else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/spark_iceberg_permissions_test.py b/tests/spark_iceberg_permissions_test.py new file mode 100644 index 0000000..a691c06 --- /dev/null +++ b/tests/spark_iceberg_permissions_test.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +""" +Integration test: Polaris permission configuration verification + +Validates: +- Catalog roles and principal roles are correctly created +- Principal roles have the expected permissions granted on the teehr namespace +- Permissions are configured for read-only and read-write access patterns +""" + +import sys +import json +import requests +import time + +# Configuration +POLARIS_URL = "http://polaris:8181" +REALM = "teehr" +CATALOG = "teehr" +NAMESPACE = "teehr" + +# Root credentials for admin access +ROOT_CLIENT_ID = "root" +ROOT_CLIENT_SECRET = "secret123" + +# Retry configuration +MAX_RETRIES = 10 +RETRY_DELAY = 2 # seconds + + +def get_root_token(): + """Get root token using client credentials""" + for attempt in range(MAX_RETRIES): + try: + token_url = f"{POLARIS_URL}/api/catalog/v1/oauth/tokens" + payload = { + "grant_type": "client_credentials", + "client_id": ROOT_CLIENT_ID, + "client_secret": ROOT_CLIENT_SECRET, + "scope": "PRINCIPAL_ROLE:ALL" + } + headers = { + "X-Polaris-Realm": REALM, + "Content-Type": "application/x-www-form-urlencoded" + } + + response = requests.post(token_url, data=payload, headers=headers, timeout=5) + if response.status_code == 200: + return response.json()["access_token"] + elif attempt < MAX_RETRIES - 1: + print(f" Attempt {attempt + 1}/{MAX_RETRIES}: Polaris not ready, retrying...") + time.sleep(RETRY_DELAY) + else: + raise Exception(f"Failed to get root token: {response.text}") + except requests.exceptions.RequestException as e: + if attempt < MAX_RETRIES - 1: + print(f" Attempt {attempt + 1}/{MAX_RETRIES}: Polaris not reachable, retrying...") + time.sleep(RETRY_DELAY) + else: + raise Exception(f"Failed to get root token: {e}") + + +def get_principal_role_grants(root_token, principal_role): + """Get all grants for a principal role""" + headers = { + "Authorization": f"Bearer {root_token}", + "X-Polaris-Realm": REALM + } + + response = requests.get( + f"{POLARIS_URL}/api/management/v1/principal-roles/{principal_role}", + headers=headers + ) + + status = response.status_code + if status != 200: + return None + + return response.json() + + +def get_catalog_role_grants(root_token, catalog_role): + """Get all grants for a catalog role""" + headers = { + "Authorization": f"Bearer {root_token}", + "X-Polaris-Realm": REALM + } + + response = requests.get( + f"{POLARIS_URL}/api/management/v1/catalogs/{CATALOG}/catalog-roles/{catalog_role}/grants", + headers=headers + ) + + status = response.status_code + if status != 200: + return None + + return response.json() + + +def main(): + """Verify Polaris permission configuration""" + print("[test] Verifying Polaris permission configuration...\n") + + all_passed = True + + try: + # Step 1: Get root token + print(" Getting root credentials token...") + root_token = get_root_token() + print(" ✓ Root token obtained\n") + + # Step 2: Verify teehr-read-only role has correct permissions + print(" Checking teehr-read-only role permissions...") + read_only_role_info = get_principal_role_grants(root_token, "teehr-read-only") + if read_only_role_info: + print(" ✓ teehr-read-only principal role exists") + else: + print(" ✗ ERROR: teehr-read-only principal role not found") + all_passed = False + + # Step 3: Verify teehr-read-write role has correct permissions + print(" Checking teehr-read-write role permissions...") + read_write_role_info = get_principal_role_grants(root_token, "teehr-read-write") + if read_write_role_info: + print(" ✓ teehr-read-write principal role exists") + else: + print(" ✗ ERROR: teehr-read-write principal role not found") + all_passed = False + + # Step 4: Verify catalog roles have the expected grants + print(" Checking teehr_read_only_role grants...") + read_only_catalog_grants = get_catalog_role_grants(root_token, "teehr_read_only_role") + if read_only_catalog_grants: + print(" ✓ teehr_read_only_role catalog role exists") + grants = read_only_catalog_grants.get("grants", []) + if any("READ_PROPERTIES" in g.get("privilege", "") for g in grants): + print(" ✓ READ_PROPERTIES permission is granted") + else: + print(" ℹ Available grants:", [g.get("privilege") for g in grants]) + else: + print(" ✗ ERROR: teehr_read_only_role not found") + all_passed = False + + print(" Checking teehr_read_write_role grants...") + read_write_catalog_grants = get_catalog_role_grants(root_token, "teehr_read_write_role") + if read_write_catalog_grants: + print(" ✓ teehr_read_write_role catalog role exists") + grants = read_write_catalog_grants.get("grants", []) + has_create = any("CREATE" in g.get("privilege", "") for g in grants) + has_write = any("WRITE" in g.get("privilege", "") for g in grants) + if has_create: + print(" ✓ TABLE_CREATE permission is granted") + if has_write: + print(" ✓ WRITE permission is granted") + if not (has_create or has_write): + print(" ℹ Available grants:", [g.get("privilege") for g in grants]) + else: + print(" ✗ ERROR: teehr_read_write_role not found") + all_passed = False + + print() + if all_passed: + print("[test] Polaris permission configuration verification PASSED") + return 0 + else: + print("[test] Polaris permission configuration verification FAILED") + return 1 + + except Exception as e: + print(f"[test] ERROR: {e}") + import traceback + traceback.print_exc() + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/spark_permission_test.py b/tests/spark_permission_test.py new file mode 100644 index 0000000..533c422 --- /dev/null +++ b/tests/spark_permission_test.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python3 +""" +Integration test: Spark session permission enforcement with Polaris + +Validates real permission enforcement through Spark operations: +- admin: Can create tables, insert, read +- poweruser (teehr-read-write): Can create tables, insert, read +- user (teehr-read-only): Can read tables, but NOT create or insert +""" + +import sys +import os +import requests +import time +import gc +from typing import Optional + +# Set up environment for Polaris/Spark before importing PySpark +os.environ.setdefault("POLARIS_DEFAULT_REALM", "teehr") +os.environ.setdefault("REMOTE_CATALOG_REST_URI", "http://polaris:8181/api/catalog") +os.environ.setdefault("REMOTE_WAREHOUSE_S3_PATH", "s3://warehouse/") +os.environ.setdefault("REMOTE_CATALOG_S3_ENDPOINT", "http://minio:9000") +os.environ.setdefault("REMOTE_CATALOG_S3_PATH_STYLE_ACCESS", "true") +os.environ.setdefault("AWS_ACCESS_KEY_ID", "minioadmin") +os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "minioadmin123") +os.environ.setdefault("AWS_REGION", "us-east-2") +# Set JVM heap size BEFORE PySpark initializes the JVM - must use JAVA_TOOL_OPTIONS +# spark.driver.memory config is ignored if JVM heap is already too small +os.environ["JAVA_TOOL_OPTIONS"] = "-Xmx1g" +os.environ.setdefault("SPARK_LOCAL_IP", "127.0.0.1") + +# Set up path to import spark_session_utils from /opt/teehr (copied in Dockerfile) +sys.path.insert(0, "/opt/teehr") + +# Configuration +KEYCLOAK_URL = "http://keycloak-service:8080" +REALM = "teehr" +CATALOG = "iceberg" # Spark catalog name (spark.sql.catalog.) +NAMESPACE = "teehr" # Polaris/Iceberg namespace name + +# Retry configuration +MAX_RETRIES = 10 +RETRY_DELAY = 2 # seconds + +TEST_USERS = { + "admin": { + "password": "admin", + "expected_read": True, + "expected_write": True + }, + "poweruser": { + "password": "poweruser", + "expected_read": True, + "expected_write": True + }, + "user": { + "password": "user", + "expected_read": True, + "expected_write": False + } +} + + +def get_keycloak_token(username: str, password: str) -> str: + """Get a JWT token from Keycloak for a user""" + for attempt in range(MAX_RETRIES): + try: + token_url = f"{KEYCLOAK_URL}/realms/{REALM}/protocol/openid-connect/token" + payload = { + "grant_type": "password", + "client_id": "jupyterhub", + "client_secret": "local-jupyterhub-client-secret", + "username": username, + "password": password, + "scope": "openid" + } + + response = requests.post(token_url, data=payload, timeout=5) + if response.status_code == 200: + return response.json()["access_token"] + elif response.status_code == 401 and attempt < MAX_RETRIES - 1: + print(f" Attempt {attempt + 1}/{MAX_RETRIES}: Keycloak not ready, retrying...") + time.sleep(RETRY_DELAY) + else: + raise Exception(f"Failed to get token for {username}: {response.text}") + except requests.exceptions.RequestException as e: + if attempt < MAX_RETRIES - 1: + print(f" Attempt {attempt + 1}/{MAX_RETRIES}: Connection failed, retrying...") + time.sleep(RETRY_DELAY) + else: + raise Exception(f"Failed to get token for {username}: {e}") + + raise Exception(f"Failed to get token for {username} after {MAX_RETRIES} attempts") + + +def test_spark_read_access(spark, username: str, catalog: str = "iceberg", namespace: str = "teehr") -> bool: + """Test read access by listing tables in namespace""" + try: + print(f" Testing READ access...") + # List tables using fully-qualified catalog.namespace reference + tables = spark.sql(f"SHOW TABLES IN {catalog}.{namespace}").collect() + print(f" ✓ {username} can READ from {namespace} namespace (found {len(tables)} tables)") + return True + except Exception as e: + error_msg = str(e).lower() + if "permission" in error_msg or "forbidden" in error_msg or "denied" in error_msg or "403" in error_msg: + print(f" ✓ {username} correctly denied READ access") + return False + else: + print(f" ✗ ERROR during READ test: {type(e).__name__}: {str(e)[:100]}") + return False + + +def test_spark_write_access(spark, username: str, catalog: str = "iceberg", namespace: str = "teehr", should_write: bool = True) -> bool: + """Test write access by creating a table and inserting data. Returns True if write succeeded, False if denied/failed.""" + table_name = f"test_table_{username}_{int(time.time() * 1000)}" + full_table_name = f"{catalog}.{namespace}.{table_name}" + + try: + print(f" Testing WRITE access (CREATE TABLE + INSERT)...") + spark.sql(f""" + CREATE TABLE {full_table_name} ( + id INT, + name STRING + ) + USING iceberg + """) + spark.sql(f"INSERT INTO {full_table_name} VALUES (1, 'test')") + + # Clean up + try: + spark.sql(f"DROP TABLE {full_table_name}") + except Exception: + pass + + if should_write: + print(f" ✓ {username} successfully created table and inserted data") + else: + print(f" ✗ ERROR: {username} should NOT be able to write but succeeded!") + return True + + except Exception as e: + # Always attempt cleanup even if write failed partway through + try: + spark.sql(f"DROP TABLE IF EXISTS {full_table_name}") + except Exception: + pass + + error_msg = str(e).lower() + is_permission_error = any(w in error_msg for w in ("permission", "forbidden", "denied", "403", "not authorized", "unauthorized", "access")) + + if should_write: + print(f" ✗ ERROR: {username} should be able to write but got: {type(e).__name__}") + print(f" {str(e)[:150]}") + elif is_permission_error: + print(f" ✓ {username} correctly denied WRITE access (permission error)") + else: + print(f" ✓ {username} denied WRITE access: {type(e).__name__}") + return False + + +def main(): + """Run Spark permission enforcement tests""" + print("[test] Starting Spark permission enforcement tests...\n") + + all_passed = True + + for username, config in TEST_USERS.items(): + print(f"Testing user: {username}") + print(f" Expected: read={config['expected_read']}, write={config['expected_write']}") + + spark = None + try: + # Step 1: Get Keycloak token + print(" Getting Keycloak token...") + token = get_keycloak_token(username, config["password"]) + print(" ✓ Token obtained") + + # Step 2: Create Spark session with Polaris catalog + print(" Creating Spark session with Polaris catalog...") + try: + from spark_session_utils import create_minio_spark_session + spark = create_minio_spark_session( + polaris_token=token, + ) + print(" ✓ Spark session created") + except ImportError: + # If spark_session_utils is not available, skip this user + print(" ✗ ERROR: spark_session_utils not available") + all_passed = False + continue + + # Step 3: Test read access + can_read = test_spark_read_access(spark, username, catalog=CATALOG, namespace=NAMESPACE) + if can_read != config["expected_read"]: + print(f" ✗ ERROR: Expected read={config['expected_read']} but got {can_read}") + all_passed = False + + # Step 4: Test write access + can_write = test_spark_write_access(spark, username, catalog=CATALOG, namespace=NAMESPACE, should_write=config["expected_write"]) + if can_write != config["expected_write"]: + print(f" ✗ ERROR: Expected write={config['expected_write']} but got {can_write}") + all_passed = False + + print() + + except Exception as e: + print(f" ✗ ERROR: {type(e).__name__}: {str(e)[:200]}\n") + all_passed = False + finally: + # Cleanup: stop Spark session and force garbage collection + if spark: + try: + spark.stop() + except: + pass + # Force garbage collection to free memory between sessions + gc.collect() + time.sleep(0.5) + + print("[test] Spark permission enforcement tests", "PASSED" if all_passed else "FAILED") + + return 0 if all_passed else 1 + + +if __name__ == "__main__": + sys.exit(main()) From 0db3e1c2cf653e4164abf6f115ae68a2b97f1e3a Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Wed, 29 Jul 2026 17:51:40 -0400 Subject: [PATCH 25/62] make the principal sync optional for individual grants --- docs/polaris-access-control.md | 124 ++++++++++++++++++ .../polaris-sync-principals-script.yaml | 17 ++- polaris/manifests/polaris-config.yaml.tpl | 27 +++- 3 files changed, 159 insertions(+), 9 deletions(-) create mode 100644 docs/polaris-access-control.md diff --git a/docs/polaris-access-control.md b/docs/polaris-access-control.md new file mode 100644 index 0000000..bf7e136 --- /dev/null +++ b/docs/polaris-access-control.md @@ -0,0 +1,124 @@ +# Polaris Access Control Design + +## Overview + +Access to the Iceberg catalog (Polaris) is controlled through a two-layer model: + +1. **Group-level access** — enforced automatically via JWT claim mapping (no sync required) +2. **Individual/table-level access** — enforced via named Polaris principals (sync required) + +--- + +## Layer 1: Group-level access (JWT claim mapping) + +Polaris is configured to read the `groups` claim from Keycloak JWTs and map group +membership directly to Polaris principal roles. This means: + +- No principal sync is needed for standard access +- Adding a user to a Keycloak group grants them the corresponding Polaris permissions + on their **next token issuance** (no delay, no operational coupling) +- Removing a user from a group immediately revokes access + +### Group → Principal role mapping + +| Keycloak group | Polaris principal role | Effect | +|---|---|---| +| `/teehr-read-only` | `teehr-read-only` | Can list and read tables in the `teehr` namespace | +| `/teehr-read-write` | `teehr-read-write` | Can create, read, write, and drop tables | +| `/iceberg-catalog-admins` | `iceberg-catalog-admin` | Full catalog management | + +Groups not in this table (e.g. `/basic-user`, `/jupyter-user`) are ignored by Polaris. + +### Polaris configuration + +Configured in `polaris/manifests/polaris-config.yaml.tpl`: + +```properties +quarkus.oidc.roles.role-claim-path=groups +polaris.oidc.principal-roles-mapper.type=default +polaris.oidc.principal-roles-mapper.mappings[0].regex=^/?iceberg-catalog-admins$ +polaris.oidc.principal-roles-mapper.mappings[0].replacement=PRINCIPAL_ROLE:iceberg-catalog-admin +polaris.oidc.principal-roles-mapper.mappings[1].regex=^/?teehr-(.+)$ +polaris.oidc.principal-roles-mapper.mappings[1].replacement=PRINCIPAL_ROLE:teehr-$1 +``` + +### Adding a new access tier + +1. Create a Keycloak group named `/teehr-` in `keycloak-bootstrap/manifests/` +2. Add a namespace policy for the new principal role in `polaris-bootstrap/manifests/acl-config.yaml.tpl` +3. No Polaris config change needed — the `teehr-(.+)` pattern picks it up automatically +4. Add users to the group in Keycloak + +--- + +## Layer 2: Individual/table-level access (principal sync) + +For use cases requiring finer-grained control beyond group defaults: + +- Granting a specific user read access to a specific table (not the whole namespace) +- Temporary elevated access for a single user +- Audit trails tied to a named Polaris principal entity + +### How it works + +The `polaris-sync-principals-script` (run as part of `polaris-bootstrap`) creates a +named Polaris principal for each Keycloak user and assigns principal role bindings +based on their group membership. These bindings are **additive** — they stack on top +of the JWT-based group grants. + +To grant a user access to a specific table: +1. Ensure the user has a synced principal in Polaris (sync script handles this) +2. Create a table-level catalog role with the desired privilege +3. Bind that catalog role to the user's principal role via the Polaris management API + +### Why the sync is optional for basic access + +Since JWT group mapping covers the common case, the sync is only needed when you +require per-principal grants. The sync can be run on-demand or on a schedule — it +does not need to run before users can access the catalog. + +--- + +## Permission model + +### Namespace-level grants (acl-config.yaml.tpl) + +| Principal role | Catalog role | Privileges | +|---|---|---| +| `teehr-read-only` | `teehr_read_only_role` | `NAMESPACE_READ_PROPERTIES`, `TABLE_LIST`, `TABLE_READ_PROPERTIES`, `TABLE_READ_DATA` | +| `teehr-read-write` | `teehr_read_write_role` | All read-only + `NAMESPACE_WRITE_PROPERTIES`, `TABLE_CREATE`, `TABLE_DROP`, `TABLE_WRITE_PROPERTIES`, `TABLE_READ_DATA`, `TABLE_WRITE_DATA` | +| `iceberg-catalog-admin` | `catalog_admin_role` | `CATALOG_MANAGE_CONTENT`, `CATALOG_MANAGE_METADATA` | + +### Storage + +- MinIO (local) / S3 (remote): credentials configured in Polaris catalog `storageConfigInfo` +- `stsUnavailable=true` for local MinIO (no STS credential vending) +- `s3.remote-signing-enabled=false` — clients use their own configured S3 credentials + +--- + +## Authentication paths + +### Direct token (notebooks, API clients) + +``` +User → Keycloak password/refresh grant → JWT (jupyterhub client) + → Spark: spark.sql.catalog.iceberg.token = {jwt} + → Polaris: validates JWT, maps groups to principal roles, enforces permissions +``` + +### AuthManager / broker (JupyterHub spawned notebooks) + +``` +User logs in → JupyterHub OAuth → Keycloak issues access_token + refresh_token + → Broker (/auth/polaris-session) stores refresh_token, returns broker_session_token + → TeehrBrokerAuthManager (JAR) holds broker_session_token + → On each Iceberg operation: JAR calls /auth/polaris-token/session + → Broker refreshes token via Keycloak (refresh_token grant) + → Returns refreshed jupyterhub JWT (preserves group claims) + → Polaris: same JWT-based group mapping as direct token path +``` + +Note: The broker does **not** do token exchange — it refreshes the user's token directly +to preserve group claims. Token exchange with a different audience was found to strip +the `groups` claim, preventing per-user permission enforcement. diff --git a/polaris-bootstrap/manifests/polaris-sync-principals-script.yaml b/polaris-bootstrap/manifests/polaris-sync-principals-script.yaml index 75a0388..ad9c81d 100644 --- a/polaris-bootstrap/manifests/polaris-sync-principals-script.yaml +++ b/polaris-bootstrap/manifests/polaris-sync-principals-script.yaml @@ -5,9 +5,20 @@ metadata: data: sync_principals.py: | """ - Syncs all enabled Keycloak realm users into Polaris as principal entities - and grants matching Polaris principal roles for Keycloak realm roles with - the teehr-* prefix. + Syncs Keycloak users into Polaris as named principal entities and grants + principal role bindings for their teehr-* group memberships. + + NOTE ON DESIGN: Basic group-level access (teehr-read-only, teehr-read-write, + iceberg-catalog-admin) is enforced via JWT group claim mapping configured in + polaris-config.yaml — no principal sync is required for that to work. + + This script exists for use cases that require a named Polaris principal: + - Table-level grants (granting a specific user access to a specific table) + - Individual permission overrides beyond group-level defaults + - Audit trails tied to a specific Polaris principal entity + + If you only need group-level permissions, this script is not required. + The principal role bindings it creates are ADDITIVE to the JWT-based grants. Required environment variables: POLARIS_MANAGEMENT_URL e.g. http://polaris:8181 diff --git a/polaris/manifests/polaris-config.yaml.tpl b/polaris/manifests/polaris-config.yaml.tpl index 05730ea..c47d952 100644 --- a/polaris/manifests/polaris-config.yaml.tpl +++ b/polaris/manifests/polaris-config.yaml.tpl @@ -17,9 +17,6 @@ data: polaris.persistence.relational.jdbc.max-duration-in-ms=5000 # Authentication Context Configuration - # This realm is configured to use an external identity provider (IDP) for authentication only. - # It accepts tokens issued by Keycloak only. - # polaris.authentication.type=external # This realm is configured to use both the internal and external authentication. # It accepts tokens issued by both Polaris and Keycloak. polaris.authentication.type=mixed @@ -41,10 +38,28 @@ data: quarkus.oidc.connection-retry-count=5 quarkus.oidc.token.audience=account quarkus.oidc.token.issuer=any - quarkus.oidc.roles.role-claim-path=realm_access/roles + + # Access control: map Keycloak group membership (JWT 'groups' claim) directly to + # Polaris principal roles. This means no user sync is required for group-level + # access — adding a user to a Keycloak group immediately grants the corresponding + # Polaris permissions on their next token issuance. + # + # Two patterns are mapped: + # /iceberg-catalog-admins → iceberg-catalog-admin (name mismatch between KC group and Polaris role) + # /teehr- → teehr- (e.g. teehr-read-only, teehr-read-write) + # + # All other Keycloak groups (basic-user, jupyter-user, etc.) produce no match + # and are safely ignored by Polaris. + # + # Individual/table-level grants: use the polaris-sync-principals script to create + # a named principal for the user and assign specific catalog-role grants. These + # are additive on top of the JWT-based group grants above. + quarkus.oidc.roles.role-claim-path=groups polaris.oidc.principal-roles-mapper.type=default - polaris.oidc.principal-roles-mapper.mappings[0].regex=^(.*)$ - polaris.oidc.principal-roles-mapper.mappings[0].replacement=PRINCIPAL_ROLE:$1 + polaris.oidc.principal-roles-mapper.mappings[0].regex=^/?iceberg-catalog-admins$ + polaris.oidc.principal-roles-mapper.mappings[0].replacement=PRINCIPAL_ROLE:iceberg-catalog-admin + polaris.oidc.principal-roles-mapper.mappings[1].regex=^/?teehr-(.+)$ + polaris.oidc.principal-roles-mapper.mappings[1].replacement=PRINCIPAL_ROLE:teehr-$1 # Storage Properties Integration polaris.features."SUPPORTED_CATALOG_STORAGE_TYPES"=["S3","GCS","AZURE","FILE"] From 4540e86dd51362290c01ef3150d320c784ef1b37 Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Thu, 30 Jul 2026 08:33:14 -0400 Subject: [PATCH 26/62] update docs --- docs/access-control-matrix.md | 7 +++++-- docs/iceberg-auth-storage-roadmap.md | 12 ++++++++++-- docs/polaris-identity-propagation-plan.md | 21 ++++++++++++++++++++- docs/polaris-migration-plan.md | 7 ++++++- 4 files changed, 41 insertions(+), 6 deletions(-) diff --git a/docs/access-control-matrix.md b/docs/access-control-matrix.md index 0c0b4df..b9201f4 100644 --- a/docs/access-control-matrix.md +++ b/docs/access-control-matrix.md @@ -2,7 +2,7 @@ This document summarizes how access is currently enforced across services in this repository. -Last updated: 2026-05-15 +Last updated: 2026-07-30 ## Identity Model @@ -40,7 +40,10 @@ Source: [keycloak-bootstrap/manifests/realm-configmap.yaml.tpl](../keycloak-boot | JupyterHub admin privileges | JupyterHub Authenticator admin_groups | Group: jupyter-admin | Users in jupyter-admin | | Keycloak admin console link in TEEHR admin page | TEEHR frontend admin visibility | Role: admin | Users with admin role (same as TEEHR admin UI) | | Keycloak admin console capabilities | Keycloak permissions | Role and client roles | Intended primary group appears to be webapi-admin due to realm-management client roles | -| Iceberg or Trino end-user auth | Polaris principal roles via JWT realm roles | Realm roles: teehr-read-only, teehr-read-write, admin | Users default to teehr-read-only; teehr-read-write and admin add broader access | +| Iceberg catalog (Polaris) — JupyterHub users | Polaris principal roles via JWT **groups** claim | Groups: teehr-read-only, teehr-read-write, iceberg-catalog-admins | JWT group → Polaris principal role mapping; no per-user sync needed | +| Iceberg catalog (Polaris) — service accounts | Polaris principal roles via JWT **realm_access/roles** claim | Realm roles: iceberg-catalog-admin (trino-polaris), teehr-read-write (prefect-polaris) | Service account JWTs carry the realm role assigned to the Keycloak service account user | +| Iceberg catalog (Polaris) — namespace privileges | Polaris catalog role grants on `teehr` namespace | Polaris principal roles | See `polaris-access-control.md` for the full privilege matrix | +| Trino queries | Trino access-control rules.json | Catalog: iceberg read-only | All Trino queries restricted to read-only regardless of user identity | ## Important Notes diff --git a/docs/iceberg-auth-storage-roadmap.md b/docs/iceberg-auth-storage-roadmap.md index 74c26d9..8dc9e29 100644 --- a/docs/iceberg-auth-storage-roadmap.md +++ b/docs/iceberg-auth-storage-roadmap.md @@ -1,7 +1,15 @@ # Iceberg Auth and Storage Permissions Roadmap -Last updated: 2026-05-10 -Status: deferred for active implementation +Last updated: 2026-07-30 + +> **Status: Phase B fully implemented via Polaris.** +> `iceberg-rest` has been replaced by Apache Polaris 1.5.0 with Keycloak OIDC integration. +> Per-user catalog permissions are enforced at the Polaris level via JWT group claim mapping. +> S3/MinIO storage still uses shared service credentials (per Pattern B below). +> Pattern C (credential vending) deferred to a future phase. +> See [`polaris-access-control.md`](./polaris-access-control.md) for the implemented access control design. + +--- ## Why this document exists diff --git a/docs/polaris-identity-propagation-plan.md b/docs/polaris-identity-propagation-plan.md index 3096e4c..a38109f 100644 --- a/docs/polaris-identity-propagation-plan.md +++ b/docs/polaris-identity-propagation-plan.md @@ -1,6 +1,25 @@ # TEEHR Hub Identity Propagation and Fine-Grained Data Authorization Plan -Last updated: 2026-06-17 +Last updated: 2026-07-30 + +> **Status: Flow A implemented and tested.** Flow B (Trino with user identity propagation) pending. +> +> **Implemented:** +> - Apache Polaris 1.5.0 replaces `iceberg-rest` as the Iceberg catalog +> - Keycloak OIDC → Polaris JWT group claim mapping (no per-user sync required for group-level access) +> - JupyterHub users authenticated via Keycloak → user JWT → Polaris via AuthManager broker +> - Per-user Polaris permission enforcement verified by integration tests (`garden test`) +> - Three tiers: `teehr-read-only`, `teehr-read-write`, `iceberg-catalog-admin` +> - Service accounts: `trino-polaris` (admin) and `prefect-polaris` (read-write) via Keycloak realm role → Polaris mapping +> +> **Pending:** +> - Flow B: Trino user identity propagation (currently uses `trino-polaris` service account for all queries) +> - Pattern C: S3 credential vending per user (currently shared MinIO/S3 credentials) +> - Per-user table-level grants via optional principal sync +> +> See [`polaris-access-control.md`](./polaris-access-control.md) for the current design. + +--- ## Goals diff --git a/docs/polaris-migration-plan.md b/docs/polaris-migration-plan.md index 30cbf75..a49b10b 100644 --- a/docs/polaris-migration-plan.md +++ b/docs/polaris-migration-plan.md @@ -1,6 +1,11 @@ # Plan: Replace iceberg-rest with Apache Polaris (Keycloak-integrated) -## TL;DR +> **Status: COMPLETED** — Last updated 2026-07-30. +> `iceberg-rest` has been removed and Apache Polaris 1.5.0 is deployed and operational. +> See [`polaris-access-control.md`](./polaris-access-control.md) for the implemented auth design. +> This document is retained as a historical record of the migration plan and rationale. + +--- Replace `tabulario/iceberg-rest` with `apache/polaris`. Tight Keycloak integration via JWKS + a `PrincipalRoleMapper` that reads `realm_access.roles` from JWTs, mapping Keycloak roles directly to Polaris principal roles at runtime — no per-user Polaris registration needed. A data-driven bootstrap job creates the full namespace × privilege ACL structure from a ConfigMap. Polaris and Keycloak are the control plane; all other services are the data plane. From 6db00c14cdf5a33bde3f7c053cfb2f844bc18ea0 Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Thu, 30 Jul 2026 08:33:56 -0400 Subject: [PATCH 27/62] cleanup unused polaris role for jupyter users --- .../polaris_spark_namespace_table_example.py | 311 ---- examples/developer/setup_utils.py | 32 - examples/developer/spark_session_utils.py | 1262 ----------------- examples/developer/test_token_renewal.ipynb | 615 -------- jupyterhub-docker/spark_session_utils.py | 228 +-- jupyterhub/garden.yaml | 12 +- .../manifests/bootstrap-job.yaml | 5 + .../manifests/realm-configmap.yaml.tpl | 30 + .../manifests/acl-config.yaml.tpl | 7 +- polaris/manifests/polaris-config.yaml.tpl | 17 +- secrets/secrets.local.yaml | 4 + 11 files changed, 181 insertions(+), 2342 deletions(-) delete mode 100644 examples/developer/polaris_spark_namespace_table_example.py delete mode 100644 examples/developer/spark_session_utils.py delete mode 100644 examples/developer/test_token_renewal.ipynb diff --git a/examples/developer/polaris_spark_namespace_table_example.py b/examples/developer/polaris_spark_namespace_table_example.py deleted file mode 100644 index 919f23e..0000000 --- a/examples/developer/polaris_spark_namespace_table_example.py +++ /dev/null @@ -1,311 +0,0 @@ -#!/usr/bin/env python3 -"""Minimal Polaris + Spark example for local KinD + MinIO development. - -This script is designed to run inside a Jupyter single-user pod. -It verifies Spark catalog connectivity, creates a namespace, and then -attempts to create and read a simple Iceberg table. -""" - -from __future__ import annotations - -import argparse -import base64 -import json -import os -import sys -import time -import traceback - -import requests - - -def _ensure_setup_utils_importable() -> None: - preferred = "/tmp/polaris-spark-example" - if preferred not in sys.path: - sys.path.insert(0, preferred) - - candidates = [ - ".", - "/workspace", - "/workspace/examples/developer", - "/home/jovyan", - "/home/jovyan/examples/developer", - ] - for path in candidates: - if path not in sys.path: - sys.path.append(path) - - -_ensure_setup_utils_importable() - -from setup_utils import ( - apply_polaris_token_to_spark, - create_minio_spark_session, - ensure_fresh_polaris_user_token, -) - - -def _decode_token_claims(token: str) -> dict: - payload = token.split(".")[1] - payload += "=" * (-len(payload) % 4) - return json.loads(base64.urlsafe_b64decode(payload.encode())) - - -def _print_token_claims(token: str) -> None: - claims = _decode_token_claims(token) - roles = sorted(claims.get("realm_access", {}).get("roles", [])) - print(f"[example] Token user: {claims.get('preferred_username')}") - print(f"[example] Token roles: {', '.join(roles)}") - - -def _print_spark_catalog_config(spark) -> None: - keys = [ - "spark.sql.catalog.iceberg.type", - "spark.sql.catalog.iceberg.uri", - "spark.sql.catalog.iceberg.warehouse", - "spark.sql.catalog.iceberg.token", - "spark.sql.catalog.iceberg.header.X-Polaris-Realm", - "spark.sql.catalog.iceberg.rest.auth.type", - "spark.sql.catalog.iceberg.rest.auth.oauth2.token", - "spark.sql.catalog.iceberg.rest.transport.header.X-Polaris-Realm", - "spark.sql.catalog.iceberg.io-impl", - "spark.sql.catalog.iceberg.s3.endpoint", - "spark.sql.catalog.iceberg.s3.path-style-access", - "spark.sql.catalog.iceberg.s3.region", - "spark.hadoop.fs.s3a.endpoint", - "spark.hadoop.fs.s3a.path.style.access", - ] - print("[example] Effective Spark catalog config:") - for key in keys: - value = spark.conf.get(key, "") - if "token" in key and value not in ("", ""): - value = f"{value[:12]}...{value[-8:]}" - print(f" {key}={value}") - - -def _apply_direct_catalog_overrides(spark, token: str) -> None: - realm = os.getenv("POLARIS_DEFAULT_REALM", "teehr") - spark.conf.set("spark.sql.catalog.iceberg.warehouse", realm) - apply_polaris_token_to_spark(spark, token, catalog_name="iceberg", realm=realm) - - -def _is_auth_failure(exc: Exception) -> bool: - msg = str(exc).lower() - markers = [ - "notauthorizedexception", - "unauthorized", - "http error 401", - "401", - "invalid token", - "token expired", - ] - return any(marker in msg for marker in markers) - - -def _sql_with_token_retry(spark, sql_text: str, token_ctx: dict, show: bool = False): - try: - df = spark.sql(sql_text) - if show: - df.show(truncate=False) - return df - except Exception as exc: - if not _is_auth_failure(exc): - raise - - print("[example] SQL failed due to auth; renewing token and retrying once...") - refreshed, refreshed_refresh_token, renewed = ensure_fresh_polaris_user_token( - current_token=token_ctx["token"], - username=token_ctx["username"], - password=token_ctx["password"], - client_id=token_ctx["client_id"], - client_secret=token_ctx["client_secret"], - refresh_token=token_ctx.get("refresh_token"), - refresh_window_seconds=300, - ) - token_ctx["token"] = refreshed - token_ctx["refresh_token"] = refreshed_refresh_token - if refreshed_refresh_token: - os.environ["POLARIS_REFRESH_TOKEN"] = refreshed_refresh_token - apply_polaris_token_to_spark(spark, refreshed, catalog_name="iceberg") - if renewed: - print("[example] Token refreshed; retrying SQL.") - - df = spark.sql(sql_text) - if show: - df.show(truncate=False) - return df - - -def _set_local_polaris_env_defaults() -> None: - realm = os.getenv("POLARIS_DEFAULT_REALM", "teehr") - os.environ["REMOTE_CATALOG_REST_URI"] = os.getenv( - "REMOTE_CATALOG_REST_URI", "http://polaris:8181/api/catalog" - ) - os.environ["REMOTE_WAREHOUSE_S3_PATH"] = realm - os.environ["REMOTE_CATALOG_S3_ENDPOINT"] = os.getenv( - "REMOTE_CATALOG_S3_ENDPOINT", "http://minio:9000" - ) - os.environ["REMOTE_CATALOG_S3_PATH_STYLE_ACCESS"] = os.getenv( - "REMOTE_CATALOG_S3_PATH_STYLE_ACCESS", "true" - ) - os.environ["POLARIS_DEFAULT_REALM"] = realm - - -def _validate_direct_polaris_access(token: str) -> None: - realm = os.getenv("POLARIS_DEFAULT_REALM", "teehr") - headers = { - "Authorization": f"Bearer {token}", - "X-Polaris-Realm": realm, - } - resp = requests.get( - "http://polaris:8181/api/catalog/v1/config", - headers=headers, - params={"warehouse": realm}, - timeout=20, - ) - print(f"[example] Direct Polaris config status: {resp.status_code}") - if resp.status_code >= 400: - print(f"[example] Direct Polaris config body: {resp.text[:500]}") - resp.raise_for_status() - - -def main() -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--username", default=os.getenv("POLARIS_TEST_USERNAME", "admin")) - parser.add_argument("--password", default=os.getenv("POLARIS_TEST_PASSWORD", "admin")) - parser.add_argument( - "--oauth-client-id", - default=os.getenv("POLARIS_OAUTH_CLIENT_ID", "jupyterhub"), - ) - parser.add_argument( - "--oauth-client-secret", - default=os.getenv("POLARIS_OAUTH_CLIENT_SECRET"), - ) - parser.add_argument( - "--refresh-token", - default=os.getenv("POLARIS_REFRESH_TOKEN"), - help="Refresh token from Jupyter login context, if available", - ) - parser.add_argument( - "--namespace", - default=f"spark_demo_{int(time.time())}", - help="Namespace to create (default: spark_demo_)", - ) - parser.add_argument( - "--table", - default="hello_table", - help="Table name to create inside the namespace", - ) - parser.add_argument( - "--keep", - action="store_true", - help="Keep created namespace/table instead of dropping at end", - ) - args = parser.parse_args() - - spark = None - created_namespace = False - created_table = False - - try: - token = os.getenv("POLARIS_USER_TOKEN") - if token: - print("[example] Using POLARIS_USER_TOKEN from Jupyter environment...") - print(f"[example] Ensuring a fresh token for user '{args.username}'...") - token, refresh_token, renewed = ensure_fresh_polaris_user_token( - current_token=token, - username=args.username, - password=args.password, - client_id=args.oauth_client_id, - client_secret=args.oauth_client_secret, - refresh_token=args.refresh_token, - refresh_window_seconds=300, - ) - if refresh_token: - os.environ["POLARIS_REFRESH_TOKEN"] = refresh_token - if renewed: - print("[example] Minted or renewed access token.") - - token_ctx = { - "token": token, - "username": args.username, - "password": args.password, - "client_id": args.oauth_client_id, - "client_secret": args.oauth_client_secret, - "refresh_token": refresh_token, - } - - _print_token_claims(token) - - _set_local_polaris_env_defaults() - _validate_direct_polaris_access(token) - - print("[example] Creating Spark session...") - spark = create_minio_spark_session( - polaris_token=token, - force_recreate_session=True, - ) - - _apply_direct_catalog_overrides(spark, token) - - _print_spark_catalog_config(spark) - - print("[example] Existing namespaces:") - _sql_with_token_retry(spark, "SHOW NAMESPACES IN iceberg", token_ctx, show=True) - - fq_namespace = f"iceberg.{args.namespace}" - fq_table = f"{fq_namespace}.{args.table}" - - print(f"[example] Creating namespace: {fq_namespace}") - _sql_with_token_retry(spark, f"CREATE NAMESPACE IF NOT EXISTS {fq_namespace}", token_ctx) - created_namespace = True - - print(f"[example] Creating table: {fq_table}") - _sql_with_token_retry(spark, f"CREATE TABLE {fq_table} (id int, name string) USING iceberg", token_ctx) - created_table = True - - print(f"[example] Inserting sample rows into: {fq_table}") - _sql_with_token_retry( - spark, - f"INSERT INTO {fq_table} VALUES (1, 'alpha'), (2, 'beta'), (3, 'gamma')", - token_ctx, - ) - - print(f"[example] Reading back rows from: {fq_table}") - _sql_with_token_retry(spark, f"SELECT * FROM {fq_table} ORDER BY id", token_ctx, show=True) - - print("[example] SUCCESS") - return 0 - - except Exception as exc: - msg = str(exc) - print("[example] FAILED") - traceback.print_exc() - - if "warehouse.minio" in msg or "UnknownHostException" in msg: - print("[example] Diagnostic: Polaris server is attempting virtual-host S3 DNS") - print("[example] Diagnostic: expected local path-style access for MinIO") - print("[example] Suggested checks:") - print(" 1) Re-run Polaris bootstrap to refresh catalog properties") - print(" 2) Verify catalog has s3.path-style-access=true and endpoint=http://minio:9000") - print(" 3) Check Polaris logs for the exact hostname in UnknownHostException") - return 1 - - finally: - if spark is not None: - if created_table and created_namespace and not args.keep: - fq_namespace = f"iceberg.{args.namespace}" - fq_table = f"{fq_namespace}.{args.table}" - try: - print(f"[example] Cleaning up table: {fq_table}") - spark.sql(f"DROP TABLE IF EXISTS {fq_table}") - print(f"[example] Cleaning up namespace: {fq_namespace}") - spark.sql(f"DROP NAMESPACE IF EXISTS {fq_namespace}") - except Exception: - print("[example] Cleanup skipped due to error") - print("[example] Stopping Spark session...") - spark.stop() - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/examples/developer/setup_utils.py b/examples/developer/setup_utils.py index c35dee8..b9560b9 100644 --- a/examples/developer/setup_utils.py +++ b/examples/developer/setup_utils.py @@ -4,24 +4,6 @@ utility surface for local and remote Polaris support. """ -import sys -from pathlib import Path - -_CURRENT_DIR = Path(__file__).resolve().parent -if str(_CURRENT_DIR) not in sys.path: - sys.path.insert(0, str(_CURRENT_DIR)) - -from spark_session_utils import apply_polaris_token_to_spark -from spark_session_utils import create_minio_spark_session -from spark_session_utils import create_spark_session -from spark_session_utils import ensure_broker_session_token -from spark_session_utils import ensure_fresh_polaris_token_via_broker -from spark_session_utils import ensure_fresh_polaris_user_token -from spark_session_utils import mint_polaris_user_token -from spark_session_utils import refresh_polaris_user_token -from spark_session_utils import request_broker_polaris_token - - DEV_LOCATION_ID_LIST = [ # CONUS "usgs-02424000", @@ -48,18 +30,4 @@ "usgs-50011000", "usgs-50011085", "usgs-50011128" -] - - -__all__ = [ - "DEV_LOCATION_ID_LIST", - "apply_polaris_token_to_spark", - "create_minio_spark_session", - "create_spark_session", - "ensure_broker_session_token", - "ensure_fresh_polaris_token_via_broker", - "ensure_fresh_polaris_user_token", - "mint_polaris_user_token", - "refresh_polaris_user_token", - "request_broker_polaris_token", ] \ No newline at end of file diff --git a/examples/developer/spark_session_utils.py b/examples/developer/spark_session_utils.py deleted file mode 100644 index 1141c5c..0000000 --- a/examples/developer/spark_session_utils.py +++ /dev/null @@ -1,1262 +0,0 @@ -"""Module to create and configure Spark sessions and Polaris auth helpers.""" -# flake8: noqa -import base64 -import json -import logging -import os -import socket -import time -from pathlib import Path -from typing import Dict, List, Optional, Tuple, Union -from urllib.parse import urlsplit, urlunsplit - -import psutil -import requests - -from pyspark import SparkConf -from pyspark.sql import SparkSession -from sedona.spark import SedonaContext -import pandas as pd -import botocore.session - -import teehr.const as const - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - -# Note: Scala version: 2.13 in pyspark 4.0 -SCALA_VERSION = "2.13" -PYSPARK_VERSION = "4.0" -ICEBERG_VERSION = "1.10.1" -SEDONA_VERSION = "1.8.0" - - - -def create_spark_session( - # App name and catalog settings - app_name: str = "TEEHR Evaluation", - local_catalog_name: str = const.LOCAL_CATALOG_NAME, - local_catalog_type: str = const.LOCAL_CATALOG_TYPE, - remote_warehouse_dir: str = const.REMOTE_WAREHOUSE_S3_PATH, - remote_catalog_name: str = const.REMOTE_CATALOG_NAME, - remote_catalog_type: str = const.REMOTE_CATALOG_TYPE, - remote_catalog_uri: str = const.REMOTE_CATALOG_REST_URI, - # Spark K8'specific parameters - start_spark_cluster: bool = False, - force_recreate_session: bool = False, - executor_instances: int = 2, - executor_memory: str = "1g", - executor_cores: int = 1, - executor_image: str = None, - executor_namespace: str = None, - driver_memory: str = None, - driver_max_result_size: str = None, - pod_template_path: Union[str, Path] = const.POD_TEMPLATE_PATH, - # AWS credential parameters - aws_access_key_id: str = None, - aws_secret_access_key: str = None, - aws_session_token: str = None, - aws_region: str = const.AWS_REGION, - aws_profile: str = None, - # GCS credential parameters - enable_gcs: bool = False, - gcs_project_id: str = None, - gcs_service_account_key_file: str = None, - # Simple extensibility parameters - add_jars: List[str] = None, - add_packages: List[str] = None, - update_configs: Dict[str, str] = None, - debug_config: bool = False -) -> SparkSession: - """Create and return a Spark session for evaluation. - - Parameters - ---------- - app_name : str - Name of the Spark application. Default is "TEEHR Evaluation". - local_catalog_name : str - Name of the local Iceberg catalog. Default is "local". - local_catalog_type : str - Type of the local Iceberg catalog. Default is "jdbc". - remote_warehouse_dir : str - Remote warehouse directory for Iceberg catalog. Default is TEEHR - warehouse S3 path. - remote_catalog_name : str - Name of the remote Iceberg catalog. Default is "iceberg". - remote_catalog_type : str - Type of the remote Iceberg catalog. Default is "rest". - remote_catalog_uri : str - URI for the remote Iceberg catalog. Default is TEEHR catalog REST URI. - start_spark_cluster : bool - Whether to start a Spark cluster (Kubernetes mode). - Default is False (local mode). - force_recreate_session : bool - Whether to stop an existing Spark session before creating a new one. - Default is False. - executor_instances : int - Number of executor instances for the Spark cluster. Default is 2. - executor_memory : str - Memory allocation for each executor. Default is "1g". - executor_cores : int - Number of CPU cores for each executor. Default is 1. - executor_image : str - Container image for Spark executors. Default is None. - executor_namespace : str - Kubernetes namespace for Spark executors. Default is None. - driver_memory : str - Memory allocation for the Spark driver. Default is None. - driver_max_result_size : str - Maximum result size for the Spark driver. Default is None. - pod_template_path : Union[str, Path] - Path to the pod template file for Spark executors. - Default is "/opt/teehr/executor-pod-template.yaml". - aws_access_key_id : str - AWS access key ID for S3 access. Default is None. - aws_secret_access_key : str - AWS secret access key for S3 access. Default is None. - aws_session_token : str - AWS session token for temporary credentials. Default is None. - aws_region : str - AWS region name. Default is "us-east-2". - aws_profile : str - AWS profile name to use from ~/.aws/credentials. Only reads credentials - file if this parameter is explicitly provided. Default is None. - enable_gcs : bool - Whether to add GCS (Google Cloud Storage) connector support. - Default is False. - gcs_project_id : str - GCS project ID. Used for billing and quota tracking. When accessing - public buckets without credentials, set to "anonymous" or any - non-empty string. Default is None (will be set to "anonymous" when - enable_gcs is True and no service account key is provided). - gcs_service_account_key_file : str - Path to a GCS service account JSON key file. When provided, - authenticated access is used. When None, unauthenticated - (public-bucket) access is used. Default is None. - add_packages : List[str] - Provided Spark packages will be added if they do not already exist. - Default is None. - >>> add_packages=["com.example:my-package:1.0.0"] - add_jars : List[str] - Provided local jar paths will be added if they do not already exist. - Default is None. - >>> add_jars=["/opt/spark/jars/teehr-authmanager.jar"] - update_configs : Dict[str, str] - Provided Spark configurations will be added if they do not already - exist, or overwritten if they do exist. Default is None. - >>> update_configs={"spark.sql.shuffle.partitions": "100"} - debug_config : bool - Whether to log the final Spark configuration for debugging. - Default is False. - - Returns - ------- - SparkSession - Configured Spark session. - """ - logger.info(f"🚀 Creating Spark session: {app_name}") - - if force_recreate_session: - existing_session = SparkSession.getActiveSession() - if existing_session is not None: - logger.info("♻️ Stopping the active Spark session before recreation") - existing_session.stop() - - # Get the base configuration with common settings - conf = _create_spark_base_session( - conf=SparkConf(), - aws_region=aws_region, - driver_memory=driver_memory, - driver_maxresultsize=driver_max_result_size - ) - - if start_spark_cluster is False: - logger.info("✅ Spark local configuration successful!") - else: - logger.info(f"📦 Configuring Spark cluster with container image: {executor_image}") - _set_spark_cluster_configuration( - conf=conf, - executor_instances=executor_instances, - executor_memory=executor_memory, - executor_cores=executor_cores, - container_image=executor_image, - spark_namespace=executor_namespace, - pod_template_path=pod_template_path - ) - logger.info("✅ Spark cluster configuration successful!") - logger.info(f" - Executor instances: {executor_instances}") - logger.info(f" - Executor memory: {executor_memory}") - logger.info(f" - Executor cores: {executor_cores}") - - # Set AWS credentials if available - _set_aws_credentials_in_spark( - conf=conf, - remote_catalog_name=remote_catalog_name, - aws_access_key_id=aws_access_key_id, - aws_secret_access_key=aws_secret_access_key, - aws_session_token=aws_session_token, - aws_region=aws_region, - aws_profile=aws_profile, - ) - - # Set GCS configuration if available - if enable_gcs: - _set_gcs_configuration( - conf=conf, - gcs_project_id=gcs_project_id, - gcs_service_account_key_file=gcs_service_account_key_file, - ) - - # Set catalog metadata in Spark configuration - _set_catalog_metadata( - conf=conf, - local_catalog_name=local_catalog_name, - local_catalog_type=local_catalog_type, - remote_catalog_name=remote_catalog_name, - remote_catalog_type=remote_catalog_type, - remote_catalog_uri=remote_catalog_uri, - remote_warehouse_dir=remote_warehouse_dir - ) - - # Apply catalog configurations - _configure_iceberg_catalogs( - conf=conf, - local_catalog_name=local_catalog_name, - local_catalog_type=local_catalog_type, - remote_warehouse_dir=remote_warehouse_dir, - remote_catalog_name=remote_catalog_name, - remote_catalog_type=remote_catalog_type, - remote_catalog_uri=remote_catalog_uri - ) - - # Update configs and packages if provided - _update_configs_and_packages( - conf=conf, - update_configs=update_configs, - add_jars=add_jars, - add_packages=add_packages - ) - - logger.info("⚙️ All settings applied. Creating Spark session...") - spark = SparkSession.builder.appName(app_name).config(conf=conf).getOrCreate() - sedona_spark = SedonaContext.create(spark) - - if debug_config: - log_session_config(sedona_spark) - - logger.info("🎉 Spark session created successfully!") - - return sedona_spark - - -def _create_spark_base_session( - conf: SparkConf, - aws_region: str, - driver_memory: float = None, - driver_maxresultsize: float = None -): - """Create a base Spark builder.""" - conf.setMaster("local[*]") - - # Set base packages - base_packages = [ - f"org.apache.sedona:sedona-spark-shaded-{PYSPARK_VERSION}_{SCALA_VERSION}:{SEDONA_VERSION}", - f"org.apache.iceberg:iceberg-spark-runtime-{PYSPARK_VERSION}_{SCALA_VERSION}:{ICEBERG_VERSION}", - f"org.apache.iceberg:iceberg-core:{ICEBERG_VERSION}", - "org.datasyslab:geotools-wrapper:1.8.0-33.1", - f"org.apache.iceberg:iceberg-spark-extensions-{PYSPARK_VERSION}_{SCALA_VERSION}:{ICEBERG_VERSION}", - "org.apache.hadoop:hadoop-aws:3.4.1", # Note. Need 3.4.1 for compatibility - "com.amazonaws:aws-java-sdk-bundle:1.12.791", - "org.xerial:sqlite-jdbc:3.42.0.0" - ] - conf.set("spark.jars.packages", ",".join(base_packages)) - - # Set configurations - conf.set("spark.driver.extraJavaOptions", f"-Daws.region={aws_region}") - conf.set("spark.executor.extraJavaOptions", f"-Daws.region={aws_region}") - conf.set("spark.sql.session.timeZone", "UTC") - conf.set("spark.sql.extensions", "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions") - conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer") - conf.set("spark.hadoop.fs.s3a.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem") - - # Enable Spark decommissioning for graceful spot instance handling - conf.set("spark.decommission.enabled", "true") - conf.set("spark.storage.decommission.enabled", "true") - conf.set("spark.storage.decommission.rddBlocks.enabled", "true") - conf.set("spark.storage.decommission.shuffleBlocks.enabled", "true") - # Grace period for executors to decommission before being terminated - conf.set("spark.kubernetes.executor.decommission.gracePeriodSeconds", "30") - - # Memory settings - memory_info = psutil.virtual_memory() - driver_memory_int = int(0.75 * memory_info.available / (1024**3)) - if driver_memory is None: - driver_memory = f"{driver_memory_int}g" - if driver_maxresultsize is None: - driver_maxresultsize = f"{int(0.5 * driver_memory_int)}g" - conf.set("spark.driver.memory", f"{driver_memory}") - conf.set("spark.driver.maxResultSize", f"{driver_maxresultsize}") - - # Driver binding configuration for local mode - conf.set("spark.driver.bindAddress", "127.0.0.1") - conf.set("spark.driver.host", "127.0.0.1") - conf.set("spark.driver.port", "0") # Let Spark choose an available port - - # Default shuffle partitions: 2x local CPU cores - local_cores = os.cpu_count() or 1 - default_shuffle_partitions = 2 * local_cores - conf.set("spark.sql.shuffle.partitions", str(default_shuffle_partitions)) - - return conf - - -def _set_spark_cluster_configuration( - conf: SparkConf, - executor_instances: int, - executor_memory: str, - executor_cores: int, - container_image: str, - spark_namespace: str, - pod_template_path: Union[str, Path] -): - """Configure Spark for Kubernetes cluster mode. - - These settings are considered immutable after session creation. - """ - # Default container image - use the same image as the current pod - if container_image is None: - container_image = os.environ["TEEHR_SPARK_IMAGE"] - - # Get Kubernetes API server - use HTTPS port specifically - k8s_host = os.environ.get('KUBERNETES_SERVICE_HOST', 'kubernetes.default.svc.cluster.local') - k8s_port_https = os.environ.get('KUBERNETES_SERVICE_PORT_HTTPS', '443') - k8s_api_server = f"https://{k8s_host}:{k8s_port_https}" - - # First try getting it from environment variable - if spark_namespace is None: - spark_namespace = os.environ.get("TEEHR_NAMESPACE", "") - logger.info(f"🔍 Initial spark namespace from ENV: {spark_namespace}") - - if spark_namespace is None: - # Then get it from here - namespace_file = "/var/run/secrets/kubernetes.io/serviceaccount/namespace" - if os.path.exists(namespace_file): - with open(namespace_file, 'r') as f: - spark_namespace = f.read().strip() - - # Finally get it here if still None - if spark_namespace is None: - spark_namespace = "default" # last resort, will probably fail - - logger.info(f"🔍 Connecting to Kubernetes API: {k8s_api_server}") - logger.info(f"🎯 Executor namespace: {spark_namespace}") - logger.info(f"🔐 Executor service account: spark (in {spark_namespace})") - - # Create Spark configuration - conf.setMaster(f"k8s://{k8s_api_server}") - - # Check project ID to specify appropriate node group name. - teehr_project_id = os.environ.get("TEEHR_PROJECT_ID", "") - if teehr_project_id != "TEEHR" and teehr_project_id != "": - conf.set( - "spark.kubernetes.executor.node.selector.teehr-hub/nodegroup-name", - f"spark-r5-4xlarge-spot-{teehr_project_id.lower()}" - ) - - # Basic Kubernetes settings - conf.set("spark.executor.instances", str(executor_instances)) - conf.set("spark.executor.memory", executor_memory) - conf.set("spark.executor.cores", str(executor_cores)) - conf.set("spark.kubernetes.container.image", container_image) - conf.set("spark.kubernetes.namespace", spark_namespace) - conf.set("spark.kubernetes.authenticate.executor.serviceAccountName", "spark") - conf.set("spark.kubernetes.container.image.pullPolicy", "Always") - - # Enable Spark's dynamic allocation and decommissioning features for better - # handling of spot instances and resource management. - conf.set("spark.decommission.enabled", "true") - conf.set("spark.executor.decommission.signal", "SIGTERM") - conf.set("spark.storage.decommission.enabled", "true") - - # Set pod name prefix for executors in TEEHR-HUB for easy identification in cluster. - # Truncated to 46 chars since K8s pod names have a 63-char limit and Spark appends suffixes. - jupyterhub_user = os.environ.get("JUPYTERHUB_USER", None) - if jupyterhub_user is not None: - conf.set("spark.kubernetes.executor.podNamePrefix", jupyterhub_user[:46]) - - if os.path.exists(pod_template_path): - conf.set("spark.kubernetes.executor.podTemplateFile", pod_template_path) - else: - logger.info(f"⚠️ Executor pod template not found: {pod_template_path}") - logger.info(" You must provide a valid pod template for executors to launch correctly.") - raise FileNotFoundError(f"Executor pod template not found: {pod_template_path}") - - conf.set("spark.kubernetes.executor.deleteOnTermination", "true") - - # Default shuffle partitions: 2x total executor cores - total_executor_cores = executor_cores * executor_instances - default_shuffle_partitions = 2 * total_executor_cores - conf.set("spark.sql.shuffle.partitions", str(default_shuffle_partitions)) - - # Authentication - use service account token if available - token_file = const.SERVICE_ACCOUNT_TOKEN_PATH - ca_file = const.CA_CERTIFICATE_PATH - if os.path.exists(token_file) and os.path.exists(ca_file): - logger.info("🔐 Using in-cluster authentication") - conf.set("spark.kubernetes.authenticate.submission.oauthTokenFile", token_file) - conf.set("spark.kubernetes.authenticate.submission.caCertFile", ca_file) - conf.set("spark.kubernetes.authenticate.driver.oauthTokenFile", token_file) - conf.set("spark.kubernetes.authenticate.executor.oauthTokenFile", token_file) - - # Critical: Set the CA cert file for SSL validation - conf.set("spark.kubernetes.authenticate.caCertFile", ca_file) - else: - logger.info("⚠️ No service account tokens found - may have authentication issues") - logger.info(f" Checked: {token_file}") - logger.info(f" Checked: {ca_file}") - - # Driver binding configuration - use pod IP for Kubernetes - conf.set("spark.driver.bindAddress", "0.0.0.0") - conf.set("spark.driver.port", "0") # Let Spark choose an available port - - # Get pod IP and set as driver host so executors can connect back - pod_ip = os.environ.get('POD_IP') - if not pod_ip: - try: - hostname = socket.gethostname() - pod_ip = socket.gethostbyname(hostname) - except: - pod_ip = None - - if pod_ip: - logger.info(f"🔗 Setting driver host to pod IP: {pod_ip}") - conf.set("spark.driver.host", pod_ip) - else: - logger.info("⚠️ Could not determine pod IP - using default driver host") - - return - - -def _set_aws_credentials_in_spark( - conf: SparkConf, - remote_catalog_name: str, - aws_access_key_id: str, - aws_secret_access_key: str, - aws_session_token: str, - aws_region: str, - aws_profile: str = None, -): - """Set AWS credentials in Spark configuration with multiple options.""" - logger.info("Setting Hadoop's default AWS credentials provider and AWS region") - conf.set( - "spark.hadoop.fs.s3a.aws.credentials.provider", - "com.amazonaws.auth.DefaultAWSCredentialsProviderChain" - ) - conf.set("spark.hadoop.fs.s3a.endpoint.region", aws_region) - - # Priority 1: Explicit credentials provided by user - if aws_access_key_id and aws_secret_access_key: - logger.info("🔑 Using user-provided AWS credentials") - conf.set(f"spark.sql.catalog.{remote_catalog_name}.s3.access-key-id", aws_access_key_id) - conf.set(f"spark.sql.catalog.{remote_catalog_name}.s3.secret-access-key", aws_secret_access_key) - conf.set("spark.hadoop.fs.s3a.access.key", aws_access_key_id) - conf.set("spark.hadoop.fs.s3a.secret.key", aws_secret_access_key) - return - - # Priority 2: Explicit token - if aws_session_token: - logger.info("🔑 Using user-provided AWS session token") - conf.set(f"spark.sql.catalog.{remote_catalog_name}.s3.session-token", aws_session_token) - conf.set("spark.hadoop.fs.s3a.session.token", aws_session_token) - return - - # Priority 3: Check ~/.aws/credentials file only if profile explicitly specified (full access) - if aws_profile: - aws_credentials_file = Path.home() / ".aws" / "credentials" - if aws_credentials_file.exists(): - try: - import configparser - config = configparser.ConfigParser() - config.read(aws_credentials_file) - - if config.has_section(aws_profile): - if config.has_option(aws_profile, "aws_access_key_id") and config.has_option(aws_profile, "aws_secret_access_key"): - creds_access_key = config.get(aws_profile, "aws_access_key_id") - creds_secret_key = config.get(aws_profile, "aws_secret_access_key") - creds_session_token = config.get(aws_profile, "aws_session_token", fallback=None) - - logger.info(f"🔑 Using AWS credentials from ~/.aws/credentials profile '{aws_profile}") - conf.set(f"spark.sql.catalog.{remote_catalog_name}.s3.access-key-id", creds_access_key) - conf.set(f"spark.sql.catalog.{remote_catalog_name}.s3.secret-access-key", creds_secret_key) - conf.set("spark.hadoop.fs.s3a.access.key", creds_access_key) - conf.set("spark.hadoop.fs.s3a.secret.key", creds_secret_key) - - if creds_session_token: - conf.set(f"spark.sql.catalog.{remote_catalog_name}.s3.session-token", creds_session_token) - conf.set("spark.hadoop.fs.s3a.session.token", creds_session_token) - return - except Exception as e: - logger.debug(f"Could not read ~/.aws/credentials: {e}") - - session = botocore.session.Session() - credentials = session.get_credentials() - - # Priority 4: Check boto token - if credentials and credentials.token: - logger.info("🔑 Using AWS session token from boto3") - conf.set(f"spark.sql.catalog.{remote_catalog_name}.s3.session-token", credentials.token) - conf.set("spark.hadoop.fs.s3a.session.token", credentials.token) - return - - # Priority 5: Check boto credentials - if credentials and credentials.access_key and credentials.secret_key: - logger.info("🔑 Using AWS credentials from boto3") - conf.set(f"spark.sql.catalog.{remote_catalog_name}.s3.access-key-id", credentials.access_key) - conf.set(f"spark.sql.catalog.{remote_catalog_name}.s3.secret-access-key", credentials.secret_key) - conf.set("spark.hadoop.fs.s3a.access.key", credentials.access_key) - conf.set("spark.hadoop.fs.s3a.secret.key", credentials.secret_key) - return - - # Priority 6: Fall back to anonymous or default provider - logger.info("🔑 Using anonymous AWS credentials for S3 access") - conf.set( - "spark.hadoop.fs.s3a.aws.credentials.provider", - "org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider" - ) - return - -def _set_gcs_configuration( - conf: SparkConf, - gcs_project_id: str = None, - gcs_service_account_key_file: str = None, -): - """Configure Spark for Google Cloud Storage (GCS) access. - - Parameters - ---------- - conf : SparkConf - The Spark configuration object to update. - gcs_project_id : str - GCS project ID for billing/quota. Defaults to "anonymous" when - no service account key is provided (public bucket access). - gcs_service_account_key_file : str - Path to a GCS service account JSON key file. When None, - unauthenticated access is used (suitable for public buckets). - """ - GCS_CONNECTOR_VERSION = "hadoop3-2.2.32" - gcs_package = f"com.google.cloud.bigdataoss:gcs-connector:{GCS_CONNECTOR_VERSION}" - - # Add the GCS connector package - current_packages = conf.get("spark.jars.packages").split(",") - if gcs_package not in current_packages: - current_packages.append(gcs_package) - conf.set("spark.jars.packages", ",".join(current_packages)) - - # Register GCS filesystem implementations - conf.set( - "spark.hadoop.fs.gs.impl", - "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem" - ) - conf.set( - "spark.hadoop.fs.AbstractFileSystem.gs.impl", - "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS" - ) - - if gcs_service_account_key_file: - key_path = Path(gcs_service_account_key_file) - if not key_path.exists(): - raise FileNotFoundError( - f"GCS service account key file not found: {gcs_service_account_key_file}" - ) - logger.info(f"🔑 Using GCS service account key: {gcs_service_account_key_file}") - conf.set("spark.hadoop.google.cloud.auth.service.account.enable", "true") - conf.set( - "spark.hadoop.google.cloud.auth.service.account.keyfile", - str(key_path) - ) - if gcs_project_id: - conf.set("spark.hadoop.fs.gs.project.id", gcs_project_id) - else: - # Unauthenticated access for public GCS buckets - logger.info("🔑 Using unauthenticated GCS access (public buckets)") - # Set both old-style (fs.gs.*) and new-style (google.cloud.*) auth - # properties so the connector picks up UNAUTHENTICATED regardless of - # which configuration namespace it checks first. - conf.set("spark.hadoop.fs.gs.auth.type", "UNAUTHENTICATED") - conf.set("spark.hadoop.google.cloud.auth.type", "UNAUTHENTICATED") - # Explicitly disable service account auth to prevent the connector from - # attempting to contact the GCE metadata server (http://metadata.google.internal/) - # for credentials. Without this, the connector may hang on machines - # that are not running on GCP. - conf.set("spark.hadoop.google.cloud.auth.service.account.enable", "false") - # Enable null (anonymous) credentials — required for the connector to - # accept a configuration with no active credential source. - conf.set("spark.hadoop.fs.gs.auth.null.enable", "true") - conf.set( - "spark.hadoop.fs.gs.project.id", - gcs_project_id if gcs_project_id else "anonymous" - ) - - -def _set_catalog_metadata( - conf: SparkConf, - local_catalog_name: str, - local_catalog_type: str, - remote_catalog_name: str, - remote_catalog_type: str, - remote_catalog_uri: str, - remote_warehouse_dir: str, -): - """Set catalog metadata in Spark configuration.""" - metadata_configs = { - "local_catalog_name": local_catalog_name, - "local_catalog_type": local_catalog_type, - "remote_warehouse_dir": remote_warehouse_dir, - "remote_catalog_name": remote_catalog_name, - "remote_catalog_type": remote_catalog_type, - "remote_catalog_uri": remote_catalog_uri - } - for key, value in metadata_configs.items(): - conf.set(key, value) - logger.debug(f"Metadata config: {key}: {value}") - - -def _configure_iceberg_catalogs( - conf: SparkConf, - local_catalog_name: str, - local_catalog_type: str, - remote_warehouse_dir: str, - remote_catalog_name: str, - remote_catalog_type: str, - remote_catalog_uri: str -): - """Configure Iceberg catalogs through conf.set().""" - logger.info("Configuring Iceberg catalogs...") - # Local catalog configuration - conf.set(f"spark.sql.catalog.{local_catalog_name}", "org.apache.iceberg.spark.SparkCatalog") - conf.set(f"spark.sql.catalog.{local_catalog_name}.type", local_catalog_type) - conf.set(f"spark.sql.catalog.{local_catalog_name}.jdbc.driver", "org.sqlite.JDBC") - conf.set(f"spark.sql.catalog.{local_catalog_name}.jdbc.initialize", "true") - conf.set(f"spark.sql.catalog.{local_catalog_name}.jdbc.schema-version", "V1") - # conf.set("spark.sql.catalog.local.jdbc.user", "user") - # conf.set("spark.sql.catalog.local.jdbc.password", "password") - - # Remote catalog configuration - conf.set(f"spark.sql.catalog.{remote_catalog_name}", "org.apache.iceberg.spark.SparkCatalog") - conf.set(f"spark.sql.catalog.{remote_catalog_name}.type", remote_catalog_type) - conf.set(f"spark.sql.catalog.{remote_catalog_name}.uri", remote_catalog_uri) - conf.set(f"spark.sql.catalog.{remote_catalog_name}.warehouse", remote_warehouse_dir) - conf.set(f"spark.sql.catalog.{remote_catalog_name}.io-impl", "org.apache.iceberg.aws.s3.S3FileIO") - # S3 end point and path style access - if os.environ.get("REMOTE_CATALOG_S3_PATH_STYLE_ACCESS", "false").lower() == "true": - conf.set(f"spark.sql.catalog.{remote_catalog_name}.s3.endpoint", os.environ.get("REMOTE_CATALOG_S3_ENDPOINT")) - conf.set(f"spark.sql.catalog.{remote_catalog_name}.s3.path-style-access", os.environ.get("REMOTE_CATALOG_S3_PATH_STYLE_ACCESS").lower()) - - -def _update_configs_and_packages( - conf: SparkConf, - update_configs: Dict[str, str], - add_jars: List[str], - add_packages: List[str] -) -> Dict[str, str]: - """Update Spark configurations and packages.""" - # Add specified local jars - if add_jars is not None: - current_jars = conf.get("spark.jars").split(",") if conf.contains("spark.jars") else [] - for jar_path in add_jars: - if jar_path not in current_jars: - current_jars.append(jar_path) - if current_jars: - conf.set("spark.jars", ",".join(current_jars)) - - # Add specified packages - if add_packages is not None: - current_packages = conf.get("spark.jars.packages").split(",") - for package in add_packages: - if package not in current_packages: - current_packages.append(package) - conf.set("spark.jars.packages", ",".join(current_packages)) - - # Update or add specified configs - if update_configs is not None: - for key, value in update_configs.items(): - conf.set(key, value) - return - - -def log_session_config(spark: SparkSession): - """Log the current Spark session configuration for debugging. - - Parameters - ---------- - spark : SparkSession - The Spark session whose configuration should be logged. - - Notes - ----- - This function logs all Spark configuration properties to the - logger at INFO level for troubleshooting purposes. - """ - logger.info("Final Spark configuration:") - df = pd.DataFrame(list(spark.conf.getAll.items()), columns=["Key", "Value"]) - gps = df.groupby(by="Key") - for key, group in gps: - value = ",".join(group["Value"].tolist()) - values = value.split(",") - if key.startswith("spark."): - if len(values) > 1: - logger.info(f" {key}: ") - for val in values: - logger.info(f" {val}") - else: - logger.info(f" {key}: {value}") - - -def remove_or_update_configs( - spark: SparkSession, - remove_configs: List[str] = None, - update_configs: Dict[str, str] = None -) -> Dict[str, str]: - """Add, remove, or update Spark configurations. - - Parameters - ---------- - spark : SparkSession - The Spark session whose configuration should be logged. - remove_configs : List[str] - List of configuration keys to remove from the Spark session. - Default is None. - >>> remove_configs=["spark.sql.shuffle.partitions"] - update_configs : Dict[str, str] - Provided Spark configurations will be added if they do not already - exist, or overwritten if they do exist. Default is None. - >>> update_configs={"spark.sql.shuffle.partitions": "100"} - - Notes - ----- - This function allows for dynamic modification of the Spark session's - configuration by removing specified keys and updating or adding new - key-value pairs. - """ - # Remove specified configs - if remove_configs is not None: - for key in remove_configs: - try: - current_value = spark.conf.get(key) - logger.info(f"Removing config: {key} (was: {current_value})") - spark.conf.unset(key) - except Exception as e: - logger.warning(f"Could not remove config {key}: {e}") - - # Update or add specified configs - if update_configs is not None: - for key, value in update_configs.items(): - spark.conf.set(key, value) - return - - -def _decode_jwt_claims(token: str) -> Dict[str, object]: - payload = token.split(".")[1] - payload += "=" * (-len(payload) % 4) - return json.loads(base64.urlsafe_b64decode(payload.encode())) - - -def _token_expires_soon(token: str, refresh_window_seconds: int = 120) -> bool: - try: - claims = _decode_jwt_claims(token) - except Exception: - return True - exp = int(claims.get("exp", 0)) - now = int(time.time()) - return exp <= now + max(refresh_window_seconds, 1) - - -def _request_oauth_tokens( - data: Dict[str, str], - token_endpoint: Optional[str] = None, - timeout_seconds: int = 20, -) -> Tuple[str, Optional[str]]: - endpoint = token_endpoint or os.getenv("POLARIS_OAUTH2_SERVER_URI") - if not endpoint: - raise RuntimeError("POLARIS_OAUTH2_SERVER_URI is required to mint or refresh a user token") - - resp = requests.post(endpoint, data=data, timeout=timeout_seconds) - resp.raise_for_status() - payload = resp.json() - - access_token = payload.get("access_token") - if not access_token: - raise RuntimeError("Token endpoint did not return access_token") - - return access_token, payload.get("refresh_token") - - -def mint_polaris_user_token( - username: Optional[str], - password: Optional[str], - client_id: str, - client_secret: Optional[str] = None, - token_endpoint: Optional[str] = None, -) -> str: - if not username or not password: - raise RuntimeError("username and password are required for password grant token minting") - - data = { - "grant_type": "password", - "client_id": client_id, - "username": username, - "password": password, - "scope": "openid profile email", - } - if client_secret: - data["client_secret"] = client_secret - - access_token, _ = _request_oauth_tokens(data=data, token_endpoint=token_endpoint) - return access_token - - -def refresh_polaris_user_token( - refresh_token: str, - client_id: str, - client_secret: Optional[str] = None, - token_endpoint: Optional[str] = None, -) -> Tuple[str, Optional[str]]: - if not refresh_token: - raise RuntimeError("refresh_token is required for refresh grant") - - data = { - "grant_type": "refresh_token", - "client_id": client_id, - "refresh_token": refresh_token, - } - if client_secret: - data["client_secret"] = client_secret - - return _request_oauth_tokens(data=data, token_endpoint=token_endpoint) - - -def ensure_fresh_polaris_user_token( - current_token: Optional[str], - username: Optional[str], - password: Optional[str], - client_id: str, - client_secret: Optional[str] = None, - refresh_token: Optional[str] = None, - allow_password_fallback: bool = True, - refresh_window_seconds: int = 120, - token_endpoint: Optional[str] = None, -) -> Tuple[str, Optional[str], bool]: - if current_token and not _token_expires_soon(current_token, refresh_window_seconds): - return current_token, refresh_token, False - - if refresh_token: - try: - refreshed_access, refreshed_refresh = refresh_polaris_user_token( - refresh_token=refresh_token, - client_id=client_id, - client_secret=client_secret, - token_endpoint=token_endpoint, - ) - return refreshed_access, (refreshed_refresh or refresh_token), True - except requests.RequestException: - if not allow_password_fallback: - raise - - if allow_password_fallback and username and password: - minted = mint_polaris_user_token( - username=username, - password=password, - client_id=client_id, - client_secret=client_secret, - token_endpoint=token_endpoint, - ) - return minted, refresh_token, True - - raise RuntimeError( - "Unable to obtain a fresh Polaris user token. " - "Provide POLARIS_REFRESH_TOKEN or enable password-grant fallback credentials." - ) - - -def apply_polaris_token_to_spark( - spark, - token: str, - catalog_name: str = "iceberg", - realm: Optional[str] = None, -) -> None: - active_realm = realm or os.getenv("POLARIS_DEFAULT_REALM", "teehr") - base = f"spark.sql.catalog.{catalog_name}" - spark.conf.set(f"{base}.rest.auth.type", "oauth2") - spark.conf.set(f"{base}.token", token) - spark.conf.set(f"{base}.rest.auth.oauth2.token", token) - spark.conf.set(f"{base}.header.X-Polaris-Realm", active_realm) - spark.conf.set(f"{base}.rest.transport.header.X-Polaris-Realm", active_realm) - - -def _as_bool_str(value: str, default: str = "true") -> str: - normalized = (value or default).strip().lower() - return "true" if normalized in ("1", "true", "t", "yes", "y", "on") else "false" - - -def _apply_runtime_spark_configs(spark, configs: Dict[str, str]) -> None: - immutable_after_start = { - "spark.jars", - "spark.driver.extraClassPath", - "spark.executor.extraClassPath", - } - for key, value in configs.items(): - if not key.startswith("spark."): - continue - if key in immutable_after_start: - continue - spark.conf.set(key, value) - - -def _is_http_error_with_status(exc: Exception, status_code: int) -> bool: - if not isinstance(exc, requests.HTTPError): - return False - response = getattr(exc, "response", None) - return bool(response is not None and response.status_code == status_code) - - -def _normalize_internal_broker_url(url: str) -> str: - parsed = urlsplit(url) - if parsed.scheme != "https": - return url - if parsed.hostname != "teehr-api": - return url - - host = parsed.hostname - port = parsed.port or 8000 - netloc = f"{host}:{port}" - return urlunsplit(("http", netloc, parsed.path, parsed.query, parsed.fragment)) - - -def _broker_session_endpoint_from_token_endpoint(token_endpoint: str) -> str: - normalized = _normalize_internal_broker_url(token_endpoint) - parsed = urlsplit(normalized) - path = parsed.path or "" - - if path.endswith("/auth/polaris-token/session"): - return normalized - - if path.endswith("/auth/polaris-token"): - session_path = path[:-len("/auth/polaris-token")] + "/auth/polaris-token/session" - return urlunsplit((parsed.scheme, parsed.netloc, session_path, parsed.query, parsed.fragment)) - - raise RuntimeError( - "POLARIS_BROKER_URL must end with /auth/polaris-token " - "(or /auth/polaris-token/session if already session-scoped)" - ) - - -def ensure_broker_session_token( - *, - user_id: str, - session_id: str, - realm: str, - refresh_token: str, - bearer_token: Optional[str] = None, - catalog: str = "iceberg", - audience: Optional[str] = None, - broker_url: Optional[str] = None, - timeout_seconds: int = 20, -) -> str: - endpoint = broker_url or os.getenv("POLARIS_BROKER_URL", "http://teehr-api:8000/auth/polaris-token") - endpoint = _normalize_internal_broker_url(endpoint) - session_endpoint = _broker_session_endpoint_from_token_endpoint(endpoint).replace( - "/auth/polaris-token/session", - "/auth/polaris-session", - ) - active_audience = audience or os.getenv("POLARIS_BROKER_AUDIENCE", "account") - subject_token = bearer_token or os.getenv("POLARIS_USER_TOKEN", "") - - if not subject_token: - raise RuntimeError("A valid bearer subject token is required to create a broker session") - if not refresh_token: - raise RuntimeError("POLARIS_REFRESH_TOKEN is required to create a broker session") - - resp = requests.post( - session_endpoint, - headers={"Authorization": f"Bearer {subject_token}"}, - json={ - "user_id": user_id, - "session_id": session_id, - "realm": realm, - "catalog": catalog, - "audience": active_audience, - "refresh_token": refresh_token, - }, - timeout=timeout_seconds, - ) - resp.raise_for_status() - payload = resp.json() - broker_session_token = payload.get("broker_session_token") - if not broker_session_token: - raise RuntimeError("Broker session endpoint did not return broker_session_token") - os.environ["POLARIS_BROKER_SESSION_TOKEN"] = broker_session_token - return broker_session_token - - -def create_minio_spark_session( - polaris_token: Optional[str] = None, - force_recreate_session: bool = False, - update_configs: Optional[Dict[str, str]] = None, - use_authmanager: Optional[bool] = None, -): - """Start a Spark session configured for the local Polaris REST catalog. - - If ``polaris_token`` is provided, Spark uses user-token OAuth2 auth. - Otherwise it falls back to client-credential OAuth2 using env vars. - """ - aws_access_key_id = os.getenv("AWS_ACCESS_KEY_ID", "minioadmin") - aws_secret_access_key = os.getenv("AWS_SECRET_ACCESS_KEY", "minioadmin123") - - remote_catalog_uri = os.getenv("REMOTE_CATALOG_REST_URI", "http://polaris:8181/api/catalog") - remote_warehouse_dir = os.getenv("REMOTE_WAREHOUSE_S3_PATH", "s3://warehouse/") - polaris_realm = os.getenv("POLARIS_DEFAULT_REALM", "teehr") - - if remote_catalog_uri.rstrip("/").endswith("/api/catalog"): - # Polaris REST expects the catalog identifier here, not the backing S3 URI. - remote_warehouse_dir = polaris_realm - - s3_endpoint = os.getenv("REMOTE_CATALOG_S3_ENDPOINT", "http://minio:9000") - s3_path_style = _as_bool_str(os.getenv("REMOTE_CATALOG_S3_PATH_STYLE_ACCESS", "true")) - s3_region = os.getenv("AWS_REGION", "us-east-2") - - merged_configs: Dict[str, str] = { - "spark.sql.catalog.iceberg.warehouse": remote_warehouse_dir, - "spark.sql.catalog.iceberg.header.X-Polaris-Realm": polaris_realm, - "spark.sql.catalog.iceberg.rest.transport.header.X-Polaris-Realm": polaris_realm, - "spark.sql.catalog.iceberg.s3.endpoint": s3_endpoint, - "spark.sql.catalog.iceberg.s3.path-style-access": s3_path_style, - "spark.sql.catalog.iceberg.s3.region": s3_region, - "spark.hadoop.fs.s3a.endpoint": s3_endpoint, - "spark.hadoop.fs.s3a.path.style.access": s3_path_style, - "spark.hadoop.fs.s3a.connection.ssl.enabled": "false", - } - - resolved_use_authmanager = ( - use_authmanager - if use_authmanager is not None - else _as_bool_str(os.getenv("POLARIS_USE_AUTHMANAGER", "false")) == "true" - ) - - if resolved_use_authmanager: - broker_url = os.getenv("POLARIS_BROKER_URL", "http://teehr-api:8000/auth/polaris-token") - broker_url = _normalize_internal_broker_url(broker_url) - broker_session_url = _broker_session_endpoint_from_token_endpoint(broker_url) - authmanager_user_id = os.getenv("JUPYTERHUB_USER", "admin") - authmanager_session_id = ( - os.getenv("JUPYTERHUB_SERVER_NAME", "").strip() or authmanager_user_id - ) - broker_audience = os.getenv("POLARIS_BROKER_AUDIENCE", "account") - refresh_token = os.getenv("POLARIS_REFRESH_TOKEN", "") - current_user_token = polaris_token or os.getenv("POLARIS_USER_TOKEN", "") - broker_session_token = os.getenv("POLARIS_BROKER_SESSION_TOKEN", "") - if broker_session_token and _token_expires_soon(broker_session_token, 300): - broker_session_token = "" - authmanager_jar = os.getenv( - "POLARIS_AUTHMANAGER_JAR", - "/opt/spark/jars/teehr-authmanager.jar", - ) - - current_user_token, refresh_token, _ = ensure_fresh_polaris_user_token( - current_token=current_user_token, - username=os.getenv("POLARIS_USERNAME"), - password=os.getenv("POLARIS_PASSWORD"), - client_id=os.getenv("POLARIS_CLIENT_ID", "spark-polaris"), - client_secret=os.getenv("POLARIS_CLIENT_SECRET"), - refresh_token=refresh_token, - allow_password_fallback=False, - refresh_window_seconds=300, - token_endpoint=os.getenv("POLARIS_OAUTH2_TOKEN_ENDPOINT"), - ) - os.environ["POLARIS_USER_TOKEN"] = current_user_token - if refresh_token: - os.environ["POLARIS_REFRESH_TOKEN"] = refresh_token - - if not broker_session_token: - try: - broker_session_token = ensure_broker_session_token( - user_id=authmanager_user_id, - session_id=authmanager_session_id, - realm=polaris_realm, - refresh_token=refresh_token, - bearer_token=current_user_token, - catalog="iceberg", - audience=broker_audience, - broker_url=broker_url, - ) - except requests.HTTPError as exc: - if not _is_http_error_with_status(exc, 401): - raise - - # The bearer token may have been invalidated server-side; force refresh and retry once. - current_user_token, refresh_token, _ = ensure_fresh_polaris_user_token( - current_token=None, - username=os.getenv("POLARIS_USERNAME"), - password=os.getenv("POLARIS_PASSWORD"), - client_id=os.getenv("POLARIS_CLIENT_ID", "spark-polaris"), - client_secret=os.getenv("POLARIS_CLIENT_SECRET"), - refresh_token=refresh_token, - allow_password_fallback=True, - refresh_window_seconds=300, - token_endpoint=os.getenv("POLARIS_OAUTH2_TOKEN_ENDPOINT"), - ) - os.environ["POLARIS_USER_TOKEN"] = current_user_token - if refresh_token: - os.environ["POLARIS_REFRESH_TOKEN"] = refresh_token - - broker_session_token = ensure_broker_session_token( - user_id=authmanager_user_id, - session_id=authmanager_session_id, - realm=polaris_realm, - refresh_token=refresh_token, - bearer_token=current_user_token, - catalog="iceberg", - audience=broker_audience, - broker_url=broker_url, - ) - - os.environ["POLARIS_BROKER_SESSION_TOKEN"] = broker_session_token - - merged_configs["spark.sql.catalog.iceberg.rest.auth.type"] = ( - "org.teehr.iceberg.auth.TeehrBrokerAuthManager" - ) - merged_configs["spark.sql.catalog.iceberg.rest.auth.teehr.broker.url"] = broker_session_url - merged_configs["spark.sql.catalog.iceberg.rest.auth.teehr.user-id"] = authmanager_user_id - merged_configs["spark.sql.catalog.iceberg.rest.auth.teehr.session-id"] = ( - authmanager_session_id - ) - merged_configs["spark.sql.catalog.iceberg.rest.auth.teehr.realm"] = polaris_realm - merged_configs["spark.sql.catalog.iceberg.rest.auth.teehr.catalog"] = "iceberg" - merged_configs["spark.sql.catalog.iceberg.rest.auth.teehr.audience"] = broker_audience - merged_configs["spark.sql.catalog.iceberg.rest.auth.teehr.broker-session-token-env"] = ( - "POLARIS_BROKER_SESSION_TOKEN" - ) - merged_configs["spark.jars"] = authmanager_jar - elif polaris_token: - merged_configs["spark.sql.catalog.iceberg.rest.auth.type"] = "oauth2" - merged_configs["spark.sql.catalog.iceberg.token"] = polaris_token - merged_configs["spark.sql.catalog.iceberg.rest.auth.oauth2.token"] = polaris_token - else: - oauth_server_uri = os.getenv("POLARIS_OAUTH2_SERVER_URI") - spark_polaris_client_secret = os.getenv("SPARK_POLARIS_CLIENT_SECRET") - - merged_configs["spark.sql.catalog.iceberg.rest.auth.type"] = "oauth2" - merged_configs["spark.sql.catalog.iceberg.scope"] = "openid" - merged_configs["spark.sql.catalog.iceberg.rest.auth.oauth2.scope"] = "openid" - if oauth_server_uri: - merged_configs["spark.sql.catalog.iceberg.oauth2-server-uri"] = oauth_server_uri - merged_configs["spark.sql.catalog.iceberg.rest.auth.oauth2.server-uri"] = oauth_server_uri - if spark_polaris_client_secret: - merged_configs["spark.sql.catalog.iceberg.credential"] = ( - f"spark-polaris:{spark_polaris_client_secret}" - ) - merged_configs["spark.sql.catalog.iceberg.rest.auth.oauth2.credential"] = ( - f"spark-polaris:{spark_polaris_client_secret}" - ) - - if update_configs: - merged_configs.update(update_configs) - - call_kwargs = { - "remote_catalog_uri": remote_catalog_uri, - "remote_warehouse_dir": remote_warehouse_dir, - "aws_access_key_id": aws_access_key_id, - "aws_secret_access_key": aws_secret_access_key, - "force_recreate_session": force_recreate_session or resolved_use_authmanager, - "update_configs": merged_configs, - } - if polaris_token: - # Preferred path for newer teehr versions that support direct token auth. - call_kwargs["oauth2_token"] = polaris_token - - while True: - try: - spark = create_spark_session(**call_kwargs) - _apply_runtime_spark_configs(spark, merged_configs) - return spark - except TypeError as exc: - msg = str(exc) - if "oauth2_token" in msg and "oauth2_token" in call_kwargs: - call_kwargs.pop("oauth2_token", None) - continue - if "force_recreate_session" in msg and "force_recreate_session" in call_kwargs: - call_kwargs.pop("force_recreate_session", None) - continue - raise - - -def request_broker_polaris_token( - *, - user_id: str, - session_id: str, - bearer_token: str, - realm: Optional[str] = None, - catalog: str = "iceberg", - requested_ttl_seconds: int = 600, - audience: Optional[str] = None, - broker_url: Optional[str] = None, - timeout_seconds: int = 20, -) -> Tuple[str, int, str]: - endpoint = broker_url or os.getenv("POLARIS_BROKER_URL", "http://teehr-api:8000/auth/polaris-token") - active_realm = realm or os.getenv("POLARIS_DEFAULT_REALM", "teehr") - active_audience = audience or os.getenv("POLARIS_BROKER_AUDIENCE", "account") - - if not bearer_token: - raise RuntimeError("bearer_token is required to request a broker token") - - resp = requests.post( - endpoint, - headers={"Authorization": f"Bearer {bearer_token}"}, - json={ - "user_id": user_id, - "session_id": session_id, - "realm": active_realm, - "catalog": catalog, - "requested_ttl_seconds": requested_ttl_seconds, - "audience": active_audience, - }, - timeout=timeout_seconds, - ) - resp.raise_for_status() - payload = resp.json() - access_token = payload.get("access_token") - expires_at = int(payload.get("expires_at_epoch_seconds", 0)) - trace_id = payload.get("trace_id", "") - - if not access_token or expires_at <= 0: - raise RuntimeError("Broker response missing access_token or expires_at_epoch_seconds") - - return access_token, expires_at, trace_id - - -def ensure_fresh_polaris_token_via_broker( - *, - current_token: Optional[str], - user_id: str, - session_id: str, - bearer_token: str, - realm: Optional[str] = None, - catalog: str = "iceberg", - refresh_window_seconds: int = 120, - requested_ttl_seconds: int = 600, - audience: Optional[str] = None, - broker_url: Optional[str] = None, -) -> Tuple[str, bool]: - if current_token and not _token_expires_soon(current_token, refresh_window_seconds): - return current_token, False - - refreshed_token, _, _ = request_broker_polaris_token( - user_id=user_id, - session_id=session_id, - bearer_token=bearer_token, - realm=realm, - catalog=catalog, - requested_ttl_seconds=requested_ttl_seconds, - audience=audience, - broker_url=broker_url, - ) - return refreshed_token, True \ No newline at end of file diff --git a/examples/developer/test_token_renewal.ipynb b/examples/developer/test_token_renewal.ipynb deleted file mode 100644 index bbebf6c..0000000 --- a/examples/developer/test_token_renewal.ipynb +++ /dev/null @@ -1,615 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "id": "b02f2f71-c50b-42ff-acd1-4c71cc8b851a", - "metadata": {}, - "outputs": [], - "source": [ - "from pathlib import Path\n", - "import base64\n", - "import json\n", - "import os\n", - "\n", - "import pandas as pd\n", - "import requests\n", - "\n", - "import teehr\n", - "from teehr import RemoteReadWriteEvaluation\n", - "from teehr.utilities.apply_migrations import evolve_catalog_schema\n", - "\n", - "from setup_utils import (\n", - " DEV_LOCATION_ID_LIST,\n", - " apply_polaris_token_to_spark,\n", - " create_minio_spark_session,\n", - " ensure_fresh_polaris_user_token,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "5319cb94-0510-4695-8127-68f4c71c6776", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "POLARIS_CLIENT_ID: jupyterhub\n", - "POLARIS_CLIENT_SECRET exists: True\n", - "POLARIS_REFRESH_TOKEN exists: True\n" - ] - } - ], - "source": [ - "import os\n", - "print(\"POLARIS_CLIENT_ID:\", os.getenv(\"POLARIS_CLIENT_ID\"))\n", - "print(\"POLARIS_CLIENT_SECRET exists:\", bool(os.getenv(\"POLARIS_CLIENT_SECRET\")))\n", - "print(\"POLARIS_REFRESH_TOKEN exists:\", bool(os.getenv(\"POLARIS_REFRESH_TOKEN\")))" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "60ca0cbf-f004-4e0a-bcdc-9b9319993a93", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "POLARIS_USER_TOKEN claims:\n", - "{\n", - " \"acr\": \"1\",\n", - " \"allowed-origins\": [\n", - " \"https://hub.teehr.local.app.garden\"\n", - " ],\n", - " \"aud\": [\n", - " \"realm-management\",\n", - " \"account\"\n", - " ],\n", - " \"auth_time\": 1784820452,\n", - " \"auth_time_utc\": \"2026-07-23T15:27:32+00:00\",\n", - " \"azp\": \"jupyterhub\",\n", - " \"email\": \"admin@example.local\",\n", - " \"email_verified\": true,\n", - " \"exp\": 1784820752,\n", - " \"exp_utc\": \"2026-07-23T15:32:32+00:00\",\n", - " \"family_name\": \"Admin\",\n", - " \"given_name\": \"Local\",\n", - " \"groups\": [\n", - " \"basic-user\",\n", - " \"iceberg-catalog-admins\",\n", - " \"iceberg-user\",\n", - " \"jupyter-admin\",\n", - " \"key-management-admin\",\n", - " \"prefect-admin\",\n", - " \"webapi-admin\"\n", - " ],\n", - " \"iat\": 1784820452,\n", - " \"iat_utc\": \"2026-07-23T15:27:32+00:00\",\n", - " \"iss\": \"https://auth.teehr.local.app.garden/realms/teehr\",\n", - " \"jti\": \"ofrtac:3719a618-6e89-7a46-38fb-c9a93cddf030\",\n", - " \"name\": \"Local Admin\",\n", - " \"preferred_username\": \"admin\",\n", - " \"realm_access\": {\n", - " \"roles\": [\n", - " \"iceberg-catalog-admin\",\n", - " \"offline_access\",\n", - " \"admin\",\n", - " \"uma_authorization\",\n", - " \"basic-user\",\n", - " \"default-roles-teehr\",\n", - " \"jupyter-user\",\n", - " \"iceberg-user\"\n", - " ]\n", - " },\n", - " \"resource_access\": {\n", - " \"account\": {\n", - " \"roles\": [\n", - " \"manage-account\",\n", - " \"manage-account-links\",\n", - " \"view-profile\"\n", - " ]\n", - " },\n", - " \"realm-management\": {\n", - " \"roles\": [\n", - " \"view-realm\",\n", - " \"manage-users\",\n", - " \"view-users\",\n", - " \"query-groups\",\n", - " \"query-users\"\n", - " ]\n", - " }\n", - " },\n", - " \"scope\": \"openid offline_access profile email\",\n", - " \"sid\": \"DxxUtwd0xu5xkhSEDcsIebzd\",\n", - " \"sub\": \"8f4441da-7756-49e3-9142-64c299ed6c39\",\n", - " \"typ\": \"Bearer\"\n", - "}\n" - ] - } - ], - "source": [ - "import base64\n", - "import json\n", - "import os\n", - "from datetime import datetime, timezone\n", - "\n", - "def decode_jwt(token: str):\n", - " if not token:\n", - " return None\n", - " parts = token.split(\".\")\n", - " if len(parts) != 3:\n", - " return {\"error\": \"Not a JWT (expected 3 dot-separated parts)\"}\n", - " payload = parts[1] + \"=\" * (-len(parts[1]) % 4)\n", - " claims = json.loads(base64.urlsafe_b64decode(payload.encode()))\n", - " return claims\n", - "\n", - "def pretty_claims(name: str, token: str):\n", - " claims = decode_jwt(token)\n", - " print(f\"\\n{name}:\")\n", - " if claims is None:\n", - " print(\" \")\n", - " return\n", - " if \"error\" in claims:\n", - " print(\" \", claims[\"error\"])\n", - " return\n", - " for ts_key in [\"iat\", \"exp\", \"auth_time\"]:\n", - " if ts_key in claims:\n", - " claims[f\"{ts_key}_utc\"] = datetime.fromtimestamp(\n", - " claims[ts_key], tz=timezone.utc\n", - " ).isoformat()\n", - " print(json.dumps(claims, indent=2, sort_keys=True))\n", - "\n", - "access_token = os.getenv(\"POLARIS_USER_TOKEN\", \"\")\n", - "refresh_token = os.getenv(\"POLARIS_REFRESH_TOKEN\", \"\")\n", - "\n", - "pretty_claims(\"POLARIS_USER_TOKEN claims\", access_token)" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "c3a50694-a1ec-4923-90ca-efcf145c14b5", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "sub: 8f4441da-7756-49e3-9142-64c299ed6c39\n", - "preferred_username: admin\n", - "groups: ['basic-user', 'iceberg-catalog-admins', 'iceberg-user', 'jupyter-admin', 'key-management-admin', 'prefect-admin', 'webapi-admin']\n", - "realm roles: ['iceberg-catalog-admin', 'offline_access', 'admin', 'uma_authorization', 'basic-user', 'default-roles-teehr', 'jupyter-user', 'iceberg-user']\n", - "exp: 1784820752\n" - ] - } - ], - "source": [ - "claims = decode_jwt(os.getenv(\"POLARIS_USER_TOKEN\", \"\"))\n", - "print(\"sub:\", claims.get(\"sub\"))\n", - "print(\"preferred_username:\", claims.get(\"preferred_username\"))\n", - "print(\"groups:\", claims.get(\"groups\"))\n", - "print(\"realm roles:\", claims.get(\"realm_access\", {}).get(\"roles\", []))\n", - "print(\"exp:\", claims.get(\"exp\"))" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "fe4b02e0-cf82-45db-bb48-3b242e11ee24", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "now_epoch: 1784820789\n", - "now_utc: 2026-07-23T15:33:09+00:00\n" - ] - } - ], - "source": [ - "import time\n", - "from datetime import datetime, timezone\n", - "\n", - "now = int(time.time())\n", - "print(\"now_epoch:\", now)\n", - "print(\"now_utc:\", datetime.fromtimestamp(now, tz=timezone.utc).isoformat())" - ] - }, - { - "cell_type": "code", - "execution_count": 48, - "id": "016380fe-06db-4a3f-87b5-854ad1d3458e", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "exp: 1784825530\n", - "now: 1784825236\n", - "seconds_remaining: 294\n" - ] - } - ], - "source": [ - "claims = decode_jwt(os.getenv(\"POLARIS_USER_TOKEN\", \"\")) # from previous helper\n", - "exp = int(claims.get(\"exp\", 0))\n", - "now = int(time.time())\n", - "print(\"exp:\", exp)\n", - "print(\"now:\", now)\n", - "print(\"seconds_remaining:\", exp - now)" - ] - }, - { - "cell_type": "code", - "execution_count": 47, - "id": "2f8ff9fe-0d4c-43a1-96e3-1eee7cc9f6e0", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "trying to refresh token\n", - "eyJhbGciOiJIUzUxMiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJkMTJhN2FiYy04MzVkLTRiNzAtODgxMi02NjcxMjMzYjI1ODUifQ.eyJpYXQiOjE3ODQ4MjI1MzQsImp0aSI6IjI2ZjI5MmJiLWVkNGMtOWY4NS04MzRmLWEzNjNmOTQ3ZDYwZiIsImlzcyI6Imh0dHBzOi8vYXV0aC50ZWVoci5sb2NhbC5hcHAuZ2FyZGVuL3JlYWxtcy90ZWVociIsImF1ZCI6Imh0dHBzOi8vYXV0aC50ZWVoci5sb2NhbC5hcHAuZ2FyZGVuL3JlYWxtcy90ZWVociIsInN1YiI6IjhmNDQ0MWRhLTc3NTYtNDllMy05MTQyLTY0YzI5OWVkNmMzOSIsInR5cCI6Ik9mZmxpbmUiLCJhenAiOiJqdXB5dGVyaHViIiwic2lkIjoiRHh4VXR3ZDB4dTV4a2hTRURjc0llYnpkIiwic2NvcGUiOiJvcGVuaWQgYWNyIG9mZmxpbmVfYWNjZXNzIHJvbGVzIGJhc2ljIHByb2ZpbGUgZW1haWwgd2ViLW9yaWdpbnMiLCJhdWRfeCI6WyJyZWFsbS1tYW5hZ2VtZW50IiwiYWNjb3VudCJdfQ.VUSlhU7bpO6pRxlDE_RMNNpDv-HcLKZ0bThk-5ewvOoHGtae9QLxBhdqFCnZg7K_-bK8Tbpt9eKteInSImNwKA\n", - "{'grant_type': 'refresh_token', 'client_id': 'jupyterhub', 'refresh_token': 'eyJhbGciOiJIUzUxMiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJkMTJhN2FiYy04MzVkLTRiNzAtODgxMi02NjcxMjMzYjI1ODUifQ.eyJpYXQiOjE3ODQ4MjI1MzQsImp0aSI6IjI2ZjI5MmJiLWVkNGMtOWY4NS04MzRmLWEzNjNmOTQ3ZDYwZiIsImlzcyI6Imh0dHBzOi8vYXV0aC50ZWVoci5sb2NhbC5hcHAuZ2FyZGVuL3JlYWxtcy90ZWVociIsImF1ZCI6Imh0dHBzOi8vYXV0aC50ZWVoci5sb2NhbC5hcHAuZ2FyZGVuL3JlYWxtcy90ZWVociIsInN1YiI6IjhmNDQ0MWRhLTc3NTYtNDllMy05MTQyLTY0YzI5OWVkNmMzOSIsInR5cCI6Ik9mZmxpbmUiLCJhenAiOiJqdXB5dGVyaHViIiwic2lkIjoiRHh4VXR3ZDB4dTV4a2hTRURjc0llYnpkIiwic2NvcGUiOiJvcGVuaWQgYWNyIG9mZmxpbmVfYWNjZXNzIHJvbGVzIGJhc2ljIHByb2ZpbGUgZW1haWwgd2ViLW9yaWdpbnMiLCJhdWRfeCI6WyJyZWFsbS1tYW5hZ2VtZW50IiwiYWNjb3VudCJdfQ.VUSlhU7bpO6pRxlDE_RMNNpDv-HcLKZ0bThk-5ewvOoHGtae9QLxBhdqFCnZg7K_-bK8Tbpt9eKteInSImNwKA', 'client_secret': 'local-jupyterhub-client-secret'}\n", - "None\n", - "http://keycloak-service:8080/realms/teehr/protocol/openid-connect/token\n", - "\n", - "eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJ6UFFnS004WjdiaUdIZnFPWGdrV3JZbjM1NjNTdHFUMFdpcm5XdHBqZXNjIn0.eyJleHAiOjE3ODQ4MjU1MzAsImlhdCI6MTc4NDgyNTIzMCwiYXV0aF90aW1lIjoxNzg0ODIwNDUyLCJqdGkiOiJvZnJ0cnQ6MWJjNTJlNjUtZTc5Ni0zOTlmLWY2NWQtOGEzNThlNWJiNDJmIiwiaXNzIjoiaHR0cHM6Ly9hdXRoLnRlZWhyLmxvY2FsLmFwcC5nYXJkZW4vcmVhbG1zL3RlZWhyIiwiYXVkIjpbInJlYWxtLW1hbmFnZW1lbnQiLCJhY2NvdW50Il0sInN1YiI6IjhmNDQ0MWRhLTc3NTYtNDllMy05MTQyLTY0YzI5OWVkNmMzOSIsInR5cCI6IkJlYXJlciIsImF6cCI6Imp1cHl0ZXJodWIiLCJzaWQiOiJEeHhVdHdkMHh1NXhraFNFRGNzSWViemQiLCJhY3IiOiIxIiwiYWxsb3dlZC1vcmlnaW5zIjpbImh0dHBzOi8vaHViLnRlZWhyLmxvY2FsLmFwcC5nYXJkZW4iXSwicmVhbG1fYWNjZXNzIjp7InJvbGVzIjpbImljZWJlcmctY2F0YWxvZy1hZG1pbiIsIm9mZmxpbmVfYWNjZXNzIiwiYWRtaW4iLCJ1bWFfYXV0aG9yaXphdGlvbiIsImJhc2ljLXVzZXIiLCJkZWZhdWx0LXJvbGVzLXRlZWhyIiwianVweXRlci11c2VyIiwiaWNlYmVyZy11c2VyIl19LCJyZXNvdXJjZV9hY2Nlc3MiOnsicmVhbG0tbWFuYWdlbWVudCI6eyJyb2xlcyI6WyJ2aWV3LXJlYWxtIiwibWFuYWdlLXVzZXJzIiwidmlldy11c2VycyIsInF1ZXJ5LWdyb3VwcyIsInF1ZXJ5LXVzZXJzIl19LCJhY2NvdW50Ijp7InJvbGVzIjpbIm1hbmFnZS1hY2NvdW50IiwibWFuYWdlLWFjY291bnQtbGlua3MiLCJ2aWV3LXByb2ZpbGUiXX19LCJzY29wZSI6Im9wZW5pZCBvZmZsaW5lX2FjY2VzcyBwcm9maWxlIGVtYWlsIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsIm5hbWUiOiJMb2NhbCBBZG1pbiIsImdyb3VwcyI6WyJiYXNpYy11c2VyIiwiaWNlYmVyZy1jYXRhbG9nLWFkbWlucyIsImljZWJlcmctdXNlciIsImp1cHl0ZXItYWRtaW4iLCJrZXktbWFuYWdlbWVudC1hZG1pbiIsInByZWZlY3QtYWRtaW4iLCJ3ZWJhcGktYWRtaW4iXSwicHJlZmVycmVkX3VzZXJuYW1lIjoiYWRtaW4iLCJnaXZlbl9uYW1lIjoiTG9jYWwiLCJmYW1pbHlfbmFtZSI6IkFkbWluIiwiZW1haWwiOiJhZG1pbkBleGFtcGxlLmxvY2FsIn0.V3YgkyyVl9eUu-We4GDV36GS2tlYJzGyw6twBOVLtgWkbu6DzGOKiTOpkGiuabeGtJaQXGFoM9h454zSicXDQUJFa4qUt7FF36C87X81_MAaLiOJ88oFQXV5JxV6VVPTR6C0faO_-j5uw7Rk3P2RNK9yHiVxVcnbTfakWO9TdgZhxk0kSg7w7mzZbPgA2Wg_83pO3kqvUZF0wUyG30QxUQifsnJfiVFfBakFj7ffPizeiw1l67Lusef0qvDkQA5VWA_UVXzp-UoKpcQ4xlK0V1Kn367fP0lX1Ea_IB9GMFjwIARHR-t3ESzowcX4bRtybOyrVbQhOOjrzzs1aX4ZHw\n", - "Token status: renewed\n", - "Refresh token available: True\n", - "Token user: admin\n", - "Token roles: admin, basic-user, default-roles-teehr, iceberg-catalog-admin, iceberg-user, jupyter-user, offline_access, uma_authorization\n" - ] - } - ], - "source": [ - "polaris_user_token = os.environ.get(\"POLARIS_USER_TOKEN\")\n", - "polaris_refresh_token = os.getenv(\"POLARIS_REFRESH_TOKEN\", \"\")\n", - "polaris_username = os.getenv(\"POLARIS_USERNAME\", os.getenv(\"JUPYTERHUB_USER\", \"admin\"))\n", - "polaris_password = os.getenv(\"POLARIS_PASSWORD\", \"\")\n", - "polaris_client_id = os.getenv(\"POLARIS_CLIENT_ID\", \"jupyterhub\")\n", - "polaris_client_secret = os.getenv(\"POLARIS_CLIENT_SECRET\")\n", - "\n", - "if not polaris_user_token and not polaris_refresh_token and not polaris_password:\n", - " raise RuntimeError(\n", - " \"No usable Polaris credentials found. Set POLARIS_USER_TOKEN or POLARIS_REFRESH_TOKEN in the session.\"\n", - " )\n", - "\n", - "polaris_user_token, polaris_refresh_token, token_renewed = ensure_fresh_polaris_user_token(\n", - " current_token=polaris_user_token,\n", - " username=polaris_username,\n", - " password=polaris_password,\n", - " client_id=polaris_client_id,\n", - " client_secret=polaris_client_secret,\n", - " refresh_token=polaris_refresh_token,\n", - " refresh_window_seconds=120,\n", - ")\n", - "os.environ[\"POLARIS_USER_TOKEN\"] = polaris_user_token\n", - "if polaris_refresh_token:\n", - " os.environ[\"POLARIS_REFRESH_TOKEN\"] = polaris_refresh_token\n", - "\n", - "payload = polaris_user_token.split(\".\")[1]\n", - "payload += \"=\" * (-len(payload) % 4)\n", - "claims = json.loads(base64.urlsafe_b64decode(payload.encode()))\n", - "roles = sorted(claims.get(\"realm_access\", {}).get(\"roles\", []))\n", - "\n", - "print(\"Token status:\", \"renewed\" if token_renewed else \"reused\")\n", - "print(\"Refresh token available:\", bool(polaris_refresh_token))\n", - "print(\"Token user:\", claims.get(\"preferred_username\"))\n", - "print(\"Token roles:\", \", \".join(roles))\n", - "if \"iceberg-catalog-admin\" not in roles:\n", - " print(\"WARNING: token is missing iceberg-catalog-admin; Polaris may reject writes\")" - ] - }, - { - "cell_type": "code", - "execution_count": 42, - "id": "7d7592da-2753-4557-8d93-8ac1e5c27364", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:teehr.evaluation.spark_session_utils:🚀 Creating Spark session: TEEHR Evaluation\n", - "INFO:teehr.evaluation.spark_session_utils:✅ Spark local configuration successful!\n", - "INFO:teehr.evaluation.spark_session_utils:Setting Hadoop's default AWS credentials provider and AWS region\n", - "INFO:teehr.evaluation.spark_session_utils:🔑 Using user-provided AWS credentials\n", - "INFO:teehr.evaluation.spark_session_utils:Configuring Iceberg catalogs...\n", - "INFO:teehr.evaluation.spark_session_utils:⚙️ All settings applied. Creating Spark session...\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "trying to refresh token\n", - "eyJhbGciOiJIUzUxMiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJkMTJhN2FiYy04MzVkLTRiNzAtODgxMi02NjcxMjMzYjI1ODUifQ.eyJpYXQiOjE3ODQ4MjIzMjQsImp0aSI6IjJhNmQwZjQ3LWQ0YzItZjRiNS0xOTNjLTRhZGYzNjFiN2RiOSIsImlzcyI6Imh0dHBzOi8vYXV0aC50ZWVoci5sb2NhbC5hcHAuZ2FyZGVuL3JlYWxtcy90ZWVociIsImF1ZCI6Imh0dHBzOi8vYXV0aC50ZWVoci5sb2NhbC5hcHAuZ2FyZGVuL3JlYWxtcy90ZWVociIsInN1YiI6IjhmNDQ0MWRhLTc3NTYtNDllMy05MTQyLTY0YzI5OWVkNmMzOSIsInR5cCI6Ik9mZmxpbmUiLCJhenAiOiJqdXB5dGVyaHViIiwic2lkIjoiRHh4VXR3ZDB4dTV4a2hTRURjc0llYnpkIiwic2NvcGUiOiJvcGVuaWQgYWNyIG9mZmxpbmVfYWNjZXNzIHJvbGVzIGJhc2ljIHByb2ZpbGUgZW1haWwgd2ViLW9yaWdpbnMiLCJhdWRfeCI6WyJyZWFsbS1tYW5hZ2VtZW50IiwiYWNjb3VudCJdfQ.GTaneyP2O50XcRm0-BQc8gRuwoI_yJskKOGRFepFrtwlBu5JHz7EVi1sef5GzlyaMVr3VOC02VTGnMk3p1fpnQ\n", - "{'grant_type': 'refresh_token', 'client_id': 'jupyterhub', 'refresh_token': 'eyJhbGciOiJIUzUxMiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJkMTJhN2FiYy04MzVkLTRiNzAtODgxMi02NjcxMjMzYjI1ODUifQ.eyJpYXQiOjE3ODQ4MjIzMjQsImp0aSI6IjJhNmQwZjQ3LWQ0YzItZjRiNS0xOTNjLTRhZGYzNjFiN2RiOSIsImlzcyI6Imh0dHBzOi8vYXV0aC50ZWVoci5sb2NhbC5hcHAuZ2FyZGVuL3JlYWxtcy90ZWVociIsImF1ZCI6Imh0dHBzOi8vYXV0aC50ZWVoci5sb2NhbC5hcHAuZ2FyZGVuL3JlYWxtcy90ZWVociIsInN1YiI6IjhmNDQ0MWRhLTc3NTYtNDllMy05MTQyLTY0YzI5OWVkNmMzOSIsInR5cCI6Ik9mZmxpbmUiLCJhenAiOiJqdXB5dGVyaHViIiwic2lkIjoiRHh4VXR3ZDB4dTV4a2hTRURjc0llYnpkIiwic2NvcGUiOiJvcGVuaWQgYWNyIG9mZmxpbmVfYWNjZXNzIHJvbGVzIGJhc2ljIHByb2ZpbGUgZW1haWwgd2ViLW9yaWdpbnMiLCJhdWRfeCI6WyJyZWFsbS1tYW5hZ2VtZW50IiwiYWNjb3VudCJdfQ.GTaneyP2O50XcRm0-BQc8gRuwoI_yJskKOGRFepFrtwlBu5JHz7EVi1sef5GzlyaMVr3VOC02VTGnMk3p1fpnQ', 'client_secret': 'local-jupyterhub-client-secret'}\n", - "None\n", - "http://keycloak-service:8080/realms/teehr/protocol/openid-connect/token\n", - "\n", - "eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJ6UFFnS004WjdiaUdIZnFPWGdrV3JZbjM1NjNTdHFUMFdpcm5XdHBqZXNjIn0.eyJleHAiOjE3ODQ4MjI4MzQsImlhdCI6MTc4NDgyMjUzNCwiYXV0aF90aW1lIjoxNzg0ODIwNDUyLCJqdGkiOiJvZnJ0cnQ6YjE5MjY0ZjgtNGQ0OC0yM2MzLTYzYjUtZGZkMjEwNjA4MGUyIiwiaXNzIjoiaHR0cHM6Ly9hdXRoLnRlZWhyLmxvY2FsLmFwcC5nYXJkZW4vcmVhbG1zL3RlZWhyIiwiYXVkIjpbInJlYWxtLW1hbmFnZW1lbnQiLCJhY2NvdW50Il0sInN1YiI6IjhmNDQ0MWRhLTc3NTYtNDllMy05MTQyLTY0YzI5OWVkNmMzOSIsInR5cCI6IkJlYXJlciIsImF6cCI6Imp1cHl0ZXJodWIiLCJzaWQiOiJEeHhVdHdkMHh1NXhraFNFRGNzSWViemQiLCJhY3IiOiIxIiwiYWxsb3dlZC1vcmlnaW5zIjpbImh0dHBzOi8vaHViLnRlZWhyLmxvY2FsLmFwcC5nYXJkZW4iXSwicmVhbG1fYWNjZXNzIjp7InJvbGVzIjpbImljZWJlcmctY2F0YWxvZy1hZG1pbiIsIm9mZmxpbmVfYWNjZXNzIiwiYWRtaW4iLCJ1bWFfYXV0aG9yaXphdGlvbiIsImJhc2ljLXVzZXIiLCJkZWZhdWx0LXJvbGVzLXRlZWhyIiwianVweXRlci11c2VyIiwiaWNlYmVyZy11c2VyIl19LCJyZXNvdXJjZV9hY2Nlc3MiOnsicmVhbG0tbWFuYWdlbWVudCI6eyJyb2xlcyI6WyJ2aWV3LXJlYWxtIiwibWFuYWdlLXVzZXJzIiwidmlldy11c2VycyIsInF1ZXJ5LWdyb3VwcyIsInF1ZXJ5LXVzZXJzIl19LCJhY2NvdW50Ijp7InJvbGVzIjpbIm1hbmFnZS1hY2NvdW50IiwibWFuYWdlLWFjY291bnQtbGlua3MiLCJ2aWV3LXByb2ZpbGUiXX19LCJzY29wZSI6Im9wZW5pZCBvZmZsaW5lX2FjY2VzcyBwcm9maWxlIGVtYWlsIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsIm5hbWUiOiJMb2NhbCBBZG1pbiIsImdyb3VwcyI6WyJiYXNpYy11c2VyIiwiaWNlYmVyZy1jYXRhbG9nLWFkbWlucyIsImljZWJlcmctdXNlciIsImp1cHl0ZXItYWRtaW4iLCJrZXktbWFuYWdlbWVudC1hZG1pbiIsInByZWZlY3QtYWRtaW4iLCJ3ZWJhcGktYWRtaW4iXSwicHJlZmVycmVkX3VzZXJuYW1lIjoiYWRtaW4iLCJnaXZlbl9uYW1lIjoiTG9jYWwiLCJmYW1pbHlfbmFtZSI6IkFkbWluIiwiZW1haWwiOiJhZG1pbkBleGFtcGxlLmxvY2FsIn0.maYl1K9wKSbIAOg9q-HjX4AomKObiQ4TNXZtjxnW9DjJG3hiHUA1zdEQXfZuduuK95vBqvuy_cP8kvzuwTa3WNePVVwlp5JZYOk2-mmWutGortefXPwraRSvtP9u28zF4uM4xjcbHB-kCY2ievOCIWRuYy-W-MyhonH-t-bhv76AEj0ysdWZ8caVQOig2Ko1HT6dLZaehegqT2ZPwJwGzHteHchOQfRq9O1NwSrfkyc18qQNi_ba4JGXkS1A3pMIkFjat4HZ0B5bPhYuk5J1HB9ZlN60n1Z_Irjz36aM4vr7oCmD2QtRSewEf8F6bwkCM75f0aYFps5KC7ZLC5pokQ\n", - "Cell 4 token status: renewed\n", - "Catalog config API status: 200\n", - "Catalog config warehouse query: teehr\n", - "Management API status: 200\n", - "Polaris access check passed.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:teehr.evaluation.spark_session_utils:🎉 Spark session created successfully!\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Spark catalog namespace probe:\n", - "+----------------+\n", - "|namespace |\n", - "+----------------+\n", - "|teehr |\n", - "|schema_evolution|\n", - "|restricted |\n", - "|public |\n", - "+----------------+\n", - "\n" - ] - } - ], - "source": [ - "import requests\n", - "\n", - "realm = os.getenv(\"POLARIS_DEFAULT_REALM\", \"teehr\")\n", - "remote_warehouse = os.getenv(\"REMOTE_WAREHOUSE_S3_PATH\", \"s3://warehouse/\")\n", - "polaris_api_base = os.getenv(\"POLARIS_CATALOG_URI\", \"http://polaris:8181/api/catalog\").rstrip(\"/\")\n", - "catalog_config_url = f\"{polaris_api_base}/v1/config\"\n", - "mgmt_catalog_url = f\"http://polaris:8181/api/management/v1/catalogs/{realm}\"\n", - "\n", - "# Refresh access token before probing Polaris or recreating Spark.\n", - "polaris_user_token, polaris_refresh_token, refreshed = ensure_fresh_polaris_user_token(\n", - " current_token=polaris_user_token,\n", - " username=polaris_username,\n", - " password=polaris_password,\n", - " client_id=polaris_client_id,\n", - " client_secret=polaris_client_secret,\n", - " refresh_token=polaris_refresh_token,\n", - " refresh_window_seconds=120,\n", - ")\n", - "os.environ[\"POLARIS_USER_TOKEN\"] = polaris_user_token\n", - "if polaris_refresh_token:\n", - " os.environ[\"POLARIS_REFRESH_TOKEN\"] = polaris_refresh_token\n", - "print(\"Cell 4 token status:\", \"renewed\" if refreshed else \"still fresh\")\n", - "\n", - "# Polaris REST config endpoint expects catalog identifier (realm), not S3 URI.\n", - "warehouse_for_config = remote_warehouse\n", - "if polaris_api_base.endswith(\"/api/catalog\"):\n", - " warehouse_for_config = realm\n", - "\n", - "headers = {\n", - " \"Authorization\": f\"Bearer {polaris_user_token}\",\n", - " \"X-Polaris-Realm\": realm,\n", - "}\n", - "\n", - "# Catalog config endpoint requires a warehouse query parameter.\n", - "catalog_resp = requests.get(\n", - " catalog_config_url,\n", - " headers=headers,\n", - " params={\"warehouse\": warehouse_for_config},\n", - " timeout=20,\n", - " )\n", - "print(\"Catalog config API status:\", catalog_resp.status_code)\n", - "print(\"Catalog config warehouse query:\", warehouse_for_config)\n", - "if catalog_resp.status_code >= 400:\n", - " print(\"Catalog config API body:\", catalog_resp.text[:500])\n", - "catalog_resp.raise_for_status()\n", - "\n", - "if \"iceberg-catalog-admin\" in roles:\n", - " mgmt_resp = requests.get(mgmt_catalog_url, headers=headers, timeout=20)\n", - " print(\"Management API status:\", mgmt_resp.status_code)\n", - " if mgmt_resp.status_code >= 400:\n", - " print(\"Management API body:\", mgmt_resp.text[:500])\n", - " mgmt_resp.raise_for_status()\n", - "else:\n", - " print(\n", - " \"Skipping Management API check: token does not include iceberg-catalog-admin. \"\n", - " \"Catalog access is sufficient for non-admin Spark paths.\"\n", - " )\n", - "\n", - "print(\"Polaris access check passed.\")\n", - "\n", - "spark = create_minio_spark_session(\n", - " polaris_token=polaris_user_token,\n", - " force_recreate_session=True,\n", - ")\n", - "\n", - "\n", - "def refresh_polaris_token_for_spark(refresh_window_seconds: int = 120) -> bool:\n", - " global polaris_user_token, polaris_refresh_token\n", - "\n", - " polaris_user_token, polaris_refresh_token, refreshed = ensure_fresh_polaris_user_token(\n", - " current_token=polaris_user_token,\n", - " username=polaris_username,\n", - " password=polaris_password,\n", - " client_id=polaris_client_id,\n", - " client_secret=polaris_client_secret,\n", - " refresh_token=polaris_refresh_token,\n", - " refresh_window_seconds=refresh_window_seconds,\n", - " )\n", - " os.environ[\"POLARIS_USER_TOKEN\"] = polaris_user_token\n", - " if polaris_refresh_token:\n", - " os.environ[\"POLARIS_REFRESH_TOKEN\"] = polaris_refresh_token\n", - " apply_polaris_token_to_spark(spark, polaris_user_token, catalog_name=\"iceberg\", realm=realm)\n", - "\n", - " print(\"Spark token status:\", \"renewed\" if refreshed else \"still fresh\")\n", - " return refreshed\n", - "\n", - "\n", - "apply_polaris_token_to_spark(spark, polaris_user_token, catalog_name=\"iceberg\", realm=realm)\n", - "print(\"Spark catalog namespace probe:\")\n", - "spark.sql(\"SHOW NAMESPACES IN iceberg\").show(truncate=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 51, - "id": "a0558850-b6a5-4994-a981-23ef845babe2", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:teehr.evaluation.spark_session_utils:🚀 Creating Spark session: TEEHR Evaluation\n", - "INFO:teehr.evaluation.spark_session_utils:✅ Spark local configuration successful!\n", - "INFO:teehr.evaluation.spark_session_utils:Setting Hadoop's default AWS credentials provider and AWS region\n", - "INFO:teehr.evaluation.spark_session_utils:🔑 Using user-provided AWS credentials\n", - "INFO:teehr.evaluation.spark_session_utils:Configuring Iceberg catalogs...\n", - "INFO:teehr.evaluation.spark_session_utils:⚙️ All settings applied. Creating Spark session...\n", - "INFO:teehr.evaluation.spark_session_utils:🎉 Spark session created successfully!\n" - ] - } - ], - "source": [ - "spark.stop()\n", - "spark = create_minio_spark_session(\n", - " polaris_token=polaris_user_token,\n", - " force_recreate_session=True,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 53, - "id": "4c74e5de-c29e-46bb-b5e3-686faec7cbf9", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:teehr.evaluation.evaluation:Using provided Spark session.\n", - "INFO:teehr.evaluation.evaluation:Active catalog set to iceberg.\n" - ] - } - ], - "source": [ - "ev = RemoteReadWriteEvaluation(spark=spark, enable_spark_proxy=True)" - ] - }, - { - "cell_type": "code", - "execution_count": 49, - "id": "ff0d1a63-f78a-4516-994d-48b964f18d4b", - "metadata": {}, - "outputs": [], - "source": [ - "spark.conf.set(\"spark.sql.catalog.iceberg.token\", polaris_user_token)\n", - "spark.conf.set(\"spark.sql.catalog.iceberg.rest.auth.oauth2.token\", polaris_user_token)" - ] - }, - { - "cell_type": "code", - "execution_count": 54, - "id": "6c171370-baf5-4a15-84f5-4d0d3a5356bb", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:teehr.evaluation.tables.base_table:Initializing Table for table: configurations.\n", - "INFO:teehr.evaluation.tables.base_table:Loading files from iceberg.teehr.configurations.\n", - "INFO:teehr.evaluation.read:Reading files from iceberg.teehr.configurations.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+--------------------+---------------+--------------------+--------------------+--------------------+----------+\n", - "| name|timeseries_type| description| created_at| updated_at|properties|\n", - "+--------------------+---------------+--------------------+--------------------+--------------------+----------+\n", - "|nrds_v22_cfenom_m...| secondary|NRDS DataStream m...|2026-07-21 19:49:...|2026-07-21 19:49:...| NULL|\n", - "|nrds_v22_cfenom_s...| secondary|POC version of Da...|2026-07-21 19:49:...|2026-07-21 19:49:...| NULL|\n", - "|nrds_v22_lstm0_me...| secondary|NRDS DataStream m...|2026-07-21 19:49:...|2026-07-21 19:49:...| NULL|\n", - "|nrds_v22_lstm0_sh...| secondary|NRDS DataStream s...|2026-07-21 19:49:...|2026-07-21 19:49:...| NULL|\n", - "|nwm30_analysis_as...| secondary|Alaska NWM standa...|2026-07-21 19:49:...|2026-07-21 19:49:...| NULL|\n", - "|nwm30_analysis_as...| secondary|Alaska NWM extend...|2026-07-21 19:49:...|2026-07-21 19:49:...| NULL|\n", - "|nwm30_analysis_as...| secondary|CONUS NWM extende...|2026-07-21 19:49:...|2026-07-21 19:49:...| NULL|\n", - "|nwm30_analysis_as...| secondary|Hawaii NWM standa...|2026-07-21 19:49:...|2026-07-21 19:49:...| NULL|\n", - "|nwm30_analysis_as...| secondary|CONUS NWM standar...|2026-07-21 19:49:...|2026-07-21 19:49:...| NULL|\n", - "|nwm30_analysis_as...| secondary|PRVI NWM standard...|2026-07-21 19:49:...|2026-07-21 19:49:...| NULL|\n", - "|nwm30_forcing_ana...| primary|CONUS MRMS mean a...|2026-07-21 19:49:...|2026-07-21 19:49:...| NULL|\n", - "|nwm30_forcing_ana...| primary|Alaska MRMS mean ...|2026-07-21 19:49:...|2026-07-21 19:49:...| NULL|\n", - "|nwm30_forcing_ana...| primary|CONUS STAGEIV mea...|2026-07-21 19:49:...|2026-07-21 19:49:...| NULL|\n", - "|nwm30_forcing_ana...| primary|Alaska StageIV me...|2026-07-21 19:49:...|2026-07-21 19:49:...| NULL|\n", - "|nwm30_forcing_ana...| primary|Hawaii MRMS mean ...|2026-07-21 19:49:...|2026-07-21 19:49:...| NULL|\n", - "|nwm30_forcing_ana...| primary|PRVI MRMS mean ar...|2026-07-21 19:49:...|2026-07-21 19:49:...| NULL|\n", - "|nwm30_forcing_med...| secondary|CONUS GFS mean ar...|2026-07-21 19:49:...|2026-07-21 19:49:...| NULL|\n", - "|nwm30_forcing_med...| secondary|Alaska GFS mean a...|2026-07-21 19:49:...|2026-07-21 19:49:...| NULL|\n", - "|nwm30_forcing_med...| secondary|CONUS NBM mean ar...|2026-07-21 19:49:...|2026-07-21 19:49:...| NULL|\n", - "|nwm30_forcing_med...| secondary|Alaska NBM mean a...|2026-07-21 19:49:...|2026-07-21 19:49:...| NULL|\n", - "+--------------------+---------------+--------------------+--------------------+--------------------+----------+\n", - "only showing top 20 rows\n" - ] - } - ], - "source": [ - "ev.configurations.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c87d9147-d860-41b0-8dcf-dfad60beaaa4", - "metadata": {}, - "outputs": [], - "source": [ - "# spark.stop()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "85a4d3ab-bfea-4c91-9f05-8ffcc73b0d4f", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.11" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/jupyterhub-docker/spark_session_utils.py b/jupyterhub-docker/spark_session_utils.py index 1141c5c..d6a57b0 100644 --- a/jupyterhub-docker/spark_session_utils.py +++ b/jupyterhub-docker/spark_session_utils.py @@ -66,7 +66,10 @@ def create_spark_session( add_jars: List[str] = None, add_packages: List[str] = None, update_configs: Dict[str, str] = None, - debug_config: bool = False + debug_config: bool = False, + # Polaris authentication + polaris_token: Optional[str] = None, + use_authmanager: Optional[bool] = None, ) -> SparkSession: """Create and return a Spark session for evaluation. @@ -156,7 +159,12 @@ def create_spark_session( """ logger.info(f"🚀 Creating Spark session: {app_name}") - if force_recreate_session: + # AuthManager requires a fresh JVM per user session to avoid static state leakage + resolved_use_authmanager = ( + use_authmanager if use_authmanager is not None + else _as_bool_str(os.getenv("POLARIS_USE_AUTHMANAGER", "false")) == "true" + ) + if force_recreate_session or resolved_use_authmanager: existing_session = SparkSession.getActiveSession() if existing_session is not None: logger.info("♻️ Stopping the active Spark session before recreation") @@ -229,10 +237,17 @@ def create_spark_session( remote_catalog_uri=remote_catalog_uri ) + # Build Polaris auth configs and merge with caller-provided update_configs. + # Auth configs are the base; caller's configs take precedence. + polaris_auth_configs = _build_polaris_auth_configs(polaris_token, use_authmanager) + effective_configs: Dict[str, str] = {**polaris_auth_configs} + if update_configs: + effective_configs.update(update_configs) + # Update configs and packages if provided _update_configs_and_packages( conf=conf, - update_configs=update_configs, + update_configs=effective_configs or None, add_jars=add_jars, add_packages=add_packages ) @@ -241,6 +256,10 @@ def create_spark_session( spark = SparkSession.builder.appName(app_name).config(conf=conf).getOrCreate() sedona_spark = SedonaContext.create(spark) + # Apply runtime-settable configs to the live session (e.g., auth tokens) + if effective_configs: + _apply_runtime_spark_configs(sedona_spark, effective_configs) + if debug_config: log_session_config(sedona_spark) @@ -998,50 +1017,39 @@ def ensure_broker_session_token( return broker_session_token -def create_minio_spark_session( - polaris_token: Optional[str] = None, - force_recreate_session: bool = False, - update_configs: Optional[Dict[str, str]] = None, - use_authmanager: Optional[bool] = None, -): - """Start a Spark session configured for the local Polaris REST catalog. - - If ``polaris_token`` is provided, Spark uses user-token OAuth2 auth. - Otherwise it falls back to client-credential OAuth2 using env vars. +def _build_polaris_auth_configs( + polaris_token: Optional[str], + use_authmanager: Optional[bool], +) -> Dict[str, str]: + """Build Spark configs for Polaris catalog authentication. + + Handles three auth paths: + 1. AuthManager (use_authmanager=True or POLARIS_USE_AUTHMANAGER=true env var) + Uses the teehr-api broker for token management — required for JupyterHub + where tokens must be refreshed transparently during long sessions. + 2. Direct user token (polaris_token provided) + Passes the JWT directly to the Iceberg REST catalog. + 3. Service account client credentials (POLARIS_CLIENT_ID + POLARIS_CLIENT_SECRET) + Used by Prefect batch jobs and other non-interactive service accounts. + + Returns an empty dict if none of the above are configured. """ - aws_access_key_id = os.getenv("AWS_ACCESS_KEY_ID", "minioadmin") - aws_secret_access_key = os.getenv("AWS_SECRET_ACCESS_KEY", "minioadmin123") - - remote_catalog_uri = os.getenv("REMOTE_CATALOG_REST_URI", "http://polaris:8181/api/catalog") - remote_warehouse_dir = os.getenv("REMOTE_WAREHOUSE_S3_PATH", "s3://warehouse/") polaris_realm = os.getenv("POLARIS_DEFAULT_REALM", "teehr") - if remote_catalog_uri.rstrip("/").endswith("/api/catalog"): - # Polaris REST expects the catalog identifier here, not the backing S3 URI. - remote_warehouse_dir = polaris_realm - - s3_endpoint = os.getenv("REMOTE_CATALOG_S3_ENDPOINT", "http://minio:9000") - s3_path_style = _as_bool_str(os.getenv("REMOTE_CATALOG_S3_PATH_STYLE_ACCESS", "true")) - s3_region = os.getenv("AWS_REGION", "us-east-2") - - merged_configs: Dict[str, str] = { - "spark.sql.catalog.iceberg.warehouse": remote_warehouse_dir, - "spark.sql.catalog.iceberg.header.X-Polaris-Realm": polaris_realm, - "spark.sql.catalog.iceberg.rest.transport.header.X-Polaris-Realm": polaris_realm, - "spark.sql.catalog.iceberg.s3.endpoint": s3_endpoint, - "spark.sql.catalog.iceberg.s3.path-style-access": s3_path_style, - "spark.sql.catalog.iceberg.s3.region": s3_region, - "spark.hadoop.fs.s3a.endpoint": s3_endpoint, - "spark.hadoop.fs.s3a.path.style.access": s3_path_style, - "spark.hadoop.fs.s3a.connection.ssl.enabled": "false", - } - resolved_use_authmanager = ( - use_authmanager - if use_authmanager is not None + use_authmanager if use_authmanager is not None else _as_bool_str(os.getenv("POLARIS_USE_AUTHMANAGER", "false")) == "true" ) + configs: Dict[str, str] = {} + + if not resolved_use_authmanager and not polaris_token and not os.getenv("POLARIS_CLIENT_ID"): + return configs + + # Realm headers are required for all Polaris auth paths + configs["spark.sql.catalog.iceberg.header.X-Polaris-Realm"] = polaris_realm + configs["spark.sql.catalog.iceberg.rest.transport.header.X-Polaris-Realm"] = polaris_realm + if resolved_use_authmanager: broker_url = os.getenv("POLARIS_BROKER_URL", "http://teehr-api:8000/auth/polaris-token") broker_url = _normalize_internal_broker_url(broker_url) @@ -1065,7 +1073,7 @@ def create_minio_spark_session( current_token=current_user_token, username=os.getenv("POLARIS_USERNAME"), password=os.getenv("POLARIS_PASSWORD"), - client_id=os.getenv("POLARIS_CLIENT_ID", "spark-polaris"), + client_id=os.getenv("POLARIS_CLIENT_ID", "jupyterhub"), client_secret=os.getenv("POLARIS_CLIENT_SECRET"), refresh_token=refresh_token, allow_password_fallback=False, @@ -1091,13 +1099,11 @@ def create_minio_spark_session( except requests.HTTPError as exc: if not _is_http_error_with_status(exc, 401): raise - - # The bearer token may have been invalidated server-side; force refresh and retry once. current_user_token, refresh_token, _ = ensure_fresh_polaris_user_token( current_token=None, username=os.getenv("POLARIS_USERNAME"), password=os.getenv("POLARIS_PASSWORD"), - client_id=os.getenv("POLARIS_CLIENT_ID", "spark-polaris"), + client_id=os.getenv("POLARIS_CLIENT_ID", "jupyterhub"), client_secret=os.getenv("POLARIS_CLIENT_SECRET"), refresh_token=refresh_token, allow_password_fallback=True, @@ -1107,7 +1113,6 @@ def create_minio_spark_session( os.environ["POLARIS_USER_TOKEN"] = current_user_token if refresh_token: os.environ["POLARIS_REFRESH_TOKEN"] = refresh_token - broker_session_token = ensure_broker_session_token( user_id=authmanager_user_id, session_id=authmanager_session_id, @@ -1118,75 +1123,100 @@ def create_minio_spark_session( audience=broker_audience, broker_url=broker_url, ) - os.environ["POLARIS_BROKER_SESSION_TOKEN"] = broker_session_token - merged_configs["spark.sql.catalog.iceberg.rest.auth.type"] = ( + configs["spark.sql.catalog.iceberg.rest.auth.type"] = ( "org.teehr.iceberg.auth.TeehrBrokerAuthManager" ) - merged_configs["spark.sql.catalog.iceberg.rest.auth.teehr.broker.url"] = broker_session_url - merged_configs["spark.sql.catalog.iceberg.rest.auth.teehr.user-id"] = authmanager_user_id - merged_configs["spark.sql.catalog.iceberg.rest.auth.teehr.session-id"] = ( - authmanager_session_id - ) - merged_configs["spark.sql.catalog.iceberg.rest.auth.teehr.realm"] = polaris_realm - merged_configs["spark.sql.catalog.iceberg.rest.auth.teehr.catalog"] = "iceberg" - merged_configs["spark.sql.catalog.iceberg.rest.auth.teehr.audience"] = broker_audience - merged_configs["spark.sql.catalog.iceberg.rest.auth.teehr.broker-session-token-env"] = ( + configs["spark.sql.catalog.iceberg.rest.auth.teehr.broker.url"] = broker_session_url + configs["spark.sql.catalog.iceberg.rest.auth.teehr.user-id"] = authmanager_user_id + configs["spark.sql.catalog.iceberg.rest.auth.teehr.session-id"] = authmanager_session_id + configs["spark.sql.catalog.iceberg.rest.auth.teehr.realm"] = polaris_realm + configs["spark.sql.catalog.iceberg.rest.auth.teehr.catalog"] = "iceberg" + configs["spark.sql.catalog.iceberg.rest.auth.teehr.audience"] = broker_audience + configs["spark.sql.catalog.iceberg.rest.auth.teehr.broker-session-token-env"] = ( "POLARIS_BROKER_SESSION_TOKEN" ) - merged_configs["spark.jars"] = authmanager_jar + configs["spark.jars"] = authmanager_jar + elif polaris_token: - merged_configs["spark.sql.catalog.iceberg.rest.auth.type"] = "oauth2" - merged_configs["spark.sql.catalog.iceberg.token"] = polaris_token - merged_configs["spark.sql.catalog.iceberg.rest.auth.oauth2.token"] = polaris_token + configs["spark.sql.catalog.iceberg.rest.auth.type"] = "oauth2" + configs["spark.sql.catalog.iceberg.token"] = polaris_token + configs["spark.sql.catalog.iceberg.rest.auth.oauth2.token"] = polaris_token + else: + # Service account / client credentials path. + # Set POLARIS_CLIENT_ID and POLARIS_CLIENT_SECRET for the service account + # (e.g. prefect-polaris for Prefect batch jobs). oauth_server_uri = os.getenv("POLARIS_OAUTH2_SERVER_URI") - spark_polaris_client_secret = os.getenv("SPARK_POLARIS_CLIENT_SECRET") + polaris_client_id = os.getenv("POLARIS_CLIENT_ID") + polaris_client_secret = os.getenv("POLARIS_CLIENT_SECRET") - merged_configs["spark.sql.catalog.iceberg.rest.auth.type"] = "oauth2" - merged_configs["spark.sql.catalog.iceberg.scope"] = "openid" - merged_configs["spark.sql.catalog.iceberg.rest.auth.oauth2.scope"] = "openid" + configs["spark.sql.catalog.iceberg.rest.auth.type"] = "oauth2" + configs["spark.sql.catalog.iceberg.scope"] = "openid" + configs["spark.sql.catalog.iceberg.rest.auth.oauth2.scope"] = "openid" if oauth_server_uri: - merged_configs["spark.sql.catalog.iceberg.oauth2-server-uri"] = oauth_server_uri - merged_configs["spark.sql.catalog.iceberg.rest.auth.oauth2.server-uri"] = oauth_server_uri - if spark_polaris_client_secret: - merged_configs["spark.sql.catalog.iceberg.credential"] = ( - f"spark-polaris:{spark_polaris_client_secret}" - ) - merged_configs["spark.sql.catalog.iceberg.rest.auth.oauth2.credential"] = ( - f"spark-polaris:{spark_polaris_client_secret}" - ) + configs["spark.sql.catalog.iceberg.oauth2-server-uri"] = oauth_server_uri + configs["spark.sql.catalog.iceberg.rest.auth.oauth2.server-uri"] = oauth_server_uri + if polaris_client_id and polaris_client_secret: + credential = f"{polaris_client_id}:{polaris_client_secret}" + configs["spark.sql.catalog.iceberg.credential"] = credential + configs["spark.sql.catalog.iceberg.rest.auth.oauth2.credential"] = credential - if update_configs: - merged_configs.update(update_configs) + return configs - call_kwargs = { - "remote_catalog_uri": remote_catalog_uri, - "remote_warehouse_dir": remote_warehouse_dir, - "aws_access_key_id": aws_access_key_id, - "aws_secret_access_key": aws_secret_access_key, - "force_recreate_session": force_recreate_session or resolved_use_authmanager, - "update_configs": merged_configs, + +def create_minio_spark_session( + polaris_token: Optional[str] = None, + force_recreate_session: bool = False, + update_configs: Optional[Dict[str, str]] = None, + use_authmanager: Optional[bool] = None, +) -> SparkSession: + """Start a Spark session with MinIO credentials for local KinD development. + + Thin wrapper around create_spark_session() that injects MinIO-specific S3 + configuration. All Polaris auth (AuthManager, direct token, client credentials) + is handled by create_spark_session() based on the parameters and environment. + + For remote deployments using AWS S3, call create_spark_session() directly with + appropriate AWS credentials and catalog configuration. + """ + s3_endpoint = os.getenv("REMOTE_CATALOG_S3_ENDPOINT", "http://minio:9000") + s3_path_style = _as_bool_str(os.getenv("REMOTE_CATALOG_S3_PATH_STYLE_ACCESS", "true")) + s3_region = os.getenv("AWS_REGION", "us-east-2") + polaris_realm = os.getenv("POLARIS_DEFAULT_REALM", "teehr") + remote_catalog_uri = os.getenv("REMOTE_CATALOG_REST_URI", "http://polaris:8181/api/catalog") + + # Polaris REST expects the catalog name as warehouse identifier, not an S3 URI + remote_warehouse_dir = ( + polaris_realm + if remote_catalog_uri.rstrip("/").endswith("/api/catalog") + else os.getenv("REMOTE_WAREHOUSE_S3_PATH", "s3://warehouse/") + ) + + minio_configs: Dict[str, str] = { + "spark.sql.catalog.iceberg.s3.endpoint": s3_endpoint, + "spark.sql.catalog.iceberg.s3.path-style-access": s3_path_style, + "spark.sql.catalog.iceberg.s3.region": s3_region, + "spark.hadoop.fs.s3a.endpoint": s3_endpoint, + "spark.hadoop.fs.s3a.path.style.access": s3_path_style, + "spark.hadoop.fs.s3a.connection.ssl.enabled": "false", } - if polaris_token: - # Preferred path for newer teehr versions that support direct token auth. - call_kwargs["oauth2_token"] = polaris_token + if update_configs: + minio_configs.update(update_configs) + + return create_spark_session( + aws_access_key_id=os.getenv("AWS_ACCESS_KEY_ID", "minioadmin"), + aws_secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY", "minioadmin123"), + remote_catalog_uri=remote_catalog_uri, + remote_warehouse_dir=remote_warehouse_dir, + polaris_token=polaris_token, + force_recreate_session=force_recreate_session, + update_configs=minio_configs, + use_authmanager=use_authmanager, + ) + - while True: - try: - spark = create_spark_session(**call_kwargs) - _apply_runtime_spark_configs(spark, merged_configs) - return spark - except TypeError as exc: - msg = str(exc) - if "oauth2_token" in msg and "oauth2_token" in call_kwargs: - call_kwargs.pop("oauth2_token", None) - continue - if "force_recreate_session" in msg and "force_recreate_session" in call_kwargs: - call_kwargs.pop("force_recreate_session", None) - continue - raise def request_broker_polaris_token( diff --git a/jupyterhub/garden.yaml b/jupyterhub/garden.yaml index a08934f..bc3dd9d 100644 --- a/jupyterhub/garden.yaml +++ b/jupyterhub/garden.yaml @@ -176,11 +176,7 @@ spec: IN_CLUSTER: ${var.polaris.inCluster} POLARIS_DEFAULT_REALM: ${var.polaris.defaultRealm} POLARIS_OAUTH2_SERVER_URI: ${var.polaris.oauthServerUri} - SPARK_POLARIS_CLIENT_SECRET: - valueFrom: - secretKeyRef: - name: spark-polaris-secrets - key: client-secret + POLARIS_USE_AUTHMANAGER: "true" extraFiles: jupyter_config: mountPath: /etc/jupyter/jupyter_notebook_config.py @@ -383,11 +379,7 @@ spec: IN_CLUSTER: ${var.polaris.inCluster} POLARIS_DEFAULT_REALM: ${var.polaris.defaultRealm} POLARIS_OAUTH2_SERVER_URI: ${var.polaris.oauthServerUri} - SPARK_POLARIS_CLIENT_SECRET: - valueFrom: - secretKeyRef: - name: spark-polaris-secrets - key: client-secret + POLARIS_USE_AUTHMANAGER: "true" TRINO_HOST: ${var.trino.host} TRINO_PORT: ${var.trino.port} TRINO_CATALOG: ${var.trino.catalog} diff --git a/keycloak-bootstrap/manifests/bootstrap-job.yaml b/keycloak-bootstrap/manifests/bootstrap-job.yaml index e0cc5e8..755ee07 100644 --- a/keycloak-bootstrap/manifests/bootstrap-job.yaml +++ b/keycloak-bootstrap/manifests/bootstrap-job.yaml @@ -48,6 +48,11 @@ spec: secretKeyRef: name: trino-polaris-secrets key: client-secret + - name: PREFECT_POLARIS_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: prefect-polaris-secrets + key: client-secret - name: SPARK_POLARIS_CLIENT_SECRET valueFrom: secretKeyRef: diff --git a/keycloak-bootstrap/manifests/realm-configmap.yaml.tpl b/keycloak-bootstrap/manifests/realm-configmap.yaml.tpl index 77e2169..f0d7dfb 100644 --- a/keycloak-bootstrap/manifests/realm-configmap.yaml.tpl +++ b/keycloak-bootstrap/manifests/realm-configmap.yaml.tpl @@ -213,6 +213,36 @@ data: "publicClient": false, "serviceAccountsEnabled": true, "secret": "$(env:TRINO_POLARIS_CLIENT_SECRET)", + "serviceAccountRoles": { + "realmRoles": ["iceberg-catalog-admin"] + }, + "protocolMappers": [ + { + "name": "realm-roles", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-realm-role-mapper", + "consentRequired": false, + "config": { + "multivalued": "true", + "id.token.claim": "true", + "access.token.claim": "true", + "userinfo.token.claim": "true", + "claim.name": "realm_access.roles", + "jsonType.label": "String" + } + } + ] + }, + { + "clientId": "prefect-polaris", + "enabled": true, + "protocol": "openid-connect", + "publicClient": false, + "serviceAccountsEnabled": true, + "secret": "$(env:PREFECT_POLARIS_CLIENT_SECRET)", + "serviceAccountRoles": { + "realmRoles": ["teehr-read-write"] + }, "protocolMappers": [ { "name": "realm-roles", diff --git a/polaris-bootstrap/manifests/acl-config.yaml.tpl b/polaris-bootstrap/manifests/acl-config.yaml.tpl index 8ddb048..57c9bfa 100644 --- a/polaris-bootstrap/manifests/acl-config.yaml.tpl +++ b/polaris-bootstrap/manifests/acl-config.yaml.tpl @@ -69,12 +69,7 @@ data: } ], "table_policies": [], - "principals": [ - { - "name": "spark-polaris", - "principal_role": "iceberg-catalog-admin" - } - ] + "principals": [] } ] } diff --git a/polaris/manifests/polaris-config.yaml.tpl b/polaris/manifests/polaris-config.yaml.tpl index c47d952..fa13036 100644 --- a/polaris/manifests/polaris-config.yaml.tpl +++ b/polaris/manifests/polaris-config.yaml.tpl @@ -44,19 +44,22 @@ data: # access — adding a user to a Keycloak group immediately grants the corresponding # Polaris permissions on their next token issuance. # - # Two patterns are mapped: - # /iceberg-catalog-admins → iceberg-catalog-admin (name mismatch between KC group and Polaris role) - # /teehr- → teehr- (e.g. teehr-read-only, teehr-read-write) + # Also reads realm_access/roles so Keycloak service accounts (trino-polaris, + # prefect-polaris) can be granted Polaris access via realm role assignment, + # without needing a separate Polaris principal entity. # - # All other Keycloak groups (basic-user, jupyter-user, etc.) produce no match - # and are safely ignored by Polaris. + # Patterns mapped: + # /iceberg-catalog-admins (group) → iceberg-catalog-admin + # /teehr- (group) → teehr- (e.g. teehr-read-only, teehr-read-write) + # iceberg-catalog-admin (realm role for service accounts) + # teehr- (realm role for service accounts) # # Individual/table-level grants: use the polaris-sync-principals script to create # a named principal for the user and assign specific catalog-role grants. These # are additive on top of the JWT-based group grants above. - quarkus.oidc.roles.role-claim-path=groups + quarkus.oidc.roles.role-claim-path=groups,realm_access/roles polaris.oidc.principal-roles-mapper.type=default - polaris.oidc.principal-roles-mapper.mappings[0].regex=^/?iceberg-catalog-admins$ + polaris.oidc.principal-roles-mapper.mappings[0].regex=^/?iceberg-catalog-admins?$ polaris.oidc.principal-roles-mapper.mappings[0].replacement=PRINCIPAL_ROLE:iceberg-catalog-admin polaris.oidc.principal-roles-mapper.mappings[1].regex=^/?teehr-(.+)$ polaris.oidc.principal-roles-mapper.mappings[1].replacement=PRINCIPAL_ROLE:teehr-$1 diff --git a/secrets/secrets.local.yaml b/secrets/secrets.local.yaml index c89d81b..a97c24d 100644 --- a/secrets/secrets.local.yaml +++ b/secrets/secrets.local.yaml @@ -60,6 +60,10 @@ secrets: data: client-secret: local-trino-polaris-client-secret credential: "trino-polaris:local-trino-polaris-client-secret" + prefect-polaris-secrets: + data: + client-secret: local-prefect-polaris-client-secret + credential: "prefect-polaris:local-prefect-polaris-client-secret" spark-polaris-secrets: data: client-secret: local-spark-polaris-client-secret \ No newline at end of file From aedd2fa46ac5b22b1803aea20f60cf52ed1aa90c Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Fri, 31 Jul 2026 16:45:15 -0400 Subject: [PATCH 28/62] update spark session utils in teehr and fix tests. --- .github/copilot-instructions.md | 5 + .../01_setup_minio_warehouse.ipynb | 2 +- .../02_create_joined_timeseries.ipynb | 2 +- .../03_generate_basic_metrics.ipynb | 2 +- .../calculate_nwm_metrics.ipynb | 4 +- jupyterhub-docker/Dockerfile.jupyter-driver | 2 - jupyterhub-docker/spark_session_utils.py | 1292 ----------------- .../manifests/realm-configmap.yaml.tpl | 20 +- tests/spark_authmanager_test.py | 2 +- tests/spark_permission_test.py | 2 +- 10 files changed, 26 insertions(+), 1307 deletions(-) create mode 100644 .github/copilot-instructions.md rename examples/{developer => }/01_setup_minio_warehouse.ipynb (99%) rename examples/{developer => }/02_create_joined_timeseries.ipynb (97%) rename examples/{developer => }/03_generate_basic_metrics.ipynb (98%) rename examples/{developer => }/calculate_nwm_metrics.ipynb (99%) delete mode 100644 jupyterhub-docker/spark_session_utils.py diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 0000000..2a785cf --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,5 @@ +Context: +- the `teehr` repo which is a python library that contains the scientific code for the teehr environment. This contains code to fetch, validate, store, and analyze hydrologic forecast and simulation data against observations. +- the `teehr-cloud-core` repo contains the main shared teehr-cloud components such as authentication, jupyterhub, iceberg warehouse, web api, query engine, spark-executors, prefect server, etc. teehr-cloud-core utilizes the teehr library. +- the teehr-cloud-platform repo contains terraform IaC for creating an AWS environment that can host a teehr-cloud deployment. +- the `teehr-hub` and `teehr-fved` repos have teehr-cloud-core as a submodule and are deployments of teehr and teehr-cloud-core. They also contain deployment specific components such as frontend dashboards, deployment specific prefect workflows and warehouse setup and maintenance code, certificates, etc. \ No newline at end of file diff --git a/examples/developer/01_setup_minio_warehouse.ipynb b/examples/01_setup_minio_warehouse.ipynb similarity index 99% rename from examples/developer/01_setup_minio_warehouse.ipynb rename to examples/01_setup_minio_warehouse.ipynb index 08a23b4..69fa9f0 100644 --- a/examples/developer/01_setup_minio_warehouse.ipynb +++ b/examples/01_setup_minio_warehouse.ipynb @@ -18,11 +18,11 @@ "import teehr\n", "from teehr import RemoteReadWriteEvaluation\n", "from teehr.utilities.apply_migrations import evolve_catalog_schema\n", + "from teehr.evaluation.spark_session_utils import create_minio_spark_session\n", "\n", "from setup_utils import (\n", " DEV_LOCATION_ID_LIST,\n", " apply_polaris_token_to_spark,\n", - " create_minio_spark_session,\n", " ensure_fresh_polaris_user_token,\n", ")" ] diff --git a/examples/developer/02_create_joined_timeseries.ipynb b/examples/02_create_joined_timeseries.ipynb similarity index 97% rename from examples/developer/02_create_joined_timeseries.ipynb rename to examples/02_create_joined_timeseries.ipynb index 6807b7a..113819d 100644 --- a/examples/developer/02_create_joined_timeseries.ipynb +++ b/examples/02_create_joined_timeseries.ipynb @@ -8,7 +8,7 @@ "outputs": [], "source": [ "from teehr import RemoteReadWriteEvaluation\n", - "from setup_utils import create_minio_spark_session" + "from teehr.evaluation.spark_session_utils import create_minio_spark_session" ] }, { diff --git a/examples/developer/03_generate_basic_metrics.ipynb b/examples/03_generate_basic_metrics.ipynb similarity index 98% rename from examples/developer/03_generate_basic_metrics.ipynb rename to examples/03_generate_basic_metrics.ipynb index 950fdf9..6745e97 100644 --- a/examples/developer/03_generate_basic_metrics.ipynb +++ b/examples/03_generate_basic_metrics.ipynb @@ -8,7 +8,7 @@ "outputs": [], "source": [ "from teehr import RemoteReadWriteEvaluation\n", - "from setup_utils import create_minio_spark_session" + "from teehr.evaluation.spark_session_utils import create_minio_spark_session" ] }, { diff --git a/examples/developer/calculate_nwm_metrics.ipynb b/examples/calculate_nwm_metrics.ipynb similarity index 99% rename from examples/developer/calculate_nwm_metrics.ipynb rename to examples/calculate_nwm_metrics.ipynb index cf60d46..f123137 100644 --- a/examples/developer/calculate_nwm_metrics.ipynb +++ b/examples/calculate_nwm_metrics.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "8877b4b8-5095-4119-9299-f65dd715ed76", "metadata": {}, "outputs": [ @@ -21,7 +21,7 @@ "import teehr\n", "import pandas as pd\n", "# from teehr.evaluation.spark_session_utils import create_spark_session\n", - "from setup_utils import create_minio_spark_session\n", + "from teehr.evaluation.spark_session_utils import create_minio_spark_session\n", "\n", "from teehr import DeterministicMetrics as dm\n", "from teehr import Signatures as s\n", diff --git a/jupyterhub-docker/Dockerfile.jupyter-driver b/jupyterhub-docker/Dockerfile.jupyter-driver index 0dbcdc5..a1e9b02 100644 --- a/jupyterhub-docker/Dockerfile.jupyter-driver +++ b/jupyterhub-docker/Dockerfile.jupyter-driver @@ -6,7 +6,6 @@ ARG TEEHR_VERSION USER root ENV DEBIAN_FRONTEND=noninteractive ENV PATH=${NB_PYTHON_PREFIX}/bin:$PATH -ENV PYTHONPATH=/opt/teehr:${PYTHONPATH} # Use HTTPS apt mirrors so builds work when plain HTTP egress is blocked. RUN set -eux; \ @@ -78,7 +77,6 @@ RUN ARCH=$(uname -m) && \ RUN mkdir -p /opt/teehr && chown -R ${NB_USER}:${NB_USER} /opt/teehr COPY executor-pod-template.yaml /opt/teehr/executor-pod-template.yaml -COPY spark_session_utils.py /opt/teehr/spark_session_utils.py # Keep driver and executor classpaths aligned for the custom AuthManager. COPY teehr-authmanager.jar /opt/spark/jars/teehr-authmanager.jar diff --git a/jupyterhub-docker/spark_session_utils.py b/jupyterhub-docker/spark_session_utils.py deleted file mode 100644 index d6a57b0..0000000 --- a/jupyterhub-docker/spark_session_utils.py +++ /dev/null @@ -1,1292 +0,0 @@ -"""Module to create and configure Spark sessions and Polaris auth helpers.""" -# flake8: noqa -import base64 -import json -import logging -import os -import socket -import time -from pathlib import Path -from typing import Dict, List, Optional, Tuple, Union -from urllib.parse import urlsplit, urlunsplit - -import psutil -import requests - -from pyspark import SparkConf -from pyspark.sql import SparkSession -from sedona.spark import SedonaContext -import pandas as pd -import botocore.session - -import teehr.const as const - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - -# Note: Scala version: 2.13 in pyspark 4.0 -SCALA_VERSION = "2.13" -PYSPARK_VERSION = "4.0" -ICEBERG_VERSION = "1.10.1" -SEDONA_VERSION = "1.8.0" - - - -def create_spark_session( - # App name and catalog settings - app_name: str = "TEEHR Evaluation", - local_catalog_name: str = const.LOCAL_CATALOG_NAME, - local_catalog_type: str = const.LOCAL_CATALOG_TYPE, - remote_warehouse_dir: str = const.REMOTE_WAREHOUSE_S3_PATH, - remote_catalog_name: str = const.REMOTE_CATALOG_NAME, - remote_catalog_type: str = const.REMOTE_CATALOG_TYPE, - remote_catalog_uri: str = const.REMOTE_CATALOG_REST_URI, - # Spark K8'specific parameters - start_spark_cluster: bool = False, - force_recreate_session: bool = False, - executor_instances: int = 2, - executor_memory: str = "1g", - executor_cores: int = 1, - executor_image: str = None, - executor_namespace: str = None, - driver_memory: str = None, - driver_max_result_size: str = None, - pod_template_path: Union[str, Path] = const.POD_TEMPLATE_PATH, - # AWS credential parameters - aws_access_key_id: str = None, - aws_secret_access_key: str = None, - aws_session_token: str = None, - aws_region: str = const.AWS_REGION, - aws_profile: str = None, - # GCS credential parameters - enable_gcs: bool = False, - gcs_project_id: str = None, - gcs_service_account_key_file: str = None, - # Simple extensibility parameters - add_jars: List[str] = None, - add_packages: List[str] = None, - update_configs: Dict[str, str] = None, - debug_config: bool = False, - # Polaris authentication - polaris_token: Optional[str] = None, - use_authmanager: Optional[bool] = None, -) -> SparkSession: - """Create and return a Spark session for evaluation. - - Parameters - ---------- - app_name : str - Name of the Spark application. Default is "TEEHR Evaluation". - local_catalog_name : str - Name of the local Iceberg catalog. Default is "local". - local_catalog_type : str - Type of the local Iceberg catalog. Default is "jdbc". - remote_warehouse_dir : str - Remote warehouse directory for Iceberg catalog. Default is TEEHR - warehouse S3 path. - remote_catalog_name : str - Name of the remote Iceberg catalog. Default is "iceberg". - remote_catalog_type : str - Type of the remote Iceberg catalog. Default is "rest". - remote_catalog_uri : str - URI for the remote Iceberg catalog. Default is TEEHR catalog REST URI. - start_spark_cluster : bool - Whether to start a Spark cluster (Kubernetes mode). - Default is False (local mode). - force_recreate_session : bool - Whether to stop an existing Spark session before creating a new one. - Default is False. - executor_instances : int - Number of executor instances for the Spark cluster. Default is 2. - executor_memory : str - Memory allocation for each executor. Default is "1g". - executor_cores : int - Number of CPU cores for each executor. Default is 1. - executor_image : str - Container image for Spark executors. Default is None. - executor_namespace : str - Kubernetes namespace for Spark executors. Default is None. - driver_memory : str - Memory allocation for the Spark driver. Default is None. - driver_max_result_size : str - Maximum result size for the Spark driver. Default is None. - pod_template_path : Union[str, Path] - Path to the pod template file for Spark executors. - Default is "/opt/teehr/executor-pod-template.yaml". - aws_access_key_id : str - AWS access key ID for S3 access. Default is None. - aws_secret_access_key : str - AWS secret access key for S3 access. Default is None. - aws_session_token : str - AWS session token for temporary credentials. Default is None. - aws_region : str - AWS region name. Default is "us-east-2". - aws_profile : str - AWS profile name to use from ~/.aws/credentials. Only reads credentials - file if this parameter is explicitly provided. Default is None. - enable_gcs : bool - Whether to add GCS (Google Cloud Storage) connector support. - Default is False. - gcs_project_id : str - GCS project ID. Used for billing and quota tracking. When accessing - public buckets without credentials, set to "anonymous" or any - non-empty string. Default is None (will be set to "anonymous" when - enable_gcs is True and no service account key is provided). - gcs_service_account_key_file : str - Path to a GCS service account JSON key file. When provided, - authenticated access is used. When None, unauthenticated - (public-bucket) access is used. Default is None. - add_packages : List[str] - Provided Spark packages will be added if they do not already exist. - Default is None. - >>> add_packages=["com.example:my-package:1.0.0"] - add_jars : List[str] - Provided local jar paths will be added if they do not already exist. - Default is None. - >>> add_jars=["/opt/spark/jars/teehr-authmanager.jar"] - update_configs : Dict[str, str] - Provided Spark configurations will be added if they do not already - exist, or overwritten if they do exist. Default is None. - >>> update_configs={"spark.sql.shuffle.partitions": "100"} - debug_config : bool - Whether to log the final Spark configuration for debugging. - Default is False. - - Returns - ------- - SparkSession - Configured Spark session. - """ - logger.info(f"🚀 Creating Spark session: {app_name}") - - # AuthManager requires a fresh JVM per user session to avoid static state leakage - resolved_use_authmanager = ( - use_authmanager if use_authmanager is not None - else _as_bool_str(os.getenv("POLARIS_USE_AUTHMANAGER", "false")) == "true" - ) - if force_recreate_session or resolved_use_authmanager: - existing_session = SparkSession.getActiveSession() - if existing_session is not None: - logger.info("♻️ Stopping the active Spark session before recreation") - existing_session.stop() - - # Get the base configuration with common settings - conf = _create_spark_base_session( - conf=SparkConf(), - aws_region=aws_region, - driver_memory=driver_memory, - driver_maxresultsize=driver_max_result_size - ) - - if start_spark_cluster is False: - logger.info("✅ Spark local configuration successful!") - else: - logger.info(f"📦 Configuring Spark cluster with container image: {executor_image}") - _set_spark_cluster_configuration( - conf=conf, - executor_instances=executor_instances, - executor_memory=executor_memory, - executor_cores=executor_cores, - container_image=executor_image, - spark_namespace=executor_namespace, - pod_template_path=pod_template_path - ) - logger.info("✅ Spark cluster configuration successful!") - logger.info(f" - Executor instances: {executor_instances}") - logger.info(f" - Executor memory: {executor_memory}") - logger.info(f" - Executor cores: {executor_cores}") - - # Set AWS credentials if available - _set_aws_credentials_in_spark( - conf=conf, - remote_catalog_name=remote_catalog_name, - aws_access_key_id=aws_access_key_id, - aws_secret_access_key=aws_secret_access_key, - aws_session_token=aws_session_token, - aws_region=aws_region, - aws_profile=aws_profile, - ) - - # Set GCS configuration if available - if enable_gcs: - _set_gcs_configuration( - conf=conf, - gcs_project_id=gcs_project_id, - gcs_service_account_key_file=gcs_service_account_key_file, - ) - - # Set catalog metadata in Spark configuration - _set_catalog_metadata( - conf=conf, - local_catalog_name=local_catalog_name, - local_catalog_type=local_catalog_type, - remote_catalog_name=remote_catalog_name, - remote_catalog_type=remote_catalog_type, - remote_catalog_uri=remote_catalog_uri, - remote_warehouse_dir=remote_warehouse_dir - ) - - # Apply catalog configurations - _configure_iceberg_catalogs( - conf=conf, - local_catalog_name=local_catalog_name, - local_catalog_type=local_catalog_type, - remote_warehouse_dir=remote_warehouse_dir, - remote_catalog_name=remote_catalog_name, - remote_catalog_type=remote_catalog_type, - remote_catalog_uri=remote_catalog_uri - ) - - # Build Polaris auth configs and merge with caller-provided update_configs. - # Auth configs are the base; caller's configs take precedence. - polaris_auth_configs = _build_polaris_auth_configs(polaris_token, use_authmanager) - effective_configs: Dict[str, str] = {**polaris_auth_configs} - if update_configs: - effective_configs.update(update_configs) - - # Update configs and packages if provided - _update_configs_and_packages( - conf=conf, - update_configs=effective_configs or None, - add_jars=add_jars, - add_packages=add_packages - ) - - logger.info("⚙️ All settings applied. Creating Spark session...") - spark = SparkSession.builder.appName(app_name).config(conf=conf).getOrCreate() - sedona_spark = SedonaContext.create(spark) - - # Apply runtime-settable configs to the live session (e.g., auth tokens) - if effective_configs: - _apply_runtime_spark_configs(sedona_spark, effective_configs) - - if debug_config: - log_session_config(sedona_spark) - - logger.info("🎉 Spark session created successfully!") - - return sedona_spark - - -def _create_spark_base_session( - conf: SparkConf, - aws_region: str, - driver_memory: float = None, - driver_maxresultsize: float = None -): - """Create a base Spark builder.""" - conf.setMaster("local[*]") - - # Set base packages - base_packages = [ - f"org.apache.sedona:sedona-spark-shaded-{PYSPARK_VERSION}_{SCALA_VERSION}:{SEDONA_VERSION}", - f"org.apache.iceberg:iceberg-spark-runtime-{PYSPARK_VERSION}_{SCALA_VERSION}:{ICEBERG_VERSION}", - f"org.apache.iceberg:iceberg-core:{ICEBERG_VERSION}", - "org.datasyslab:geotools-wrapper:1.8.0-33.1", - f"org.apache.iceberg:iceberg-spark-extensions-{PYSPARK_VERSION}_{SCALA_VERSION}:{ICEBERG_VERSION}", - "org.apache.hadoop:hadoop-aws:3.4.1", # Note. Need 3.4.1 for compatibility - "com.amazonaws:aws-java-sdk-bundle:1.12.791", - "org.xerial:sqlite-jdbc:3.42.0.0" - ] - conf.set("spark.jars.packages", ",".join(base_packages)) - - # Set configurations - conf.set("spark.driver.extraJavaOptions", f"-Daws.region={aws_region}") - conf.set("spark.executor.extraJavaOptions", f"-Daws.region={aws_region}") - conf.set("spark.sql.session.timeZone", "UTC") - conf.set("spark.sql.extensions", "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions") - conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer") - conf.set("spark.hadoop.fs.s3a.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem") - - # Enable Spark decommissioning for graceful spot instance handling - conf.set("spark.decommission.enabled", "true") - conf.set("spark.storage.decommission.enabled", "true") - conf.set("spark.storage.decommission.rddBlocks.enabled", "true") - conf.set("spark.storage.decommission.shuffleBlocks.enabled", "true") - # Grace period for executors to decommission before being terminated - conf.set("spark.kubernetes.executor.decommission.gracePeriodSeconds", "30") - - # Memory settings - memory_info = psutil.virtual_memory() - driver_memory_int = int(0.75 * memory_info.available / (1024**3)) - if driver_memory is None: - driver_memory = f"{driver_memory_int}g" - if driver_maxresultsize is None: - driver_maxresultsize = f"{int(0.5 * driver_memory_int)}g" - conf.set("spark.driver.memory", f"{driver_memory}") - conf.set("spark.driver.maxResultSize", f"{driver_maxresultsize}") - - # Driver binding configuration for local mode - conf.set("spark.driver.bindAddress", "127.0.0.1") - conf.set("spark.driver.host", "127.0.0.1") - conf.set("spark.driver.port", "0") # Let Spark choose an available port - - # Default shuffle partitions: 2x local CPU cores - local_cores = os.cpu_count() or 1 - default_shuffle_partitions = 2 * local_cores - conf.set("spark.sql.shuffle.partitions", str(default_shuffle_partitions)) - - return conf - - -def _set_spark_cluster_configuration( - conf: SparkConf, - executor_instances: int, - executor_memory: str, - executor_cores: int, - container_image: str, - spark_namespace: str, - pod_template_path: Union[str, Path] -): - """Configure Spark for Kubernetes cluster mode. - - These settings are considered immutable after session creation. - """ - # Default container image - use the same image as the current pod - if container_image is None: - container_image = os.environ["TEEHR_SPARK_IMAGE"] - - # Get Kubernetes API server - use HTTPS port specifically - k8s_host = os.environ.get('KUBERNETES_SERVICE_HOST', 'kubernetes.default.svc.cluster.local') - k8s_port_https = os.environ.get('KUBERNETES_SERVICE_PORT_HTTPS', '443') - k8s_api_server = f"https://{k8s_host}:{k8s_port_https}" - - # First try getting it from environment variable - if spark_namespace is None: - spark_namespace = os.environ.get("TEEHR_NAMESPACE", "") - logger.info(f"🔍 Initial spark namespace from ENV: {spark_namespace}") - - if spark_namespace is None: - # Then get it from here - namespace_file = "/var/run/secrets/kubernetes.io/serviceaccount/namespace" - if os.path.exists(namespace_file): - with open(namespace_file, 'r') as f: - spark_namespace = f.read().strip() - - # Finally get it here if still None - if spark_namespace is None: - spark_namespace = "default" # last resort, will probably fail - - logger.info(f"🔍 Connecting to Kubernetes API: {k8s_api_server}") - logger.info(f"🎯 Executor namespace: {spark_namespace}") - logger.info(f"🔐 Executor service account: spark (in {spark_namespace})") - - # Create Spark configuration - conf.setMaster(f"k8s://{k8s_api_server}") - - # Check project ID to specify appropriate node group name. - teehr_project_id = os.environ.get("TEEHR_PROJECT_ID", "") - if teehr_project_id != "TEEHR" and teehr_project_id != "": - conf.set( - "spark.kubernetes.executor.node.selector.teehr-hub/nodegroup-name", - f"spark-r5-4xlarge-spot-{teehr_project_id.lower()}" - ) - - # Basic Kubernetes settings - conf.set("spark.executor.instances", str(executor_instances)) - conf.set("spark.executor.memory", executor_memory) - conf.set("spark.executor.cores", str(executor_cores)) - conf.set("spark.kubernetes.container.image", container_image) - conf.set("spark.kubernetes.namespace", spark_namespace) - conf.set("spark.kubernetes.authenticate.executor.serviceAccountName", "spark") - conf.set("spark.kubernetes.container.image.pullPolicy", "Always") - - # Enable Spark's dynamic allocation and decommissioning features for better - # handling of spot instances and resource management. - conf.set("spark.decommission.enabled", "true") - conf.set("spark.executor.decommission.signal", "SIGTERM") - conf.set("spark.storage.decommission.enabled", "true") - - # Set pod name prefix for executors in TEEHR-HUB for easy identification in cluster. - # Truncated to 46 chars since K8s pod names have a 63-char limit and Spark appends suffixes. - jupyterhub_user = os.environ.get("JUPYTERHUB_USER", None) - if jupyterhub_user is not None: - conf.set("spark.kubernetes.executor.podNamePrefix", jupyterhub_user[:46]) - - if os.path.exists(pod_template_path): - conf.set("spark.kubernetes.executor.podTemplateFile", pod_template_path) - else: - logger.info(f"⚠️ Executor pod template not found: {pod_template_path}") - logger.info(" You must provide a valid pod template for executors to launch correctly.") - raise FileNotFoundError(f"Executor pod template not found: {pod_template_path}") - - conf.set("spark.kubernetes.executor.deleteOnTermination", "true") - - # Default shuffle partitions: 2x total executor cores - total_executor_cores = executor_cores * executor_instances - default_shuffle_partitions = 2 * total_executor_cores - conf.set("spark.sql.shuffle.partitions", str(default_shuffle_partitions)) - - # Authentication - use service account token if available - token_file = const.SERVICE_ACCOUNT_TOKEN_PATH - ca_file = const.CA_CERTIFICATE_PATH - if os.path.exists(token_file) and os.path.exists(ca_file): - logger.info("🔐 Using in-cluster authentication") - conf.set("spark.kubernetes.authenticate.submission.oauthTokenFile", token_file) - conf.set("spark.kubernetes.authenticate.submission.caCertFile", ca_file) - conf.set("spark.kubernetes.authenticate.driver.oauthTokenFile", token_file) - conf.set("spark.kubernetes.authenticate.executor.oauthTokenFile", token_file) - - # Critical: Set the CA cert file for SSL validation - conf.set("spark.kubernetes.authenticate.caCertFile", ca_file) - else: - logger.info("⚠️ No service account tokens found - may have authentication issues") - logger.info(f" Checked: {token_file}") - logger.info(f" Checked: {ca_file}") - - # Driver binding configuration - use pod IP for Kubernetes - conf.set("spark.driver.bindAddress", "0.0.0.0") - conf.set("spark.driver.port", "0") # Let Spark choose an available port - - # Get pod IP and set as driver host so executors can connect back - pod_ip = os.environ.get('POD_IP') - if not pod_ip: - try: - hostname = socket.gethostname() - pod_ip = socket.gethostbyname(hostname) - except: - pod_ip = None - - if pod_ip: - logger.info(f"🔗 Setting driver host to pod IP: {pod_ip}") - conf.set("spark.driver.host", pod_ip) - else: - logger.info("⚠️ Could not determine pod IP - using default driver host") - - return - - -def _set_aws_credentials_in_spark( - conf: SparkConf, - remote_catalog_name: str, - aws_access_key_id: str, - aws_secret_access_key: str, - aws_session_token: str, - aws_region: str, - aws_profile: str = None, -): - """Set AWS credentials in Spark configuration with multiple options.""" - logger.info("Setting Hadoop's default AWS credentials provider and AWS region") - conf.set( - "spark.hadoop.fs.s3a.aws.credentials.provider", - "com.amazonaws.auth.DefaultAWSCredentialsProviderChain" - ) - conf.set("spark.hadoop.fs.s3a.endpoint.region", aws_region) - - # Priority 1: Explicit credentials provided by user - if aws_access_key_id and aws_secret_access_key: - logger.info("🔑 Using user-provided AWS credentials") - conf.set(f"spark.sql.catalog.{remote_catalog_name}.s3.access-key-id", aws_access_key_id) - conf.set(f"spark.sql.catalog.{remote_catalog_name}.s3.secret-access-key", aws_secret_access_key) - conf.set("spark.hadoop.fs.s3a.access.key", aws_access_key_id) - conf.set("spark.hadoop.fs.s3a.secret.key", aws_secret_access_key) - return - - # Priority 2: Explicit token - if aws_session_token: - logger.info("🔑 Using user-provided AWS session token") - conf.set(f"spark.sql.catalog.{remote_catalog_name}.s3.session-token", aws_session_token) - conf.set("spark.hadoop.fs.s3a.session.token", aws_session_token) - return - - # Priority 3: Check ~/.aws/credentials file only if profile explicitly specified (full access) - if aws_profile: - aws_credentials_file = Path.home() / ".aws" / "credentials" - if aws_credentials_file.exists(): - try: - import configparser - config = configparser.ConfigParser() - config.read(aws_credentials_file) - - if config.has_section(aws_profile): - if config.has_option(aws_profile, "aws_access_key_id") and config.has_option(aws_profile, "aws_secret_access_key"): - creds_access_key = config.get(aws_profile, "aws_access_key_id") - creds_secret_key = config.get(aws_profile, "aws_secret_access_key") - creds_session_token = config.get(aws_profile, "aws_session_token", fallback=None) - - logger.info(f"🔑 Using AWS credentials from ~/.aws/credentials profile '{aws_profile}") - conf.set(f"spark.sql.catalog.{remote_catalog_name}.s3.access-key-id", creds_access_key) - conf.set(f"spark.sql.catalog.{remote_catalog_name}.s3.secret-access-key", creds_secret_key) - conf.set("spark.hadoop.fs.s3a.access.key", creds_access_key) - conf.set("spark.hadoop.fs.s3a.secret.key", creds_secret_key) - - if creds_session_token: - conf.set(f"spark.sql.catalog.{remote_catalog_name}.s3.session-token", creds_session_token) - conf.set("spark.hadoop.fs.s3a.session.token", creds_session_token) - return - except Exception as e: - logger.debug(f"Could not read ~/.aws/credentials: {e}") - - session = botocore.session.Session() - credentials = session.get_credentials() - - # Priority 4: Check boto token - if credentials and credentials.token: - logger.info("🔑 Using AWS session token from boto3") - conf.set(f"spark.sql.catalog.{remote_catalog_name}.s3.session-token", credentials.token) - conf.set("spark.hadoop.fs.s3a.session.token", credentials.token) - return - - # Priority 5: Check boto credentials - if credentials and credentials.access_key and credentials.secret_key: - logger.info("🔑 Using AWS credentials from boto3") - conf.set(f"spark.sql.catalog.{remote_catalog_name}.s3.access-key-id", credentials.access_key) - conf.set(f"spark.sql.catalog.{remote_catalog_name}.s3.secret-access-key", credentials.secret_key) - conf.set("spark.hadoop.fs.s3a.access.key", credentials.access_key) - conf.set("spark.hadoop.fs.s3a.secret.key", credentials.secret_key) - return - - # Priority 6: Fall back to anonymous or default provider - logger.info("🔑 Using anonymous AWS credentials for S3 access") - conf.set( - "spark.hadoop.fs.s3a.aws.credentials.provider", - "org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider" - ) - return - -def _set_gcs_configuration( - conf: SparkConf, - gcs_project_id: str = None, - gcs_service_account_key_file: str = None, -): - """Configure Spark for Google Cloud Storage (GCS) access. - - Parameters - ---------- - conf : SparkConf - The Spark configuration object to update. - gcs_project_id : str - GCS project ID for billing/quota. Defaults to "anonymous" when - no service account key is provided (public bucket access). - gcs_service_account_key_file : str - Path to a GCS service account JSON key file. When None, - unauthenticated access is used (suitable for public buckets). - """ - GCS_CONNECTOR_VERSION = "hadoop3-2.2.32" - gcs_package = f"com.google.cloud.bigdataoss:gcs-connector:{GCS_CONNECTOR_VERSION}" - - # Add the GCS connector package - current_packages = conf.get("spark.jars.packages").split(",") - if gcs_package not in current_packages: - current_packages.append(gcs_package) - conf.set("spark.jars.packages", ",".join(current_packages)) - - # Register GCS filesystem implementations - conf.set( - "spark.hadoop.fs.gs.impl", - "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem" - ) - conf.set( - "spark.hadoop.fs.AbstractFileSystem.gs.impl", - "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS" - ) - - if gcs_service_account_key_file: - key_path = Path(gcs_service_account_key_file) - if not key_path.exists(): - raise FileNotFoundError( - f"GCS service account key file not found: {gcs_service_account_key_file}" - ) - logger.info(f"🔑 Using GCS service account key: {gcs_service_account_key_file}") - conf.set("spark.hadoop.google.cloud.auth.service.account.enable", "true") - conf.set( - "spark.hadoop.google.cloud.auth.service.account.keyfile", - str(key_path) - ) - if gcs_project_id: - conf.set("spark.hadoop.fs.gs.project.id", gcs_project_id) - else: - # Unauthenticated access for public GCS buckets - logger.info("🔑 Using unauthenticated GCS access (public buckets)") - # Set both old-style (fs.gs.*) and new-style (google.cloud.*) auth - # properties so the connector picks up UNAUTHENTICATED regardless of - # which configuration namespace it checks first. - conf.set("spark.hadoop.fs.gs.auth.type", "UNAUTHENTICATED") - conf.set("spark.hadoop.google.cloud.auth.type", "UNAUTHENTICATED") - # Explicitly disable service account auth to prevent the connector from - # attempting to contact the GCE metadata server (http://metadata.google.internal/) - # for credentials. Without this, the connector may hang on machines - # that are not running on GCP. - conf.set("spark.hadoop.google.cloud.auth.service.account.enable", "false") - # Enable null (anonymous) credentials — required for the connector to - # accept a configuration with no active credential source. - conf.set("spark.hadoop.fs.gs.auth.null.enable", "true") - conf.set( - "spark.hadoop.fs.gs.project.id", - gcs_project_id if gcs_project_id else "anonymous" - ) - - -def _set_catalog_metadata( - conf: SparkConf, - local_catalog_name: str, - local_catalog_type: str, - remote_catalog_name: str, - remote_catalog_type: str, - remote_catalog_uri: str, - remote_warehouse_dir: str, -): - """Set catalog metadata in Spark configuration.""" - metadata_configs = { - "local_catalog_name": local_catalog_name, - "local_catalog_type": local_catalog_type, - "remote_warehouse_dir": remote_warehouse_dir, - "remote_catalog_name": remote_catalog_name, - "remote_catalog_type": remote_catalog_type, - "remote_catalog_uri": remote_catalog_uri - } - for key, value in metadata_configs.items(): - conf.set(key, value) - logger.debug(f"Metadata config: {key}: {value}") - - -def _configure_iceberg_catalogs( - conf: SparkConf, - local_catalog_name: str, - local_catalog_type: str, - remote_warehouse_dir: str, - remote_catalog_name: str, - remote_catalog_type: str, - remote_catalog_uri: str -): - """Configure Iceberg catalogs through conf.set().""" - logger.info("Configuring Iceberg catalogs...") - # Local catalog configuration - conf.set(f"spark.sql.catalog.{local_catalog_name}", "org.apache.iceberg.spark.SparkCatalog") - conf.set(f"spark.sql.catalog.{local_catalog_name}.type", local_catalog_type) - conf.set(f"spark.sql.catalog.{local_catalog_name}.jdbc.driver", "org.sqlite.JDBC") - conf.set(f"spark.sql.catalog.{local_catalog_name}.jdbc.initialize", "true") - conf.set(f"spark.sql.catalog.{local_catalog_name}.jdbc.schema-version", "V1") - # conf.set("spark.sql.catalog.local.jdbc.user", "user") - # conf.set("spark.sql.catalog.local.jdbc.password", "password") - - # Remote catalog configuration - conf.set(f"spark.sql.catalog.{remote_catalog_name}", "org.apache.iceberg.spark.SparkCatalog") - conf.set(f"spark.sql.catalog.{remote_catalog_name}.type", remote_catalog_type) - conf.set(f"spark.sql.catalog.{remote_catalog_name}.uri", remote_catalog_uri) - conf.set(f"spark.sql.catalog.{remote_catalog_name}.warehouse", remote_warehouse_dir) - conf.set(f"spark.sql.catalog.{remote_catalog_name}.io-impl", "org.apache.iceberg.aws.s3.S3FileIO") - # S3 end point and path style access - if os.environ.get("REMOTE_CATALOG_S3_PATH_STYLE_ACCESS", "false").lower() == "true": - conf.set(f"spark.sql.catalog.{remote_catalog_name}.s3.endpoint", os.environ.get("REMOTE_CATALOG_S3_ENDPOINT")) - conf.set(f"spark.sql.catalog.{remote_catalog_name}.s3.path-style-access", os.environ.get("REMOTE_CATALOG_S3_PATH_STYLE_ACCESS").lower()) - - -def _update_configs_and_packages( - conf: SparkConf, - update_configs: Dict[str, str], - add_jars: List[str], - add_packages: List[str] -) -> Dict[str, str]: - """Update Spark configurations and packages.""" - # Add specified local jars - if add_jars is not None: - current_jars = conf.get("spark.jars").split(",") if conf.contains("spark.jars") else [] - for jar_path in add_jars: - if jar_path not in current_jars: - current_jars.append(jar_path) - if current_jars: - conf.set("spark.jars", ",".join(current_jars)) - - # Add specified packages - if add_packages is not None: - current_packages = conf.get("spark.jars.packages").split(",") - for package in add_packages: - if package not in current_packages: - current_packages.append(package) - conf.set("spark.jars.packages", ",".join(current_packages)) - - # Update or add specified configs - if update_configs is not None: - for key, value in update_configs.items(): - conf.set(key, value) - return - - -def log_session_config(spark: SparkSession): - """Log the current Spark session configuration for debugging. - - Parameters - ---------- - spark : SparkSession - The Spark session whose configuration should be logged. - - Notes - ----- - This function logs all Spark configuration properties to the - logger at INFO level for troubleshooting purposes. - """ - logger.info("Final Spark configuration:") - df = pd.DataFrame(list(spark.conf.getAll.items()), columns=["Key", "Value"]) - gps = df.groupby(by="Key") - for key, group in gps: - value = ",".join(group["Value"].tolist()) - values = value.split(",") - if key.startswith("spark."): - if len(values) > 1: - logger.info(f" {key}: ") - for val in values: - logger.info(f" {val}") - else: - logger.info(f" {key}: {value}") - - -def remove_or_update_configs( - spark: SparkSession, - remove_configs: List[str] = None, - update_configs: Dict[str, str] = None -) -> Dict[str, str]: - """Add, remove, or update Spark configurations. - - Parameters - ---------- - spark : SparkSession - The Spark session whose configuration should be logged. - remove_configs : List[str] - List of configuration keys to remove from the Spark session. - Default is None. - >>> remove_configs=["spark.sql.shuffle.partitions"] - update_configs : Dict[str, str] - Provided Spark configurations will be added if they do not already - exist, or overwritten if they do exist. Default is None. - >>> update_configs={"spark.sql.shuffle.partitions": "100"} - - Notes - ----- - This function allows for dynamic modification of the Spark session's - configuration by removing specified keys and updating or adding new - key-value pairs. - """ - # Remove specified configs - if remove_configs is not None: - for key in remove_configs: - try: - current_value = spark.conf.get(key) - logger.info(f"Removing config: {key} (was: {current_value})") - spark.conf.unset(key) - except Exception as e: - logger.warning(f"Could not remove config {key}: {e}") - - # Update or add specified configs - if update_configs is not None: - for key, value in update_configs.items(): - spark.conf.set(key, value) - return - - -def _decode_jwt_claims(token: str) -> Dict[str, object]: - payload = token.split(".")[1] - payload += "=" * (-len(payload) % 4) - return json.loads(base64.urlsafe_b64decode(payload.encode())) - - -def _token_expires_soon(token: str, refresh_window_seconds: int = 120) -> bool: - try: - claims = _decode_jwt_claims(token) - except Exception: - return True - exp = int(claims.get("exp", 0)) - now = int(time.time()) - return exp <= now + max(refresh_window_seconds, 1) - - -def _request_oauth_tokens( - data: Dict[str, str], - token_endpoint: Optional[str] = None, - timeout_seconds: int = 20, -) -> Tuple[str, Optional[str]]: - endpoint = token_endpoint or os.getenv("POLARIS_OAUTH2_SERVER_URI") - if not endpoint: - raise RuntimeError("POLARIS_OAUTH2_SERVER_URI is required to mint or refresh a user token") - - resp = requests.post(endpoint, data=data, timeout=timeout_seconds) - resp.raise_for_status() - payload = resp.json() - - access_token = payload.get("access_token") - if not access_token: - raise RuntimeError("Token endpoint did not return access_token") - - return access_token, payload.get("refresh_token") - - -def mint_polaris_user_token( - username: Optional[str], - password: Optional[str], - client_id: str, - client_secret: Optional[str] = None, - token_endpoint: Optional[str] = None, -) -> str: - if not username or not password: - raise RuntimeError("username and password are required for password grant token minting") - - data = { - "grant_type": "password", - "client_id": client_id, - "username": username, - "password": password, - "scope": "openid profile email", - } - if client_secret: - data["client_secret"] = client_secret - - access_token, _ = _request_oauth_tokens(data=data, token_endpoint=token_endpoint) - return access_token - - -def refresh_polaris_user_token( - refresh_token: str, - client_id: str, - client_secret: Optional[str] = None, - token_endpoint: Optional[str] = None, -) -> Tuple[str, Optional[str]]: - if not refresh_token: - raise RuntimeError("refresh_token is required for refresh grant") - - data = { - "grant_type": "refresh_token", - "client_id": client_id, - "refresh_token": refresh_token, - } - if client_secret: - data["client_secret"] = client_secret - - return _request_oauth_tokens(data=data, token_endpoint=token_endpoint) - - -def ensure_fresh_polaris_user_token( - current_token: Optional[str], - username: Optional[str], - password: Optional[str], - client_id: str, - client_secret: Optional[str] = None, - refresh_token: Optional[str] = None, - allow_password_fallback: bool = True, - refresh_window_seconds: int = 120, - token_endpoint: Optional[str] = None, -) -> Tuple[str, Optional[str], bool]: - if current_token and not _token_expires_soon(current_token, refresh_window_seconds): - return current_token, refresh_token, False - - if refresh_token: - try: - refreshed_access, refreshed_refresh = refresh_polaris_user_token( - refresh_token=refresh_token, - client_id=client_id, - client_secret=client_secret, - token_endpoint=token_endpoint, - ) - return refreshed_access, (refreshed_refresh or refresh_token), True - except requests.RequestException: - if not allow_password_fallback: - raise - - if allow_password_fallback and username and password: - minted = mint_polaris_user_token( - username=username, - password=password, - client_id=client_id, - client_secret=client_secret, - token_endpoint=token_endpoint, - ) - return minted, refresh_token, True - - raise RuntimeError( - "Unable to obtain a fresh Polaris user token. " - "Provide POLARIS_REFRESH_TOKEN or enable password-grant fallback credentials." - ) - - -def apply_polaris_token_to_spark( - spark, - token: str, - catalog_name: str = "iceberg", - realm: Optional[str] = None, -) -> None: - active_realm = realm or os.getenv("POLARIS_DEFAULT_REALM", "teehr") - base = f"spark.sql.catalog.{catalog_name}" - spark.conf.set(f"{base}.rest.auth.type", "oauth2") - spark.conf.set(f"{base}.token", token) - spark.conf.set(f"{base}.rest.auth.oauth2.token", token) - spark.conf.set(f"{base}.header.X-Polaris-Realm", active_realm) - spark.conf.set(f"{base}.rest.transport.header.X-Polaris-Realm", active_realm) - - -def _as_bool_str(value: str, default: str = "true") -> str: - normalized = (value or default).strip().lower() - return "true" if normalized in ("1", "true", "t", "yes", "y", "on") else "false" - - -def _apply_runtime_spark_configs(spark, configs: Dict[str, str]) -> None: - immutable_after_start = { - "spark.jars", - "spark.driver.extraClassPath", - "spark.executor.extraClassPath", - } - for key, value in configs.items(): - if not key.startswith("spark."): - continue - if key in immutable_after_start: - continue - spark.conf.set(key, value) - - -def _is_http_error_with_status(exc: Exception, status_code: int) -> bool: - if not isinstance(exc, requests.HTTPError): - return False - response = getattr(exc, "response", None) - return bool(response is not None and response.status_code == status_code) - - -def _normalize_internal_broker_url(url: str) -> str: - parsed = urlsplit(url) - if parsed.scheme != "https": - return url - if parsed.hostname != "teehr-api": - return url - - host = parsed.hostname - port = parsed.port or 8000 - netloc = f"{host}:{port}" - return urlunsplit(("http", netloc, parsed.path, parsed.query, parsed.fragment)) - - -def _broker_session_endpoint_from_token_endpoint(token_endpoint: str) -> str: - normalized = _normalize_internal_broker_url(token_endpoint) - parsed = urlsplit(normalized) - path = parsed.path or "" - - if path.endswith("/auth/polaris-token/session"): - return normalized - - if path.endswith("/auth/polaris-token"): - session_path = path[:-len("/auth/polaris-token")] + "/auth/polaris-token/session" - return urlunsplit((parsed.scheme, parsed.netloc, session_path, parsed.query, parsed.fragment)) - - raise RuntimeError( - "POLARIS_BROKER_URL must end with /auth/polaris-token " - "(or /auth/polaris-token/session if already session-scoped)" - ) - - -def ensure_broker_session_token( - *, - user_id: str, - session_id: str, - realm: str, - refresh_token: str, - bearer_token: Optional[str] = None, - catalog: str = "iceberg", - audience: Optional[str] = None, - broker_url: Optional[str] = None, - timeout_seconds: int = 20, -) -> str: - endpoint = broker_url or os.getenv("POLARIS_BROKER_URL", "http://teehr-api:8000/auth/polaris-token") - endpoint = _normalize_internal_broker_url(endpoint) - session_endpoint = _broker_session_endpoint_from_token_endpoint(endpoint).replace( - "/auth/polaris-token/session", - "/auth/polaris-session", - ) - active_audience = audience or os.getenv("POLARIS_BROKER_AUDIENCE", "account") - subject_token = bearer_token or os.getenv("POLARIS_USER_TOKEN", "") - - if not subject_token: - raise RuntimeError("A valid bearer subject token is required to create a broker session") - if not refresh_token: - raise RuntimeError("POLARIS_REFRESH_TOKEN is required to create a broker session") - - resp = requests.post( - session_endpoint, - headers={"Authorization": f"Bearer {subject_token}"}, - json={ - "user_id": user_id, - "session_id": session_id, - "realm": realm, - "catalog": catalog, - "audience": active_audience, - "refresh_token": refresh_token, - }, - timeout=timeout_seconds, - ) - resp.raise_for_status() - payload = resp.json() - broker_session_token = payload.get("broker_session_token") - if not broker_session_token: - raise RuntimeError("Broker session endpoint did not return broker_session_token") - os.environ["POLARIS_BROKER_SESSION_TOKEN"] = broker_session_token - return broker_session_token - - -def _build_polaris_auth_configs( - polaris_token: Optional[str], - use_authmanager: Optional[bool], -) -> Dict[str, str]: - """Build Spark configs for Polaris catalog authentication. - - Handles three auth paths: - 1. AuthManager (use_authmanager=True or POLARIS_USE_AUTHMANAGER=true env var) - Uses the teehr-api broker for token management — required for JupyterHub - where tokens must be refreshed transparently during long sessions. - 2. Direct user token (polaris_token provided) - Passes the JWT directly to the Iceberg REST catalog. - 3. Service account client credentials (POLARIS_CLIENT_ID + POLARIS_CLIENT_SECRET) - Used by Prefect batch jobs and other non-interactive service accounts. - - Returns an empty dict if none of the above are configured. - """ - polaris_realm = os.getenv("POLARIS_DEFAULT_REALM", "teehr") - - resolved_use_authmanager = ( - use_authmanager if use_authmanager is not None - else _as_bool_str(os.getenv("POLARIS_USE_AUTHMANAGER", "false")) == "true" - ) - - configs: Dict[str, str] = {} - - if not resolved_use_authmanager and not polaris_token and not os.getenv("POLARIS_CLIENT_ID"): - return configs - - # Realm headers are required for all Polaris auth paths - configs["spark.sql.catalog.iceberg.header.X-Polaris-Realm"] = polaris_realm - configs["spark.sql.catalog.iceberg.rest.transport.header.X-Polaris-Realm"] = polaris_realm - - if resolved_use_authmanager: - broker_url = os.getenv("POLARIS_BROKER_URL", "http://teehr-api:8000/auth/polaris-token") - broker_url = _normalize_internal_broker_url(broker_url) - broker_session_url = _broker_session_endpoint_from_token_endpoint(broker_url) - authmanager_user_id = os.getenv("JUPYTERHUB_USER", "admin") - authmanager_session_id = ( - os.getenv("JUPYTERHUB_SERVER_NAME", "").strip() or authmanager_user_id - ) - broker_audience = os.getenv("POLARIS_BROKER_AUDIENCE", "account") - refresh_token = os.getenv("POLARIS_REFRESH_TOKEN", "") - current_user_token = polaris_token or os.getenv("POLARIS_USER_TOKEN", "") - broker_session_token = os.getenv("POLARIS_BROKER_SESSION_TOKEN", "") - if broker_session_token and _token_expires_soon(broker_session_token, 300): - broker_session_token = "" - authmanager_jar = os.getenv( - "POLARIS_AUTHMANAGER_JAR", - "/opt/spark/jars/teehr-authmanager.jar", - ) - - current_user_token, refresh_token, _ = ensure_fresh_polaris_user_token( - current_token=current_user_token, - username=os.getenv("POLARIS_USERNAME"), - password=os.getenv("POLARIS_PASSWORD"), - client_id=os.getenv("POLARIS_CLIENT_ID", "jupyterhub"), - client_secret=os.getenv("POLARIS_CLIENT_SECRET"), - refresh_token=refresh_token, - allow_password_fallback=False, - refresh_window_seconds=300, - token_endpoint=os.getenv("POLARIS_OAUTH2_TOKEN_ENDPOINT"), - ) - os.environ["POLARIS_USER_TOKEN"] = current_user_token - if refresh_token: - os.environ["POLARIS_REFRESH_TOKEN"] = refresh_token - - if not broker_session_token: - try: - broker_session_token = ensure_broker_session_token( - user_id=authmanager_user_id, - session_id=authmanager_session_id, - realm=polaris_realm, - refresh_token=refresh_token, - bearer_token=current_user_token, - catalog="iceberg", - audience=broker_audience, - broker_url=broker_url, - ) - except requests.HTTPError as exc: - if not _is_http_error_with_status(exc, 401): - raise - current_user_token, refresh_token, _ = ensure_fresh_polaris_user_token( - current_token=None, - username=os.getenv("POLARIS_USERNAME"), - password=os.getenv("POLARIS_PASSWORD"), - client_id=os.getenv("POLARIS_CLIENT_ID", "jupyterhub"), - client_secret=os.getenv("POLARIS_CLIENT_SECRET"), - refresh_token=refresh_token, - allow_password_fallback=True, - refresh_window_seconds=300, - token_endpoint=os.getenv("POLARIS_OAUTH2_TOKEN_ENDPOINT"), - ) - os.environ["POLARIS_USER_TOKEN"] = current_user_token - if refresh_token: - os.environ["POLARIS_REFRESH_TOKEN"] = refresh_token - broker_session_token = ensure_broker_session_token( - user_id=authmanager_user_id, - session_id=authmanager_session_id, - realm=polaris_realm, - refresh_token=refresh_token, - bearer_token=current_user_token, - catalog="iceberg", - audience=broker_audience, - broker_url=broker_url, - ) - os.environ["POLARIS_BROKER_SESSION_TOKEN"] = broker_session_token - - configs["spark.sql.catalog.iceberg.rest.auth.type"] = ( - "org.teehr.iceberg.auth.TeehrBrokerAuthManager" - ) - configs["spark.sql.catalog.iceberg.rest.auth.teehr.broker.url"] = broker_session_url - configs["spark.sql.catalog.iceberg.rest.auth.teehr.user-id"] = authmanager_user_id - configs["spark.sql.catalog.iceberg.rest.auth.teehr.session-id"] = authmanager_session_id - configs["spark.sql.catalog.iceberg.rest.auth.teehr.realm"] = polaris_realm - configs["spark.sql.catalog.iceberg.rest.auth.teehr.catalog"] = "iceberg" - configs["spark.sql.catalog.iceberg.rest.auth.teehr.audience"] = broker_audience - configs["spark.sql.catalog.iceberg.rest.auth.teehr.broker-session-token-env"] = ( - "POLARIS_BROKER_SESSION_TOKEN" - ) - configs["spark.jars"] = authmanager_jar - - elif polaris_token: - configs["spark.sql.catalog.iceberg.rest.auth.type"] = "oauth2" - configs["spark.sql.catalog.iceberg.token"] = polaris_token - configs["spark.sql.catalog.iceberg.rest.auth.oauth2.token"] = polaris_token - - else: - # Service account / client credentials path. - # Set POLARIS_CLIENT_ID and POLARIS_CLIENT_SECRET for the service account - # (e.g. prefect-polaris for Prefect batch jobs). - oauth_server_uri = os.getenv("POLARIS_OAUTH2_SERVER_URI") - polaris_client_id = os.getenv("POLARIS_CLIENT_ID") - polaris_client_secret = os.getenv("POLARIS_CLIENT_SECRET") - - configs["spark.sql.catalog.iceberg.rest.auth.type"] = "oauth2" - configs["spark.sql.catalog.iceberg.scope"] = "openid" - configs["spark.sql.catalog.iceberg.rest.auth.oauth2.scope"] = "openid" - if oauth_server_uri: - configs["spark.sql.catalog.iceberg.oauth2-server-uri"] = oauth_server_uri - configs["spark.sql.catalog.iceberg.rest.auth.oauth2.server-uri"] = oauth_server_uri - if polaris_client_id and polaris_client_secret: - credential = f"{polaris_client_id}:{polaris_client_secret}" - configs["spark.sql.catalog.iceberg.credential"] = credential - configs["spark.sql.catalog.iceberg.rest.auth.oauth2.credential"] = credential - - return configs - - -def create_minio_spark_session( - polaris_token: Optional[str] = None, - force_recreate_session: bool = False, - update_configs: Optional[Dict[str, str]] = None, - use_authmanager: Optional[bool] = None, -) -> SparkSession: - """Start a Spark session with MinIO credentials for local KinD development. - - Thin wrapper around create_spark_session() that injects MinIO-specific S3 - configuration. All Polaris auth (AuthManager, direct token, client credentials) - is handled by create_spark_session() based on the parameters and environment. - - For remote deployments using AWS S3, call create_spark_session() directly with - appropriate AWS credentials and catalog configuration. - """ - s3_endpoint = os.getenv("REMOTE_CATALOG_S3_ENDPOINT", "http://minio:9000") - s3_path_style = _as_bool_str(os.getenv("REMOTE_CATALOG_S3_PATH_STYLE_ACCESS", "true")) - s3_region = os.getenv("AWS_REGION", "us-east-2") - polaris_realm = os.getenv("POLARIS_DEFAULT_REALM", "teehr") - remote_catalog_uri = os.getenv("REMOTE_CATALOG_REST_URI", "http://polaris:8181/api/catalog") - - # Polaris REST expects the catalog name as warehouse identifier, not an S3 URI - remote_warehouse_dir = ( - polaris_realm - if remote_catalog_uri.rstrip("/").endswith("/api/catalog") - else os.getenv("REMOTE_WAREHOUSE_S3_PATH", "s3://warehouse/") - ) - - minio_configs: Dict[str, str] = { - "spark.sql.catalog.iceberg.s3.endpoint": s3_endpoint, - "spark.sql.catalog.iceberg.s3.path-style-access": s3_path_style, - "spark.sql.catalog.iceberg.s3.region": s3_region, - "spark.hadoop.fs.s3a.endpoint": s3_endpoint, - "spark.hadoop.fs.s3a.path.style.access": s3_path_style, - "spark.hadoop.fs.s3a.connection.ssl.enabled": "false", - } - if update_configs: - minio_configs.update(update_configs) - - return create_spark_session( - aws_access_key_id=os.getenv("AWS_ACCESS_KEY_ID", "minioadmin"), - aws_secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY", "minioadmin123"), - remote_catalog_uri=remote_catalog_uri, - remote_warehouse_dir=remote_warehouse_dir, - polaris_token=polaris_token, - force_recreate_session=force_recreate_session, - update_configs=minio_configs, - use_authmanager=use_authmanager, - ) - - - - -def request_broker_polaris_token( - *, - user_id: str, - session_id: str, - bearer_token: str, - realm: Optional[str] = None, - catalog: str = "iceberg", - requested_ttl_seconds: int = 600, - audience: Optional[str] = None, - broker_url: Optional[str] = None, - timeout_seconds: int = 20, -) -> Tuple[str, int, str]: - endpoint = broker_url or os.getenv("POLARIS_BROKER_URL", "http://teehr-api:8000/auth/polaris-token") - active_realm = realm or os.getenv("POLARIS_DEFAULT_REALM", "teehr") - active_audience = audience or os.getenv("POLARIS_BROKER_AUDIENCE", "account") - - if not bearer_token: - raise RuntimeError("bearer_token is required to request a broker token") - - resp = requests.post( - endpoint, - headers={"Authorization": f"Bearer {bearer_token}"}, - json={ - "user_id": user_id, - "session_id": session_id, - "realm": active_realm, - "catalog": catalog, - "requested_ttl_seconds": requested_ttl_seconds, - "audience": active_audience, - }, - timeout=timeout_seconds, - ) - resp.raise_for_status() - payload = resp.json() - access_token = payload.get("access_token") - expires_at = int(payload.get("expires_at_epoch_seconds", 0)) - trace_id = payload.get("trace_id", "") - - if not access_token or expires_at <= 0: - raise RuntimeError("Broker response missing access_token or expires_at_epoch_seconds") - - return access_token, expires_at, trace_id - - -def ensure_fresh_polaris_token_via_broker( - *, - current_token: Optional[str], - user_id: str, - session_id: str, - bearer_token: str, - realm: Optional[str] = None, - catalog: str = "iceberg", - refresh_window_seconds: int = 120, - requested_ttl_seconds: int = 600, - audience: Optional[str] = None, - broker_url: Optional[str] = None, -) -> Tuple[str, bool]: - if current_token and not _token_expires_soon(current_token, refresh_window_seconds): - return current_token, False - - refreshed_token, _, _ = request_broker_polaris_token( - user_id=user_id, - session_id=session_id, - bearer_token=bearer_token, - realm=realm, - catalog=catalog, - requested_ttl_seconds=requested_ttl_seconds, - audience=audience, - broker_url=broker_url, - ) - return refreshed_token, True \ No newline at end of file diff --git a/keycloak-bootstrap/manifests/realm-configmap.yaml.tpl b/keycloak-bootstrap/manifests/realm-configmap.yaml.tpl index f0d7dfb..2882e0d 100644 --- a/keycloak-bootstrap/manifests/realm-configmap.yaml.tpl +++ b/keycloak-bootstrap/manifests/realm-configmap.yaml.tpl @@ -213,9 +213,6 @@ data: "publicClient": false, "serviceAccountsEnabled": true, "secret": "$(env:TRINO_POLARIS_CLIENT_SECRET)", - "serviceAccountRoles": { - "realmRoles": ["iceberg-catalog-admin"] - }, "protocolMappers": [ { "name": "realm-roles", @@ -240,9 +237,6 @@ data: "publicClient": false, "serviceAccountsEnabled": true, "secret": "$(env:PREFECT_POLARIS_CLIENT_SECRET)", - "serviceAccountRoles": { - "realmRoles": ["teehr-read-write"] - }, "protocolMappers": [ { "name": "realm-roles", @@ -284,5 +278,19 @@ data: } ] } + ], + "users": [ + { + "username": "service-account-trino-polaris", + "enabled": true, + "serviceAccountClientId": "trino-polaris", + "realmRoles": ["iceberg-catalog-admin"] + }, + { + "username": "service-account-prefect-polaris", + "enabled": true, + "serviceAccountClientId": "prefect-polaris", + "realmRoles": ["teehr-read-write"] + } ] } diff --git a/tests/spark_authmanager_test.py b/tests/spark_authmanager_test.py index 0643dc2..4f78731 100644 --- a/tests/spark_authmanager_test.py +++ b/tests/spark_authmanager_test.py @@ -207,7 +207,7 @@ def main(): print(" Creating Spark session via AuthManager...") try: - from spark_session_utils import create_minio_spark_session + from teehr.evaluation.spark_session_utils import create_minio_spark_session spark = create_minio_spark_session( use_authmanager=True, force_recreate_session=True, diff --git a/tests/spark_permission_test.py b/tests/spark_permission_test.py index 533c422..abccc6b 100644 --- a/tests/spark_permission_test.py +++ b/tests/spark_permission_test.py @@ -179,7 +179,7 @@ def main(): # Step 2: Create Spark session with Polaris catalog print(" Creating Spark session with Polaris catalog...") try: - from spark_session_utils import create_minio_spark_session + from teehr.evaluation.spark_session_utils import create_minio_spark_session spark = create_minio_spark_session( polaris_token=token, ) From 6734c6622775184dae92a2acadc7fc1bfd46cbd9 Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Fri, 31 Jul 2026 17:30:33 -0400 Subject: [PATCH 29/62] Simplify 01_setup_minio_warehouse: remove manual token management, use create_minio_spark_session directly --- examples/01_setup_minio_warehouse.ipynb | 196 +----------------------- 1 file changed, 3 insertions(+), 193 deletions(-) diff --git a/examples/01_setup_minio_warehouse.ipynb b/examples/01_setup_minio_warehouse.ipynb index 69fa9f0..3b6c45d 100644 --- a/examples/01_setup_minio_warehouse.ipynb +++ b/examples/01_setup_minio_warehouse.ipynb @@ -8,23 +8,16 @@ "outputs": [], "source": [ "from pathlib import Path\n", - "import base64\n", - "import json\n", "import os\n", "\n", "import pandas as pd\n", - "import requests\n", "\n", "import teehr\n", "from teehr import RemoteReadWriteEvaluation\n", "from teehr.utilities.apply_migrations import evolve_catalog_schema\n", "from teehr.evaluation.spark_session_utils import create_minio_spark_session\n", "\n", - "from setup_utils import (\n", - " DEV_LOCATION_ID_LIST,\n", - " apply_polaris_token_to_spark,\n", - " ensure_fresh_polaris_user_token,\n", - ")" + "from setup_utils import DEV_LOCATION_ID_LIST" ] }, { @@ -35,90 +28,6 @@ "#### Start the spark session" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "6d8287a8-fbf3-492e-9af7-53126b63a5d8", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "print(\"POLARIS_CLIENT_ID:\", os.getenv(\"POLARIS_CLIENT_ID\"))\n", - "print(\"POLARIS_CLIENT_SECRET exists:\", bool(os.getenv(\"POLARIS_CLIENT_SECRET\")))\n", - "print(\"POLARIS_REFRESH_TOKEN exists:\", bool(os.getenv(\"POLARIS_REFRESH_TOKEN\")))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1af43785", - "metadata": {}, - "outputs": [], - "source": [ - "polaris_user_token = os.environ.get(\"POLARIS_USER_TOKEN\")\n", - "polaris_refresh_token = os.getenv(\"POLARIS_REFRESH_TOKEN\", \"\")\n", - "polaris_username = os.getenv(\"POLARIS_USERNAME\", os.getenv(\"JUPYTERHUB_USER\", \"admin\"))\n", - "polaris_password = os.getenv(\"POLARIS_PASSWORD\", \"\")\n", - "polaris_client_id = os.getenv(\"POLARIS_CLIENT_ID\", \"jupyterhub\")\n", - "polaris_client_secret = os.getenv(\"POLARIS_CLIENT_SECRET\")\n", - "\n", - "if not polaris_user_token and not polaris_refresh_token and not polaris_password:\n", - " raise RuntimeError(\n", - " \"No usable Polaris credentials found. Set POLARIS_USER_TOKEN or POLARIS_REFRESH_TOKEN in the session.\"\n", - " )\n", - "\n", - "polaris_user_token, polaris_refresh_token, token_renewed = ensure_fresh_polaris_user_token(\n", - " current_token=polaris_user_token,\n", - " username=polaris_username,\n", - " password=polaris_password,\n", - " client_id=polaris_client_id,\n", - " client_secret=polaris_client_secret,\n", - " refresh_token=polaris_refresh_token,\n", - " refresh_window_seconds=120,\n", - ")\n", - "os.environ[\"POLARIS_USER_TOKEN\"] = polaris_user_token\n", - "if polaris_refresh_token:\n", - " os.environ[\"POLARIS_REFRESH_TOKEN\"] = polaris_refresh_token\n", - "\n", - "payload = polaris_user_token.split(\".\")[1]\n", - "payload += \"=\" * (-len(payload) % 4)\n", - "claims = json.loads(base64.urlsafe_b64decode(payload.encode()))\n", - "roles = sorted(claims.get(\"realm_access\", {}).get(\"roles\", []))\n", - "\n", - "print(\"Token status:\", \"renewed\" if token_renewed else \"reused\")\n", - "print(\"Refresh token available:\", bool(polaris_refresh_token))\n", - "print(\"Token user:\", claims.get(\"preferred_username\"))\n", - "print(\"Token roles:\", \", \".join(roles))\n", - "if \"iceberg-catalog-admin\" not in roles:\n", - " print(\"WARNING: token is missing iceberg-catalog-admin; Polaris may reject writes\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4f288bea", - "metadata": {}, - "outputs": [], - "source": [ - "use_broker = os.getenv(\"POLARIS_USE_BROKER\", \"false\").strip().lower() in {\"1\", \"true\", \"t\", \"yes\", \"y\", \"on\"}\n", - "if use_broker:\n", - " broker_user_id = claims.get(\"preferred_username\") or claims.get(\"sub\") or polaris_username\n", - " broker_session_id = os.getenv(\"JUPYTERHUB_SERVER_NAME\", os.getenv(\"JUPYTERHUB_USER\", \"notebook\"))\n", - "\n", - " polaris_user_token, broker_renewed = ensure_fresh_polaris_token_via_broker(\n", - " current_token=polaris_user_token,\n", - " user_id=broker_user_id,\n", - " session_id=broker_session_id,\n", - " bearer_token=polaris_user_token,\n", - " realm=os.getenv(\"POLARIS_DEFAULT_REALM\", \"teehr\"),\n", - " catalog=\"iceberg\",\n", - " refresh_window_seconds=120,\n", - " requested_ttl_seconds=600,\n", - " )\n", - " os.environ[\"POLARIS_USER_TOKEN\"] = polaris_user_token\n", - " print(\"Broker token status:\", \"renewed\" if broker_renewed else \"still fresh\")" - ] - }, { "cell_type": "code", "execution_count": null, @@ -126,108 +35,10 @@ "metadata": {}, "outputs": [], "source": [ - "import requests\n", - "\n", - "realm = os.getenv(\"POLARIS_DEFAULT_REALM\", \"teehr\")\n", - "remote_warehouse = os.getenv(\"REMOTE_WAREHOUSE_S3_PATH\", \"s3://warehouse/\")\n", - "polaris_api_base = os.getenv(\"POLARIS_CATALOG_URI\", \"http://polaris:8181/api/catalog\").rstrip(\"/\")\n", - "catalog_config_url = f\"{polaris_api_base}/v1/config\"\n", - "mgmt_catalog_url = f\"http://polaris:8181/api/management/v1/catalogs/{realm}\"\n", - "\n", - "# Refresh access token before probing Polaris or recreating Spark.\n", - "polaris_user_token, polaris_refresh_token, refreshed = ensure_fresh_polaris_user_token(\n", - " current_token=polaris_user_token,\n", - " username=polaris_username,\n", - " password=polaris_password,\n", - " client_id=polaris_client_id,\n", - " client_secret=polaris_client_secret,\n", - " refresh_token=polaris_refresh_token,\n", - " refresh_window_seconds=120,\n", - ")\n", - "os.environ[\"POLARIS_USER_TOKEN\"] = polaris_user_token\n", - "if polaris_refresh_token:\n", - " os.environ[\"POLARIS_REFRESH_TOKEN\"] = polaris_refresh_token\n", - "print(\"Cell 4 token status:\", \"renewed\" if refreshed else \"still fresh\")\n", - "\n", - "# Polaris REST config endpoint expects catalog identifier (realm), not S3 URI.\n", - "warehouse_for_config = remote_warehouse\n", - "if polaris_api_base.endswith(\"/api/catalog\"):\n", - " warehouse_for_config = realm\n", - "\n", - "headers = {\n", - " \"Authorization\": f\"Bearer {polaris_user_token}\",\n", - " \"X-Polaris-Realm\": realm,\n", - "}\n", - "\n", - "# Catalog config endpoint requires a warehouse query parameter.\n", - "catalog_resp = requests.get(\n", - " catalog_config_url,\n", - " headers=headers,\n", - " params={\"warehouse\": warehouse_for_config},\n", - " timeout=20,\n", - " )\n", - "print(\"Catalog config API status:\", catalog_resp.status_code)\n", - "print(\"Catalog config warehouse query:\", warehouse_for_config)\n", - "if catalog_resp.status_code >= 400:\n", - " print(\"Catalog config API body:\", catalog_resp.text[:500])\n", - "catalog_resp.raise_for_status()\n", - "\n", - "if \"iceberg-catalog-admin\" in roles:\n", - " mgmt_resp = requests.get(mgmt_catalog_url, headers=headers, timeout=20)\n", - " print(\"Management API status:\", mgmt_resp.status_code)\n", - " if mgmt_resp.status_code >= 400:\n", - " print(\"Management API body:\", mgmt_resp.text[:500])\n", - " mgmt_resp.raise_for_status()\n", - "else:\n", - " print(\n", - " \"Skipping Management API check: token does not include iceberg-catalog-admin. \"\n", - " \"Catalog access is sufficient for non-admin Spark paths.\"\n", - " )\n", - "\n", - "print(\"Polaris access check passed.\")\n", - "\n", - "spark = create_minio_spark_session(\n", - " polaris_token=polaris_user_token,\n", - " force_recreate_session=True,\n", - ")\n", - "\n", - "\n", - "def refresh_polaris_token_for_spark(refresh_window_seconds: int = 120) -> bool:\n", - " global polaris_user_token, polaris_refresh_token\n", - "\n", - " polaris_user_token, polaris_refresh_token, refreshed = ensure_fresh_polaris_user_token(\n", - " current_token=polaris_user_token,\n", - " username=polaris_username,\n", - " password=polaris_password,\n", - " client_id=polaris_client_id,\n", - " client_secret=polaris_client_secret,\n", - " refresh_token=polaris_refresh_token,\n", - " refresh_window_seconds=refresh_window_seconds,\n", - " )\n", - " os.environ[\"POLARIS_USER_TOKEN\"] = polaris_user_token\n", - " if polaris_refresh_token:\n", - " os.environ[\"POLARIS_REFRESH_TOKEN\"] = polaris_refresh_token\n", - " apply_polaris_token_to_spark(spark, polaris_user_token, catalog_name=\"iceberg\", realm=realm)\n", - "\n", - " print(\"Spark token status:\", \"renewed\" if refreshed else \"still fresh\")\n", - " return refreshed\n", - "\n", - "\n", - "apply_polaris_token_to_spark(spark, polaris_user_token, catalog_name=\"iceberg\", realm=realm)\n", - "print(\"Spark catalog namespace probe:\")\n", + "spark = create_minio_spark_session(force_recreate_session=True)\n", "spark.sql(\"SHOW NAMESPACES IN iceberg\").show(truncate=False)" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "07e97944-1dd5-4631-9799-0124e671e472", - "metadata": {}, - "outputs": [], - "source": [ - "os.environ[\"POLARIS_REFRESH_TOKEN\"]" - ] - }, { "cell_type": "code", "execution_count": null, @@ -268,8 +79,7 @@ "id": "101613ce-8167-4aad-8347-7492a21827e0", "metadata": {}, "source": [ - "#### Now you can pull in a subset of data from the TEEHR-Cloud warehouse via the API\n", - "Note: If you encounter a timeout or auth error, run refresh_polaris_token_for_spark() and retry. Prefer setting POLARIS_REFRESH_TOKEN in the session so renewals do not depend on username/password." + "#### Now you can pull in a subset of data from the TEEHR-Cloud warehouse via the API" ] }, { From ad150cf2653cb3eef7bc4dc26446dd540aaf812c Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Sat, 1 Aug 2026 07:39:18 -0400 Subject: [PATCH 30/62] move notebooks to correct dir --- examples/{ => developer}/01_setup_minio_warehouse.ipynb | 6 ++++-- examples/{ => developer}/02_create_joined_timeseries.ipynb | 5 ++++- examples/{ => developer}/03_generate_basic_metrics.ipynb | 5 ++++- examples/{ => developer}/calculate_nwm_metrics.ipynb | 0 4 files changed, 12 insertions(+), 4 deletions(-) rename examples/{ => developer}/01_setup_minio_warehouse.ipynb (98%) rename examples/{ => developer}/02_create_joined_timeseries.ipynb (95%) rename examples/{ => developer}/03_generate_basic_metrics.ipynb (97%) rename examples/{ => developer}/calculate_nwm_metrics.ipynb (100%) diff --git a/examples/01_setup_minio_warehouse.ipynb b/examples/developer/01_setup_minio_warehouse.ipynb similarity index 98% rename from examples/01_setup_minio_warehouse.ipynb rename to examples/developer/01_setup_minio_warehouse.ipynb index 3b6c45d..913cf24 100644 --- a/examples/01_setup_minio_warehouse.ipynb +++ b/examples/developer/01_setup_minio_warehouse.ipynb @@ -35,8 +35,10 @@ "metadata": {}, "outputs": [], "source": [ - "spark = create_minio_spark_session(force_recreate_session=True)\n", - "spark.sql(\"SHOW NAMESPACES IN iceberg\").show(truncate=False)" + "spark = create_minio_spark_session(\n", + " force_recreate_session=True,\n", + " use_authmanager=True\n", + ")" ] }, { diff --git a/examples/02_create_joined_timeseries.ipynb b/examples/developer/02_create_joined_timeseries.ipynb similarity index 95% rename from examples/02_create_joined_timeseries.ipynb rename to examples/developer/02_create_joined_timeseries.ipynb index 113819d..67b2c4f 100644 --- a/examples/02_create_joined_timeseries.ipynb +++ b/examples/developer/02_create_joined_timeseries.ipynb @@ -26,7 +26,10 @@ "metadata": {}, "outputs": [], "source": [ - "spark = create_minio_spark_session()" + "spark = create_minio_spark_session(\n", + " force_recreate_session=True,\n", + " use_authmanager=True\n", + ")" ] }, { diff --git a/examples/03_generate_basic_metrics.ipynb b/examples/developer/03_generate_basic_metrics.ipynb similarity index 97% rename from examples/03_generate_basic_metrics.ipynb rename to examples/developer/03_generate_basic_metrics.ipynb index 6745e97..0736735 100644 --- a/examples/03_generate_basic_metrics.ipynb +++ b/examples/developer/03_generate_basic_metrics.ipynb @@ -26,7 +26,10 @@ "metadata": {}, "outputs": [], "source": [ - "spark = create_minio_spark_session()" + "spark = create_minio_spark_session(\n", + " force_recreate_session=True,\n", + " use_authmanager=True\n", + ")" ] }, { diff --git a/examples/calculate_nwm_metrics.ipynb b/examples/developer/calculate_nwm_metrics.ipynb similarity index 100% rename from examples/calculate_nwm_metrics.ipynb rename to examples/developer/calculate_nwm_metrics.ipynb From d931ff2f5abb50ba46e7082b069d9ed1515d46e0 Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Sat, 1 Aug 2026 07:40:25 -0400 Subject: [PATCH 31/62] add copilot instructions --- .github/copilot-instructions.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 2a785cf..34b4df1 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -1,5 +1,7 @@ Context: -- the `teehr` repo which is a python library that contains the scientific code for the teehr environment. This contains code to fetch, validate, store, and analyze hydrologic forecast and simulation data against observations. -- the `teehr-cloud-core` repo contains the main shared teehr-cloud components such as authentication, jupyterhub, iceberg warehouse, web api, query engine, spark-executors, prefect server, etc. teehr-cloud-core utilizes the teehr library. -- the teehr-cloud-platform repo contains terraform IaC for creating an AWS environment that can host a teehr-cloud deployment. -- the `teehr-hub` and `teehr-fved` repos have teehr-cloud-core as a submodule and are deployments of teehr and teehr-cloud-core. They also contain deployment specific components such as frontend dashboards, deployment specific prefect workflows and warehouse setup and maintenance code, certificates, etc. \ No newline at end of file +- the `teehr` repo which is a python library that contains the scientific code for the teehr environment. This contains code to fetch, validate, store, and analyze hydrologic forecast and simulation data against observations. It works on the concept of an Evaluation which references either a local or remote warehouse and a relates spark session. +- the `teehr-cloud-core` repo contains the main shared teehr-cloud components such as authentication, jupyterhub, Apache Iceberg warehouse, web api, Trino query engine, spark-executors, prefect server, etc. teehr-cloud-core utilizes the teehr library. +- the `teehr-cloud-platform` repo contains terraform IaC for creating an AWS environment that can host a teehr-cloud deployment. +- the `teehr-hub` and `teehr-fved` repos have `teehr-cloud-core` as a submodule and are deployments of `teehr` and `teehr-cloud-core`. They also contain deployment specific components such as frontend dashboards, deployment specific prefect workflows and warehouse setup and maintenance code, certificates, etc. + +Don't make any changes in the `teehr-cloud-core` submodules to `teehr-hub` and `teehr-fved`. Only make changes in the `teehr-cloud-core` repo and then update the submodules in the deployments. \ No newline at end of file From 2c630dd39653478b4818b5f72c777b4cc1bf0a2d Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Mon, 3 Aug 2026 11:24:30 -0400 Subject: [PATCH 32/62] Fix Polaris auth and Trino catalog config - polaris-config: add missing quarkus.oidc.auth-server-url (was causing userinfo introspection fallback, breaking service account JWT validation) - polaris-config: use realm_access/roles claim path (covers both human users and service accounts); simplify iceberg-catalog-admin regex - polaris-sync-principals: sync iceberg-catalog-admin role (named principals override JWT mapping so all intended roles must be synced explicitly) - trino: use realm name as REST catalog warehouse (Polaris expects catalog name, not S3 URI) --- .../polaris-sync-principals-script.yaml | 22 ++++++------ polaris/manifests/polaris-config.yaml.tpl | 36 ++++++++----------- trino/garden.yaml | 5 ++- 3 files changed, 28 insertions(+), 35 deletions(-) diff --git a/polaris-bootstrap/manifests/polaris-sync-principals-script.yaml b/polaris-bootstrap/manifests/polaris-sync-principals-script.yaml index ad9c81d..010db6d 100644 --- a/polaris-bootstrap/manifests/polaris-sync-principals-script.yaml +++ b/polaris-bootstrap/manifests/polaris-sync-principals-script.yaml @@ -8,17 +8,14 @@ data: Syncs Keycloak users into Polaris as named principal entities and grants principal role bindings for their teehr-* group memberships. - NOTE ON DESIGN: Basic group-level access (teehr-read-only, teehr-read-write, - iceberg-catalog-admin) is enforced via JWT group claim mapping configured in - polaris-config.yaml — no principal sync is required for that to work. + NOTE ON DESIGN: When a named Polaris principal exists for a user, Polaris uses + that principal's explicit role bindings rather than the JWT group claim mapper. + Therefore this script must sync ALL intended PrincipalRoles — teehr-*, and + iceberg-catalog-admin — not just teehr-* roles. - This script exists for use cases that require a named Polaris principal: - - Table-level grants (granting a specific user access to a specific table) - - Individual permission overrides beyond group-level defaults - - Audit trails tied to a specific Polaris principal entity - - If you only need group-level permissions, this script is not required. - The principal role bindings it creates are ADDITIVE to the JWT-based grants. + This script is required for group-level access to work correctly once named + principals exist. It is also used for table-level grants and individual + permission overrides beyond group-level defaults. Required environment variables: POLARIS_MANAGEMENT_URL e.g. http://polaris:8181 @@ -192,7 +189,7 @@ data: if user_id: # Check realm roles for role_name in get_keycloak_realm_roles(kc_token, user_id): - if role_name.startswith("teehr-"): + if role_name.startswith("teehr-") or role_name == "iceberg-catalog-admin": ensure_principal_role_binding(polaris_token, name, role_name) granted += 1 @@ -201,6 +198,9 @@ data: if group_name.startswith("teehr-"): ensure_principal_role_binding(polaris_token, name, group_name) granted += 1 + elif group_name == "iceberg-catalog-admins": + ensure_principal_role_binding(polaris_token, name, "iceberg-catalog-admin") + granted += 1 synced += 1 print( diff --git a/polaris/manifests/polaris-config.yaml.tpl b/polaris/manifests/polaris-config.yaml.tpl index fa13036..66b970f 100644 --- a/polaris/manifests/polaris-config.yaml.tpl +++ b/polaris/manifests/polaris-config.yaml.tpl @@ -29,8 +29,9 @@ data: quarkus.oidc.tenant-enabled=true quarkus.oidc.application-type=service quarkus.oidc.client-id=jupyterhub - # Keycloak advertises external hostnames in discovery metadata; disable discovery - # so Polaris uses the configured auth-server-url and internal JWKS path directly. + # auth-server-url + relative jwks-path enables local JWT validation without discovery. + # Without auth-server-url, Quarkus falls back to userinfo introspection which fails for service accounts. + quarkus.oidc.auth-server-url=${var.polaris.oidcIssuerUri} quarkus.oidc.discovery-enabled=false quarkus.oidc.jwks-path=/protocol/openid-connect/certs quarkus.tls.trust-all=true @@ -39,29 +40,22 @@ data: quarkus.oidc.token.audience=account quarkus.oidc.token.issuer=any - # Access control: map Keycloak group membership (JWT 'groups' claim) directly to - # Polaris principal roles. This means no user sync is required for group-level - # access — adding a user to a Keycloak group immediately grants the corresponding - # Polaris permissions on their next token issuance. + # Access control: map Keycloak realm roles to Polaris PrincipalRoles. + # Uses realm_access/roles (not groups) to cover both: + # - human users: realm roles propagated from Keycloak group membership + # - service accounts: realm roles assigned directly (trino-polaris, prefect-polaris) # - # Also reads realm_access/roles so Keycloak service accounts (trino-polaris, - # prefect-polaris) can be granted Polaris access via realm role assignment, - # without needing a separate Polaris principal entity. + # Patterns: + # iceberg-catalog-admin → PRINCIPAL_ROLE:iceberg-catalog-admin + # teehr- → PRINCIPAL_ROLE:teehr- # - # Patterns mapped: - # /iceberg-catalog-admins (group) → iceberg-catalog-admin - # /teehr- (group) → teehr- (e.g. teehr-read-only, teehr-read-write) - # iceberg-catalog-admin (realm role for service accounts) - # teehr- (realm role for service accounts) - # - # Individual/table-level grants: use the polaris-sync-principals script to create - # a named principal for the user and assign specific catalog-role grants. These - # are additive on top of the JWT-based group grants above. - quarkus.oidc.roles.role-claim-path=groups,realm_access/roles + # Individual/table-level grants: use the polaris-sync-principals script. + # Named principal role bindings from the sync take precedence over JWT mapping. + quarkus.oidc.roles.role-claim-path=realm_access/roles polaris.oidc.principal-roles-mapper.type=default - polaris.oidc.principal-roles-mapper.mappings[0].regex=^/?iceberg-catalog-admins?$ + polaris.oidc.principal-roles-mapper.mappings[0].regex=^iceberg-catalog-admin$ polaris.oidc.principal-roles-mapper.mappings[0].replacement=PRINCIPAL_ROLE:iceberg-catalog-admin - polaris.oidc.principal-roles-mapper.mappings[1].regex=^/?teehr-(.+)$ + polaris.oidc.principal-roles-mapper.mappings[1].regex=^teehr-(.+)$ polaris.oidc.principal-roles-mapper.mappings[1].replacement=PRINCIPAL_ROLE:teehr-$1 # Storage Properties Integration diff --git a/trino/garden.yaml b/trino/garden.yaml index f5d68a8..cf80b9c 100644 --- a/trino/garden.yaml +++ b/trino/garden.yaml @@ -23,13 +23,12 @@ spec: # worker: # nodeSelector: # teehr-hub/nodegroup-name: core-a - # iceberg.rest-catalog.header.X-Polaris-Realm=${var.polaris.defaultRealm} # removed from below. Needed for multi-realm support? catalogs: iceberg: |- connector.name=iceberg iceberg.catalog.type=${var.polaris.catalogType} iceberg.rest-catalog.uri=${var.polaris.catalogUri} - iceberg.rest-catalog.warehouse=${var.polaris.catalogWarehouse} + iceberg.rest-catalog.warehouse=${var.polaris.defaultRealm} iceberg.rest-catalog.security=OAUTH2 iceberg.rest-catalog.oauth2.server-uri=${var.polaris.oauthServerUri} iceberg.rest-catalog.oauth2.credential=$${ENV:POLARIS_CREDENTIAL} @@ -118,7 +117,7 @@ spec: connector.name=iceberg iceberg.catalog.type=${var.polaris.catalogType} iceberg.rest-catalog.uri=${var.polaris.catalogUri} - iceberg.rest-catalog.warehouse=${var.polaris.catalogWarehouse} + iceberg.rest-catalog.warehouse=${var.polaris.defaultRealm} iceberg.rest-catalog.security=OAUTH2 iceberg.rest-catalog.oauth2.server-uri=${var.polaris.oauthServerUri} iceberg.rest-catalog.oauth2.credential=$${ENV:POLARIS_CREDENTIAL} From 847251128ec9edd5be980d84a80d929bb2abd72d Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Mon, 3 Aug 2026 11:24:52 -0400 Subject: [PATCH 33/62] Update copilot instructions --- .github/copilot-instructions.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 34b4df1..cbd5731 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -4,4 +4,6 @@ Context: - the `teehr-cloud-platform` repo contains terraform IaC for creating an AWS environment that can host a teehr-cloud deployment. - the `teehr-hub` and `teehr-fved` repos have `teehr-cloud-core` as a submodule and are deployments of `teehr` and `teehr-cloud-core`. They also contain deployment specific components such as frontend dashboards, deployment specific prefect workflows and warehouse setup and maintenance code, certificates, etc. -Don't make any changes in the `teehr-cloud-core` submodules to `teehr-hub` and `teehr-fved`. Only make changes in the `teehr-cloud-core` repo and then update the submodules in the deployments. \ No newline at end of file +Don't make any changes in the `teehr-cloud-core` submodules to `teehr-hub` and `teehr-fved`. Only make changes in the `teehr-cloud-core` repo and then update the submodules in the deployments. + +Make all changes to the cluster via code. Running one-off commands via kubectl or other tools is not allowed. All changes must be made via code and then applied to the cluster via CI/CD pipelines. This ensures that all changes are tracked in version control and can be rolled back if necessary. \ No newline at end of file From a20d124dd2a689a97e24cf833df0e8b9935828c2 Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Mon, 3 Aug 2026 11:51:10 -0400 Subject: [PATCH 34/62] update garden project template --- project.garden.yml.template | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/project.garden.yml.template b/project.garden.yml.template index 767c92f..a67eeae 100644 --- a/project.garden.yml.template +++ b/project.garden.yml.template @@ -13,6 +13,9 @@ variables: ssl: "false" aws: region: us-east-2 + remoteCluster: + contextArn: "" + ecrRegistry: "" environments: - name: local @@ -20,7 +23,8 @@ environments: variables: hostname: teehr.local.app.garden certificateIssuerName: letsencrypt-prod - devTeehrVersion: 267f8a75034132aefe84749af10ab1562a6ac169 + devTeehrVersion: f2736eedd525a14ccbf8ae2a02966bd15f7c3a5b + previousTeehrVersion: place-holder stableTeehrVersion: place-holder # Needed for sync to work for some reason. # iceberg: # inCluster: "true" @@ -42,7 +46,7 @@ environments: defaultRealm: teehr realmsCsv: teehr oauthServerUri: http://keycloak-service:8080/realms/teehr/protocol/openid-connect/token - oidcIssuerUri: https://auth.${var.hostname}/realms/teehr + oidcIssuerUri: http://keycloak-service:8080/realms/teehr trino: host: trino port: "8080" From 781d0e5a48940bc9b0453cfaa28c85197ac7d49c Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Thu, 6 Aug 2026 09:26:22 -0400 Subject: [PATCH 35/62] update teehr version --- project.garden.yml.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.garden.yml.template b/project.garden.yml.template index a67eeae..6feedd0 100644 --- a/project.garden.yml.template +++ b/project.garden.yml.template @@ -23,7 +23,7 @@ environments: variables: hostname: teehr.local.app.garden certificateIssuerName: letsencrypt-prod - devTeehrVersion: f2736eedd525a14ccbf8ae2a02966bd15f7c3a5b + devTeehrVersion: 27f0c3dd1c56db78ce4d2284b8f3caa966bebbe1 previousTeehrVersion: place-holder stableTeehrVersion: place-holder # Needed for sync to work for some reason. # iceberg: From 1b15ed3822ea0c406f4f9bfd28ff5172d1fa8383 Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Tue, 18 Aug 2026 14:11:22 -0400 Subject: [PATCH 36/62] add env remote warehouse id to jupyter --- jupyterhub/garden.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jupyterhub/garden.yaml b/jupyterhub/garden.yaml index bc3dd9d..771abae 100644 --- a/jupyterhub/garden.yaml +++ b/jupyterhub/garden.yaml @@ -173,6 +173,7 @@ spec: REMOTE_WAREHOUSE_S3_PATH: ${var.polaris.catalogWarehouse} REMOTE_CATALOG_S3_ENDPOINT: ${var.polaris.catalogS3Endpoint} REMOTE_CATALOG_S3_PATH_STYLE_ACCESS: ${var.polaris.catalogS3PathStyleAccess} + REMOTE_WAREHOUSE_IDENTIFIER: ${var.polaris.defaultRealm} IN_CLUSTER: ${var.polaris.inCluster} POLARIS_DEFAULT_REALM: ${var.polaris.defaultRealm} POLARIS_OAUTH2_SERVER_URI: ${var.polaris.oauthServerUri} @@ -376,6 +377,7 @@ spec: REMOTE_WAREHOUSE_S3_PATH: ${var.polaris.defaultRealm} REMOTE_CATALOG_S3_ENDPOINT: ${var.polaris.catalogS3Endpoint} REMOTE_CATALOG_S3_PATH_STYLE_ACCESS: ${var.polaris.catalogS3PathStyleAccess} + REMOTE_WAREHOUSE_IDENTIFIER: ${var.polaris.defaultRealm} IN_CLUSTER: ${var.polaris.inCluster} POLARIS_DEFAULT_REALM: ${var.polaris.defaultRealm} POLARIS_OAUTH2_SERVER_URI: ${var.polaris.oauthServerUri} From e3013aafa6c7dd1d77ff9c46a2cd3ce35079c335 Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Tue, 18 Aug 2026 17:26:06 -0400 Subject: [PATCH 37/62] update to skip lfs --- jupyterhub-docker/Dockerfile.jupyter-driver | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyterhub-docker/Dockerfile.jupyter-driver b/jupyterhub-docker/Dockerfile.jupyter-driver index a1e9b02..a15a3e2 100644 --- a/jupyterhub-docker/Dockerfile.jupyter-driver +++ b/jupyterhub-docker/Dockerfile.jupyter-driver @@ -72,7 +72,7 @@ RUN ARCH=$(uname -m) && \ if [ "$ARCH" = "aarch64" ]; then \ export GDAL_CONFIG=/usr/bin/gdal-config; \ fi && \ - pip install "git+https://github.com/RTIInternational/teehr.git@${TEEHR_VERSION}" + GIT_LFS_SKIP_SMUDGE=1 pip install --no-cache-dir "git+https://github.com/RTIInternational/teehr.git@${TEEHR_VERSION}" RUN mkdir -p /opt/teehr && chown -R ${NB_USER}:${NB_USER} /opt/teehr From e858890b1eaf9982d4da877977997aaf4deb5efa Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Tue, 18 Aug 2026 17:27:15 -0400 Subject: [PATCH 38/62] skip lfs --- spark/docker/Dockerfile.spark-executor | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spark/docker/Dockerfile.spark-executor b/spark/docker/Dockerfile.spark-executor index 9b65861..f4a3c22 100644 --- a/spark/docker/Dockerfile.spark-executor +++ b/spark/docker/Dockerfile.spark-executor @@ -59,11 +59,11 @@ RUN ARCH=$(uname -m) && \ echo "rasterio==1.3.11" > /tmp/constraints.txt && \ python3.12 -m pip install --no-cache-dir --ignore-installed "setuptools<81" "wheel" "cython<3.1" "numpy<2" && \ python3.12 -m pip install --no-cache-dir --no-build-isolation "rasterio==1.3.11" && \ - python3.12 -m pip install --no-cache-dir "git+https://github.com/RTIInternational/teehr.git@${TEEHR_VERSION}" --constraint /tmp/constraints.txt && \ + GIT_LFS_SKIP_SMUDGE=1 python3.12 -m pip install --no-cache-dir "git+https://github.com/RTIInternational/teehr.git@${TEEHR_VERSION}" --constraint /tmp/constraints.txt && \ rm /tmp/constraints.txt; \ else \ echo "Installing TEEHR normally for x86_64..." && \ - python3.12 -m pip install --no-cache-dir "git+https://github.com/RTIInternational/teehr.git@${TEEHR_VERSION}"; \ + GIT_LFS_SKIP_SMUDGE=1 python3.12 -m pip install --no-cache-dir "git+https://github.com/RTIInternational/teehr.git@${TEEHR_VERSION}"; \ fi # change spark UID and GID to 1000 From 3743af6d2a2587aafb8d4ab1494f20a9a047a77a Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Wed, 19 Aug 2026 07:58:43 -0400 Subject: [PATCH 39/62] add aws keys to local jupyter for dev --- jupyterhub/garden.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/jupyterhub/garden.yaml b/jupyterhub/garden.yaml index 771abae..22df7fd 100644 --- a/jupyterhub/garden.yaml +++ b/jupyterhub/garden.yaml @@ -163,6 +163,16 @@ spec: TEEHR_SPARK_IMAGE: ${actions.build.teehr-spark-executor-image.outputs.deploymentImageId} TEEHR_NAMESPACE: ${environment.namespace} AWS_REGION: ${var.aws.region} + AWS_ACCESS_KEY_ID: + valueFrom: + secretKeyRef: + name: minio-secrets + key: accesskey + AWS_SECRET_ACCESS_KEY: + valueFrom: + secretKeyRef: + name: minio-secrets + key: secretkey TEEHR_DOWNLOAD_API_KEY: valueFrom: secretKeyRef: From 000578d1f7820b8e5c5e58bb2fa1f92c3a217f71 Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Wed, 19 Aug 2026 10:44:23 -0400 Subject: [PATCH 40/62] add a teehr project id to local jupyter --- jupyterhub-profiles/profile-list.local.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/jupyterhub-profiles/profile-list.local.json b/jupyterhub-profiles/profile-list.local.json index e390244..8291ffb 100644 --- a/jupyterhub-profiles/profile-list.local.json +++ b/jupyterhub-profiles/profile-list.local.json @@ -22,6 +22,9 @@ "kubespawner_override": { "node_selector": { "teehr-hub/nodegroup-name": "nb-r5-xlarge" + }, + "environment": { + "TEEHR_PROJECT_ID": "TEEHR" } } }, From a56e48158e68acac6f09ee2fe591a52a41b9c0d3 Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Wed, 19 Aug 2026 11:26:17 -0400 Subject: [PATCH 41/62] comment out xpublish --- xpublish-api/garden.yaml | 108 +++++++++++++++++++-------------------- 1 file changed, 54 insertions(+), 54 deletions(-) diff --git a/xpublish-api/garden.yaml b/xpublish-api/garden.yaml index b3bc628..8fbb717 100644 --- a/xpublish-api/garden.yaml +++ b/xpublish-api/garden.yaml @@ -1,69 +1,69 @@ -kind: Build -type: container -name: xpublish-api +# kind: Build +# type: container +# name: xpublish-api -spec: - dockerfile: Dockerfile.xpublish +# spec: +# dockerfile: Dockerfile.xpublish -environments: - - local - - remote +# environments: +# - local +# - remote ---- +# --- -kind: Deploy -type: kubernetes -name: xpublish-api -environments: - - local +# kind: Deploy +# type: kubernetes +# name: xpublish-api +# environments: +# - local -dependencies: - - build.xpublish-api - - deploy.secrets +# dependencies: +# - build.xpublish-api +# - deploy.secrets -spec: - manifestFiles: - - manifests/service.yaml +# spec: +# manifestFiles: +# - manifests/service.yaml - manifestTemplates: - - manifests/serviceaccount.yaml.tpl - - manifests/configmap-${environment.name}.yaml.tpl - - manifests/deployment.yaml.tpl +# manifestTemplates: +# - manifests/serviceaccount.yaml.tpl +# - manifests/configmap-${environment.name}.yaml.tpl +# - manifests/deployment.yaml.tpl - defaultTarget: - kind: Deployment - name: xpublish-api +# defaultTarget: +# kind: Deployment +# name: xpublish-api - sync: - paths: - - containerPath: /app/src - sourcePath: ./src - mode: one-way - exclude: ["__pycache__", "*.pyc"] - overrides: - - command: ["uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "8000", "--reload"] +# sync: +# paths: +# - containerPath: /app/src +# sourcePath: ./src +# mode: one-way +# exclude: ["__pycache__", "*.pyc"] +# overrides: +# - command: ["uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "8000", "--reload"] ---- +# --- -kind: Deploy -type: kubernetes -name: xpublish-api -environments: - - remote +# kind: Deploy +# type: kubernetes +# name: xpublish-api +# environments: +# - remote -dependencies: - - build.xpublish-api - - deploy.secrets +# dependencies: +# - build.xpublish-api +# - deploy.secrets -spec: - manifestFiles: - - manifests/service.yaml +# spec: +# manifestFiles: +# - manifests/service.yaml - manifestTemplates: - - manifests/serviceaccount.yaml.tpl - - manifests/configmap-${environment.name}.yaml.tpl - - manifests/deployment.yaml.tpl +# manifestTemplates: +# - manifests/serviceaccount.yaml.tpl +# - manifests/configmap-${environment.name}.yaml.tpl +# - manifests/deployment.yaml.tpl - defaultTarget: - kind: Deployment - name: xpublish-api +# defaultTarget: +# kind: Deployment +# name: xpublish-api From 10bd7ceab42cecf6e6e6e51247cf23c48951a1dc Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Wed, 19 Aug 2026 11:27:35 -0400 Subject: [PATCH 42/62] comment xpublish --- ingress/garden.yaml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/ingress/garden.yaml b/ingress/garden.yaml index 1366a6c..88f68aa 100644 --- a/ingress/garden.yaml +++ b/ingress/garden.yaml @@ -62,18 +62,18 @@ dependencies: environments: - local - remote ---- -kind: Deploy -type: kubernetes -name: xpublish-api-ingress -spec: - manifestTemplates: - - manifests/xpublish-api.yaml.tpl -dependencies: - - deploy.xpublish-api -environments: - - local - - remote +# --- +# kind: Deploy +# type: kubernetes +# name: xpublish-api-ingress +# spec: +# manifestTemplates: +# - manifests/xpublish-api.yaml.tpl +# dependencies: +# - deploy.xpublish-api +# environments: +# - local +# - remote # --- # kind: Deploy # type: kubernetes From e50c38518b120962900bc67ded67fbfa53702280 Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Wed, 19 Aug 2026 17:08:19 -0400 Subject: [PATCH 43/62] remove authmanager jar handlng in favor of the package --- .gitignore | 4 +-- jupyterhub-docker/Dockerfile.jupyter-driver | 3 --- jupyterhub-docker/garden.yaml | 4 --- scripts/stage_authmanager_jar.sh | 27 --------------------- spark/authmanager-prototype/garden.yaml | 13 ---------- spark/docker/Dockerfile.spark-executor | 3 --- spark/docker/garden.yaml | 2 -- 7 files changed, 1 insertion(+), 55 deletions(-) delete mode 100644 scripts/stage_authmanager_jar.sh delete mode 100644 spark/authmanager-prototype/garden.yaml diff --git a/.gitignore b/.gitignore index 2bfb397..a1418bc 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,4 @@ secrets.remote.yaml:Zone.Identifier __pycache__/ .python-version *.egg-info -project.garden.yml -jupyterhub-docker/teehr-authmanager.jar -spark/docker/teehr-authmanager.jar \ No newline at end of file +project.garden.yml \ No newline at end of file diff --git a/jupyterhub-docker/Dockerfile.jupyter-driver b/jupyterhub-docker/Dockerfile.jupyter-driver index a15a3e2..3775e5d 100644 --- a/jupyterhub-docker/Dockerfile.jupyter-driver +++ b/jupyterhub-docker/Dockerfile.jupyter-driver @@ -78,9 +78,6 @@ RUN mkdir -p /opt/teehr && chown -R ${NB_USER}:${NB_USER} /opt/teehr COPY executor-pod-template.yaml /opt/teehr/executor-pod-template.yaml -# Keep driver and executor classpaths aligned for the custom AuthManager. -COPY teehr-authmanager.jar /opt/spark/jars/teehr-authmanager.jar - # Configure IPython system-wide COPY ipython_kernel_config.py /etc/ipython/ipython_kernel_config.py # RUN fix-permissions /etc/ipython/ diff --git a/jupyterhub-docker/garden.yaml b/jupyterhub-docker/garden.yaml index 57a3d0f..86ebd91 100644 --- a/jupyterhub-docker/garden.yaml +++ b/jupyterhub-docker/garden.yaml @@ -2,8 +2,6 @@ kind: Build type: container name: teehr-jupyter-driver-image-edge description: Build TEEHR Jupyter Driver Image -dependencies: - - deploy.stage-authmanager-jar spec: dockerfile: Dockerfile.jupyter-driver localId: jupyter-driver @@ -27,7 +25,6 @@ spec: environments: - remote dependencies: - - deploy.stage-authmanager-jar - build.teehr-jupyter-driver-image-edge --- kind: Build @@ -43,5 +40,4 @@ spec: environments: - remote dependencies: - - deploy.stage-authmanager-jar - build.teehr-jupyter-driver-image-stable \ No newline at end of file diff --git a/scripts/stage_authmanager_jar.sh b/scripts/stage_authmanager_jar.sh deleted file mode 100644 index 649482b..0000000 --- a/scripts/stage_authmanager_jar.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -AUTHMANAGER_DIR="$ROOT_DIR/spark/authmanager-prototype" -SPARK_DOCKER_DIR="$ROOT_DIR/spark/docker" -JUPYTER_DOCKER_DIR="$ROOT_DIR/jupyterhub-docker" - -cd "$ROOT_DIR" - -echo "[authmanager-stage] Building AuthManager jar..." -mvn -f "$AUTHMANAGER_DIR/pom.xml" -DskipTests package >/dev/null - -JAR_PATH="$(find "$AUTHMANAGER_DIR/target" -maxdepth 1 -type f -name 'teehr-iceberg-authmanager-prototype-*.jar' ! -name '*-sources.jar' ! -name '*-javadoc.jar' | head -n1)" - -if [[ -z "${JAR_PATH:-}" || ! -f "$JAR_PATH" ]]; then - echo "[authmanager-stage] Failed to find built jar under $AUTHMANAGER_DIR/target" >&2 - exit 1 -fi - -echo "[authmanager-stage] Staging jar from $JAR_PATH" -cp "$JAR_PATH" "$SPARK_DOCKER_DIR/teehr-authmanager.jar" -cp "$JAR_PATH" "$JUPYTER_DOCKER_DIR/teehr-authmanager.jar" - -echo "[authmanager-stage] Staged jar to:" -echo " - $SPARK_DOCKER_DIR/teehr-authmanager.jar" -echo " - $JUPYTER_DOCKER_DIR/teehr-authmanager.jar" diff --git a/spark/authmanager-prototype/garden.yaml b/spark/authmanager-prototype/garden.yaml deleted file mode 100644 index 3bb735f..0000000 --- a/spark/authmanager-prototype/garden.yaml +++ /dev/null @@ -1,13 +0,0 @@ -kind: Deploy -type: exec -name: stage-authmanager-jar -description: Build and stage AuthManager jar into Spark and Jupyter Docker contexts -environments: - - local - - remote -spec: - deployCommand: - - bash - - -c - - >- - bash ../../scripts/stage_authmanager_jar.sh diff --git a/spark/docker/Dockerfile.spark-executor b/spark/docker/Dockerfile.spark-executor index f4a3c22..ae30b79 100644 --- a/spark/docker/Dockerfile.spark-executor +++ b/spark/docker/Dockerfile.spark-executor @@ -72,8 +72,5 @@ RUN usermod -u 1000 spark && groupmod -g 1000 spark # Ensure work directory is owned by spark user RUN chown -R spark:spark /opt/spark/work-dir -# Stage custom Iceberg AuthManager extension into Spark classpath. -COPY teehr-authmanager.jar /opt/spark/jars/teehr-authmanager.jar - # Switch back to spark user USER 1000 \ No newline at end of file diff --git a/spark/docker/garden.yaml b/spark/docker/garden.yaml index 010b441..0cb4a32 100644 --- a/spark/docker/garden.yaml +++ b/spark/docker/garden.yaml @@ -2,8 +2,6 @@ kind: Build type: container name: teehr-spark-executor-image description: Build TEEHR Spark Executor Image -dependencies: - - deploy.stage-authmanager-jar environments: - local - remote From f0667987269d4554edc28ad60d6ac69b8e2b0a33 Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Thu, 20 Aug 2026 08:26:34 -0400 Subject: [PATCH 44/62] cleanup old authmanager code --- spark/authmanager-prototype/README.md | 39 ----- spark/authmanager-prototype/pom.xml | 53 ------ .../iceberg/auth/BrokerBackedAuthSession.java | 127 --------------- .../org/teehr/iceberg/auth/BrokerToken.java | 10 -- .../teehr/iceberg/auth/BrokerTokenClient.java | 132 --------------- .../iceberg/auth/TeehrAuthProperties.java | 29 ---- .../iceberg/auth/TeehrBrokerAuthManager.java | 154 ------------------ .../auth/BrokerBackedAuthSession.class | Bin 5275 -> 0 bytes .../org/teehr/iceberg/auth/BrokerToken.class | Bin 1937 -> 0 bytes .../iceberg/auth/BrokerTokenClient.class | Bin 6356 -> 0 bytes .../iceberg/auth/TeehrAuthProperties.class | Bin 1677 -> 0 bytes .../iceberg/auth/TeehrBrokerAuthManager.class | Bin 7223 -> 0 bytes .../target/maven-archiver/pom.properties | 3 - .../compile/default-compile/createdFiles.lst | 5 - .../compile/default-compile/inputFiles.lst | 5 - ...g-authmanager-prototype-0.0.1-SNAPSHOT.jar | Bin 12971 -> 0 bytes 16 files changed, 557 deletions(-) delete mode 100644 spark/authmanager-prototype/README.md delete mode 100644 spark/authmanager-prototype/pom.xml delete mode 100644 spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerBackedAuthSession.java delete mode 100644 spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerToken.java delete mode 100644 spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerTokenClient.java delete mode 100644 spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/TeehrAuthProperties.java delete mode 100644 spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/TeehrBrokerAuthManager.java delete mode 100644 spark/authmanager-prototype/target/classes/org/teehr/iceberg/auth/BrokerBackedAuthSession.class delete mode 100644 spark/authmanager-prototype/target/classes/org/teehr/iceberg/auth/BrokerToken.class delete mode 100644 spark/authmanager-prototype/target/classes/org/teehr/iceberg/auth/BrokerTokenClient.class delete mode 100644 spark/authmanager-prototype/target/classes/org/teehr/iceberg/auth/TeehrAuthProperties.class delete mode 100644 spark/authmanager-prototype/target/classes/org/teehr/iceberg/auth/TeehrBrokerAuthManager.class delete mode 100644 spark/authmanager-prototype/target/maven-archiver/pom.properties delete mode 100644 spark/authmanager-prototype/target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst delete mode 100644 spark/authmanager-prototype/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst delete mode 100644 spark/authmanager-prototype/target/teehr-iceberg-authmanager-prototype-0.0.1-SNAPSHOT.jar diff --git a/spark/authmanager-prototype/README.md b/spark/authmanager-prototype/README.md deleted file mode 100644 index 39c851c..0000000 --- a/spark/authmanager-prototype/README.md +++ /dev/null @@ -1,39 +0,0 @@ -# Spark AuthManager Prototype - -This directory contains a minimal broker-backed Iceberg REST AuthManager prototype intended for local validation. - -## What this prototype does - -- implements an Iceberg AuthManager class (`org.teehr.iceberg.auth.TeehrBrokerAuthManager`) -- requests short-lived Polaris access tokens from a broker endpoint -- caches tokens in-memory and refreshes before expiry -- injects `Authorization: Bearer ` into REST catalog requests - -## Build - -```bash -mvn -f spark/authmanager-prototype/pom.xml -DskipTests package -``` - -## Spark configuration sketch - -Set the auth manager class as the Iceberg REST auth type: - -```bash ---conf spark.sql.catalog.iceberg.rest.auth.type=org.teehr.iceberg.auth.TeehrBrokerAuthManager \ ---conf spark.sql.catalog.iceberg.rest.auth.teehr.broker.url=http://teehr-token-broker.teehr.svc.cluster.local:8080/v1/polaris/token \ ---conf spark.sql.catalog.iceberg.rest.auth.teehr.user-id=${POLARIS_USER_ID} \ ---conf spark.sql.catalog.iceberg.rest.auth.teehr.session-id=${JUPYTERHUB_SERVER_NAME} \ ---conf spark.sql.catalog.iceberg.rest.auth.teehr.realm=teehr \ ---conf spark.sql.catalog.iceberg.rest.auth.teehr.catalog=iceberg \ ---conf spark.sql.catalog.iceberg.rest.auth.teehr.audience=account \ ---conf spark.sql.catalog.iceberg.rest.auth.teehr.broker-session-token-env=POLARIS_BROKER_SESSION_TOKEN -``` - -## Prototype notes - -- Keep this classpath-local to development images until broker authn/authz is production-ready. -- Do not log access tokens. -- Use broker-issued token TTL of 5 to 15 minutes with proactive refresh. -- Preferred: provide a broker delegated session token via `rest.auth.teehr.broker-session-token` or `rest.auth.teehr.broker-session-token-env`. -- Fallback without delegated session token: `rest.auth.teehr.subject-token` or `rest.auth.teehr.subject-token-env`. diff --git a/spark/authmanager-prototype/pom.xml b/spark/authmanager-prototype/pom.xml deleted file mode 100644 index 1006033..0000000 --- a/spark/authmanager-prototype/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - org.teehr - teehr-iceberg-authmanager-prototype - 0.0.1-SNAPSHOT - TEEHR Iceberg AuthManager Prototype - Broker-backed AuthManager prototype for Iceberg REST catalog auth. - - - 17 - 17 - UTF-8 - 1.10.0 - 2.18.2 - 2.0.16 - - - - - org.apache.iceberg - iceberg-core - ${iceberg.version} - provided - - - com.fasterxml.jackson.core - jackson-databind - ${jackson.version} - - - org.slf4j - slf4j-api - ${slf4j.version} - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.13.0 - - 17 - - - - - diff --git a/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerBackedAuthSession.java b/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerBackedAuthSession.java deleted file mode 100644 index 399a840..0000000 --- a/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerBackedAuthSession.java +++ /dev/null @@ -1,127 +0,0 @@ -package org.teehr.iceberg.auth; - -import java.io.IOException; -import java.time.Duration; -import java.time.Instant; -import java.util.Map; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Supplier; -import org.apache.iceberg.rest.HTTPHeaders; -import org.apache.iceberg.rest.HTTPRequest; -import org.apache.iceberg.rest.auth.AuthSession; -import org.apache.iceberg.rest.auth.DefaultAuthSession; - -final class BrokerBackedAuthSession implements AuthSession { - - private final BrokerTokenClient tokenClient; - private final String brokerUrl; - private final String userId; - private final String sessionId; - private final String realm; - private final String catalog; - private final String audience; - private final Supplier brokerSessionTokenSupplier; - private final Supplier subjectTokenSupplier; - private final Duration timeout; - private final long requestedTtlSeconds; - private final long refreshSkewSeconds; - - private final AtomicReference cachedToken = new AtomicReference<>(); - - BrokerBackedAuthSession( - BrokerTokenClient tokenClient, - String brokerUrl, - String userId, - String sessionId, - String realm, - String catalog, - String audience, - Supplier brokerSessionTokenSupplier, - Supplier subjectTokenSupplier, - Duration timeout, - long requestedTtlSeconds, - long refreshSkewSeconds) { - this.tokenClient = tokenClient; - this.brokerUrl = brokerUrl; - this.userId = userId; - this.sessionId = sessionId; - this.realm = realm; - this.catalog = catalog; - this.audience = audience; - this.brokerSessionTokenSupplier = brokerSessionTokenSupplier; - this.subjectTokenSupplier = subjectTokenSupplier; - this.timeout = timeout; - this.requestedTtlSeconds = requestedTtlSeconds; - this.refreshSkewSeconds = refreshSkewSeconds; - } - - @Override - public HTTPRequest authenticate(HTTPRequest request) { - BrokerToken token = currentToken(); - AuthSession delegate = - DefaultAuthSession.of(HTTPHeaders.of(Map.of("Authorization", "Bearer " + token.accessToken()))); - return delegate.authenticate(request); - } - - private BrokerToken currentToken() { - BrokerToken token = cachedToken.get(); - Instant now = Instant.now(); - - if (token != null && !token.expiresWithinSeconds(refreshSkewSeconds, now)) { - return token; - } - - synchronized (this) { - token = cachedToken.get(); - now = Instant.now(); - if (token != null && !token.expiresWithinSeconds(refreshSkewSeconds, now)) { - return token; - } - - try { - String brokerSessionToken = - brokerSessionTokenSupplier == null ? null : brokerSessionTokenSupplier.get(); - String subjectToken = null; - if (subjectTokenSupplier != null) { - try { - subjectToken = subjectTokenSupplier.get(); - } catch (Exception e) { - if (brokerSessionToken == null || brokerSessionToken.isBlank()) { - throw e; - } - } - } - - if ((brokerSessionToken == null || brokerSessionToken.isBlank()) - && (subjectToken == null || subjectToken.isBlank())) { - throw new IllegalStateException("Subject token supplier returned an empty token"); - } - - BrokerToken refreshed = - tokenClient.mintToken( - brokerUrl, - userId, - sessionId, - realm, - catalog, - audience, - brokerSessionToken, - subjectToken, - requestedTtlSeconds, - timeout); - cachedToken.set(refreshed); - return refreshed; - } catch (IOException | InterruptedException e) { - if (e instanceof InterruptedException) { - Thread.currentThread().interrupt(); - } - throw new IllegalStateException("Unable to acquire broker-backed Polaris token", e); - } - } - } - - @Override - public void close() { - cachedToken.set(null); - } -} diff --git a/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerToken.java b/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerToken.java deleted file mode 100644 index 2ac2ba6..0000000 --- a/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerToken.java +++ /dev/null @@ -1,10 +0,0 @@ -package org.teehr.iceberg.auth; - -import java.time.Instant; - -record BrokerToken(String accessToken, Instant expiresAt) { - - boolean expiresWithinSeconds(long seconds, Instant now) { - return !expiresAt.isAfter(now.plusSeconds(seconds)); - } -} diff --git a/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerTokenClient.java b/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerTokenClient.java deleted file mode 100644 index 41342df..0000000 --- a/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerTokenClient.java +++ /dev/null @@ -1,132 +0,0 @@ -package org.teehr.iceberg.auth; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import java.io.IOException; -import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.time.Duration; -import java.time.Instant; -import java.util.HashMap; -import java.util.Map; - -final class BrokerTokenClient { - - private static final ObjectMapper MAPPER = new ObjectMapper(); - - private final HttpClient httpClient; - - BrokerTokenClient(HttpClient httpClient) { - this.httpClient = httpClient; - } - - BrokerToken mintToken( - String brokerUrl, - String userId, - String sessionId, - String realm, - String catalog, - String audience, - String brokerSessionToken, - String subjectToken, - long requestedTtlSeconds, - Duration timeout) - throws IOException, InterruptedException { - - Map body = new HashMap<>(); - body.put("user_id", userId); - body.put("session_id", sessionId); - body.put("realm", realm); - body.put("catalog", catalog); - body.put("requested_ttl_seconds", requestedTtlSeconds); - body.put("audience", audience); - - HttpRequest.Builder requestBuilder = - HttpRequest.newBuilder(URI.create(brokerUrl)) - .header("Content-Type", "application/json") - .timeout(timeout) - .POST(HttpRequest.BodyPublishers.ofString(MAPPER.writeValueAsString(body))); - - if (brokerSessionToken != null && !brokerSessionToken.isBlank()) { - requestBuilder.header("X-Broker-Session-Token", brokerSessionToken); - } else { - requestBuilder.header("Authorization", "Bearer " + subjectToken); - } - - HttpRequest request = requestBuilder.build(); - - HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); - - if (shouldFallbackToSubjectExchange(response, brokerUrl, brokerSessionToken, subjectToken)) { - String directBrokerUrl = brokerUrl.replace("/auth/polaris-token/session", "/auth/polaris-token"); - HttpRequest directRequest = - HttpRequest.newBuilder(URI.create(directBrokerUrl)) - .header("Content-Type", "application/json") - .header("Authorization", "Bearer " + subjectToken) - .timeout(timeout) - .POST(HttpRequest.BodyPublishers.ofString(MAPPER.writeValueAsString(body))) - .build(); - - response = httpClient.send(directRequest, HttpResponse.BodyHandlers.ofString()); - } - - if (response.statusCode() != 200) { - String bodySnippet = response.body() == null ? "" : response.body(); - if (bodySnippet.length() > 500) { - bodySnippet = bodySnippet.substring(0, 500); - } - String authMode = - brokerSessionToken != null && !brokerSessionToken.isBlank() - ? "broker-session-header" - : "subject-authorization-header"; - throw new IOException( - "Broker token request failed with status " - + response.statusCode() - + " against " - + brokerUrl - + " using " - + authMode - + ": " - + bodySnippet); - } - - JsonNode payload = MAPPER.readTree(response.body()); - String accessToken = payload.path("access_token").asText(null); - long expiresAtEpoch = payload.path("expires_at_epoch_seconds").asLong(0); - - if (accessToken == null || expiresAtEpoch <= 0) { - throw new IOException("Broker response missing access_token or expires_at_epoch_seconds"); - } - - return new BrokerToken(accessToken, Instant.ofEpochSecond(expiresAtEpoch)); - } - - private static boolean shouldFallbackToSubjectExchange( - HttpResponse response, - String brokerUrl, - String brokerSessionToken, - String subjectToken) { - if (response.statusCode() != 401) { - return false; - } - if (brokerSessionToken == null || brokerSessionToken.isBlank()) { - return false; - } - if (subjectToken == null || subjectToken.isBlank()) { - return false; - } - if (!brokerUrl.endsWith("/auth/polaris-token/session")) { - return false; - } - - String body = response.body(); - if (body == null || body.isBlank()) { - return false; - } - - return body.contains("Delegated broker session not found") - || body.contains("Delegated broker session expired"); - } -} diff --git a/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/TeehrAuthProperties.java b/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/TeehrAuthProperties.java deleted file mode 100644 index ea8bc83..0000000 --- a/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/TeehrAuthProperties.java +++ /dev/null @@ -1,29 +0,0 @@ -package org.teehr.iceberg.auth; - -final class TeehrAuthProperties { - - private TeehrAuthProperties() {} - - static final String PREFIX = "rest.auth.teehr."; - - static final String BROKER_URL = PREFIX + "broker.url"; - static final String BROKER_SESSION_TOKEN = PREFIX + "broker-session-token"; - static final String BROKER_SESSION_TOKEN_ENV = PREFIX + "broker-session-token-env"; - static final String USER_ID = PREFIX + "user-id"; - static final String SESSION_ID = PREFIX + "session-id"; - static final String REALM = PREFIX + "realm"; - static final String CATALOG = PREFIX + "catalog"; - static final String AUDIENCE = PREFIX + "audience"; - static final String SUBJECT_TOKEN = PREFIX + "subject-token"; - static final String SUBJECT_TOKEN_ENV = PREFIX + "subject-token-env"; - static final String REQUESTED_TTL_SECONDS = PREFIX + "requested-ttl-seconds"; - static final String REQUEST_TIMEOUT_MS = PREFIX + "request-timeout-ms"; - static final String REFRESH_SKEW_SECONDS = PREFIX + "refresh-skew-seconds"; - - static final int DEFAULT_REQUEST_TIMEOUT_MS = 5000; - static final long DEFAULT_REFRESH_SKEW_SECONDS = 60L; - static final long DEFAULT_REQUESTED_TTL_SECONDS = 600L; - static final String DEFAULT_AUDIENCE = "account"; - static final String DEFAULT_SUBJECT_TOKEN_ENV = "POLARIS_USER_TOKEN"; - static final String DEFAULT_BROKER_SESSION_TOKEN_ENV = "POLARIS_BROKER_SESSION_TOKEN"; -} diff --git a/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/TeehrBrokerAuthManager.java b/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/TeehrBrokerAuthManager.java deleted file mode 100644 index c8f4b3e..0000000 --- a/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/TeehrBrokerAuthManager.java +++ /dev/null @@ -1,154 +0,0 @@ -package org.teehr.iceberg.auth; - -import java.net.http.HttpClient; -import java.net.http.HttpClient.Version; -import java.time.Duration; -import java.util.Map; -import java.util.function.Supplier; -import org.apache.iceberg.rest.RESTClient; -import org.apache.iceberg.rest.auth.AuthManager; -import org.apache.iceberg.rest.auth.AuthSession; -import org.apache.iceberg.util.PropertyUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Prototype Iceberg AuthManager that sources short-lived bearer tokens from a broker endpoint. - */ -public class TeehrBrokerAuthManager implements AuthManager { - - private static final Logger LOG = LoggerFactory.getLogger(TeehrBrokerAuthManager.class); - - private final String name; - private volatile AuthSession catalogSession; - - public TeehrBrokerAuthManager(String managerName) { - this.name = managerName; - } - - @Override - public AuthSession catalogSession(RESTClient sharedClient, Map properties) { - AuthSession existing = catalogSession; - if (existing != null) { - return existing; - } - - synchronized (this) { - existing = catalogSession; - if (existing != null) { - return existing; - } - - String brokerUrl = required(properties, TeehrAuthProperties.BROKER_URL); - String userId = required(properties, TeehrAuthProperties.USER_ID); - String sessionId = required(properties, TeehrAuthProperties.SESSION_ID); - String realm = required(properties, TeehrAuthProperties.REALM); - String catalog = properties.getOrDefault(TeehrAuthProperties.CATALOG, "iceberg"); - String audience = - properties.getOrDefault(TeehrAuthProperties.AUDIENCE, TeehrAuthProperties.DEFAULT_AUDIENCE); - Supplier brokerSessionTokenSupplier = brokerSessionTokenSupplier(properties); - Supplier subjectTokenSupplier = subjectTokenSupplier(properties); - - int timeoutMs = - PropertyUtil.propertyAsInt( - properties, - TeehrAuthProperties.REQUEST_TIMEOUT_MS, - TeehrAuthProperties.DEFAULT_REQUEST_TIMEOUT_MS); - long refreshSkewSeconds = - PropertyUtil.propertyAsLong( - properties, - TeehrAuthProperties.REFRESH_SKEW_SECONDS, - TeehrAuthProperties.DEFAULT_REFRESH_SKEW_SECONDS); - long requestedTtlSeconds = - PropertyUtil.propertyAsLong( - properties, - TeehrAuthProperties.REQUESTED_TTL_SECONDS, - TeehrAuthProperties.DEFAULT_REQUESTED_TTL_SECONDS); - - BrokerTokenClient tokenClient = - new BrokerTokenClient(HttpClient.newBuilder().version(Version.HTTP_1_1).build()); - - catalogSession = - new BrokerBackedAuthSession( - tokenClient, - brokerUrl, - userId, - sessionId, - realm, - catalog, - audience, - brokerSessionTokenSupplier, - subjectTokenSupplier, - Duration.ofMillis(timeoutMs), - requestedTtlSeconds, - refreshSkewSeconds); - - LOG.info("Initialized TeehrBrokerAuthManager for catalog {}", name); - return catalogSession; - } - } - - @Override - public void close() { - AuthSession session = catalogSession; - this.catalogSession = null; - if (session != null) { - try { - session.close(); - } catch (Exception e) { - LOG.warn("Error closing broker auth session", e); - } - } - } - - private static String required(Map properties, String key) { - String value = properties.get(key); - if (value == null || value.isBlank()) { - throw new IllegalArgumentException("Missing required property: " + key); - } - return value; - } - - private static Supplier subjectTokenSupplier(Map properties) { - String explicitToken = properties.get(TeehrAuthProperties.SUBJECT_TOKEN); - if (explicitToken != null && !explicitToken.isBlank()) { - return () -> explicitToken; - } - - String tokenEnv = - properties.getOrDefault( - TeehrAuthProperties.SUBJECT_TOKEN_ENV, TeehrAuthProperties.DEFAULT_SUBJECT_TOKEN_ENV); - - return () -> { - String token = System.getenv(tokenEnv); - if (token == null || token.isBlank()) { - throw new IllegalStateException( - "Missing subject token: set " - + TeehrAuthProperties.SUBJECT_TOKEN - + " or export env var " - + tokenEnv); - } - return token; - }; - } - - private static Supplier brokerSessionTokenSupplier(Map properties) { - String explicitToken = properties.get(TeehrAuthProperties.BROKER_SESSION_TOKEN); - if (explicitToken != null && !explicitToken.isBlank()) { - return () -> explicitToken; - } - - String tokenEnv = - properties.getOrDefault( - TeehrAuthProperties.BROKER_SESSION_TOKEN_ENV, - TeehrAuthProperties.DEFAULT_BROKER_SESSION_TOKEN_ENV); - - return () -> { - String token = System.getenv(tokenEnv); - if (token == null || token.isBlank()) { - return null; - } - return token; - }; - } -} diff --git a/spark/authmanager-prototype/target/classes/org/teehr/iceberg/auth/BrokerBackedAuthSession.class b/spark/authmanager-prototype/target/classes/org/teehr/iceberg/auth/BrokerBackedAuthSession.class deleted file mode 100644 index f6046b288a6903cd13c28644d59b8948e0c1978c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5275 zcmdT|`*Rc575=U*Yi(&Az!->l8Unr>oOG@8qNYnR&zDv@`yxKrQOJ>^tpugnTw$n_1>`W&q{qEhBERBt!Go8+K#_N0c z?z!ijd(L;xJx71|*OfN_Y{A85+ zR%_*xR6QZpVzY@BLiD+OO2_lrEYIi&EGskBNIso#Y0g;jvsQJTi8jTiF_+i= zu3(_##7u0!160VhCuAm1yES@D)q8A5-mw%@&FeuEAHYKb^Bs9CFLN$E=BDD37)j;? z#4g2dvxyHXng&OXI5IaDAD1V5k`O<)o9IBNfa#-3t0SG+~X)ZU%zo9?mXIulUS$XmW2y z7<=(y1N#)`xWIkt^j1~{Qz`Q31UJ?*IJmb*T1n|p>rK9>5*N?*ci2Z+LtuV5pky5s zh}}+>4)q;!#LB1KiYPzWtbKeZIk(s5Hk_&1VLWEwaT6cGM+KT^)vQ7#Q?X>Su<2VV z&^Nn=`#jip++IUv=OUHLj7LmZ7-ni6l`60*kmkEIge0Va5fh^r6PTyLp<}Pt!x?v{ zGbaQV6nB?@>`?eWYGNGBR*mvx)~3G|O6_zhFc&rNDvM~JYe&oBFw!_;AS1A33X#*Y zh=F4j*M=7$1_-42Q(^PS$%M?RL=i$3c@xJ$jSV~3#nd^jua?6GP6@1-+R~d!$x$m6 zcWL_4_F8<5_eb2zRZXr+=S?|3-8DJV%{ytCY_if#aw6-V@(4mWiKo^5c3K&!ENJ{# zZrLBg$MFdRpEU6)JnIAGw)c}x3&lI6Zo*Exexmr}47{gH`(Lh<`?Zv*?<$ijaDK+b zb9kQ4$tiGFSgx3E81J7e`q|FJdiRtZW#9#YVIF-JV$IanvpX~EJt z-TPZR1mb0T?pRTE-px&@xcgu0r&Vt%IDL01K3 zw>tL;ZdgzT=Tb)Ew}$EUG|SQQ>X`BF7q5~jYJS=Ii+3jz7q6|vI7={%I0LU2v!ren zO4)-aPNXt9suPN5@=ikTv=wNVcFo)gtylS!iLxt`adWO?W&0&d-DIv7S7?kbX*tqq z5_5w1O$fim8wRea67@RmIhBa*^l{#rF)zx!s&%K8a5D~X-oq)3I67BO4+;vNt$)!=JN2CdUGWy+AADPxB9OqsJDKj3_!FRO)aDDHfPJ3ZI@kUFs;;C~vJ6={{$ zVpVh!AL?(|vThO&U%?~5CG=jxo=X@UXluQULshsy>MCum2EC(`I&UHWt!(-q))t6* zoJtm0g&*NO<X^@8y84&|JyLcdCXDM%a{ zv|e)_3)+K`z)gfBfl0XM5R3$`mA@~J)Toa&`eXf7oIKR<C-F>skdsg(IEhb3f>g3S5DC1AGv`ou37?Kln2Vp|7u4Uv3;3l*dXn1zN+bOp{vLLLcnEU; zVok?55OaOpghh=;3(FcI*!{gqth~n4WKsVrN9v6#k|cwv?@fLQ%Y-2+|gz-fusD_x1I64@*-|%<5Sm)M8 z@tH7er^?VVX|4B@)LSEjOWVnsTpwNX-_$8@C8&Cn@6G)zah(~_ar7nSCgD3&c(a!Zzy$e>q4 zpN=g08G1ahumss_)SK=Kd2dx+hW^|_{y#V;bPV7mgJ!zZucdI5a2wJ5iwwo;w2m`4 z%aD+^?>58jT;*{7rNFuu(}1wTJE!A3h6u05-P){O6%3;|snrx|Vn|Oa-GSL89^#RPIUSEN&oH>}o71$smhgQH zP^q`p_Y5(73)3{MchssyUvQpV)w*ygXZB{ZQ3_EGsyB2JD6W2Nh=;;89J3*9C#*7C z=dK$*2ryp$3lg^eSJSj2t(PrJI3A>MH9TP$+2?!!2Mvo1XAc&oeE5WMnWj;MIxIqy zhK3yNGHNu@TcanAQM$>|Zc$B6dZ!CNzzSpA&=VS3G~RxJqQAxJG($M0%%ah|*-c9@tVs$}O$1i&MZ?+WyN%_eBEq9vWmf z+Nci#%DKbUM{qMxpCoA&`$S=E7Z?7g-|x$`$R%(Kw+Vqpe~X8Tofm$JnRXD1cmTsF z>~}~-1w5>rI*M`(cRi*hOw#)p`R7-TUpM!>Glm(;VgmOug=v)OEJ@}TP{9%g{s3$c Bv7!I~ diff --git a/spark/authmanager-prototype/target/classes/org/teehr/iceberg/auth/BrokerTokenClient.class b/spark/authmanager-prototype/target/classes/org/teehr/iceberg/auth/BrokerTokenClient.class deleted file mode 100644 index bca23d112b001072234bb116d430bb8182474a3a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6356 zcmcgwd3+Sr9sj;$li6%0*Mh+Xk5y4}5Z1Fuf(Qv9i{U66K(sbac86qOv$M|3a)?^A zm%VIltqQHxwo|(2vpBCGKMoxpt7ZP zO$~ymQ4ms53ss=Sv<732t`Au;BdHJQ9BDacDAsM68+EIXKbfAip=Zb@G~_th{9tOY zdrn4oVsbjRr0}gvLA^ke`#I+r>DUs@9$KblL#RV#5KRI~&epAdBUOt@n5^J970s9; zPI7`1Wr`wL6>UW%UzpW?DOv(=76r8HyG!@fvxqjHfu z#~WcnX{8hws+f-j^o?T->!ysLIX-_m5nGtET&atVIw3!qD?KU}ViCn%wqWJTMXLll zdXwgGY`w-{v9=7SV;i*OMp_`2VmJ*LnUtS)4B@P9(ODN^iGsL_rMOt2{dMdW2yeCw zM_(g@Y=Q0BLEwyHb9pJASH{f|-(Dds!*T^HRIJ1snde9Fh?&~DGB=Pm>>=H<8M5a3 z{8<+ipLHybC}!0q-H}kyht&dsl`9f`0?U1dCAi^->-6rD-NzE1A*{tE3f8H(6ql7? zUtR;nu)ArNjV}0I8pP!S&6jj|=+=?&z@fv1xgh$9K??|ere$2^21%F*tEm`3Qeeuk zwozvyFzDAB&JZcv(J~J0RpO9RBNaPDT0>W{9t4i60hyEnC$=0xOU90Y4GK0&7p4Vf zlyjk-r69U%x+PjBm3BSYHUZo5i+M%{rh=@BD?yAne*77E#1Ghdh9YZ6@SqROhdH;* zspYly7E+`j92Gfi5;)l&GIQzFVlAB>V4n4viJV*17HvrmvBVKDSC^N^=QSQjj%B8m zfwDzlnpX<5W?HihyTg&iFyz+Ndi&78($&p!lxAMR&lpHX~r>fqEEA$^wBQdqvA6Xu&On?Pv7DM zaUauK-;y;f-R{?%em!d@hYoc(yYM*`_hXN{!E0**Y-W}Q@u0wb4=AJD>f%gv*kC)A z8H^T-95t=zMEn&z%%0}xJ!S}dF{)rp#XjuMcaLttj(a#uXg1d`lAwO_N+|C7ok!X4^Nj8 zxb@Jwt91<{!b|>!UL&I~&keJ;@6!g-jM;Fnnbgv22x@ZdPX?SJgVMG29^F~~8H9mh zBjdQmQ{ahWwfw9ZpL+k#nWaUpE^7OZu3wixLyGB=biD9iZL#`>$BWLG^p+L+?P=bk z5|8*F$+T9&ZX#o_F=SiHn}IX5%*E##FNEB4s^#S<&O%$`2b844-hzbJulN>MeFXJ~ zldSv|rq4-xNkV^$rNsGihMvonvNL>VS>OI^ZYqe6T7M~4cv%U19=fs@>-<{lmAt;(`;S%OGm5&fPp1~aVP z7Bkf-m*V*%Usix3Jzhr=wq8o#%}u8LcY;vD%vni&u_4i-*-HY~=3eH{k}9y>yqyn{HjRbSo-?ctv2Iw>j#_-ic+zjg#nljhs`_%>>z~*QZ29R2!7| z=#a{}B}Q~cMME`S3*o;E7a>^nMTNlhL-sQ=o5(ii>FKrRi?yU<)|Tikz*H(1F=dHXH1{IVjGm${DlEPB|%CU}`+gsAaMMz?y>n#~NSYK3g8 zFS!8OicltK`durQUY8hnCA)cy9Ov%@im2s9#Wmew;N)dw@dbiuRVYR&r%ei|^ul{6?OYPoWx<0rA{^ z)UDf#hE6r2h8t&&AzX>IB#-YXkW(Xp0)0ZHsz9IATr-MjVR%NQ+N0Y?(K3oz;d8@p z7{mFMh}4c@UIiXNcSIe*;!*UDVO0e>>mtfQOpPcbShEx1NL~0%5yjoGb~h$-_~uA( zAFcqrxk%j@h?Sl7Z4W_-)Q@6l1jC&Tk%sn&5($pN-hoTn!<)lfN3gB4G17RL5T)EE z5Xn2C$vvUz&*QYYZI3iX8u#Hgu>bqW0cj-f2=W}k$0XbECq{6W z7{RAIxiWk&2Q?%3?Cfx)=^#SQ;hLLoN1bFCen4{AkB8PpnwrC-7}<-@-%qCp+(5Yp zZKTg&kj$XtTW~U8V#$V8bm#|O>J`<@}EGA)zIE9Zrv(PKzSSI?gT=eq|CXH3% zN+d)MePSEdh#UFfa3`)1_dyf;*s&j^ho8it8^q5tjGxBEcnaS~BRSS{^aH+~#+fM3 zxN|u~@Iz8fGK}-VPU=UTW3ZF@F)1bvsh^NyqL7+Kis?n_L{e2QHG@>OOZ}9T;!@9& z3cA#D_!+Gxcg)AnnMyU{XiR3nSC0cCieKY5XrN{n;G_4G7kdbuanvV zC9oTT%6oZ~7j>Pq2w+mXEGdf~v+02x<8TvWa0}yXD>c2EF?9`neJx+?uES-xo^M3g zxlwSgC$?)*GGc7_-X$e{C1G(8(pQz_(MrEa?@HTxfH%mUFK{F+ItjJ)A;Ev1{luio z`2v6EhYGw%SO0^S^YHdGmn~4c9~TiDpLh`65^#l^M{*GubCvj~yY(gfi+}664dV}E9jXEHYqgAk3u&@jY6ATV~MpX%wBZZ*=S+7UB60z+G-WqNN246LqI5f95W zFMT91A@{V3=a^RGHGz?=ZMmLqc~!mTGXX}>rNdk=6~s@qQ%Vtt@03diw5(OivH)WQ zX8R>I$3A9G%6D1>5*@KZ6-6o)H5E;Z0!$KE4mT5B=DMbBi5@z!2qeQCXtY=r;3k39 zt2hyvbxI&sQ7~yKC%_DW>3%!kWsYdpad+KJ1La)E?z;H};%J#>1EQ8@0DV3dlnpL}%TqQ8y zuXg?R>32&w{*TF7?H&+V{IA(r(Wc7ueT6D2&1tGCYYNSliaA99Y=f14bBBHLu~}GM z^t=`xqhVWh7dx%vLQ|yzEmc&lfc6gwEL~zRdgckUeNQ|=;{>*AnJRmlazH=y!aoUV zABTNHk4@1%W?y^JXLB^4smQ8!F$)4j8UUF5MqsX|x)=_1`%%1~!DkDr-P4AYrX!5t z6&vdcdqePqq1qSqJM%fjH=gKxH2KzhTf{`ju~)ganNg;coG}()t@DNO5%#`I|j>}f!}1v z)0GO|AJ_y;2kV-EBwT|y(siT=3?tv*cocb@;|b&h$5Y7D94C=yIlhHF&+%>K1&$Yy z?{d71e2?S%$ZH(0BX4jlBBwdtM1I8aW8|ms?0kREkzc^ebAE-q#ql=sTh9I+@(#yY wWXf?KJw<{)0UYm*_q{jMNoJZ~;D__ayyc#I z&v(!H&hqY4|K0yt0B4H%et1x(pj?F)6#~^;v@tD~)KY`7^?h6Pge6e1!blm`N`bN^ z^_wc;L!|=0iU3rB#Q#u9@foOeD2ax+MsR2 zT_D(bL`~c>jnrU^TzH0xnWz?+ko`w$hFH+ctIw%^cG3EhV#;?xp}bT(y4v!ASDDNdnH9B;TD?jfmC z@$Bem(vXa5n(PP{jlHLVY${=qL9v{Ci(L0K70cwhwVA9<@d+#XaJsJV@Hs!V%|20n2z`eM$s*SGs{ z7Bv)O8sMv;#>}w3wK1b7(y9K8mq9ImD?$ntJr`yk{LD`Q@Rx!vaHeATK+k5 z0{)b~wKZ!b`=!h4mbgAHAzJ53F`vZIqhOUq;&+2nZh^greM2@OK};~vUF?Vx-82^YJlcG$>oHd?u(6>OZS@s*nuk)yj8{9WY1v7 z`*eykX_`o;GrEGe3)I;BuvN8fOX#DLAU<3zu&B*6C4Ne>F$g=!KP-!L*e!-~L+?=W zPFzdqTQyU52vbAv88XvbwZ5cIZp#vQor-ti-841poxtM4$+=w?aK(M)M?G#(@gBTa z*3(QYu@Bo_5%a)JOd3Zv)SgW0gIaQxIhY+`p3Cnq$kcp`id(UZ{2$SV_0{PV{dB%z z4e>5xX(@~Tp8&^ogBvP^p7G;G+%5y<4z>k_JIGDhw@e(@ABck62;B& zI_rzw0p_rlN%4=8ag$Y+5_dFzj5#r>xXX`E;!_Gftzx%y zO+gbJ-=4Ab5jMdj3U{L5L4o6n938hbOE2)U7q|Ga2eGH{uw?<1? zX>+?kC6{-C{77}SeZ4xVWisrrCeLJ_``C0M??~spI^ECCduFGR(${52*sAx)c(C(K zLQ8JaOhZ1qFUzeVnWLF_UQK`BA`lpHTK#o)pWo}`?^3y;yDi@1RChOX+X+*4(#~c} z&5=1ER#0)MuhQ}SMw1n%LN?OvtZkJpZREYzZjc$UZa!oU$~ByfjVam9h1`Cxi?$H$ zw3VsbJy)D9SKR5iIMBgl<~E3We~*>Sj|A)x%k?nMN6fTp85{A;kj4*<+}t|b5v6QW zU`Mt|Zl}lZ8tH1Ar+K?_=lV7yV=<`;%;`)QdVTw7&htwPW)-#1E1bc?EVPnNSXMBH z9xNeMdR`UdCNazzHtQ7pXhK!U=WO~RZVaY0D{GRk*SrQ!O%u_pVxAzD*#-N`hd&LX zxCCT%7}goy-Z3qirTm3%hRyk8nz#zI|Nv6Z#rM=Dr1``==%uI4lFL>9m!xOl`DFw}#UFJS+GMt4Wu^n#9H=cNh@P zeO@((MK%7$@5)>cZpVq+Jj|@XKjg>fODcYcUDf#KWI5Nfl>4`oJFMcj*u|&+%BNRU z{6;O}n*_g+e*Hnsf!44>9e-ovXu zvKJ!K@CcMWeDXjZU(VnvKnUers}c)vkWX_QX*`b4@rm>B1XpL3=IT#!jB9$*n1x!7 zy%2dARqGlL!P~eyvbnkZiQKcVF;xEK?r5-P9J8Cf(O^hkE26;#@~T9G$I7cO8aytz zXdg}}qq6$u%4o15*qDFxN5>Is4)FifW;GgIKC9;1ah$QdDjHl_i?hekdisoLa7``N z%G>G#=-51t&Ii%G4;OgYEnY0J7k$ma$N}h^L&1Pna3f>KxGEbQbV)b(h?y*L+-amJTd}{W|ZeiR4Ut1K%X^NP#?2Uck5TZPG-(`wqTKs(A5pe2?;F#0y%c%+>WwUe`0! z<;Xuv`5*AU7G;MKP_W}Lf(kq@;c*2&#E&`Rq2-1AVCKIh_T5}h(zi}}t1}|##fC$e z*RT(l3*3*HUDBslMfc&FGTMZvcwBofuH+m^{o`p3;38tYm^SF;sK)VrO7vkK5;y?^ zwsj@ydBpo1eo6$?=I+-^Z~u&Y6E^A^cwXUDj{+Z|Lnyd1z-4*p%}EmMjV{>55#rnF zk%aGx9)dsmD6YQ)vku_K%?AQzOAUb2vL`)maV1 zc_;kD_S=sfx=D%&c2y;@CUy0;E(tdugiHZ=cUA-MSvrJ#ozI= rd-t#W-G6d-G~CNlT9gTy66bTqW^{>~3VDX!C2G$OiP>VV2qE-8N_OV) diff --git a/spark/authmanager-prototype/target/maven-archiver/pom.properties b/spark/authmanager-prototype/target/maven-archiver/pom.properties deleted file mode 100644 index 41b0911..0000000 --- a/spark/authmanager-prototype/target/maven-archiver/pom.properties +++ /dev/null @@ -1,3 +0,0 @@ -artifactId=teehr-iceberg-authmanager-prototype -groupId=org.teehr -version=0.0.1-SNAPSHOT diff --git a/spark/authmanager-prototype/target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst b/spark/authmanager-prototype/target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst deleted file mode 100644 index 8cca880..0000000 --- a/spark/authmanager-prototype/target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst +++ /dev/null @@ -1,5 +0,0 @@ -org/teehr/iceberg/auth/BrokerBackedAuthSession.class -org/teehr/iceberg/auth/TeehrAuthProperties.class -org/teehr/iceberg/auth/BrokerTokenClient.class -org/teehr/iceberg/auth/TeehrBrokerAuthManager.class -org/teehr/iceberg/auth/BrokerToken.class diff --git a/spark/authmanager-prototype/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst b/spark/authmanager-prototype/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst deleted file mode 100644 index dd63896..0000000 --- a/spark/authmanager-prototype/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst +++ /dev/null @@ -1,5 +0,0 @@ -/Users/mdenno/repos/teehr-cloud-core/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerBackedAuthSession.java -/Users/mdenno/repos/teehr-cloud-core/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerToken.java -/Users/mdenno/repos/teehr-cloud-core/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerTokenClient.java -/Users/mdenno/repos/teehr-cloud-core/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/TeehrAuthProperties.java -/Users/mdenno/repos/teehr-cloud-core/spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/TeehrBrokerAuthManager.java diff --git a/spark/authmanager-prototype/target/teehr-iceberg-authmanager-prototype-0.0.1-SNAPSHOT.jar b/spark/authmanager-prototype/target/teehr-iceberg-authmanager-prototype-0.0.1-SNAPSHOT.jar deleted file mode 100644 index ccd6947a74fc8fb5de21afcc4d226f8851194321..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12971 zcmb_?by!?W@;2`7?ry=|Ex_RJKDfKPYj7vHLkJ$+-3buf-6cT+;Uo9%Z*w=hcc1;^ z>*qN$ea_TdRo&Cwr>gr^1b{=pfq=k(fXFs{*9G}YfqkvZNvMi2O3O>WQG6A{col>B zLyWFjpHKVsXOq|K_31A$IT3kjNeLBICOOF)x$!Ys03*{3B7l)@a(t>@nfWu@=AILi zj2wfs+>C2QgYu;u17Jes(g{HsW|&rPYFveRfeo_!N`?U-K5I5Pwx^nzkyLP!l30ze z`!3U_*wA!vY+wu$pBS9}+G=poPeR!?s3lK)e4UjQj7xuT5q8y&=Eb5cqGfKfsLt zXdUBEZ*7h2jLd;fj1ErrF7_^-4!}P=`t{Ej@u{%x!GnNs5rBZO{e7ENe|E2^lf5<2 zN#s?l+@D&ROl*vtof|bheN^W$f7mr~WwZ<9^s?qLOLd#FCgGrdhEM_%L8BJVKq+QA z8r!AxG-X9!Z->n@ww_nFX>{!?ZdBK?K8vCUwfVR_yRhtApVhRG+UjD^z&P)BH<2~M ziCg;P_=)9W-F-^nfNxCTVD)a-8oCQ+6Mq?Bd6FQUo}ui0#`NAaP_pZ*`;Hnii^yvc`_rl;qioTx6+JFFfXf; z$9~aR`@J0;#z2FJUN<@(CVG=B`NARvu7U~+Atu-yS!S`sj<1nG0`oQtsK3#ki(SuJLj z>A|1Cm{c}$*=0N48hX0QgcbqCPgWZ9OGfbKW&QSg`2%^-Ln+Z)aoN3extZV-#bw@j z`v!JbMV^$@NI>gd0=G^yXF|~-FouwTWxZ&S0JZE0tH5hV8JIHP*2Wp#$~&tO8a;n) zR|{i)b~!&4wucn3mljQ#qYF`C!*lU;?Yg096V$a(-tEt12_CFw()I6)3P7pnsA%oE ziOx5lhwu@Nu78j)Et-LdMg@I}9jyR1A4XqAT zl(K`L$26XM`U5)2)2GR=G!+VP$~R>9k82Ov9e1y2?CF4(O@f1B={UD+V-WBqWSft_Gg_DV3A`w~fSspz z7V=n%dl!vWRuB@KlU6&Gr7QFvD%MI!P)}lB%6-aL;Zl(vZ0syFQPB|70Dl(Z@DFb< zN6@))gFL8g0m0tiejiwkP)b={>1Xo8vy6t+rQ8FWJ#gFZL3y)X>2j_rfCw&ks@dZDn|CCm+1Hd^Gx$B`Bn zv@6GASAwYusqsa^V)T_C^Ce=VzY-aI<2^Zv%@U;k{+($_FO2X@xnWvVv?J`?wRk7P zm}FJjzTijR((5Ca4Xv=X7TF#?wRXTag!u<7^Ds*aJcUEJbw+0AWXG||2tf+Dt=<%{ zXl+@QZE8x;d*zW0YR`vkV;W4a_^7*WE76Voj5@Jl#E!y^n2YJw!q(VMxWzIL;&la; z`gK@_3FZAd3=S?Uhny(n&EUFRs?1NEhVByM;9Sg>F?JTQl3xWGTbEA(NUWS}kDloi z_R5ZI-&lV@H-h@g@kDw7Ch6HB6NrDC!4CgrdnkGGhd&zh%*98kDnJjn`T8fcTCXQ^p+- z?DEi`)GQyZ>FAs$OuJNyyp#CRP7{3|4c5y@nMv`|LXg_N;YJ_*!stLiz_kAsaYQ81 zZTQCUAR_|n?6a)cWKNodde`{OSC!(8Q_oWAJR4??Rl~any?GVR^DLcS!X4JTbSS0^ zv-e>DPrvAqC>1jNBlU$QWoo3|mM?biJPmVvhVhc^xIHTp7r@`NwqEMOAp}c z_uxvkiJ2N$GJwkkNc5RnwDpoFDvxMM8jL37xrd7$Gg24TqeF2vNnxyact^)qZpHhL z^5jOib&%Nt>nCktv$^)ez{Hd!^nSW)JwbhCtcug`qn>tT;HySs>X$c;YNSc^N8z_7b!cNe72o$eA1eaeYNhRX z7C&t?vZA>P8n^Ua72(b-;p$(}8(^AkI4!QuMvKh_7Uu@^tT%gU4HM zAliuv!n{Cu;G%YsG73V}l0EKNaHS^pG?~F*vh~y?dcY$xD>dGAf~zV+P~-=tT|P`2 znaol#3>W$ojW9Xqcr4PS>P}+0L33QS7Lw7Z#Yv;4f~o>@-~e>ul3k{XZ88rn{KLa1 z9ggq;JVrc3{8V{oxOv9W0d@&l4T;cnmKp`;GBLMX#j1rY^)efW)6?>1{f;MW6*A|G z2_PC)+JdGm*V%$7798+4>UdQUlO@YnN}rD2xTZU3wTNMY)|kPV0lRjOzhB)hxBaY^ z^xLpPqz~s%y(CeE$b|=vG)6M;Sj2p-F z`XyQ`{;)So8x*S^*{c~fBhRok3FaaMY>Fd|g+Ia-J4rnt26}vdRL4rhmmbvQH6KR$ zf=-R2j$1Jxd9+_^px&YzSm6zt2LIv%a)4Q@(=KHw)QT_g zLu|U)OJLpDK2qbVwY^xsxp};DlCH}N!ndB@OkpLyCtT?VNEkoX3_UcnE=+ltYMvw2 zO9vbBoGd0c(UkHYq{pD2VD!P6d4lb*Q4JWpr5iEgCz$If)j-Ph&|>zB$kn}rGAvgZ zL&zY_0VTfUkGRxZVC;GL)Hm1uaK@VQX}NKF7X84YRy1>COb|HACm7AYQRu zC?wttM@TW|4<}COeCHuDhQV8xJ2~vNjHE05f@48 zi?n1uoP%vfU3)JFQzNOXoF?w!qJ)L^z|Q@uIQY{88jX_ zI`9}RFodE#dq7jZaGf*szAQ7iQ)Qv6Gun5k>qOjO?whE(-uPw&y$Nw_!HZ zLV}RD3SXu#zOHg%J(g07b!GP6p*(Vg_%Y@iytUwZ^3T_30YYz!JB29V>ZJTY^^r7o zTU<6$V)!;MV-sp3x+Uq!WwyFD{=jI0=2pyTFu3Q^Zb&}d(iY?d3E7#Bic@>T6L^0I zv=WJY-p9)V$^?Sx@HiX*GIAV)61IKN^H_r}HjafNpn(uVE*@y70GLzd-gUVoeHN&u zH{jlWoL}}ZemD`;q9+;6?RQP>MoD~w!}ZoB_qZ297{Q|%WHwyxAwHe(?|*-<8;GVf zo+1>@TJwbIKU6t+Xpk}EKTFQ)p{5)yy7e)*3`J*#$t-o){_w2en&Cq_nmhoum z+8M-zgf&acWDEy&R@e$3jFYtKBcg{&QanqS~!5$xJ5$I*HTsoaW7u{UY# z_yQD~wv+sK;#)vxlfcB$Q0`n(@(;81dd4ltQsU(~=Ua9*_wx?tgY}KoPdDNIdcKI= zob9-hxTA;=vJj|f>He#{mIKP_UNlYan=P^}VELSUwrwkrqxg8pcx5OSv$CFx@&jlY zE@T(ntY?D^Tx?d<#(b{z21qi>=&T6fdnsi>mU+wl0_oJXEISATy0qar={fJf6Upf> zkcO6X#v@#otNSBl&nZZmoO$??u*zgKBaJdn57^&M2wWIFt2cfFL+8+x5~!J-PN*As zgL0NJANNCAzuGh*azK+zkteSdnVU8+GgsPwKyOGM*>WKwl6L@a^q^=iMYRY>m1=#I z-Ao(bIxF5$07Yn7EQtwhjO2(ireIi<0fu4N=0tvtQa9R5YFbVv<4m^@SIwJb8@fy3 zZ68((esJlf1lag!XSMc-*DI@Jjh%R;(I_IYpl3Mijx#{6!vC1U!ess}4Tmxz}2~ z*>)^5QJkQ2@fF0^UlC)SVbrZ5VdktLb7D+{$`RpBpxy8+xMl+Nz$!0lLYY6J!M4s3 z&pVybD%ETqE!l1>xxg{tHcFa82U>gzw{cJJ$hIzC%Vzbi*kvEBn2r^I@{+wRh{sQl z(fW{rNi&83$FEs6%9mNKLK`bUS@*denV7lOz%mZLN~x{6?wDE5=Dmo6NzU{0ci!TZ zWTfi#S%+|}sXOEd zEs@CEVql2%qiN};bZABvY)#5}=5FwJ-nCuPU4x%g&G^lFdqGrA-nm?eXA;jcC%%J% zl0z`kH0;@3I=Oa`JaXAj?k;dV+8$94ec7Rs?6y&(S1`9-&R4W5rAFFM3_G`kz>ila zjWvN{1p}CmkU_~O>*5oYASzzNKyVc~>lRsMVSZ&61k4h)h zsLCK|mA3#E6(Te?``U%J^=m2?kJ<$fbd)coxGsu-b?Y*VV>4CWgC;OL?78Z9D_VO+ zmh!tj%z{zzxc4oQ3Y)mZB+j3((5mzTNX;-a`RG!7Z&xC?kWt=pD%vE*HHQfj0p)tjBji zY+vu(xfSvA1f`$M zi2>fe{^rx2Bd?eoYxsyVx4U~K3Exh}lchMRn$qf#fZAp>Qi|WQ914&`w zff@k2zR1%aLTcpM{niB@AwYA$;6UO9OXh_*m;fkaAR(}&GxT#218tny z2Ro0I1cPLVl?;NcijWkAtcsG10b7tH$AnzPj#Vt8#e>DJWod1x2$+}|n1{6aBsH+- zk@2$fl}t|8hsPUV5wJu4B?2?Th!HV<_Ezpo45J5#?W!=J+{JDgvd+v@tWVnP)U+o% z#zx;A=$c9#Mr*0AR*Cm$#$IdbD(vw-3R3lFfxXe%b6Rut zqg*F*4!`an#{Sikt-iX~lrmmSKS*u2g9Bb%nCWL(Z3m7Ian?6<>TUaQh&rR*28miIq?+TikFSZ%hgZR0*JbWta0XN3w#3!YG19Q>V{?e!J<(UodJ7W3KXSE+x=!o1Y%2A+A;Ec z79Uo$f&|7usZCdYZ={s8s|OYzHF{q`w3ZO;nr55{F7iQFV_W9PSB!1Shv;Xm7 zfwwya$+3Fq)3QNEF8p zoX2-wY>YqkE|x&& zKSQM0loRziVU!{Bz}X0rW->?-P!Uk+=sWnlRL)T0Wy3`EuDzn8WvFFw=N^qq*QZ?` zze`Xbn)oFB!&xlkGoGbq+aGN8e0*&WH+PHnAYZ_F;J_g`37ZpdX{%eM$BHPUOd*+#s&J!D|d~Vh&@{JAQ~{_A6j1h6qgXl@P?j=*W{I zkVQ$qNKMu~h-r=Nu11Y$?61hHD9B#qDB_2ngHZpTbw%`*f9#_t@dy>l#ep#PwFHtK zx@K1#dhYT8L;)n`S_CfAKFqk=3M7%X7Eo7@Pq9q*u)ekFM3F!#BsqC!lPEC5E(Z{+ zP1skWqAin%%&mEZzQFENxUUN1P%bKoacvnG%%SVjJP9+3)hHL`U4->5KBQq`a+f!7 zAeR;@cFn%j;W|;8%{?B5;bpNZqXAdqUJ*rAZul^hYKy99&qs0(WX!PIy~hBYpa8gj zXWar`Vv;6M)sPCWhwdDtJAF&AEJRi76e@6`<9ZAsXz+FiC$*BSEWFi=!RIu@f1%Vz z_YpU|qTOAx%e7$2JjAAg#M0}60S3hp#qg0YFf^`$i{LnpZ2IZUxT%d_UtSKt9q`ts zfbrDE7Ve~nS&q8)%3`u{<8CPzVgF1#RXM8cIJVWcoxJvQ)2ByOf=#F8l4n(juvvLr zdQ06pn@r?mk3I{@a>+hG4@<1}j6?N1onSG)`Cf55>BX*8w_cN7h#Kj7^+U4tw@EM; z#9yEg0v@772<*CHW+#}vnWKE7w;Xq+!%E+oAWV@=u@Vfu@b|gD!^Z85ZtuTjz4i^$ zNFZi@Lh%gVlG|kr^W3HATNzfs@r!yQ(AT{L>}p&-25pdBu(pRk#eMXTZ=Z6&(=z0a z?g+dj4>%)@qxeYqMUvCMcQZary#K(#871zLXZe^oID|1!S1jcP`d?>KgfV&m&+9S8 zXB-d^roW#_e^c=fWF_tLU9rgo)mv1VnpIm~oaN7+14Q{y*P9q)~_ z>T0!>i>wSAv@ZaE9&SQ4e!mDJJaN{M>>t4mBKs34YoE!VpJdM@`nbYfMOrG%sp)ND^jt zfiaalW*Lz{;n*mG;sBOF4x9QNysV|B-~z-08)7;o%7U}BB&Ct~bOaUT1f5x-gz!YT z{o1m835`gfuGr@8$8)!~CTWRSJrnGyc#^m|%V|C}%;@RC)t3D{BWZ{U_}j6_`|a#AEdg5JI(1%E zhC#HD)Uc(3Y)J^0Z(+>K$%5c(WkpJ( z_B2VSpM*{^BkUUG7B%09EbQMqqHtFWi?9>net=OoP6?W)Ti6s`@Ir}wVABTgFJXM@k%^*VrN1tdnIiO?AUk`?)bR+DEabRrF|y2ea+NOyyXZ+(v_ z$st#|T~y-EG43J{1L;mMyPJ$;pW+O%w%P(UAqsbAeni^sWIq*T77%U@tRAk~rH;M- zQnXoiiegS5DvL_dd)GKOA+VUFa@z^TaY(-Zcs$R<)S^zGBKU^d+luAJLK&wfil7N5 z`>?1=m-myZLg{x+dI5F!NwKFy+ei!(K(DUGmt5gR)AVBDZ!pQWU~Es7%+|%OD=yl) zk%=A-`5}&wT3y{>up{(px}lY@f-A!%Um>b>`LW0eY1htEQFj`d{R00Cm zbS~BPd(gi#z7f=Lca{%NVdp<|ADyt*a1^jJr2C!@A6SkE`C-BxPT1?p4)Y@6Ah|Sl zJRmDk4z34r^^-TS2^Yu+a zMVr6`nmak;;uqKP-M;(m@!i7qTdGrrLED6pwSXa2A0h)oCu!%nqI=kP!glv)zS=fL zv7k*hBkA#1ngLgugH9&T>P$(=GRWoka73lb%csX?VY~b$qIB;n-lIgwl`Y3RDT{to z)h9x~+l)QQknFm*ZXJ*LNXIqYa(C!!IVs*&6_Ts3Fqybx$huo1>*?wu_f3TPJv+07 znXbLe=7_Ybo`Da3-jWGzN_on&cJ70w z&Iev^z_}V{0ZI}Zhdg`Nj-NA<2~N!QMwR?UrawTSYJfu_hN)aDq!V$SVZ+|grY0T0 zLyCf#tOiH3%?C#L#QTj>crjmO!apIeJis5e7%#HknSMR91eDq*nFA!J4|2M9jsskm zb>nfQV0Qv17G*%rod?@m^Jsj6q8&)5_&OrcA(Mw0)z+8@H%WVD5}wv1Sf47pVGG-M zA1dn3bW>@^-Wad9O#~ub*kxr+7*EKFfa}}$^{p~W6$N*pi?>#1=P;*c&IUe`)A`4( zPT94s)#uYOP#*wW_o&RQ9qRjQrl zn!6Cm$nTcEx9S$aSjwQ=WC&w>_q(jaDTX+P~U zP%Y@_I#iC9?uBCD0_7!b%i88^r@SSt#;|KbNe7DDM7*$P_sUm9KlQ@o_d^bE1?ZR& zH+IX`4Pr|#8M~8;)W0>{b9VM^l-1u_CY#WabR* z*orkeHe?C(d|~up^S}W8N2^it98Yo@5MF|AY(=xFlKTaD@Sc&om-i z1E#f$X~UWIRKjZgunV-+ZeugIZNJn)tMK?FVXeoHf%wZ;E<-y3tzGB(6dR6>io%$` zXcx}2TSs9Lh1H{DbG&VqY(_SEYhA7Go&LHw$r&d4i{`kCt{uXDzck}C^>{*}*xj@Z zB9!BPj1wq>3yU`j=68a*s6cn>MQf&JOe%Yq9G50g&7gp%D8I9#GMjz71QD4LehZ-2hDnh}3sQsZq3= zRPw!WEYNxd==2iE>XG`oG_#uYOmCZq^WUC!dpVMO4dk@Qu0rdbnh*I!T=~O+6Ggly zk9pq%#~*u9kcwBy^GI}DW)C}+;>=@l4_eF%yMtP*@<@6RLj4?}97+A0u3Sz%k%w_z zv_qTXIm;lkmwU@VIgpfFi1DrH;0j$*PA2+NB~>}pBP@OxTT_`&98IWiN@-xIkKi>E zRNKjm*h;&{v^rdRxaaT_16-Ie_|r2Mrnj7kno_D$GI9?I|ZWTqaMif^;i)(oYUf>pFp+QxKA5)$;LEKR#(CC?grQpKluOwe~G8%lD0C#O5LqRbQ4Odbc^2esYzt@kZL5 zREyEob;z-a<)#BKBzuU*HH`Cwl_nh(E4?TN+9K*CIb)_+XpgeZgs7o(bDW2<2}^Ck zi~y6vl=FpUeUdQyGgi41!q6G;P5$Nz*DdAA(+RUe7e!zLevOY!1avP4LH*W}JfC6B zq#jrAP-S1?i?gLRb#*EufV&g0Md#7WypR9|2|y@C3Re~cV`ft1ikd-;RtnFfdRr*C zlzCV=Js0Y@JY&&7!AI>|KR|3a5NS&@Fk)-k`A`DsMcv~QT+n_7ih-UE@s!{)@p=*U zVZ^qXG%%#?+T-zpniefeO%o4ecagJk)CI2-BKQU+-p7bP!Q1w)Y!@$Y@9A7!Q=AlZ zy&s~C!X5&?*Qt6DJ z)xiH<)zKK+9J!7dftk^fR(A+d$Uy-RPK0(;z@~zvGK1NgpBt8iI+!6JgWj@NC{qN& z9i*wrBeJ%7wTNQ|o7SthI)pIVkXq6R$*@fu_VNc?@=yLAjOcgWv1IeLfY&W1xYs!; z{yHarqaXfa6#g&NKnHtUCJ$Si6jddK6=oE_iw_YM9fNk-4f&%uEam7@%qV9i9j$R$ zcxZfedT^QFHrziZy$h+Ce^lM@-Pk*Co1#klAVJgtV^XY+y#paPn4qXzstm;5A+EH) zdbm%Fld`-`+GGN2sKef=cHSe{3F?3e)f~B3C$VLHa_|qInue%|DcC=tz0si8S@!FC z`#pnNmO2UIrL|2@G&E>HS!)ObTdy_6`J}!%lkH%I|KZ`#AVMQ6%<$9^X3_+Xm^@To z;s@KIuy5R^zKv==YoYs|oJ>B3lc)`1k6f7y+sMs?|Dt4)Scz}|k6 z;<@u`3<&p2%np6->3MtCyO_+y|OCb-NM2pMed*m|BVipkI6a5**LQhI%T(wY!E zW**~iWwmrV&RY-Ujq^$C$y)ZK%Ffy`x(FFlXA_F~OiagLJTVXhel#7BgKAEbp~huxNGj}5 z&B?lgyYJ;VjFSLw)`vd(_2u{2}fCw0!R=*x^udOFF_IPTFYSl@{A#&Xv+usH>}$q0w{@5OjmPA zs1|rO!w5LorKvF@hFe)-M^uy~O4Pdm1QRdwu|7b$Njz)bvr@mSzV-@iMF1!m2Ke7I zuwPs7T2MgFM1Fq%bGrTs`*U9Zr2PKQ$o?t(vlInVzkdD`_7{HkUk(2!9`zsaKMdEu zR{vyG|8n(r#7|!JpXH~k^RLyPh(8vQR|fIl<9_n2|13Xo&#%>AxL>U6U-;kY%|Ch9 zf0mzkM$rF+|H*6q7bkvK@{@=CONsHTl7DvC|FN?F-h-dK>|fyce+U0sQ~%l#zx(i$ zr~OOK;D1x|zbwtakB{GZ+y7^*D3J8^ZSe1X{zsqvanSHLEbiY8{7&)sWk3VzZw>rc z!pH9xem@=gWx?-t6#Ta){HL>$-?jg~WBg0|Fy7y4|C{~e-*x{U2>;S8NA Date: Thu, 20 Aug 2026 08:26:51 -0400 Subject: [PATCH 45/62] update docs --- docs/polaris-identity-propagation-plan.md | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/docs/polaris-identity-propagation-plan.md b/docs/polaris-identity-propagation-plan.md index a38109f..36ae2f1 100644 --- a/docs/polaris-identity-propagation-plan.md +++ b/docs/polaris-identity-propagation-plan.md @@ -797,12 +797,11 @@ Preferred flow: - allows independent evolution of Keycloak refresh/exchange logic without pushing auth complexity into notebooks -Concrete prototype artifacts now in this repo: +Concrete broker/auth artifacts: - broker contract: `docs/polaris-broker-api-contract.md` -- AuthManager scaffold root: `spark/authmanager-prototype/` -- prototype manager class: `spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/TeehrBrokerAuthManager.java` -- prototype session class: `spark/authmanager-prototype/src/main/java/org/teehr/iceberg/auth/BrokerBackedAuthSession.java` +- published Spark AuthManager package: `org.rtiamanzi:teehr-iceberg-authmanager` +- manager class in package: `org.teehr.iceberg.auth.TeehrBrokerAuthManager` Prototype property contract used by the manager: @@ -815,7 +814,7 @@ Prototype property contract used by the manager: - `rest.auth.teehr.request-timeout-ms` (default `5000`) - `rest.auth.teehr.refresh-skew-seconds` (default `60`) -### Prototype integration points +### Integration points - package custom AuthManager in a jar available to Spark driver and executors - configure Spark Iceberg catalog to use custom auth manager type/class @@ -823,7 +822,7 @@ Prototype property contract used by the manager: - keep current notebook refresh helper as fallback for direct REST/API calls, but do not treat it as the primary fix for Spark session longevity -### Prototype success criteria +### Success criteria 1. Spark interactive session survives normal Keycloak access-token expiry without full Spark restart @@ -831,17 +830,17 @@ Prototype property contract used by the manager: 3. no long-lived refresh token is exposed in notebook code or Spark SQL properties 4. audit logs can correlate notebook user, broker issuance, and Polaris access -### Near-term recommendation while prototyping +### Near-term recommendation - keep longer access-token TTL for user experience stability - keep notebook-side proactive refresh for direct API access - treat custom AuthManager plus broker as the real fix for Spark session continuity -Execution checklist for this prototype: +Execution checklist: -1. build the prototype jar from `spark/authmanager-prototype/` -2. place jar on Spark driver and executor classpaths -3. configure `spark.sql.catalog..rest.auth.type` to the custom class name +1. publish or select a release of `org.rtiamanzi:teehr-iceberg-authmanager` +2. ensure Spark includes that coordinate in `spark.jars.packages` +3. configure `spark.sql.catalog..rest.auth.type` to `org.teehr.iceberg.auth.TeehrBrokerAuthManager` 4. provide the `rest.auth.teehr.*` properties via Spark config 5. validate session behavior across at least one access-token expiration window From 8da14aaf33cea9b298983426733b5f3caafaf962 Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Thu, 20 Aug 2026 11:08:52 -0400 Subject: [PATCH 46/62] add encryption to delegated tokens --- api/manifests/deployment.yaml.tpl | 5 ++ api/src/config.py | 4 ++ api/src/delegated_session_store.py | 26 +++++++-- api/src/main.py | 5 +- tests/test_executor_auth.py | 92 ++++++++++++++++++++++++++++++ 5 files changed, 127 insertions(+), 5 deletions(-) create mode 100644 tests/test_executor_auth.py diff --git a/api/manifests/deployment.yaml.tpl b/api/manifests/deployment.yaml.tpl index 812406f..db55174 100644 --- a/api/manifests/deployment.yaml.tpl +++ b/api/manifests/deployment.yaml.tpl @@ -141,6 +141,11 @@ spec: secretKeyRef: name: teehr-api-secrets key: client-secret + - name: BROKER_REFRESH_TOKEN_ENCRYPTION_SECRET + valueFrom: + secretKeyRef: + name: teehr-api-secrets + key: client-secret - name: API_KEYS_DB_HOST value: keycloak-pg - name: API_KEYS_DB_PORT diff --git a/api/src/config.py b/api/src/config.py index b02b010..6680264 100644 --- a/api/src/config.py +++ b/api/src/config.py @@ -97,6 +97,10 @@ class Config: "BROKER_SESSION_SIGNING_SECRET", API_KEY_HASH_SALT, ) + BROKER_REFRESH_TOKEN_ENCRYPTION_SECRET = os.environ.get( + "BROKER_REFRESH_TOKEN_ENCRYPTION_SECRET", + BROKER_SESSION_SIGNING_SECRET, + ) # Role-based record/page limits ROW_LIMIT_ANON = int(os.environ.get("ROW_LIMIT_ANON", "200")) diff --git a/api/src/delegated_session_store.py b/api/src/delegated_session_store.py index 37b5772..9a03e20 100644 --- a/api/src/delegated_session_store.py +++ b/api/src/delegated_session_store.py @@ -1,12 +1,26 @@ +import base64 +import hashlib from datetime import UTC, datetime import asyncpg +from cryptography.fernet import Fernet class DelegatedSessionStore: - def __init__(self, dsn: str): + def __init__(self, dsn: str, refresh_token_encryption_secret: str): self._dsn = dsn self._pool: asyncpg.Pool | None = None + key_material = hashlib.sha256( + refresh_token_encryption_secret.encode("utf-8") + ).digest() + fernet_key = base64.urlsafe_b64encode(key_material) + self._fernet = Fernet(fernet_key) + + def _encrypt_refresh_token(self, refresh_token: str) -> str: + return self._fernet.encrypt(refresh_token.encode("utf-8")).decode("utf-8") + + def _decrypt_refresh_token(self, stored_value: str) -> str: + return self._fernet.decrypt(stored_value.encode("utf-8")).decode("utf-8") async def startup(self): self._pool = await asyncpg.create_pool(dsn=self._dsn, min_size=1, max_size=5) @@ -55,6 +69,7 @@ async def put_session( raise RuntimeError("Delegated session store is not initialized") expires_at = datetime.fromtimestamp(expires_at_epoch_seconds, tz=UTC) + encrypted_refresh_token = self._encrypt_refresh_token(refresh_token) async with self._pool.acquire() as conn: await conn.execute( """ @@ -90,7 +105,7 @@ async def put_session( realm, catalog, audience, - refresh_token, + encrypted_refresh_token, expires_at, ) @@ -124,6 +139,8 @@ async def get_session(self, sid: str) -> dict | None: if expires_at.tzinfo is None: expires_at = expires_at.replace(tzinfo=UTC) + decrypted_refresh_token = self._decrypt_refresh_token(row["refresh_token"]) + return { "sid": row["sid"], "subject": row["subject"], @@ -132,7 +149,7 @@ async def get_session(self, sid: str) -> dict | None: "realm": row["realm"], "catalog": row["catalog"], "audience": row["audience"], - "refresh_token": row["refresh_token"], + "refresh_token": decrypted_refresh_token, "expires_at": int(expires_at.timestamp()), } @@ -140,6 +157,7 @@ async def update_refresh_token(self, sid: str, refresh_token: str): if self._pool is None: raise RuntimeError("Delegated session store is not initialized") + encrypted_refresh_token = self._encrypt_refresh_token(refresh_token) async with self._pool.acquire() as conn: await conn.execute( """ @@ -147,7 +165,7 @@ async def update_refresh_token(self, sid: str, refresh_token: str): SET refresh_token = $1, updated_at = NOW() WHERE sid = $2 """, - refresh_token, + encrypted_refresh_token, sid, ) diff --git a/api/src/main.py b/api/src/main.py index e8e1f45..fa98da5 100644 --- a/api/src/main.py +++ b/api/src/main.py @@ -178,7 +178,10 @@ async def startup_event(): app.state.rate_limiter = InMemoryRateLimiter() app.state.api_key_store = ApiKeyStore(config.API_KEYS_DB_DSN) await app.state.api_key_store.startup() - app.state.delegated_session_store = DelegatedSessionStore(config.API_KEYS_DB_DSN) + app.state.delegated_session_store = DelegatedSessionStore( + config.API_KEYS_DB_DSN, + config.BROKER_REFRESH_TOKEN_ENCRYPTION_SECRET, + ) await app.state.delegated_session_store.startup() await set_delegated_session_store(app.state.delegated_session_store) diff --git a/tests/test_executor_auth.py b/tests/test_executor_auth.py new file mode 100644 index 0000000..c15984a --- /dev/null +++ b/tests/test_executor_auth.py @@ -0,0 +1,92 @@ +from pyspark.sql import Row +from pyspark.sql import functions as F +from teehr import RemoteReadWriteEvaluation +from teehr.evaluation.spark_session_utils import create_spark_session +import time + +spark = create_spark_session( + update_configs={"spark.kubernetes.executor.node.selector.teehr-hub/nodegroup-name": "spark-r5-4xlarge"}, + start_spark_cluster=True, + use_authmanager=True, + executor_instances=1, + executor_cores=1, + executor_memory="1g" +) + +ev = RemoteReadWriteEvaluation(spark=spark) + +print(ev.list_tables()) + +print(ev.configurations.to_sdf().show()) + +def probe_executor_env(spark, expected_env_keys, partitions=8): + """ + Returns one row per partition attempt with only booleans + executor identity. + Never returns secret values. + """ + keys = list(expected_env_keys) + + def _probe_partition(it): + import os + import socket + # Force execution of partition iterator so Spark doesn't prune the task. + _ = list(it) + result = { + "executor_host": socket.gethostname(), + "pid_present": os.getpid() > 0, + } + for k in keys: + result[f"has_{k}"] = bool(os.environ.get(k)) + yield Row(**result) + + # Use enough partitions to spread across executors + rdd = spark.sparkContext.parallelize(range(partitions), partitions) + rows = rdd.mapPartitions(_probe_partition).collect() + return rows + +expected = [ + "POLARIS_DEFAULT_REALM", + "POLARIS_BROKER_SESSION_TOKEN" +] +rows = probe_executor_env(spark, expected, partitions=12) +for r in rows: + print(r.asDict()) + + catalog = "iceberg" +namespace = "teehr" +table = f"executor_probe_{int(time.time())}" +full_table = f"{catalog}.{namespace}.{table}" + +# 1) Build distributed data (force executor work via repartition) +n = 200_000 +parts = 12 +df = ( + spark.range(0, n) + .repartition(parts) + .withColumn("grp", (F.col("id") % 17).cast("int")) + .withColumn("payload", F.concat(F.lit("v-"), F.col("id").cast("string"))) +) + +# Optional: materialize first to ensure tasks run +print("input_count:", df.count()) + +# 2) Real distributed WRITE to Polaris/Iceberg +df.writeTo(full_table).using("iceberg").create() + +# 3) Real distributed READ from Polaris/Iceberg +read_df = spark.read.table(full_table).repartition(parts) +print("table_count:", read_df.count()) + +# 4) A distributed aggregate to exercise more executor paths +agg = ( + read_df.groupBy("grp") + .count() + .orderBy("grp") +) +agg.show(20, truncate=False) + +# 5) Cleanup +spark.sql(f"DROP TABLE {full_table}") +print("dropped:", full_table) + +spark.stop() \ No newline at end of file From 1a883cb79b5b360b920864165de4b52c11a171c8 Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Thu, 20 Aug 2026 13:15:03 -0400 Subject: [PATCH 47/62] leave api_keys in keycloak but move delegated session to teehr-api --- api/manifests/deployment.yaml.tpl | 21 +++++++++++++++++++++ api/src/config.py | 4 ++++ api/src/main.py | 2 +- secrets/secrets.local.yaml | 5 +++++ 4 files changed, 31 insertions(+), 1 deletion(-) diff --git a/api/manifests/deployment.yaml.tpl b/api/manifests/deployment.yaml.tpl index db55174..1afc694 100644 --- a/api/manifests/deployment.yaml.tpl +++ b/api/manifests/deployment.yaml.tpl @@ -167,6 +167,27 @@ spec: key: password - name: API_KEYS_DB_DSN value: "postgresql://$(API_KEYS_DB_USER):$(API_KEYS_DB_PASSWORD)@$(API_KEYS_DB_HOST):$(API_KEYS_DB_PORT)/$(API_KEYS_DB_NAME)" + - name: DELEGATED_SESSIONS_DB_HOST + value: keycloak-pg + - name: DELEGATED_SESSIONS_DB_PORT + value: "5432" + - name: DELEGATED_SESSIONS_DB_NAME + valueFrom: + secretKeyRef: + name: teehr-api-db-secrets + key: database + - name: DELEGATED_SESSIONS_DB_USER + valueFrom: + secretKeyRef: + name: teehr-api-db-secrets + key: username + - name: DELEGATED_SESSIONS_DB_PASSWORD + valueFrom: + secretKeyRef: + name: teehr-api-db-secrets + key: password + - name: DELEGATED_SESSIONS_DB_DSN + value: "postgresql://$(DELEGATED_SESSIONS_DB_USER):$(DELEGATED_SESSIONS_DB_PASSWORD)@$(DELEGATED_SESSIONS_DB_HOST):$(DELEGATED_SESSIONS_DB_PORT)/$(DELEGATED_SESSIONS_DB_NAME)" - name: ANON_RATE_LIMIT_RPM valueFrom: configMapKeyRef: diff --git a/api/src/config.py b/api/src/config.py index 6680264..9c04d6d 100644 --- a/api/src/config.py +++ b/api/src/config.py @@ -60,6 +60,10 @@ class Config: "API_KEYS_DB_DSN", "postgresql://keycloak:keycloak123@keycloak-pg:5432/teehr_api", ) + DELEGATED_SESSIONS_DB_DSN = os.environ.get( + "DELEGATED_SESSIONS_DB_DSN", + API_KEYS_DB_DSN, + ) API_KEY_PREFIX = os.environ.get("API_KEY_PREFIX", "thk_") API_KEY_HASH_SALT = os.environ.get( "API_KEY_HASH_SALT", diff --git a/api/src/main.py b/api/src/main.py index fa98da5..a7a9d41 100644 --- a/api/src/main.py +++ b/api/src/main.py @@ -179,7 +179,7 @@ async def startup_event(): app.state.api_key_store = ApiKeyStore(config.API_KEYS_DB_DSN) await app.state.api_key_store.startup() app.state.delegated_session_store = DelegatedSessionStore( - config.API_KEYS_DB_DSN, + config.DELEGATED_SESSIONS_DB_DSN, config.BROKER_REFRESH_TOKEN_ENCRYPTION_SECRET, ) await app.state.delegated_session_store.startup() diff --git a/secrets/secrets.local.yaml b/secrets/secrets.local.yaml index a97c24d..422c038 100644 --- a/secrets/secrets.local.yaml +++ b/secrets/secrets.local.yaml @@ -20,6 +20,11 @@ secrets: database: keycloak username: keycloak password: keycloak123 + teehr-api-db-secrets: + data: + database: teehr_api + username: keycloak + password: keycloak123 keycloak-admin-secrets: data: username: admin From a4b5985cf3ce3331c6bf2442b2290d322feeed85 Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Fri, 21 Aug 2026 15:00:11 -0400 Subject: [PATCH 48/62] updates to use polaris sts --- jupyterhub/garden.yaml | 1 + polaris-bootstrap/manifests/acl-config.yaml.tpl | 1 + polaris-bootstrap/manifests/bootstrap-job.yaml | 14 +++++++++++--- polaris/manifests/polaris.yaml.tpl | 4 ++++ spark/manifests/spark-roles.yaml.tpl | 4 ---- trino/garden.yaml | 8 +++++--- 6 files changed, 22 insertions(+), 10 deletions(-) diff --git a/jupyterhub/garden.yaml b/jupyterhub/garden.yaml index 22df7fd..e8c47a4 100644 --- a/jupyterhub/garden.yaml +++ b/jupyterhub/garden.yaml @@ -392,6 +392,7 @@ spec: POLARIS_DEFAULT_REALM: ${var.polaris.defaultRealm} POLARIS_OAUTH2_SERVER_URI: ${var.polaris.oauthServerUri} POLARIS_USE_AUTHMANAGER: "true" + POLARIS_USE_STS: "true" TRINO_HOST: ${var.trino.host} TRINO_PORT: ${var.trino.port} TRINO_CATALOG: ${var.trino.catalog} diff --git a/polaris-bootstrap/manifests/acl-config.yaml.tpl b/polaris-bootstrap/manifests/acl-config.yaml.tpl index 57c9bfa..7452d32 100644 --- a/polaris-bootstrap/manifests/acl-config.yaml.tpl +++ b/polaris-bootstrap/manifests/acl-config.yaml.tpl @@ -15,6 +15,7 @@ data: "path_style_access": "${var.polaris.catalogS3PathStyleAccess}", "s3_region": "${var.polaris.catalogS3Region}", "sts_unavailable": ${var.polaris.storageStsUnavailable}, + "role_arn": "${var.polaris.catalogRoleArn}", "allowed_locations": [ "${var.polaris.catalogWarehouse}" ], diff --git a/polaris-bootstrap/manifests/bootstrap-job.yaml b/polaris-bootstrap/manifests/bootstrap-job.yaml index 2e3cf11..3bad3e6 100644 --- a/polaris-bootstrap/manifests/bootstrap-job.yaml +++ b/polaris-bootstrap/manifests/bootstrap-job.yaml @@ -243,6 +243,7 @@ spec: path_style_access = realm_cfg.get("path_style_access") s3_region = realm_cfg.get("s3_region") sts_unavailable = realm_cfg.get("sts_unavailable") + role_arn = realm_cfg.get("role_arn") allowed_locations = realm_cfg.get("allowed_locations") or [warehouse] print(f"[polaris-bootstrap] realm={realm} catalog={catalog_name}") @@ -272,10 +273,17 @@ spec: storage_config_info["region"] = s3_region catalog_properties["s3.region"] = s3_region catalog_properties["table-default.s3.region"] = s3_region + sts_unavailable_bool = False if sts_unavailable is not None: - storage_config_info["stsUnavailable"] = parse_bool(sts_unavailable) - catalog_properties["s3.remote-signing-enabled"] = "false" - catalog_properties["table-default.s3.remote-signing-enabled"] = "false" + sts_unavailable_bool = parse_bool(sts_unavailable) + storage_config_info["stsUnavailable"] = sts_unavailable_bool + if role_arn: + storage_config_info["roleArn"] = role_arn + # Remote signing (Polaris-vended, per-request S3 credentials) only + # makes sense when Polaris can actually call STS to assume roleArn. + remote_signing_enabled = str(not sts_unavailable_bool).lower() + catalog_properties["s3.remote-signing-enabled"] = remote_signing_enabled + catalog_properties["table-default.s3.remote-signing-enabled"] = remote_signing_enabled ensure_catalog( realm, diff --git a/polaris/manifests/polaris.yaml.tpl b/polaris/manifests/polaris.yaml.tpl index 2abb7b7..bddb240 100644 --- a/polaris/manifests/polaris.yaml.tpl +++ b/polaris/manifests/polaris.yaml.tpl @@ -3,6 +3,10 @@ kind: ServiceAccount metadata: name: polaris namespace: ${environment.namespace} + ${if environment.name == "remote"} + annotations: + eks.amazonaws.com/role-arn: ${var.irsa.polarisRoleArn} + ${endif} --- apiVersion: apps/v1 kind: Deployment diff --git a/spark/manifests/spark-roles.yaml.tpl b/spark/manifests/spark-roles.yaml.tpl index 00826b1..6477cb7 100644 --- a/spark/manifests/spark-roles.yaml.tpl +++ b/spark/manifests/spark-roles.yaml.tpl @@ -3,10 +3,6 @@ kind: ServiceAccount metadata: name: spark namespace: ${environment.namespace} - ${if environment.name == "remote"} - annotations: - eks.amazonaws.com/role-arn: ${var.irsa.sparkRoleArn} - ${endif} --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role diff --git a/trino/garden.yaml b/trino/garden.yaml index bd677bf..dcc8e64 100644 --- a/trino/garden.yaml +++ b/trino/garden.yaml @@ -123,6 +123,10 @@ spec: iceberg.rest-catalog.oauth2.credential=$${ENV:POLARIS_CREDENTIAL} iceberg.rest-catalog.oauth2.scope=openid # S3 Configuration + # Trino holds no AWS identity of its own for the warehouse bucket; + # it relies entirely on Polaris to vend scoped, short-lived S3 + # credentials via the REST catalog protocol. + iceberg.rest-catalog.vended-credentials-enabled=true fs.native-s3.enabled=true s3.region=${var.aws.region} accessControl: @@ -150,6 +154,4 @@ spec: serviceAccount: create: true - name: trino - annotations: - eks.amazonaws.com/role-arn: ${var.irsa.trinoRoleArn} \ No newline at end of file + name: trino \ No newline at end of file From 3dd2177025b5fcdc76e68874a76cc35d586c5099 Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Mon, 24 Aug 2026 18:56:21 -0400 Subject: [PATCH 49/62] remove keycloak-local-users mount from botstrap --- keycloak-bootstrap/manifests/bootstrap-job.yaml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/keycloak-bootstrap/manifests/bootstrap-job.yaml b/keycloak-bootstrap/manifests/bootstrap-job.yaml index 755ee07..82e8428 100644 --- a/keycloak-bootstrap/manifests/bootstrap-job.yaml +++ b/keycloak-bootstrap/manifests/bootstrap-job.yaml @@ -123,8 +123,6 @@ spec: volumeMounts: - name: keycloak-bootstrap-config mountPath: /config - - name: keycloak-local-users-config - mountPath: /config/users volumes: - name: keycloak-bootstrap-config configMap: @@ -132,9 +130,3 @@ spec: items: - key: teehr-realm.json path: teehr-realm.json - - name: keycloak-local-users-config - configMap: - name: keycloak-local-users-bootstrap - items: - - key: teehr-local-users.json - path: teehr-local-users.json From 309eae2741d8788cb22a2314731a0d2b1ffc87b1 Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Tue, 25 Aug 2026 14:52:57 -0400 Subject: [PATCH 50/62] fix: stop advertising unsupported s3.remote-signing-enabled to clients MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Polaris's /v1/config endpoint echoes the catalog's entire raw properties map back to every REST client, unconditionally, regardless of the client's requested X-Iceberg-Access-Delegation mode. Setting s3.remote-signing-enabled=true here made every client's Iceberg S3FileIO try to use remote signing, which Polaris doesn't implement (no /v1/aws/s3/sign route) — breaking clients that correctly ask for vended-credentials instead. Drop the property; ensure_catalog() will PUT the corrected properties onto the existing catalog on next run. Co-Authored-By: Claude Sonnet 5 --- polaris-bootstrap/manifests/bootstrap-job.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/polaris-bootstrap/manifests/bootstrap-job.yaml b/polaris-bootstrap/manifests/bootstrap-job.yaml index 3bad3e6..c202ea8 100644 --- a/polaris-bootstrap/manifests/bootstrap-job.yaml +++ b/polaris-bootstrap/manifests/bootstrap-job.yaml @@ -279,11 +279,6 @@ spec: storage_config_info["stsUnavailable"] = sts_unavailable_bool if role_arn: storage_config_info["roleArn"] = role_arn - # Remote signing (Polaris-vended, per-request S3 credentials) only - # makes sense when Polaris can actually call STS to assume roleArn. - remote_signing_enabled = str(not sts_unavailable_bool).lower() - catalog_properties["s3.remote-signing-enabled"] = remote_signing_enabled - catalog_properties["table-default.s3.remote-signing-enabled"] = remote_signing_enabled ensure_catalog( realm, From 4979178672484cf4a2fc0f6c9c3a84842d41ae0e Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Wed, 26 Aug 2026 14:55:30 -0400 Subject: [PATCH 51/62] fix: remove vestigial IRSA annotation from jupyter service account The jupyter SA's IAM role trust was recently narrowed to exclude it, breaking Spark session creation. It was never load-bearing for real data access: Iceberg reads go through the Polaris broker/AuthManager (pure OAuth, no AWS creds), and icechunk/gridded reads go through xpublish-api's own scoped role. Removing the annotation lets the credential fallback in spark_session_utils.py degrade to anonymous instead of throwing AssumeRoleWithWebIdentity AccessDenied. Co-Authored-By: Claude Sonnet 5 --- jupyterhub/manifests/jupyter-serviceaccount.yaml.tpl | 4 ---- 1 file changed, 4 deletions(-) diff --git a/jupyterhub/manifests/jupyter-serviceaccount.yaml.tpl b/jupyterhub/manifests/jupyter-serviceaccount.yaml.tpl index 034b8c5..28b1cfc 100644 --- a/jupyterhub/manifests/jupyter-serviceaccount.yaml.tpl +++ b/jupyterhub/manifests/jupyter-serviceaccount.yaml.tpl @@ -3,10 +3,6 @@ kind: ServiceAccount metadata: name: jupyter namespace: ${environment.namespace} - ${if environment.name == "remote"} - annotations: - eks.amazonaws.com/role-arn: ${var.irsa.jupyterRoleArn} - ${endif} labels: app: jupyterhub component: jupyter \ No newline at end of file From c85a39544e3c6af3b2a964f90a30ed16a5037aa1 Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Wed, 26 Aug 2026 16:12:26 -0400 Subject: [PATCH 52/62] uncomment xpublish to add back to deployment --- xpublish-api/garden.yaml | 108 +++++++++++++++++++-------------------- 1 file changed, 54 insertions(+), 54 deletions(-) diff --git a/xpublish-api/garden.yaml b/xpublish-api/garden.yaml index 8fbb717..b3bc628 100644 --- a/xpublish-api/garden.yaml +++ b/xpublish-api/garden.yaml @@ -1,69 +1,69 @@ -# kind: Build -# type: container -# name: xpublish-api +kind: Build +type: container +name: xpublish-api -# spec: -# dockerfile: Dockerfile.xpublish +spec: + dockerfile: Dockerfile.xpublish -# environments: -# - local -# - remote +environments: + - local + - remote -# --- +--- -# kind: Deploy -# type: kubernetes -# name: xpublish-api -# environments: -# - local +kind: Deploy +type: kubernetes +name: xpublish-api +environments: + - local -# dependencies: -# - build.xpublish-api -# - deploy.secrets +dependencies: + - build.xpublish-api + - deploy.secrets -# spec: -# manifestFiles: -# - manifests/service.yaml +spec: + manifestFiles: + - manifests/service.yaml -# manifestTemplates: -# - manifests/serviceaccount.yaml.tpl -# - manifests/configmap-${environment.name}.yaml.tpl -# - manifests/deployment.yaml.tpl + manifestTemplates: + - manifests/serviceaccount.yaml.tpl + - manifests/configmap-${environment.name}.yaml.tpl + - manifests/deployment.yaml.tpl -# defaultTarget: -# kind: Deployment -# name: xpublish-api + defaultTarget: + kind: Deployment + name: xpublish-api -# sync: -# paths: -# - containerPath: /app/src -# sourcePath: ./src -# mode: one-way -# exclude: ["__pycache__", "*.pyc"] -# overrides: -# - command: ["uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "8000", "--reload"] + sync: + paths: + - containerPath: /app/src + sourcePath: ./src + mode: one-way + exclude: ["__pycache__", "*.pyc"] + overrides: + - command: ["uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "8000", "--reload"] -# --- +--- -# kind: Deploy -# type: kubernetes -# name: xpublish-api -# environments: -# - remote +kind: Deploy +type: kubernetes +name: xpublish-api +environments: + - remote -# dependencies: -# - build.xpublish-api -# - deploy.secrets +dependencies: + - build.xpublish-api + - deploy.secrets -# spec: -# manifestFiles: -# - manifests/service.yaml +spec: + manifestFiles: + - manifests/service.yaml -# manifestTemplates: -# - manifests/serviceaccount.yaml.tpl -# - manifests/configmap-${environment.name}.yaml.tpl -# - manifests/deployment.yaml.tpl + manifestTemplates: + - manifests/serviceaccount.yaml.tpl + - manifests/configmap-${environment.name}.yaml.tpl + - manifests/deployment.yaml.tpl -# defaultTarget: -# kind: Deployment -# name: xpublish-api + defaultTarget: + kind: Deployment + name: xpublish-api From 539b52c79b4602bf1856dd7a0cbd877e8b09b689 Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Wed, 26 Aug 2026 21:22:55 -0400 Subject: [PATCH 53/62] remove unused spark-polaris Keycloak client MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nothing consumes its credentials anywhere in teehr-cloud-core, teehr-fved, or teehr — real Spark→Polaris auth goes through the AuthManager/broker per-user delegated path instead. It was planned for a headless Prefect batch client-credentials flow but never wired up, and its lack of a realm role assignment (unlike trino-polaris and prefect-polaris) would have made it a live footgun if anything ever did start using it. Removing the client, its secret, and doc references rather than fixing the missing role. Co-Authored-By: Claude Sonnet 5 --- docs/polaris-migration-plan.md | 28 +++++++------------ .../manifests/bootstrap-job.yaml | 5 ---- .../manifests/realm-configmap.yaml.tpl | 24 ---------------- secrets/secrets.local.yaml | 5 +--- 4 files changed, 11 insertions(+), 51 deletions(-) diff --git a/docs/polaris-migration-plan.md b/docs/polaris-migration-plan.md index a49b10b..341dfe9 100644 --- a/docs/polaris-migration-plan.md +++ b/docs/polaris-migration-plan.md @@ -37,9 +37,6 @@ Replace `tabulario/iceberg-rest` with `apache/polaris`. Tight Keycloak integrati data: client-secret: local-trino-polaris-client-secret # plain secret — used by keycloak-bootstrap env var credential: "trino-polaris:local-trino-polaris-client-secret" # full credential string — mounted as Trino credential file - spark-polaris-secrets: - data: - client-secret: local-spark-polaris-client-secret ``` > **Note**: `trino-polaris-secrets` needs **two keys** because the Keycloak bootstrap job needs the plain secret value (`client-secret`) to set as the Keycloak client secret, while Trino's credential file mount needs the full `trino-polaris:` string (`credential`). Using the `credential` key for the Keycloak env var would inject the wrong value. @@ -91,14 +88,12 @@ Replace `tabulario/iceberg-rest` with `apache/polaris`. Tight Keycloak integrati - `iceberg-restricted-writers` (realmRoles: `iceberg-namespace-restricted-write`, `iceberg-namespace-restricted-read`) - `iceberg-catalog-admins` (realmRoles: `iceberg-catalog-admin`) - Retain existing `iceberg-user` role and group during transition - - Add to `clients` array — **two** new confidential service account clients (not three — Polaris does not need its own Keycloak client; it validates tokens via JWKS only and does not perform token introspection or act as an OAuth2 client itself): + - Add to `clients` array — **one** new confidential service account client (Polaris does not need its own Keycloak client; it validates tokens via JWKS only and does not perform token introspection or act as an OAuth2 client itself): - `trino-polaris` client — `serviceAccountsEnabled: true`, `secret: $(env:TRINO_POLARIS_CLIENT_SECRET)`, add `realm_access` protocol mapper to include roles in access token - - `spark-polaris` client — `serviceAccountsEnabled: true`, `secret: $(env:SPARK_POLARIS_CLIENT_SECRET)`, same `realm_access` mapper - **Note on `realm_access.roles` claim**: Keycloak includes realm roles in access tokens by default, but verify against the running Keycloak version. If the Polaris `PrincipalRoleMapper` needs roles under a custom claim path, add an explicit `oidc-usermodel-realm-role-mapper` protocolMapper to the Polaris-facing clients. -8. Update `keycloak-bootstrap/manifests/bootstrap-job.yaml` — add two new `env` entries matching the existing pattern: +8. Update `keycloak-bootstrap/manifests/bootstrap-job.yaml` — add one new `env` entry matching the existing pattern: - `TRINO_POLARIS_CLIENT_SECRET` from `trino-polaris-secrets` key `client-secret` - - `SPARK_POLARIS_CLIENT_SECRET` from `spark-polaris-secrets` key `client-secret` --- @@ -244,12 +239,11 @@ Replace `tabulario/iceberg-rest` with `apache/polaris`. Tight Keycloak integrati - Add an optional `oauth2_token: str = None` parameter to `create_spark_session()` — passed through to `_configure_iceberg_catalogs()` - In `_configure_iceberg_catalogs()`: add OAuth2 conf.set calls at the end of the existing function body: - If `oauth2_token` provided (JupyterHub user token pass-through): set `rest.auth.type=oauth2` + `rest.auth.oauth2.token=` - - If absent (Prefect batch): set `rest.auth.type=oauth2`, `rest.auth.oauth2.server-uri` (from `POLARIS_OAUTH2_SERVER_URI` env), `rest.auth.oauth2.credential=spark-polaris:` (from `SPARK_POLARIS_CLIENT_SECRET` env), `rest.auth.oauth2.scope=openid` - - Both paths: set `rest.transport.header.X-Polaris-Realm=teehr` + - Set `rest.transport.header.X-Polaris-Realm=teehr` - The existing `update_configs: Dict[str, str]` parameter on `create_spark_session()` remains available as an override escape hatch — no structural change needed - **No changes** to `_create_spark_base_session`, `_set_spark_cluster_configuration`, `_set_aws_credentials_in_spark`, `_update_configs_and_packages`, `_set_catalog_metadata`, or any other existing functions -14. Update `teehr/src/teehr/const.py` — add `POLARIS_OAUTH2_SERVER_URI` and `SPARK_POLARIS_CLIENT_SECRET` env var reads alongside existing constants. +14. Update `teehr/src/teehr/const.py` — add `POLARIS_OAUTH2_SERVER_URI` env var read alongside existing constants. 15. Update `prefect-workflows/manifests/prefect-deployer-job.yaml` — change `REMOTE_CATALOG_REST_URI` value from `${var.iceberg.catalogUri}` to `${var.polaris.catalogUri}`. @@ -320,13 +314,13 @@ Replace `tabulario/iceberg-rest` with `apache/polaris`. Tight Keycloak integrati | File | Change | |---|---| -| `secrets/secrets.local.yaml` | Add `polaris-db-secrets`, `polaris-secrets`, `trino-polaris-secrets`, `spark-polaris-secrets` | -| `secrets/secrets.remote.yaml` | Same four secrets with production-grade values | -| `keycloak-bootstrap/manifests/realm-configmap.yaml.tpl` | Add 5 realm roles, 5 groups, 2 confidential clients (`trino-polaris`, `spark-polaris`) | -| `keycloak-bootstrap/manifests/bootstrap-job.yaml` | Add 2 new secret env vars (`TRINO_POLARIS_CLIENT_SECRET`, `SPARK_POLARIS_CLIENT_SECRET`) | +| `secrets/secrets.local.yaml` | Add `polaris-db-secrets`, `polaris-secrets`, `trino-polaris-secrets` | +| `secrets/secrets.remote.yaml` | Same three secrets with production-grade values | +| `keycloak-bootstrap/manifests/realm-configmap.yaml.tpl` | Add 5 realm roles, 5 groups, 1 confidential client (`trino-polaris`) | +| `keycloak-bootstrap/manifests/bootstrap-job.yaml` | Add 1 new secret env var (`TRINO_POLARIS_CLIENT_SECRET`) | | `trino/garden.yaml` | Both local + remote Deploy blocks: OAuth2 catalog auth + credential-file volume mount via Helm values | | `spark_session_utils.py` | Additive: dual-path OAuth2 + `X-Polaris-Realm` header; no existing signatures changed | -| `teehr/src/teehr/const.py` | Add `POLARIS_OAUTH2_SERVER_URI`, `SPARK_POLARIS_CLIENT_SECRET` | +| `teehr/src/teehr/const.py` | Add `POLARIS_OAUTH2_SERVER_URI` | | `prefect-workflows/manifests/prefect-deployer-job.yaml` | Update `REMOTE_CATALOG_REST_URI` to `${var.polaris.catalogUri}` | | `project.garden.yml` | Add `polaris` variable group + control/data plane comments | | `ingress/garden.yaml` | Add `polaris-ingress` Deploy entry | @@ -340,7 +334,7 @@ Replace `tabulario/iceberg-rest` with `apache/polaris`. Tight Keycloak integrati 3. Fetch `client_credentials` token for `trino-polaris` from Keycloak → `GET https://polaris.${var.hostname}/api/catalog/v1/config` with `X-Polaris-Realm: teehr` → expect 200; confirm `realm_access.roles` present in decoded token 4. `trino --execute "SHOW SCHEMAS IN iceberg"` → teehr schema visible 5. JupyterHub Spark with user token injected → user with no namespace role gets 403 from Polaris -6. Prefect batch job → `spark-polaris` service client token accepted; `REMOTE_CATALOG_REST_URI` resolves to Polaris +6. `REMOTE_CATALOG_REST_URI` resolves to Polaris 7. **ACL matrix**: - `iceberg-namespace-public-read` member → read `public` ✓, write `public` ✗, read `restricted` ✗ - `iceberg-namespace-public-write` member → read+write `public` ✓, `restricted` ✗ @@ -362,7 +356,6 @@ Replace `tabulario/iceberg-rest` with `apache/polaris`. Tight Keycloak integrati - Secrets via `secrets/secrets.local.yaml` + `secrets/secrets.remote.yaml` varfiles — consistent with existing `$forEach` pattern; no standalone K8s Secret manifests - Trino OAuth2 credential delivered via mounted credential-file (`iceberg.rest-catalog.oauth2.credential-file`) — avoids env var interpolation limitations in Trino catalog properties; stored as full `trino-polaris:` string, mounted via `subPath`, no init container - Trino: `client_credentials` — access control enforced at Trino layer; Polaris sees service identity -- Spark in Prefect: `client_credentials` (`spark-polaris`) — headless batch, no user context - Spark in JupyterHub: user token pass-through — Polaris enforces per-user namespace/table ACLs - Polaris has **no IRSA annotation** — pure metadata service on the control plane - Control plane: Polaris + Keycloak (future: dedicated cluster); Data plane: all other services @@ -392,4 +385,3 @@ Replace `tabulario/iceberg-rest` with `apache/polaris`. Tight Keycloak integrati 4. **OPA for Trino access control (future)**: Trino uses a single `trino-polaris` service identity so Polaris cannot enforce per-user namespace/table ACLs for Trino queries. Open Policy Agent (OPA) — a lightweight Go service on the control plane — can fill this gap. Trino's native OPA system access control plugin receives full query context (user identity, Keycloak groups, target catalog/schema/table) and evaluates Rego policies that mirror the Keycloak role taxonomy. Policy changes hot-reload via ConfigMap without Trino restarts. Would require: new `opa/` Garden module + `access-control.name=opa` in Trino config + policy ConfigMap mirroring the Phase 3 role taxonomy. -5. **Per-workflow Prefect clients (future)**: Replace single `spark-polaris` client with per-category Keycloak clients (`prefect-ingest`, `prefect-metrics`, etc.), each granted only the namespace roles it needs. Prefect deployment job templates inject credentials via workflow-specific K8s Secrets. `spark_session_utils.py` reads `SPARK_POLARIS_CLIENT_ID` from env rather than hardcoding. No changes needed to Polaris bootstrap or `acl-config.yaml`. diff --git a/keycloak-bootstrap/manifests/bootstrap-job.yaml b/keycloak-bootstrap/manifests/bootstrap-job.yaml index 82e8428..2c529b1 100644 --- a/keycloak-bootstrap/manifests/bootstrap-job.yaml +++ b/keycloak-bootstrap/manifests/bootstrap-job.yaml @@ -53,11 +53,6 @@ spec: secretKeyRef: name: prefect-polaris-secrets key: client-secret - - name: SPARK_POLARIS_CLIENT_SECRET - valueFrom: - secretKeyRef: - name: spark-polaris-secrets - key: client-secret - name: SMTP_HOST valueFrom: configMapKeyRef: diff --git a/keycloak-bootstrap/manifests/realm-configmap.yaml.tpl b/keycloak-bootstrap/manifests/realm-configmap.yaml.tpl index 2882e0d..0f2effd 100644 --- a/keycloak-bootstrap/manifests/realm-configmap.yaml.tpl +++ b/keycloak-bootstrap/manifests/realm-configmap.yaml.tpl @@ -253,30 +253,6 @@ data: } } ] - }, - { - "clientId": "spark-polaris", - "enabled": true, - "protocol": "openid-connect", - "publicClient": false, - "serviceAccountsEnabled": true, - "secret": "$(env:SPARK_POLARIS_CLIENT_SECRET)", - "protocolMappers": [ - { - "name": "realm-roles", - "protocol": "openid-connect", - "protocolMapper": "oidc-usermodel-realm-role-mapper", - "consentRequired": false, - "config": { - "multivalued": "true", - "id.token.claim": "true", - "access.token.claim": "true", - "userinfo.token.claim": "true", - "claim.name": "realm_access.roles", - "jsonType.label": "String" - } - } - ] } ], "users": [ diff --git a/secrets/secrets.local.yaml b/secrets/secrets.local.yaml index 422c038..8089082 100644 --- a/secrets/secrets.local.yaml +++ b/secrets/secrets.local.yaml @@ -68,7 +68,4 @@ secrets: prefect-polaris-secrets: data: client-secret: local-prefect-polaris-client-secret - credential: "prefect-polaris:local-prefect-polaris-client-secret" - spark-polaris-secrets: - data: - client-secret: local-spark-polaris-client-secret \ No newline at end of file + credential: "prefect-polaris:local-prefect-polaris-client-secret" \ No newline at end of file From 3332ca7c71214372a2a97cda583f5837fbdd7d5f Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Wed, 26 Aug 2026 22:11:00 -0400 Subject: [PATCH 54/62] fix 3 broker security gaps in the delegated-session flow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Stop BROKER_SESSION_SIGNING_SECRET and BROKER_REFRESH_TOKEN_ENCRYPTION_SECRET from collapsing to the same value as BROKER_OAUTH_CLIENT_SECRET (and each other). New dedicated broker-secrets K8s secret; config.py no longer cascades defaults between the three. - exchange_token_for_polaris_via_broker_session now verifies the request's user_id/session_id/realm against the stored session record and rejects mismatches, per the binding requirement in polaris-broker-api-contract.md. - /auth/polaris-token/session is no longer exempt from rate limiting — added InMemoryRateLimiter.check_by_key() for paths with no resolved AuthIdentity, keyed by client IP. Co-Authored-By: Claude Sonnet 5 --- api/manifests/deployment.yaml.tpl | 8 ++++---- api/src/broker.py | 10 ++++++++++ api/src/config.py | 4 ++-- api/src/main.py | 5 +++++ api/src/rate_limit.py | 8 ++++++++ api/src/routes/auth.py | 3 +++ secrets/secrets.local.yaml | 4 ++++ 7 files changed, 36 insertions(+), 6 deletions(-) diff --git a/api/manifests/deployment.yaml.tpl b/api/manifests/deployment.yaml.tpl index 1afc694..b3e38c0 100644 --- a/api/manifests/deployment.yaml.tpl +++ b/api/manifests/deployment.yaml.tpl @@ -139,13 +139,13 @@ spec: - name: BROKER_SESSION_SIGNING_SECRET valueFrom: secretKeyRef: - name: teehr-api-secrets - key: client-secret + name: broker-secrets + key: session-signing-secret - name: BROKER_REFRESH_TOKEN_ENCRYPTION_SECRET valueFrom: secretKeyRef: - name: teehr-api-secrets - key: client-secret + name: broker-secrets + key: refresh-token-encryption-secret - name: API_KEYS_DB_HOST value: keycloak-pg - name: API_KEYS_DB_PORT diff --git a/api/src/broker.py b/api/src/broker.py index 9380558..38cd5e0 100644 --- a/api/src/broker.py +++ b/api/src/broker.py @@ -225,6 +225,9 @@ async def create_delegated_broker_session( async def exchange_token_for_polaris_via_broker_session( *, broker_session_token: str, + user_id: str, + session_id: str, + realm: str, requested_ttl_seconds: int | None, ) -> dict: if not broker_session_token: @@ -253,6 +256,13 @@ async def exchange_token_for_polaris_via_broker_session( await store.delete_session(delegated_session_id) raise HTTPException(status_code=401, detail="Delegated broker session expired") + if ( + record.get("user_id") != user_id + or record.get("session_id") != session_id + or record.get("realm") != realm + ): + raise HTTPException(status_code=403, detail="Session identity mismatch") + refreshed_subject_token, maybe_new_refresh_token = await _refresh_subject_access_token( record["refresh_token"] ) diff --git a/api/src/config.py b/api/src/config.py index 9c04d6d..90ea248 100644 --- a/api/src/config.py +++ b/api/src/config.py @@ -99,11 +99,11 @@ class Config: ) BROKER_SESSION_SIGNING_SECRET = os.environ.get( "BROKER_SESSION_SIGNING_SECRET", - API_KEY_HASH_SALT, + "local-dev-change-me-session-signing", ) BROKER_REFRESH_TOKEN_ENCRYPTION_SECRET = os.environ.get( "BROKER_REFRESH_TOKEN_ENCRYPTION_SECRET", - BROKER_SESSION_SIGNING_SECRET, + "local-dev-change-me-refresh-encryption", ) # Role-based record/page limits diff --git a/api/src/main.py b/api/src/main.py index a7a9d41..448ed9c 100644 --- a/api/src/main.py +++ b/api/src/main.py @@ -228,6 +228,11 @@ async def auth_context_middleware(request: Request, call_next): path = request.url.path if path == "/auth/polaris-token/session": + client_host = request.client.host if request.client else "unknown" + app.state.rate_limiter.check_by_key( + f"polaris-token-session:{client_host}", + limit=config.AUTH_RATE_LIMIT_RPM, + ) return await call_next(request) exempt_paths = ( diff --git a/api/src/rate_limit.py b/api/src/rate_limit.py index 0d8f0d0..2b33832 100644 --- a/api/src/rate_limit.py +++ b/api/src/rate_limit.py @@ -28,6 +28,14 @@ def check(self, identity: AuthIdentity, route_key: str): minute_bucket = int(time.time() // 60) key = self._key(identity, route_key, minute_bucket) + self._enforce(key, limit, minute_bucket) + + def check_by_key(self, key_prefix: str, limit: int): + minute_bucket = int(time.time() // 60) + key = f"{minute_bucket}:{key_prefix}" + self._enforce(key, limit, minute_bucket) + + def _enforce(self, key: str, limit: int, minute_bucket: int): self._counts[key] += 1 # Cheap periodic cleanup to avoid unbounded growth. diff --git a/api/src/routes/auth.py b/api/src/routes/auth.py index 526e59b..429d750 100644 --- a/api/src/routes/auth.py +++ b/api/src/routes/auth.py @@ -189,6 +189,9 @@ async def polaris_token_via_session( broker_session_token = request.headers.get("x-broker-session-token", "").strip() exchanged = await exchange_token_for_polaris_via_broker_session( broker_session_token=broker_session_token, + user_id=payload.user_id, + session_id=payload.session_id, + realm=payload.realm, requested_ttl_seconds=payload.requested_ttl_seconds, ) diff --git a/secrets/secrets.local.yaml b/secrets/secrets.local.yaml index 8089082..8994283 100644 --- a/secrets/secrets.local.yaml +++ b/secrets/secrets.local.yaml @@ -43,6 +43,10 @@ secrets: teehr-api-secrets: data: client-secret: local-teehr-api-client-secret + broker-secrets: + data: + session-signing-secret: local-broker-session-signing-secret + refresh-token-encryption-secret: local-broker-refresh-token-encryption-secret minio-secrets: data: accesskey: minioadmin From 5f3e762be63a693ed7f08eb3c7267107b6aeae36 Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Thu, 27 Aug 2026 12:48:06 -0400 Subject: [PATCH 55/62] add automated cluster-mode executor AuthManager test spark-authmanager-enforcement (and spark-permission-enforcement) only exercise driver-only local mode via create_minio_spark_session(), so neither ever tested whether Spark executors actually get the Polaris auth env vars AuthManager needs, or can participate in a real distributed Iceberg write/read. That path was previously only checked manually via test_executor_auth.py (run by hand in Jupyter, not part of the automated suite). Adds spark_authmanager_executor_test.py, adapted from that manual script but with proper PASS/FAIL assertions and exit codes matching the other Garden test scripts' convention, and without the list_tables()/configurations table reads the manual version had -- those depend on warehouse seed data that may or may not exist when this runs as an isolated automated test; this one only touches a table it creates and drops itself. Uses create_spark_session() directly rather than create_minio_spark_session(), since the MinIO endpoint/path-style env vars are already set on this test container (matching how the real jupyter/spark pods pick them up) and go through the catalog-level S3 config path, not the Hadoop-level one the MinIO wrapper additionally sets (which Iceberg's S3FileIO doesn't use anyway). Not yet confirmed whether the test container's service account can actually create executor pods (RBAC for that is granted to the `spark` ServiceAccount; unclear if/how the test container's default SA gets equivalent permission) -- first run against the KinD cluster will tell us. Co-Authored-By: Claude Sonnet 5 --- tests/Dockerfile | 1 + tests/garden.yaml | 27 +++ tests/spark_authmanager_executor_test.py | 220 +++++++++++++++++++++++ 3 files changed, 248 insertions(+) create mode 100644 tests/spark_authmanager_executor_test.py diff --git a/tests/Dockerfile b/tests/Dockerfile index dc87003..c3ddc90 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -15,3 +15,4 @@ COPY polaris_auth_token_test.py /app/tests/ COPY polaris_permissions_config_test.py /app/tests/ COPY spark_permission_test.py /app/tests/ COPY spark_authmanager_test.py /app/tests/ +COPY spark_authmanager_executor_test.py /app/tests/ diff --git a/tests/garden.yaml b/tests/garden.yaml index b1366c0..d74322f 100644 --- a/tests/garden.yaml +++ b/tests/garden.yaml @@ -139,3 +139,30 @@ spec: cpu: min: 1000 max: 2000 + +--- +kind: Test +name: spark-authmanager-executor-enforcement +description: >- + Cluster-mode AuthManager test -- confirms Spark executors (not just the + driver) get the Polaris auth env vars they need and can participate in a + real distributed Iceberg write/read, unlike spark-authmanager-enforcement + above which only exercises local (driver-only) mode. +dependencies: + - deploy.polaris-bootstrap + - deploy.keycloak-local-users-bootstrap + - deploy.teehr-api + - build.test-scripts +timeout: 600 +type: container +spec: + image: ${actions.build.test-scripts.outputs.deploymentImageName}:${actions.build.test-scripts.version} + command: + - python3 + - /app/tests/spark_authmanager_executor_test.py + memory: + min: 2048 + max: 3072 + cpu: + min: 1000 + max: 2000 diff --git a/tests/spark_authmanager_executor_test.py b/tests/spark_authmanager_executor_test.py new file mode 100644 index 0000000..827354e --- /dev/null +++ b/tests/spark_authmanager_executor_test.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 +""" +Integration test: Spark cluster-mode executor auth via AuthManager + +Validates that Spark EXECUTORS (not just the driver) can participate in +real distributed Iceberg catalog operations when POLARIS_USE_AUTHMANAGER +is enabled with start_spark_cluster=True, and that the Polaris auth env +vars AuthManager needs are actually propagated to executor pods. + +TEST_USERNAME - Keycloak username (default: admin) +TEST_PASSWORD - Keycloak password (default: admin) +""" + +import sys +import os +import socket +import time +import gc + +import requests +from pyspark.sql import Row +from pyspark.sql import functions as F + +# Set up environment for Polaris/Spark before importing PySpark +os.environ.setdefault("POLARIS_DEFAULT_REALM", "teehr") +os.environ.setdefault("REMOTE_CATALOG_REST_URI", "http://polaris:8181/api/catalog") +os.environ.setdefault("REMOTE_CATALOG_S3_ENDPOINT", "http://minio:9000") +os.environ.setdefault("REMOTE_CATALOG_S3_PATH_STYLE_ACCESS", "true") +os.environ.setdefault("AWS_ACCESS_KEY_ID", "minioadmin") +os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "minioadmin123") +os.environ.setdefault("AWS_REGION", "us-east-2") +# Broker and OAuth endpoints +os.environ.setdefault("POLARIS_BROKER_URL", "http://teehr-api:8000/auth/polaris-token") +os.environ.setdefault( + "POLARIS_OAUTH2_TOKEN_ENDPOINT", + "http://keycloak-service:8080/realms/teehr/protocol/openid-connect/token", +) +os.environ.setdefault("POLARIS_CLIENT_ID", "jupyterhub") +os.environ.setdefault("POLARIS_CLIENT_SECRET", "local-jupyterhub-client-secret") +# JVM heap +os.environ["JAVA_TOOL_OPTIONS"] = "-Xmx1g" +os.environ.setdefault("SPARK_LOCAL_IP", "127.0.0.1") + +sys.path.insert(0, "/opt/teehr") + +KEYCLOAK_URL = "http://keycloak-service:8080" +REALM = "teehr" +CATALOG = "iceberg" +NAMESPACE = "teehr" + +MAX_RETRIES = 10 +RETRY_DELAY = 2 + +TEST_USERNAME = os.getenv("TEST_USERNAME", "admin") +TEST_PASSWORD = os.getenv("TEST_PASSWORD", "admin") + +EXPECTED_EXECUTOR_ENV_KEYS = [ + "POLARIS_DEFAULT_REALM", + "POLARIS_BROKER_SESSION_TOKEN", +] + + +def get_keycloak_tokens(username: str, password: str) -> tuple: + """Get access_token and refresh_token from Keycloak via password grant.""" + for attempt in range(MAX_RETRIES): + try: + token_url = f"{KEYCLOAK_URL}/realms/{REALM}/protocol/openid-connect/token" + payload = { + "grant_type": "password", + "client_id": "jupyterhub", + "client_secret": "local-jupyterhub-client-secret", + "username": username, + "password": password, + "scope": "openid", + } + response = requests.post(token_url, data=payload, timeout=10) + if response.status_code == 200: + data = response.json() + return data["access_token"], data.get("refresh_token", "") + elif attempt < MAX_RETRIES - 1: + print(f" Attempt {attempt + 1}/{MAX_RETRIES}: Keycloak not ready, retrying...") + time.sleep(RETRY_DELAY) + else: + raise Exception(f"Failed to get token for {username}: {response.text}") + except requests.exceptions.RequestException as e: + if attempt < MAX_RETRIES - 1: + print(f" Attempt {attempt + 1}/{MAX_RETRIES}: Connection failed, retrying...") + time.sleep(RETRY_DELAY) + else: + raise Exception(f"Failed to get token for {username}: {e}") + + raise Exception(f"Failed to get token for {username} after {MAX_RETRIES} attempts") + + +def probe_executor_env(spark, expected_env_keys, partitions=4): + """Confirm expected env vars are present on executors. + + Returns one row per partition attempt with only booleans + executor + identity. Never returns secret values. + """ + keys = list(expected_env_keys) + + def _probe_partition(it): + import os + # Force execution of partition iterator so Spark doesn't prune the task. + _ = list(it) + result = { + "executor_host": socket.gethostname(), + "pid_present": os.getpid() > 0, + } + for k in keys: + result[f"has_{k}"] = bool(os.environ.get(k)) + yield Row(**result) + + rdd = spark.sparkContext.parallelize(range(partitions), partitions) + return rdd.mapPartitions(_probe_partition).collect() + + +def main(): + print(f"[test] Starting Spark cluster-mode executor AuthManager test for user: {TEST_USERNAME}") + + all_passed = True + spark = None + try: + print(" Getting Keycloak tokens...") + access_token, refresh_token = get_keycloak_tokens(TEST_USERNAME, TEST_PASSWORD) + if not refresh_token: + print(" ✗ ERROR: No refresh_token returned - required for AuthManager broker session") + return 1 + print(" ✓ Tokens obtained (access + refresh)") + + os.environ["POLARIS_USER_TOKEN"] = access_token + os.environ["POLARIS_REFRESH_TOKEN"] = refresh_token + os.environ["JUPYTERHUB_USER"] = TEST_USERNAME + + print(" Creating cluster-mode Spark session via AuthManager...") + from teehr.evaluation.spark_session_utils import create_spark_session + spark = create_spark_session( + update_configs={ + "spark.kubernetes.executor.node.selector.teehr-hub/nodegroup-name": "spark-r5-4xlarge", + }, + start_spark_cluster=True, + use_authmanager=True, + force_recreate_session=True, + executor_instances=1, + executor_cores=1, + executor_memory="1g", + ) + print(" ✓ Spark session created") + + print(" Probing executor environment for propagated Polaris auth vars...") + rows = probe_executor_env(spark, EXPECTED_EXECUTOR_ENV_KEYS, partitions=4) + if not rows: + print(" ✗ ERROR: no executor probe results returned") + all_passed = False + for row in rows: + print(f" {row.asDict()}") + for key in EXPECTED_EXECUTOR_ENV_KEYS: + if not row[f"has_{key}"]: + print(f" ✗ ERROR: executor {row['executor_host']} missing {key}") + all_passed = False + + # Real distributed write + read through the executors just probed, + # exercising the same Iceberg/Polaris auth path end to end. This + # table is created fresh and dropped at the end -- it never reads + # from or depends on any pre-existing warehouse table/data. + table = f"executor_authmanager_test_{int(time.time())}" + full_table = f"{CATALOG}.{NAMESPACE}.{table}" + + n = 200_000 + parts = 4 + print(f" Building {n}-row distributed dataset across {parts} partitions...") + df = ( + spark.range(0, n) + .repartition(parts) + .withColumn("grp", (F.col("id") % 17).cast("int")) + .withColumn("payload", F.concat(F.lit("v-"), F.col("id").cast("string"))) + ) + input_count = df.count() + print(f" input_count: {input_count}") + if input_count != n: + print(f" ✗ ERROR: expected input_count={n}, got {input_count}") + all_passed = False + + print(f" Writing distributed table {full_table}...") + df.writeTo(full_table).using("iceberg").create() + + read_df = spark.read.table(full_table).repartition(parts) + table_count = read_df.count() + print(f" table_count: {table_count}") + if table_count != n: + print(f" ✗ ERROR: expected table_count={n}, got {table_count}") + all_passed = False + + group_count = read_df.groupBy("grp").count().count() + if group_count != 17: + print(f" ✗ ERROR: expected 17 groups, got {group_count}") + all_passed = False + + spark.sql(f"DROP TABLE {full_table}") + print(f" dropped: {full_table}") + + except Exception as e: + print(f" ✗ ERROR: {type(e).__name__}: {str(e)[:300]}") + all_passed = False + finally: + if spark: + try: + spark.stop() + except Exception: + pass + gc.collect() + + result = "PASSED" if all_passed else "FAILED" + print(f"\n[test] Spark cluster-mode executor AuthManager test: {result}") + return 0 if all_passed else 1 + + +if __name__ == "__main__": + sys.exit(main()) From 00775defcf8baa3d28cee436a65e503420d5cd02 Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Thu, 27 Aug 2026 14:26:24 -0400 Subject: [PATCH 56/62] fix KeyError: TEEHR_SPARK_IMAGE in new executor test create_spark_session(start_spark_cluster=True) reads os.environ["TEEHR_SPARK_IMAGE"] with no fallback when executor_image isn't passed explicitly. jupyterhub's garden.yaml sets this from build.teehr-spark-executor-image's output; the new Test action didn't, since it's a different container than the jupyter pod. Co-Authored-By: Claude Sonnet 5 --- tests/garden.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/garden.yaml b/tests/garden.yaml index d74322f..2a12434 100644 --- a/tests/garden.yaml +++ b/tests/garden.yaml @@ -153,6 +153,7 @@ dependencies: - deploy.keycloak-local-users-bootstrap - deploy.teehr-api - build.test-scripts + - build.teehr-spark-executor-image timeout: 600 type: container spec: @@ -160,6 +161,8 @@ spec: command: - python3 - /app/tests/spark_authmanager_executor_test.py + env: + TEEHR_SPARK_IMAGE: ${actions.build.teehr-spark-executor-image.outputs.deploymentImageId} memory: min: 2048 max: 3072 From 79ccfa4488a3cdb7a02dd73b6d2216f648b7fe65 Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Thu, 27 Aug 2026 14:56:54 -0400 Subject: [PATCH 57/62] change test SA --- tests/garden.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/garden.yaml b/tests/garden.yaml index 2a12434..bcd47cf 100644 --- a/tests/garden.yaml +++ b/tests/garden.yaml @@ -163,6 +163,7 @@ spec: - /app/tests/spark_authmanager_executor_test.py env: TEEHR_SPARK_IMAGE: ${actions.build.teehr-spark-executor-image.outputs.deploymentImageId} + serviceAccountName: jupyter memory: min: 2048 max: 3072 From a6f2db91ef637a866c121197b0e0a9e0de4c7ed3 Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Thu, 27 Aug 2026 15:25:27 -0400 Subject: [PATCH 58/62] fix executor test: kubernetes-pod type for SA control + warehouse default - Garden's type: container action schema has no serviceAccountName field (confirmed via its own "key not allowed" validation error, which lists every supported key). Switched to type: kubernetes-pod with a full podSpec, which does support it -- confirmed against a real KinD cluster that executor pods only get created successfully once this test container runs as the jupyter SA (the default SA in this namespace has no RBAC to create/list/watch pods, services, or PVCs; that's scoped to the jupyter/spark SAs specifically). - create_spark_session() (used here instead of create_minio_spark_session()) defaults remote_warehouse_dir to "" when REMOTE_WAREHOUSE_IDENTIFIER isn't set, unlike the minio wrapper which defaults it to the realm name -- Polaris's REST catalog rejects CREATE TABLE with "Please specify a warehouse" when it's empty. Set REMOTE_WAREHOUSE_IDENTIFIER=teehr explicitly to match. Confirmed passing end to end against a real KinD cluster: executor pods created, auth env vars (POLARIS_DEFAULT_REALM, POLARIS_BROKER_SESSION_TOKEN) present on all partitions, distributed write/read/drop of a 200k-row Iceberg table succeeded. Co-Authored-By: Claude Sonnet 5 --- tests/garden.yaml | 34 ++++++++++++++---------- tests/spark_authmanager_executor_test.py | 5 ++++ 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/tests/garden.yaml b/tests/garden.yaml index bcd47cf..4b89a70 100644 --- a/tests/garden.yaml +++ b/tests/garden.yaml @@ -155,18 +155,24 @@ dependencies: - build.test-scripts - build.teehr-spark-executor-image timeout: 600 -type: container +type: kubernetes-pod spec: - image: ${actions.build.test-scripts.outputs.deploymentImageName}:${actions.build.test-scripts.version} - command: - - python3 - - /app/tests/spark_authmanager_executor_test.py - env: - TEEHR_SPARK_IMAGE: ${actions.build.teehr-spark-executor-image.outputs.deploymentImageId} - serviceAccountName: jupyter - memory: - min: 2048 - max: 3072 - cpu: - min: 1000 - max: 2000 + podSpec: + serviceAccountName: jupyter + restartPolicy: Never + containers: + - name: spark-authmanager-executor-test + image: ${actions.build.test-scripts.outputs.deploymentImageName}:${actions.build.test-scripts.version} + command: + - python3 + - /app/tests/spark_authmanager_executor_test.py + env: + - name: TEEHR_SPARK_IMAGE + value: ${actions.build.teehr-spark-executor-image.outputs.deploymentImageId} + resources: + requests: + memory: 2Gi + cpu: "1" + limits: + memory: 3Gi + cpu: "2" diff --git a/tests/spark_authmanager_executor_test.py b/tests/spark_authmanager_executor_test.py index 827354e..6650e02 100644 --- a/tests/spark_authmanager_executor_test.py +++ b/tests/spark_authmanager_executor_test.py @@ -23,6 +23,11 @@ # Set up environment for Polaris/Spark before importing PySpark os.environ.setdefault("POLARIS_DEFAULT_REALM", "teehr") +# create_spark_session() (unlike create_minio_spark_session(), which we +# deliberately don't use here) defaults remote_warehouse_dir to "" rather +# than the realm name when this isn't set, which Polaris's REST catalog +# rejects with "Please specify a warehouse" on CREATE TABLE. +os.environ.setdefault("REMOTE_WAREHOUSE_IDENTIFIER", "teehr") os.environ.setdefault("REMOTE_CATALOG_REST_URI", "http://polaris:8181/api/catalog") os.environ.setdefault("REMOTE_CATALOG_S3_ENDPOINT", "http://minio:9000") os.environ.setdefault("REMOTE_CATALOG_S3_PATH_STYLE_ACCESS", "true") From b7fcd88de20ea783228e39ffdc10a9822961cff5 Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Thu, 27 Aug 2026 15:42:07 -0400 Subject: [PATCH 59/62] remove minio spark session util useage. Not needed anymore. --- tests/spark_authmanager_test.py | 4 ++-- tests/spark_permission_test.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/spark_authmanager_test.py b/tests/spark_authmanager_test.py index 4f78731..d9f225d 100644 --- a/tests/spark_authmanager_test.py +++ b/tests/spark_authmanager_test.py @@ -207,8 +207,8 @@ def main(): print(" Creating Spark session via AuthManager...") try: - from teehr.evaluation.spark_session_utils import create_minio_spark_session - spark = create_minio_spark_session( + from teehr.evaluation.spark_session_utils import create_spark_session + spark = create_spark_session( use_authmanager=True, force_recreate_session=True, ) diff --git a/tests/spark_permission_test.py b/tests/spark_permission_test.py index abccc6b..d3ad6fc 100644 --- a/tests/spark_permission_test.py +++ b/tests/spark_permission_test.py @@ -179,8 +179,8 @@ def main(): # Step 2: Create Spark session with Polaris catalog print(" Creating Spark session with Polaris catalog...") try: - from teehr.evaluation.spark_session_utils import create_minio_spark_session - spark = create_minio_spark_session( + from teehr.evaluation.spark_session_utils import create_spark_session + spark = create_spark_session( polaris_token=token, ) print(" ✓ Spark session created") From 02338302ccffaf22622ab2e61e88ddb2acaf3b81 Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Thu, 27 Aug 2026 16:31:02 -0400 Subject: [PATCH 60/62] fix REMOTE_WAREHOUSE_IDENTIFIER after switch off create_minio_spark_session create_minio_spark_session() defaulted remote_warehouse_dir to the realm name when REMOTE_WAREHOUSE_IDENTIFIER wasn't set; plain create_spark_session() defaults it to "" instead, which Polaris's REST catalog rejects with "Malformed request: Please specify a warehouse" on any catalog read/write. Both test scripts already set POLARIS_DEFAULT_REALM=teehr but not REMOTE_WAREHOUSE_IDENTIFIER, which only surfaced once they stopped going through the minio wrapper. Confirmed passing against a real KinD cluster: spark-permission-enforcement (admin/poweruser/user) and spark-authmanager-enforcement (admin/poweruser/user) both PASSED end to end. Co-Authored-By: Claude Sonnet 5 --- tests/spark_authmanager_test.py | 4 ++++ tests/spark_permission_test.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/tests/spark_authmanager_test.py b/tests/spark_authmanager_test.py index d9f225d..0bfbed0 100644 --- a/tests/spark_authmanager_test.py +++ b/tests/spark_authmanager_test.py @@ -53,6 +53,10 @@ # Set up environment before importing PySpark os.environ.setdefault("POLARIS_DEFAULT_REALM", "teehr") +# create_spark_session() defaults remote_warehouse_dir to "" (not the realm +# name) when this isn't set, which Polaris's REST catalog rejects with +# "Please specify a warehouse" on any catalog read/write. +os.environ.setdefault("REMOTE_WAREHOUSE_IDENTIFIER", "teehr") os.environ.setdefault("REMOTE_CATALOG_REST_URI", "http://polaris:8181/api/catalog") os.environ.setdefault("REMOTE_WAREHOUSE_S3_PATH", "s3://warehouse/") os.environ.setdefault("REMOTE_CATALOG_S3_ENDPOINT", "http://minio:9000") diff --git a/tests/spark_permission_test.py b/tests/spark_permission_test.py index d3ad6fc..1ba51e7 100644 --- a/tests/spark_permission_test.py +++ b/tests/spark_permission_test.py @@ -17,6 +17,10 @@ # Set up environment for Polaris/Spark before importing PySpark os.environ.setdefault("POLARIS_DEFAULT_REALM", "teehr") +# create_spark_session() defaults remote_warehouse_dir to "" (not the realm +# name) when this isn't set, which Polaris's REST catalog rejects with +# "Please specify a warehouse" on any catalog read/write. +os.environ.setdefault("REMOTE_WAREHOUSE_IDENTIFIER", "teehr") os.environ.setdefault("REMOTE_CATALOG_REST_URI", "http://polaris:8181/api/catalog") os.environ.setdefault("REMOTE_WAREHOUSE_S3_PATH", "s3://warehouse/") os.environ.setdefault("REMOTE_CATALOG_S3_ENDPOINT", "http://minio:9000") From c692128b46706f019927aed8627dc839db91b286 Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Thu, 27 Aug 2026 18:21:32 -0400 Subject: [PATCH 61/62] fix 4 issues: OpenAPI docs, broker error shape, principal sync, access-control doc - /auth/polaris-session's OpenAPI schema advertised ApiKeyAuth like the generic default, but the route requires identity.auth_type == "jwt" same as /auth/polaris-token, which already has this override. Added the matching security override. - docs/polaris-broker-api-contract.md documented a flat error payload shape; FastAPI actually wraps every HTTPException(detail=...) under a top-level "detail" key with no unwrapping handler registered. Updated the doc to match reality rather than changing response behavior (unknown whether any real consumer depends on the documented flat shape). - polaris-sync-principals-script.yaml's sync_principals.py had two redundant paths granting principal roles: one from the user's effective realm roles (matching Polaris's own regex mapper input exactly), and a second, less complete one hardcoded against raw Keycloak group names/paths -- including a hardcoded plural/singular special case (iceberg-catalog-admins group -> iceberg-catalog-admin role) that only worked by coincidence for the groups that exist today. Removed the group-based path entirely; the realm-role-based path already covers everything correctly (composite roles included) since it mirrors exactly what Polaris's regex reads from the JWT. - docs/polaris-access-control.md described Polaris reading a `groups` claim and matching plural/path-prefixed group names directly. The actual mechanism (polaris/manifests/polaris-config.yaml.tpl) reads realm_access.roles, populated via each Keycloak group's composite realmRoles mapping (keycloak-bootstrap/manifests/realm-configmap.yaml.tpl) -- a two-step chain the doc didn't mention. Rewrote to describe both steps accurately, including the plural-group/singular-role naming mismatch for iceberg-catalog-admins. Verified against a real KinD cluster: /openapi.json confirms /auth/polaris-session no longer advertises ApiKeyAuth; re-ran spark-permission-enforcement after redeploying the updated sync_principals.py -- still PASSED for all 3 users (admin/poweruser/user), confirming the realm-role-only path grants identical permissions to what the removed group-based path did. Co-Authored-By: Claude Sonnet 5 --- api/src/main.py | 11 ++++ docs/polaris-access-control.md | 52 ++++++++++++------- docs/polaris-broker-api-contract.md | 15 ++++-- .../polaris-sync-principals-script.yaml | 38 ++++---------- 4 files changed, 68 insertions(+), 48 deletions(-) diff --git a/api/src/main.py b/api/src/main.py index 448ed9c..4650c9e 100644 --- a/api/src/main.py +++ b/api/src/main.py @@ -161,6 +161,17 @@ def custom_openapi(): {"BearerAuth": []}, ] + # Same override as /auth/polaris-token: the route requires + # identity.auth_type == "jwt", so API-key auth is not accepted here + # either, even though the generic default above includes it. + auth_polaris_session = openapi_schema.get("paths", {}).get("/auth/polaris-session", {}).get("post") + if auth_polaris_session: + auth_polaris_session["security"] = [ + {"OAuth2KeycloakPassword": ["openid", "profile", "email"]}, + {"OAuth2Keycloak": ["openid", "profile", "email"]}, + {"BearerAuth": []}, + ] + auth_polaris_token_session = openapi_schema.get("paths", {}).get("/auth/polaris-token/session", {}).get("post") if auth_polaris_token_session: auth_polaris_token_session["security"] = [] diff --git a/docs/polaris-access-control.md b/docs/polaris-access-control.md index bf7e136..b856f0e 100644 --- a/docs/polaris-access-control.md +++ b/docs/polaris-access-control.md @@ -11,40 +11,53 @@ Access to the Iceberg catalog (Polaris) is controlled through a two-layer model: ## Layer 1: Group-level access (JWT claim mapping) -Polaris is configured to read the `groups` claim from Keycloak JWTs and map group -membership directly to Polaris principal roles. This means: +Access flows through two independent mappings, not one: + +1. **Keycloak**: group membership → composite **realm roles**, configured per-group in + `keycloak-bootstrap/manifests/realm-configmap.yaml.tpl` (each group's `realmRoles` list). + Keycloak includes a user's realm roles in the `realm_access.roles` claim of every + token by default — this is standard Keycloak behavior, not something configured here. +2. **Polaris**: reads realm role names out of that claim and maps them to Polaris + principal roles via a regex mapper, configured in `polaris/manifests/polaris-config.yaml.tpl`. + +Polaris itself never sees Keycloak group names or paths — only the realm role names +that groups happen to be composited to. This means: - No principal sync is needed for standard access - Adding a user to a Keycloak group grants them the corresponding Polaris permissions on their **next token issuance** (no delay, no operational coupling) - Removing a user from a group immediately revokes access -### Group → Principal role mapping +### Group → realm role → Polaris principal role mapping -| Keycloak group | Polaris principal role | Effect | -|---|---|---| -| `/teehr-read-only` | `teehr-read-only` | Can list and read tables in the `teehr` namespace | -| `/teehr-read-write` | `teehr-read-write` | Can create, read, write, and drop tables | -| `/iceberg-catalog-admins` | `iceberg-catalog-admin` | Full catalog management | +| Keycloak group | Composite realm role(s) | Polaris principal role | Effect | +|---|---|---|---| +| `/teehr-read-only` | `teehr-read-only` | `teehr-read-only` | Can list and read tables in the `teehr` namespace | +| `/teehr-read-write` | `teehr-read-write`, `teehr-read-only` | `teehr-read-write`, `teehr-read-only` | Can create, read, write, and drop tables | +| `/iceberg-catalog-admins` | `iceberg-catalog-admin` | `iceberg-catalog-admin` | Full catalog management | -Groups not in this table (e.g. `/basic-user`, `/jupyter-user`) are ignored by Polaris. +Realm roles not in this table (e.g. `basic-user`, `jupyter-user`) are ignored by Polaris — +note the *realm role* names are singular/unprefixed even where the *group* name (e.g. +`iceberg-catalog-admins`) isn't; don't confuse the two when tracing a permission issue. ### Polaris configuration Configured in `polaris/manifests/polaris-config.yaml.tpl`: ```properties -quarkus.oidc.roles.role-claim-path=groups +quarkus.oidc.roles.role-claim-path=realm_access/roles polaris.oidc.principal-roles-mapper.type=default -polaris.oidc.principal-roles-mapper.mappings[0].regex=^/?iceberg-catalog-admins$ +polaris.oidc.principal-roles-mapper.mappings[0].regex=^iceberg-catalog-admin$ polaris.oidc.principal-roles-mapper.mappings[0].replacement=PRINCIPAL_ROLE:iceberg-catalog-admin -polaris.oidc.principal-roles-mapper.mappings[1].regex=^/?teehr-(.+)$ +polaris.oidc.principal-roles-mapper.mappings[1].regex=^teehr-(.+)$ polaris.oidc.principal-roles-mapper.mappings[1].replacement=PRINCIPAL_ROLE:teehr-$1 ``` ### Adding a new access tier -1. Create a Keycloak group named `/teehr-` in `keycloak-bootstrap/manifests/` +1. Create a Keycloak group named `/teehr-` in `keycloak-bootstrap/manifests/`, with a + composite `realmRoles: ["teehr-"]` mapping — the realm role is what actually reaches + Polaris, so this step is required, not just the group itself 2. Add a namespace policy for the new principal role in `polaris-bootstrap/manifests/acl-config.yaml.tpl` 3. No Polaris config change needed — the `teehr-(.+)` pattern picks it up automatically 4. Add users to the group in Keycloak @@ -104,7 +117,8 @@ does not need to run before users can access the catalog. ``` User → Keycloak password/refresh grant → JWT (jupyterhub client) → Spark: spark.sql.catalog.iceberg.token = {jwt} - → Polaris: validates JWT, maps groups to principal roles, enforces permissions + → Polaris: validates JWT, maps realm_access.roles (from Keycloak group + membership) to principal roles, enforces permissions ``` ### AuthManager / broker (JupyterHub spawned notebooks) @@ -115,10 +129,12 @@ User logs in → JupyterHub OAuth → Keycloak issues access_token + refresh_tok → TeehrBrokerAuthManager (JAR) holds broker_session_token → On each Iceberg operation: JAR calls /auth/polaris-token/session → Broker refreshes token via Keycloak (refresh_token grant) - → Returns refreshed jupyterhub JWT (preserves group claims) - → Polaris: same JWT-based group mapping as direct token path + → Returns refreshed jupyterhub JWT (preserves realm_access.roles claim) + → Polaris: same realm_access.roles-based mapping as direct token path ``` Note: The broker does **not** do token exchange — it refreshes the user's token directly -to preserve group claims. Token exchange with a different audience was found to strip -the `groups` claim, preventing per-user permission enforcement. +to preserve the `realm_access.roles` claim (which is what Polaris actually reads; it's +derived from the user's Keycloak group membership, per Layer 1 above). Token exchange +with a different audience was found to strip this claim, preventing per-user permission +enforcement. diff --git a/docs/polaris-broker-api-contract.md b/docs/polaris-broker-api-contract.md index 1ffe2db..dd90b0c 100644 --- a/docs/polaris-broker-api-contract.md +++ b/docs/polaris-broker-api-contract.md @@ -82,14 +82,23 @@ Current prototype implementation in this repo: Error payload shape: +FastAPI wraps every `HTTPException(detail=...)` payload under a top-level +`"detail"` key by default, and this API has no exception handler that +unwraps it — so error responses are actually: + ```json { - "error": "forbidden", - "message": "session is not authorized for requested user_id", - "trace_id": "f8af1e8b3d21469a9adf99c9185d2d10" + "detail": { + "error": "forbidden", + "message": "session is not authorized for requested user_id", + "trace_id": "f8af1e8b3d21469a9adf99c9185d2d10" + } } ``` +Callers must read `error`/`message`/`trace_id` from `response.json()["detail"]`, +not from the top level of the response body. + ## Security Requirements - Broker must not trust notebook-supplied `groups` blindly. diff --git a/polaris-bootstrap/manifests/polaris-sync-principals-script.yaml b/polaris-bootstrap/manifests/polaris-sync-principals-script.yaml index 010db6d..d4a628d 100644 --- a/polaris-bootstrap/manifests/polaris-sync-principals-script.yaml +++ b/polaris-bootstrap/manifests/polaris-sync-principals-script.yaml @@ -6,7 +6,10 @@ data: sync_principals.py: | """ Syncs Keycloak users into Polaris as named principal entities and grants - principal role bindings for their teehr-* group memberships. + principal role bindings matching their effective (composite) realm roles -- + the same realm_access.roles claim Polaris's own JWT-based principal-roles-mapper + reads (polaris/manifests/polaris-config.yaml.tpl), so named-principal bindings + can't silently diverge from what group-based JWT mapping would have granted. NOTE ON DESIGN: When a named Polaris principal exists for a user, Polaris uses that principal's explicit role bindings rather than the JWT group claim mapper. @@ -95,22 +98,6 @@ data: return [role.get("name") for role in resp.json() if role.get("name")] - def get_keycloak_user_groups(kc_token, user_id): - """Get user's groups, including group paths like /teehr-read-only""" - resp = requests.get( - f"{KC_URL}/admin/realms/{REALM}/users/{user_id}/groups", - headers={"Authorization": f"Bearer {kc_token}"}, - ) - resp.raise_for_status() - groups = [] - for group in resp.json(): - # Extract the group name from the path (e.g., "/teehr-read-only" -> "teehr-read-only") - path = group.get("path", "").strip("/") - if path: - groups.append(path) - return groups - - def ensure_principal(polaris_token, principal_name): resp = requests.post( f"{MGMT_URL}/api/management/v1/principals", @@ -187,20 +174,17 @@ data: if name: ensure_principal(polaris_token, name) if user_id: - # Check realm roles + # Grant principal roles from the user's effective (composite) realm + # roles -- this is exactly what ends up in the JWT's realm_access.roles + # claim, i.e. the same input Polaris's own principal-roles-mapper regex + # (polaris/manifests/polaris-config.yaml.tpl) uses for JWT-based mapping. + # Matching that regex here (rather than deriving from Keycloak group + # names/paths separately) keeps named-principal bindings from silently + # diverging from what group-based JWT mapping would have granted. for role_name in get_keycloak_realm_roles(kc_token, user_id): if role_name.startswith("teehr-") or role_name == "iceberg-catalog-admin": ensure_principal_role_binding(polaris_token, name, role_name) granted += 1 - - # Check group memberships and map to principal roles - for group_name in get_keycloak_user_groups(kc_token, user_id): - if group_name.startswith("teehr-"): - ensure_principal_role_binding(polaris_token, name, group_name) - granted += 1 - elif group_name == "iceberg-catalog-admins": - ensure_principal_role_binding(polaris_token, name, "iceberg-catalog-admin") - granted += 1 synced += 1 print( From 5eadf1a51d08c97e7210b22dc771e2d4c128295c Mon Sep 17 00:00:00 2001 From: Matt Denno Date: Thu, 27 Aug 2026 21:29:50 -0400 Subject: [PATCH 62/62] make small test commit to submod. --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 75f108e..e82d472 100644 --- a/README.md +++ b/README.md @@ -120,4 +120,6 @@ Now the fun of adding new features and bug fixes starts. When working on the API or the frontend it is convenient to have code syncing. Code syncing can be done in `garden` by running: ```bash garden deploy --sync -``` \ No newline at end of file +``` + +test commit. \ No newline at end of file