diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 0000000..cbd5731 --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,9 @@ +Context: +- the `teehr` repo which is a python library that contains the scientific code for the teehr environment. This contains code to fetch, validate, store, and analyze hydrologic forecast and simulation data against observations. It works on the concept of an Evaluation which references either a local or remote warehouse and a relates spark session. +- the `teehr-cloud-core` repo contains the main shared teehr-cloud components such as authentication, jupyterhub, Apache Iceberg warehouse, web api, Trino query engine, spark-executors, prefect server, etc. teehr-cloud-core utilizes the teehr library. +- the `teehr-cloud-platform` repo contains terraform IaC for creating an AWS environment that can host a teehr-cloud deployment. +- the `teehr-hub` and `teehr-fved` repos have `teehr-cloud-core` as a submodule and are deployments of `teehr` and `teehr-cloud-core`. They also contain deployment specific components such as frontend dashboards, deployment specific prefect workflows and warehouse setup and maintenance code, certificates, etc. + +Don't make any changes in the `teehr-cloud-core` submodules to `teehr-hub` and `teehr-fved`. Only make changes in the `teehr-cloud-core` repo and then update the submodules in the deployments. + +Make all changes to the cluster via code. Running one-off commands via kubectl or other tools is not allowed. All changes must be made via code and then applied to the cluster via CI/CD pipelines. This ensures that all changes are tracked in version control and can be rolled back if necessary. \ No newline at end of file diff --git a/README.md b/README.md index 75f108e..e82d472 100644 --- a/README.md +++ b/README.md @@ -120,4 +120,6 @@ Now the fun of adding new features and bug fixes starts. When working on the API or the frontend it is convenient to have code syncing. Code syncing can be done in `garden` by running: ```bash garden deploy --sync -``` \ No newline at end of file +``` + +test commit. \ No newline at end of file diff --git a/api/manifests/configmap.yaml.tpl b/api/manifests/configmap.yaml.tpl index b0e5750..edddbff 100644 --- a/api/manifests/configmap.yaml.tpl +++ b/api/manifests/configmap.yaml.tpl @@ -15,7 +15,18 @@ data: KEYCLOAK_ISSUER_URL: "https://auth.${var.hostname}/realms/teehr" KEYCLOAK_JWKS_URL: "http://keycloak-service:8080/realms/teehr/protocol/openid-connect/certs" KEYCLOAK_AUDIENCE: "teehr-api" - KEYCLOAK_ALLOWED_AUDIENCES: "teehr-api,teehr-frontend" + KEYCLOAK_ALLOWED_AUDIENCES: "teehr-api,teehr-frontend,jupyterhub" + BROKER_TOKEN_EXCHANGE_ENABLED: "true" + BROKER_TOKEN_ENDPOINT: "http://keycloak-service:8080/realms/teehr/protocol/openid-connect/token" + BROKER_OAUTH_CLIENT_ID: "teehr-api" + BROKER_OAUTH_CLIENT_SECRET: "" + BROKER_TARGET_AUDIENCE: "account" + BROKER_DEFAULT_SCOPE: "openid profile email" + BROKER_MIN_TTL_SECONDS: "120" + BROKER_MAX_TTL_SECONDS: "900" + BROKER_REQUEST_TIMEOUT_SECONDS: "10" + BROKER_SUBJECT_CLIENT_ID: "jupyterhub" + BROKER_DELEGATED_SESSION_TTL_SECONDS: "43200" ANON_RATE_LIMIT_RPM: "20" AUTH_RATE_LIMIT_RPM: "120" ROW_LIMIT_ANON: "200" diff --git a/api/manifests/deployment.yaml.tpl b/api/manifests/deployment.yaml.tpl index 49993db..b3e38c0 100644 --- a/api/manifests/deployment.yaml.tpl +++ b/api/manifests/deployment.yaml.tpl @@ -76,6 +76,76 @@ spec: configMapKeyRef: name: teehr-api-config key: KEYCLOAK_ALLOWED_AUDIENCES + - name: BROKER_TOKEN_EXCHANGE_ENABLED + valueFrom: + configMapKeyRef: + name: teehr-api-config + key: BROKER_TOKEN_EXCHANGE_ENABLED + - name: BROKER_TOKEN_ENDPOINT + valueFrom: + configMapKeyRef: + name: teehr-api-config + key: BROKER_TOKEN_ENDPOINT + - name: BROKER_OAUTH_CLIENT_ID + valueFrom: + configMapKeyRef: + name: teehr-api-config + key: BROKER_OAUTH_CLIENT_ID + - name: BROKER_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: teehr-api-secrets + key: client-secret + - name: BROKER_TARGET_AUDIENCE + valueFrom: + configMapKeyRef: + name: teehr-api-config + key: BROKER_TARGET_AUDIENCE + - name: BROKER_DEFAULT_SCOPE + valueFrom: + configMapKeyRef: + name: teehr-api-config + key: BROKER_DEFAULT_SCOPE + - name: BROKER_MIN_TTL_SECONDS + valueFrom: + configMapKeyRef: + name: teehr-api-config + key: BROKER_MIN_TTL_SECONDS + - name: BROKER_MAX_TTL_SECONDS + valueFrom: + configMapKeyRef: + name: teehr-api-config + key: BROKER_MAX_TTL_SECONDS + - name: BROKER_REQUEST_TIMEOUT_SECONDS + valueFrom: + configMapKeyRef: + name: teehr-api-config + key: BROKER_REQUEST_TIMEOUT_SECONDS + - name: BROKER_SUBJECT_CLIENT_ID + valueFrom: + configMapKeyRef: + name: teehr-api-config + key: BROKER_SUBJECT_CLIENT_ID + - name: BROKER_DELEGATED_SESSION_TTL_SECONDS + valueFrom: + configMapKeyRef: + name: teehr-api-config + key: BROKER_DELEGATED_SESSION_TTL_SECONDS + - name: BROKER_SUBJECT_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: jupyterhub + key: OAUTH_CLIENT_SECRET + - name: BROKER_SESSION_SIGNING_SECRET + valueFrom: + secretKeyRef: + name: broker-secrets + key: session-signing-secret + - name: BROKER_REFRESH_TOKEN_ENCRYPTION_SECRET + valueFrom: + secretKeyRef: + name: broker-secrets + key: refresh-token-encryption-secret - name: API_KEYS_DB_HOST value: keycloak-pg - name: API_KEYS_DB_PORT @@ -97,6 +167,27 @@ spec: key: password - name: API_KEYS_DB_DSN value: "postgresql://$(API_KEYS_DB_USER):$(API_KEYS_DB_PASSWORD)@$(API_KEYS_DB_HOST):$(API_KEYS_DB_PORT)/$(API_KEYS_DB_NAME)" + - name: DELEGATED_SESSIONS_DB_HOST + value: keycloak-pg + - name: DELEGATED_SESSIONS_DB_PORT + value: "5432" + - name: DELEGATED_SESSIONS_DB_NAME + valueFrom: + secretKeyRef: + name: teehr-api-db-secrets + key: database + - name: DELEGATED_SESSIONS_DB_USER + valueFrom: + secretKeyRef: + name: teehr-api-db-secrets + key: username + - name: DELEGATED_SESSIONS_DB_PASSWORD + valueFrom: + secretKeyRef: + name: teehr-api-db-secrets + key: password + - name: DELEGATED_SESSIONS_DB_DSN + value: "postgresql://$(DELEGATED_SESSIONS_DB_USER):$(DELEGATED_SESSIONS_DB_PASSWORD)@$(DELEGATED_SESSIONS_DB_HOST):$(DELEGATED_SESSIONS_DB_PORT)/$(DELEGATED_SESSIONS_DB_NAME)" - name: ANON_RATE_LIMIT_RPM valueFrom: configMapKeyRef: diff --git a/api/src/auth.py b/api/src/auth.py index bf2293e..c821a39 100644 --- a/api/src/auth.py +++ b/api/src/auth.py @@ -35,6 +35,8 @@ class AuthIdentity: auth_type: str roles: list[str] = field(default_factory=list) scopes: list[str] = field(default_factory=list) + preferred_username: str | None = None + groups: list[str] = field(default_factory=list) @property def is_authenticated(self) -> bool: @@ -129,11 +131,17 @@ async def validate(self, token: str) -> AuthIdentity: if not subject: raise HTTPException(status_code=401, detail="JWT missing subject") + groups = claims.get("groups", []) + if not isinstance(groups, list): + groups = [] + return AuthIdentity( subject=subject, auth_type="jwt", roles=roles, scopes=scopes, + preferred_username=claims.get("preferred_username"), + groups=[str(group) for group in groups], ) except httpx.HTTPError as exc: logger.error("Keycloak connectivity error during token validation: %s", str(exc)) @@ -172,6 +180,16 @@ async def get_request_identity(request: Request) -> AuthIdentity: return identity +def extract_bearer_token_from_request(request: Request) -> str: + auth_header = request.headers.get("authorization", "") + if not auth_header.lower().startswith("bearer "): + raise HTTPException(status_code=401, detail="Bearer token required") + token = auth_header.split(" ", 1)[1].strip() + if not token: + raise HTTPException(status_code=401, detail="Bearer token required") + return token + + async def get_authenticated_identity( identity: AuthIdentity = Depends(get_request_identity), ) -> AuthIdentity: diff --git a/api/src/broker.py b/api/src/broker.py new file mode 100644 index 0000000..38cd5e0 --- /dev/null +++ b/api/src/broker.py @@ -0,0 +1,293 @@ +import time +import uuid +from asyncio import Lock + +import httpx +from fastapi import HTTPException +from jose import JWTError, jwt + +from .config import config +from .delegated_session_store import DelegatedSessionStore + + +TOKEN_EXCHANGE_GRANT = "urn:ietf:params:oauth:grant-type:token-exchange" +ACCESS_TOKEN_TYPE = "urn:ietf:params:oauth:token-type:access_token" + + +_DELEGATED_SESSION_STORE: DelegatedSessionStore | None = None +_DELEGATED_SESSION_STORE_LOCK = Lock() + + +async def set_delegated_session_store(store: DelegatedSessionStore): + async with _DELEGATED_SESSION_STORE_LOCK: + global _DELEGATED_SESSION_STORE + _DELEGATED_SESSION_STORE = store + + +async def _get_delegated_session_store() -> DelegatedSessionStore: + async with _DELEGATED_SESSION_STORE_LOCK: + if _DELEGATED_SESSION_STORE is None: + raise HTTPException(status_code=503, detail="Delegated session store unavailable") + return _DELEGATED_SESSION_STORE + + +def clamp_requested_ttl(requested_ttl_seconds: int | None) -> int: + requested = requested_ttl_seconds or config.BROKER_MAX_TTL_SECONDS + return max(config.BROKER_MIN_TTL_SECONDS, min(requested, config.BROKER_MAX_TTL_SECONDS)) + + +async def exchange_token_for_polaris( + *, + subject_token: str, + audience: str, + requested_ttl_seconds: int | None, +) -> dict: + if not config.BROKER_TOKEN_EXCHANGE_ENABLED: + raise HTTPException(status_code=503, detail="Token broker is disabled") + + trace_id = uuid.uuid4().hex + clamped_ttl = clamp_requested_ttl(requested_ttl_seconds) + + payload = { + "grant_type": TOKEN_EXCHANGE_GRANT, + "client_id": config.BROKER_OAUTH_CLIENT_ID, + "subject_token": subject_token, + "subject_token_type": ACCESS_TOKEN_TYPE, + "requested_token_type": ACCESS_TOKEN_TYPE, + "audience": audience, + "scope": config.BROKER_DEFAULT_SCOPE, + } + if config.BROKER_OAUTH_CLIENT_SECRET: + payload["client_secret"] = config.BROKER_OAUTH_CLIENT_SECRET + + try: + async with httpx.AsyncClient(timeout=config.BROKER_REQUEST_TIMEOUT_SECONDS) as client: + response = await client.post(config.BROKER_TOKEN_ENDPOINT, data=payload) + + if response.status_code >= 400: + upstream_error = None + upstream_error_description = None + try: + upstream_payload = response.json() + upstream_error = upstream_payload.get("error") + upstream_error_description = upstream_payload.get("error_description") + except ValueError: + upstream_error_description = response.text[:500] + + raise HTTPException( + status_code=502, + detail={ + "error": "token_exchange_failed", + "message": "Broker failed to exchange token with identity provider", + "trace_id": trace_id, + "upstream_status": response.status_code, + "upstream_error": upstream_error, + "upstream_error_description": upstream_error_description, + }, + ) + + token_payload = response.json() + access_token = token_payload.get("access_token") + if not access_token: + raise HTTPException( + status_code=502, + detail={ + "error": "invalid_upstream_response", + "message": "Identity provider did not return access_token", + "trace_id": trace_id, + }, + ) + + expires_in = int(token_payload.get("expires_in", clamped_ttl)) + expires_at_epoch_seconds = int(time.time()) + max(expires_in, 1) + + # Prefer JWT exp claim when present. + try: + claims = jwt.get_unverified_claims(access_token) + exp = int(claims.get("exp", 0)) + if exp > 0: + expires_at_epoch_seconds = exp + except (JWTError, ValueError, TypeError): + pass + + return { + "access_token": access_token, + "token_type": token_payload.get("token_type", "Bearer"), + "expires_in_seconds": max(expires_at_epoch_seconds - int(time.time()), 1), + "expires_at_epoch_seconds": expires_at_epoch_seconds, + "trace_id": trace_id, + } + except httpx.HTTPError as exc: + raise HTTPException( + status_code=503, + detail={ + "error": "broker_connectivity_error", + "message": "Unable to contact identity provider for token exchange", + "trace_id": trace_id, + }, + ) from exc + + +async def _refresh_subject_access_token(refresh_token: str) -> tuple[str, str | None]: + payload = { + "grant_type": "refresh_token", + "client_id": config.BROKER_SUBJECT_CLIENT_ID, + "refresh_token": refresh_token, + } + if config.BROKER_SUBJECT_CLIENT_SECRET: + payload["client_secret"] = config.BROKER_SUBJECT_CLIENT_SECRET + + async with httpx.AsyncClient(timeout=config.BROKER_REQUEST_TIMEOUT_SECONDS) as client: + response = await client.post(config.BROKER_TOKEN_ENDPOINT, data=payload) + + if response.status_code >= 400: + try: + upstream_payload = response.json() + detail = upstream_payload.get("error_description") or upstream_payload.get("error") + except ValueError: + detail = response.text[:500] + raise HTTPException( + status_code=401, + detail={ + "error": "subject_refresh_failed", + "message": "Unable to refresh delegated subject token", + "upstream_status": response.status_code, + "upstream_detail": detail, + }, + ) + + token_payload = response.json() + access_token = token_payload.get("access_token") + if not access_token: + raise HTTPException( + status_code=502, + detail={ + "error": "invalid_upstream_response", + "message": "Identity provider did not return refreshed access_token", + }, + ) + + return access_token, token_payload.get("refresh_token") + + +async def create_delegated_broker_session( + *, + subject: str, + user_id: str, + session_id: str, + realm: str, + catalog: str, + audience: str, + refresh_token: str, +) -> dict: + if not refresh_token: + raise HTTPException(status_code=400, detail="refresh_token is required") + + now = int(time.time()) + ttl = max(config.BROKER_DELEGATED_SESSION_TTL_SECONDS, 300) + expires_at = now + ttl + delegated_session_id = uuid.uuid4().hex + + claims = { + "sid": delegated_session_id, + "sub": subject, + "uid": user_id, + "exp": expires_at, + "iat": now, + "typ": "teehr-broker-session", + } + + broker_session_token = jwt.encode( + claims, + config.BROKER_SESSION_SIGNING_SECRET, + algorithm="HS256", + ) + + store = await _get_delegated_session_store() + await store.put_session( + sid=delegated_session_id, + subject=subject, + user_id=user_id, + session_id=session_id, + realm=realm, + catalog=catalog, + audience=audience, + refresh_token=refresh_token, + expires_at_epoch_seconds=expires_at, + ) + + return { + "broker_session_token": broker_session_token, + "expires_at_epoch_seconds": expires_at, + } + + +async def exchange_token_for_polaris_via_broker_session( + *, + broker_session_token: str, + user_id: str, + session_id: str, + realm: str, + requested_ttl_seconds: int | None, +) -> dict: + if not broker_session_token: + raise HTTPException(status_code=401, detail="broker session token required") + + try: + claims = jwt.decode( + broker_session_token, + config.BROKER_SESSION_SIGNING_SECRET, + algorithms=["HS256"], + ) + except JWTError as exc: + raise HTTPException(status_code=401, detail="Invalid broker session token") from exc + + delegated_session_id = claims.get("sid") + if not delegated_session_id: + raise HTTPException(status_code=401, detail="Invalid broker session token") + + store = await _get_delegated_session_store() + record = await store.get_session(delegated_session_id) + + if not record: + raise HTTPException(status_code=401, detail="Delegated broker session not found") + + if int(record.get("expires_at", 0)) <= int(time.time()): + await store.delete_session(delegated_session_id) + raise HTTPException(status_code=401, detail="Delegated broker session expired") + + if ( + record.get("user_id") != user_id + or record.get("session_id") != session_id + or record.get("realm") != realm + ): + raise HTTPException(status_code=403, detail="Session identity mismatch") + + refreshed_subject_token, maybe_new_refresh_token = await _refresh_subject_access_token( + record["refresh_token"] + ) + + if maybe_new_refresh_token: + await store.update_refresh_token(delegated_session_id, maybe_new_refresh_token) + + # Return the refreshed subject token directly — it preserves the user's + # group claims and is accepted by Polaris for per-user permission enforcement. + # Token exchange with a different audience would strip group claims and + # prevent Polaris from mapping the user to their correct principal roles. + trace_id = uuid.uuid4().hex + expires_at_epoch_seconds = int(time.time()) + clamp_requested_ttl(requested_ttl_seconds) + try: + token_claims = jwt.get_unverified_claims(refreshed_subject_token) + exp = int(token_claims.get("exp", 0)) + if exp > 0: + expires_at_epoch_seconds = exp + except (JWTError, ValueError, TypeError): + pass + + return { + "access_token": refreshed_subject_token, + "token_type": "Bearer", + "expires_in_seconds": max(expires_at_epoch_seconds - int(time.time()), 1), + "expires_at_epoch_seconds": expires_at_epoch_seconds, + "trace_id": trace_id, + } diff --git a/api/src/config.py b/api/src/config.py index 4092da6..90ea248 100644 --- a/api/src/config.py +++ b/api/src/config.py @@ -40,7 +40,7 @@ class Config: KEYCLOAK_AUDIENCE = os.environ.get("KEYCLOAK_AUDIENCE", "teehr-api") KEYCLOAK_ALLOWED_AUDIENCES = os.environ.get( "KEYCLOAK_ALLOWED_AUDIENCES", - "teehr-api,teehr-frontend", + "teehr-api,teehr-frontend,jupyterhub", ) KEYCLOAK_AUTH_URL = os.environ.get( "KEYCLOAK_AUTH_URL", @@ -60,6 +60,10 @@ class Config: "API_KEYS_DB_DSN", "postgresql://keycloak:keycloak123@keycloak-pg:5432/teehr_api", ) + DELEGATED_SESSIONS_DB_DSN = os.environ.get( + "DELEGATED_SESSIONS_DB_DSN", + API_KEYS_DB_DSN, + ) API_KEY_PREFIX = os.environ.get("API_KEY_PREFIX", "thk_") API_KEY_HASH_SALT = os.environ.get( "API_KEY_HASH_SALT", @@ -70,6 +74,38 @@ class Config: ANON_RATE_LIMIT_RPM = int(os.environ.get("ANON_RATE_LIMIT_RPM", "20")) AUTH_RATE_LIMIT_RPM = int(os.environ.get("AUTH_RATE_LIMIT_RPM", "120")) + # Polaris token broker settings + BROKER_TOKEN_EXCHANGE_ENABLED = ( + os.environ.get("BROKER_TOKEN_EXCHANGE_ENABLED", "true").strip().lower() + in {"1", "true", "t", "yes", "y", "on"} + ) + BROKER_TOKEN_ENDPOINT = os.environ.get( + "BROKER_TOKEN_ENDPOINT", + KEYCLOAK_TOKEN_URL, + ) + BROKER_OAUTH_CLIENT_ID = os.environ.get("BROKER_OAUTH_CLIENT_ID", "teehr-api") + BROKER_OAUTH_CLIENT_SECRET = os.environ.get("BROKER_OAUTH_CLIENT_SECRET", "") + BROKER_TARGET_AUDIENCE = os.environ.get("BROKER_TARGET_AUDIENCE", "account") + BROKER_DEFAULT_SCOPE = os.environ.get("BROKER_DEFAULT_SCOPE", "openid profile email") + BROKER_MIN_TTL_SECONDS = int(os.environ.get("BROKER_MIN_TTL_SECONDS", "120")) + BROKER_MAX_TTL_SECONDS = int(os.environ.get("BROKER_MAX_TTL_SECONDS", "900")) + BROKER_REQUEST_TIMEOUT_SECONDS = int( + os.environ.get("BROKER_REQUEST_TIMEOUT_SECONDS", "10") + ) + BROKER_SUBJECT_CLIENT_ID = os.environ.get("BROKER_SUBJECT_CLIENT_ID", "jupyterhub") + BROKER_SUBJECT_CLIENT_SECRET = os.environ.get("BROKER_SUBJECT_CLIENT_SECRET", "") + BROKER_DELEGATED_SESSION_TTL_SECONDS = int( + os.environ.get("BROKER_DELEGATED_SESSION_TTL_SECONDS", "43200") + ) + BROKER_SESSION_SIGNING_SECRET = os.environ.get( + "BROKER_SESSION_SIGNING_SECRET", + "local-dev-change-me-session-signing", + ) + BROKER_REFRESH_TOKEN_ENCRYPTION_SECRET = os.environ.get( + "BROKER_REFRESH_TOKEN_ENCRYPTION_SECRET", + "local-dev-change-me-refresh-encryption", + ) + # Role-based record/page limits ROW_LIMIT_ANON = int(os.environ.get("ROW_LIMIT_ANON", "200")) ROW_LIMIT_API_KEY = int(os.environ.get("ROW_LIMIT_API_KEY", "50000")) diff --git a/api/src/delegated_session_store.py b/api/src/delegated_session_store.py new file mode 100644 index 0000000..9a03e20 --- /dev/null +++ b/api/src/delegated_session_store.py @@ -0,0 +1,183 @@ +import base64 +import hashlib +from datetime import UTC, datetime + +import asyncpg +from cryptography.fernet import Fernet + + +class DelegatedSessionStore: + def __init__(self, dsn: str, refresh_token_encryption_secret: str): + self._dsn = dsn + self._pool: asyncpg.Pool | None = None + key_material = hashlib.sha256( + refresh_token_encryption_secret.encode("utf-8") + ).digest() + fernet_key = base64.urlsafe_b64encode(key_material) + self._fernet = Fernet(fernet_key) + + def _encrypt_refresh_token(self, refresh_token: str) -> str: + return self._fernet.encrypt(refresh_token.encode("utf-8")).decode("utf-8") + + def _decrypt_refresh_token(self, stored_value: str) -> str: + return self._fernet.decrypt(stored_value.encode("utf-8")).decode("utf-8") + + async def startup(self): + self._pool = await asyncpg.create_pool(dsn=self._dsn, min_size=1, max_size=5) + async with self._pool.acquire() as conn: + await conn.execute( + """ + CREATE TABLE IF NOT EXISTS delegated_sessions ( + sid TEXT PRIMARY KEY, + subject TEXT NOT NULL, + user_id TEXT NOT NULL, + session_id TEXT NOT NULL, + realm TEXT NOT NULL, + catalog TEXT NOT NULL, + audience TEXT NOT NULL, + refresh_token TEXT NOT NULL, + expires_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + ); + """ + ) + await conn.execute( + """ + CREATE INDEX IF NOT EXISTS delegated_sessions_expires_at_idx + ON delegated_sessions (expires_at); + """ + ) + + async def shutdown(self): + if self._pool: + await self._pool.close() + + async def put_session( + self, + *, + sid: str, + subject: str, + user_id: str, + session_id: str, + realm: str, + catalog: str, + audience: str, + refresh_token: str, + expires_at_epoch_seconds: int, + ): + if self._pool is None: + raise RuntimeError("Delegated session store is not initialized") + + expires_at = datetime.fromtimestamp(expires_at_epoch_seconds, tz=UTC) + encrypted_refresh_token = self._encrypt_refresh_token(refresh_token) + async with self._pool.acquire() as conn: + await conn.execute( + """ + INSERT INTO delegated_sessions ( + sid, + subject, + user_id, + session_id, + realm, + catalog, + audience, + refresh_token, + expires_at, + updated_at + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, NOW()) + ON CONFLICT (sid) + DO UPDATE SET + subject = EXCLUDED.subject, + user_id = EXCLUDED.user_id, + session_id = EXCLUDED.session_id, + realm = EXCLUDED.realm, + catalog = EXCLUDED.catalog, + audience = EXCLUDED.audience, + refresh_token = EXCLUDED.refresh_token, + expires_at = EXCLUDED.expires_at, + updated_at = NOW() + """, + sid, + subject, + user_id, + session_id, + realm, + catalog, + audience, + encrypted_refresh_token, + expires_at, + ) + + async def get_session(self, sid: str) -> dict | None: + if self._pool is None: + raise RuntimeError("Delegated session store is not initialized") + + async with self._pool.acquire() as conn: + row = await conn.fetchrow( + """ + SELECT + sid, + subject, + user_id, + session_id, + realm, + catalog, + audience, + refresh_token, + expires_at + FROM delegated_sessions + WHERE sid = $1 + """, + sid, + ) + + if row is None: + return None + + expires_at = row["expires_at"] + if expires_at.tzinfo is None: + expires_at = expires_at.replace(tzinfo=UTC) + + decrypted_refresh_token = self._decrypt_refresh_token(row["refresh_token"]) + + return { + "sid": row["sid"], + "subject": row["subject"], + "user_id": row["user_id"], + "session_id": row["session_id"], + "realm": row["realm"], + "catalog": row["catalog"], + "audience": row["audience"], + "refresh_token": decrypted_refresh_token, + "expires_at": int(expires_at.timestamp()), + } + + async def update_refresh_token(self, sid: str, refresh_token: str): + if self._pool is None: + raise RuntimeError("Delegated session store is not initialized") + + encrypted_refresh_token = self._encrypt_refresh_token(refresh_token) + async with self._pool.acquire() as conn: + await conn.execute( + """ + UPDATE delegated_sessions + SET refresh_token = $1, updated_at = NOW() + WHERE sid = $2 + """, + encrypted_refresh_token, + sid, + ) + + async def delete_session(self, sid: str): + if self._pool is None: + raise RuntimeError("Delegated session store is not initialized") + + async with self._pool.acquire() as conn: + await conn.execute( + """ + DELETE FROM delegated_sessions + WHERE sid = $1 + """, + sid, + ) diff --git a/api/src/main.py b/api/src/main.py index 186c9cd..4650c9e 100644 --- a/api/src/main.py +++ b/api/src/main.py @@ -10,7 +10,9 @@ from .api_key_store import ApiKeyStore from .auth import KeycloakJWTValidator, resolve_identity +from .broker import set_delegated_session_store from .config import config +from .delegated_session_store import DelegatedSessionStore from .models import HealthResponse from .rate_limit import InMemoryRateLimiter from .routes import router @@ -72,6 +74,20 @@ def custom_openapi(): } }, } + security_schemes["OAuth2KeycloakPassword"] = { + "type": "oauth2", + "description": "Login with Keycloak username/password using the password grant", + "flows": { + "password": { + "tokenUrl": config.KEYCLOAK_TOKEN_URL, + "scopes": { + "openid": "OpenID Connect scope", + "profile": "User profile", + "email": "User email", + }, + } + }, + } security_schemes["ApiKeyAuth"] = { "type": "apiKey", "in": "header", @@ -96,6 +112,7 @@ def custom_openapi(): operation.setdefault( "security", [ + {"OAuth2KeycloakPassword": ["openid", "profile", "email"]}, {"OAuth2Keycloak": ["openid", "profile", "email"]}, {"BearerAuth": []}, {"ApiKeyAuth": []}, @@ -106,6 +123,7 @@ def custom_openapi(): auth_me = openapi_schema.get("paths", {}).get("/auth/me", {}).get("get") if auth_me: auth_me["security"] = [ + {"OAuth2KeycloakPassword": ["openid", "profile", "email"]}, {"OAuth2Keycloak": ["openid", "profile", "email"]}, {"BearerAuth": []}, {"ApiKeyAuth": []}, @@ -114,6 +132,7 @@ def custom_openapi(): auth_keys_get = openapi_schema.get("paths", {}).get("/auth/api-keys", {}).get("get") if auth_keys_get: auth_keys_get["security"] = [ + {"OAuth2KeycloakPassword": ["openid", "profile", "email"]}, {"OAuth2Keycloak": ["openid", "profile", "email"]}, {"BearerAuth": []}, ] @@ -121,6 +140,7 @@ def custom_openapi(): auth_keys_post = openapi_schema.get("paths", {}).get("/auth/api-keys", {}).get("post") if auth_keys_post: auth_keys_post["security"] = [ + {"OAuth2KeycloakPassword": ["openid", "profile", "email"]}, {"OAuth2Keycloak": ["openid", "profile", "email"]}, {"BearerAuth": []}, ] @@ -128,10 +148,34 @@ def custom_openapi(): auth_keys_delete = openapi_schema.get("paths", {}).get("/auth/api-keys/{key_id}", {}).get("delete") if auth_keys_delete: auth_keys_delete["security"] = [ + {"OAuth2KeycloakPassword": ["openid", "profile", "email"]}, {"OAuth2Keycloak": ["openid", "profile", "email"]}, {"BearerAuth": []}, ] + auth_polaris_token = openapi_schema.get("paths", {}).get("/auth/polaris-token", {}).get("post") + if auth_polaris_token: + auth_polaris_token["security"] = [ + {"OAuth2KeycloakPassword": ["openid", "profile", "email"]}, + {"OAuth2Keycloak": ["openid", "profile", "email"]}, + {"BearerAuth": []}, + ] + + # Same override as /auth/polaris-token: the route requires + # identity.auth_type == "jwt", so API-key auth is not accepted here + # either, even though the generic default above includes it. + auth_polaris_session = openapi_schema.get("paths", {}).get("/auth/polaris-session", {}).get("post") + if auth_polaris_session: + auth_polaris_session["security"] = [ + {"OAuth2KeycloakPassword": ["openid", "profile", "email"]}, + {"OAuth2Keycloak": ["openid", "profile", "email"]}, + {"BearerAuth": []}, + ] + + auth_polaris_token_session = openapi_schema.get("paths", {}).get("/auth/polaris-token/session", {}).get("post") + if auth_polaris_token_session: + auth_polaris_token_session["security"] = [] + app.openapi_schema = openapi_schema return app.openapi_schema @@ -145,10 +189,17 @@ async def startup_event(): app.state.rate_limiter = InMemoryRateLimiter() app.state.api_key_store = ApiKeyStore(config.API_KEYS_DB_DSN) await app.state.api_key_store.startup() + app.state.delegated_session_store = DelegatedSessionStore( + config.DELEGATED_SESSIONS_DB_DSN, + config.BROKER_REFRESH_TOKEN_ENCRYPTION_SECRET, + ) + await app.state.delegated_session_store.startup() + await set_delegated_session_store(app.state.delegated_session_store) @app.on_event("shutdown") async def shutdown_event(): + await app.state.delegated_session_store.shutdown() await app.state.api_key_store.shutdown() # CORS middleware to allow frontend requests - MUST be first middleware @@ -187,6 +238,14 @@ async def auth_context_middleware(request: Request, call_next): return await call_next(request) path = request.url.path + if path == "/auth/polaris-token/session": + client_host = request.client.host if request.client else "unknown" + app.state.rate_limiter.check_by_key( + f"polaris-token-session:{client_host}", + limit=config.AUTH_RATE_LIMIT_RPM, + ) + return await call_next(request) + exempt_paths = ( path == "/health" or path == "/openapi.json" @@ -195,7 +254,10 @@ async def auth_context_middleware(request: Request, call_next): ) # Keep auth optional while attaching identity for routes that need it. - request.state.identity = await resolve_identity(request) + try: + request.state.identity = await resolve_identity(request) + except HTTPException as exc: + return JSONResponse(status_code=exc.status_code, content={"detail": exc.detail}) # Require authentication for all non-diagnostic API paths. if not exempt_paths and not request.state.identity.is_authenticated: diff --git a/api/src/rate_limit.py b/api/src/rate_limit.py index 0d8f0d0..2b33832 100644 --- a/api/src/rate_limit.py +++ b/api/src/rate_limit.py @@ -28,6 +28,14 @@ def check(self, identity: AuthIdentity, route_key: str): minute_bucket = int(time.time() // 60) key = self._key(identity, route_key, minute_bucket) + self._enforce(key, limit, minute_bucket) + + def check_by_key(self, key_prefix: str, limit: int): + minute_bucket = int(time.time() // 60) + key = f"{minute_bucket}:{key_prefix}" + self._enforce(key, limit, minute_bucket) + + def _enforce(self, key: str, limit: int, minute_bucket: int): self._counts[key] += 1 # Cheap periodic cleanup to avoid unbounded growth. diff --git a/api/src/routes/auth.py b/api/src/routes/auth.py index cb42cad..429d750 100644 --- a/api/src/routes/auth.py +++ b/api/src/routes/auth.py @@ -1,7 +1,19 @@ from fastapi import APIRouter, Depends, HTTPException, Request from pydantic import BaseModel, Field -from ..auth import AuthIdentity, get_admin_identity, get_request_identity +from ..auth import ( + AuthIdentity, + extract_bearer_token_from_request, + get_admin_identity, + get_authenticated_identity, + get_request_identity, +) +from ..broker import exchange_token_for_polaris +from ..broker import ( + create_delegated_broker_session, + exchange_token_for_polaris_via_broker_session, +) +from ..config import config router = APIRouter(prefix="/auth", tags=["Auth"]) @@ -11,6 +23,45 @@ class ApiKeyCreateRequest(BaseModel): scopes: list[str] = Field(default_factory=list) +class PolarisTokenRequest(BaseModel): + user_id: str = Field(min_length=1) + session_id: str = Field(min_length=1) + realm: str = Field(min_length=1) + catalog: str = Field(default="iceberg", min_length=1) + groups: list[str] = Field(default_factory=list) + requested_ttl_seconds: int = Field(default=600, ge=1) + audience: str = Field(default_factory=lambda: config.BROKER_TARGET_AUDIENCE) + + +class PolarisTokenIssuedFor(BaseModel): + user_id: str + session_id: str + realm: str + + +class PolarisTokenResponse(BaseModel): + access_token: str + token_type: str + expires_at_epoch_seconds: int + expires_in_seconds: int + issued_for: PolarisTokenIssuedFor + trace_id: str + + +class PolarisSessionRequest(BaseModel): + user_id: str = Field(min_length=1) + session_id: str = Field(min_length=1) + realm: str = Field(min_length=1) + catalog: str = Field(default="iceberg", min_length=1) + audience: str = Field(default_factory=lambda: config.BROKER_TARGET_AUDIENCE) + refresh_token: str = Field(min_length=1) + + +class PolarisSessionResponse(BaseModel): + broker_session_token: str + expires_at_epoch_seconds: int + + @router.get("/me") async def me(identity: AuthIdentity = Depends(get_request_identity)): return { @@ -54,3 +105,105 @@ async def revoke_api_key( revoked = await request.app.state.api_key_store.revoke_key(identity.subject, key_id) if not revoked: raise HTTPException(status_code=404, detail="API key not found") + + +@router.post("/polaris-token", response_model=PolarisTokenResponse) +async def polaris_token( + request: Request, + payload: PolarisTokenRequest, + identity: AuthIdentity = Depends(get_authenticated_identity), +): + if identity.auth_type != "jwt": + raise HTTPException(status_code=403, detail="JWT identity required") + + allowed_user_ids = {identity.subject} + if identity.preferred_username: + allowed_user_ids.add(identity.preferred_username) + + if payload.user_id not in allowed_user_ids: + raise HTTPException( + status_code=403, + detail="Requested user_id does not match authenticated identity", + ) + + bearer_token = extract_bearer_token_from_request(request) + exchanged = await exchange_token_for_polaris( + subject_token=bearer_token, + audience=payload.audience, + requested_ttl_seconds=payload.requested_ttl_seconds, + ) + + return PolarisTokenResponse( + access_token=exchanged["access_token"], + token_type=exchanged["token_type"], + expires_at_epoch_seconds=exchanged["expires_at_epoch_seconds"], + expires_in_seconds=exchanged["expires_in_seconds"], + issued_for=PolarisTokenIssuedFor( + user_id=payload.user_id, + session_id=payload.session_id, + realm=payload.realm, + ), + trace_id=exchanged["trace_id"], + ) + + +@router.post("/polaris-session", response_model=PolarisSessionResponse) +async def polaris_session( + payload: PolarisSessionRequest, + identity: AuthIdentity = Depends(get_authenticated_identity), +): + if identity.auth_type != "jwt": + raise HTTPException(status_code=403, detail="JWT identity required") + + allowed_user_ids = {identity.subject} + if identity.preferred_username: + allowed_user_ids.add(identity.preferred_username) + + if payload.user_id not in allowed_user_ids: + raise HTTPException( + status_code=403, + detail="Requested user_id does not match authenticated identity", + ) + + created = await create_delegated_broker_session( + subject=identity.subject, + user_id=payload.user_id, + session_id=payload.session_id, + realm=payload.realm, + catalog=payload.catalog, + audience=payload.audience, + refresh_token=payload.refresh_token, + ) + + return PolarisSessionResponse( + broker_session_token=created["broker_session_token"], + expires_at_epoch_seconds=created["expires_at_epoch_seconds"], + ) + + +@router.post("/polaris-token/session", response_model=PolarisTokenResponse) +async def polaris_token_via_session( + request: Request, + payload: PolarisTokenRequest, +): + broker_session_token = request.headers.get("x-broker-session-token", "").strip() + exchanged = await exchange_token_for_polaris_via_broker_session( + broker_session_token=broker_session_token, + user_id=payload.user_id, + session_id=payload.session_id, + realm=payload.realm, + requested_ttl_seconds=payload.requested_ttl_seconds, + ) + + return PolarisTokenResponse( + access_token=exchanged["access_token"], + token_type=exchanged["token_type"], + expires_at_epoch_seconds=exchanged["expires_at_epoch_seconds"], + expires_in_seconds=exchanged["expires_in_seconds"], + issued_for=PolarisTokenIssuedFor( + user_id=payload.user_id, + session_id=payload.session_id, + realm=payload.realm, + ), + trace_id=exchanged["trace_id"], + ) diff --git a/docs/access-control-matrix.md b/docs/access-control-matrix.md index 52d1852..b9201f4 100644 --- a/docs/access-control-matrix.md +++ b/docs/access-control-matrix.md @@ -2,7 +2,7 @@ This document summarizes how access is currently enforced across services in this repository. -Last updated: 2026-05-15 +Last updated: 2026-07-30 ## Identity Model @@ -20,7 +20,8 @@ Source: [keycloak-bootstrap/manifests/realm-configmap.yaml.tpl](../keycloak-boot | Group | Realm role(s) granted | Extra client roles | |---|---|---| | basic-user | basic-user | None | -| iceberg-user | iceberg-user | None | +| teehr-read-only | teehr-read-only | None | +| teehr-read-write | teehr-read-write, teehr-read-only | None | | jupyter-user | jupyter-user | None | | jupyter-admin | jupyter-user | None | | key-management-admin | admin | None | @@ -39,23 +40,27 @@ Source: [keycloak-bootstrap/manifests/realm-configmap.yaml.tpl](../keycloak-boot | JupyterHub admin privileges | JupyterHub Authenticator admin_groups | Group: jupyter-admin | Users in jupyter-admin | | Keycloak admin console link in TEEHR admin page | TEEHR frontend admin visibility | Role: admin | Users with admin role (same as TEEHR admin UI) | | Keycloak admin console capabilities | Keycloak permissions | Role and client roles | Intended primary group appears to be webapi-admin due to realm-management client roles | -| Iceberg or Trino end-user auth | Not fully wired in this repo | No clear active user-facing Keycloak gate | Undetermined from current implementation | +| Iceberg catalog (Polaris) — JupyterHub users | Polaris principal roles via JWT **groups** claim | Groups: teehr-read-only, teehr-read-write, iceberg-catalog-admins | JWT group → Polaris principal role mapping; no per-user sync needed | +| Iceberg catalog (Polaris) — service accounts | Polaris principal roles via JWT **realm_access/roles** claim | Realm roles: iceberg-catalog-admin (trino-polaris), teehr-read-write (prefect-polaris) | Service account JWTs carry the realm role assigned to the Keycloak service account user | +| Iceberg catalog (Polaris) — namespace privileges | Polaris catalog role grants on `teehr` namespace | Polaris principal roles | See `polaris-access-control.md` for the full privilege matrix | +| Trino queries | Trino access-control rules.json | Catalog: iceberg read-only | All Trino queries restricted to read-only regardless of user identity | ## Important Notes 1. key-management-admin and prefect-admin both grant the admin realm role, so both can administer API keys in the current API and frontend implementation. 2. Prefect is intentionally stricter: it checks membership in the prefect-admin group directly, not just the admin role. 3. JupyterHub authorization is also group-based, not based on the admin realm role. -4. Iceberg REST auth rollout is deferred per plan and appears not fully enforced for end-user role mapping yet. +4. Polaris access is now centered on a single `iceberg.teehr` namespace with default read-only access for all users. ## Local Test Users -Local environments now seed two Keycloak users automatically via a local-only bootstrap job: +Local environments now seed three Keycloak users automatically via a local-only bootstrap job: | User type | Default username | Default password | Group membership | |---|---|---|---| -| Admin test user | admin | admin |basic-user, iceberg-user, jupyter-admin, key-management-admin, prefect-admin, webapi-admin | -| Regular test user | user | user |basic-user, jupyter-user | +| Admin test user | admin | admin |basic-user, teehr-read-only, teehr-read-write, jupyter-admin, key-management-admin, prefect-admin, webapi-admin | +| PowerUser test user | poweruser | poweruser |basic-user, teehr-read-write, jupyter-user | +| Regular test user | user | user |basic-user, teehr-read-only, jupyter-user | To add more personas as permissions evolve, update user entries and group assignments in `keycloak-bootstrap/manifests/local-users-configmap.yaml.tpl`. diff --git a/docs/iceberg-auth-storage-roadmap.md b/docs/iceberg-auth-storage-roadmap.md index 74c26d9..8dc9e29 100644 --- a/docs/iceberg-auth-storage-roadmap.md +++ b/docs/iceberg-auth-storage-roadmap.md @@ -1,7 +1,15 @@ # Iceberg Auth and Storage Permissions Roadmap -Last updated: 2026-05-10 -Status: deferred for active implementation +Last updated: 2026-07-30 + +> **Status: Phase B fully implemented via Polaris.** +> `iceberg-rest` has been replaced by Apache Polaris 1.5.0 with Keycloak OIDC integration. +> Per-user catalog permissions are enforced at the Polaris level via JWT group claim mapping. +> S3/MinIO storage still uses shared service credentials (per Pattern B below). +> Pattern C (credential vending) deferred to a future phase. +> See [`polaris-access-control.md`](./polaris-access-control.md) for the implemented access control design. + +--- ## Why this document exists diff --git a/docs/polaris-access-control.md b/docs/polaris-access-control.md new file mode 100644 index 0000000..b856f0e --- /dev/null +++ b/docs/polaris-access-control.md @@ -0,0 +1,140 @@ +# Polaris Access Control Design + +## Overview + +Access to the Iceberg catalog (Polaris) is controlled through a two-layer model: + +1. **Group-level access** — enforced automatically via JWT claim mapping (no sync required) +2. **Individual/table-level access** — enforced via named Polaris principals (sync required) + +--- + +## Layer 1: Group-level access (JWT claim mapping) + +Access flows through two independent mappings, not one: + +1. **Keycloak**: group membership → composite **realm roles**, configured per-group in + `keycloak-bootstrap/manifests/realm-configmap.yaml.tpl` (each group's `realmRoles` list). + Keycloak includes a user's realm roles in the `realm_access.roles` claim of every + token by default — this is standard Keycloak behavior, not something configured here. +2. **Polaris**: reads realm role names out of that claim and maps them to Polaris + principal roles via a regex mapper, configured in `polaris/manifests/polaris-config.yaml.tpl`. + +Polaris itself never sees Keycloak group names or paths — only the realm role names +that groups happen to be composited to. This means: + +- No principal sync is needed for standard access +- Adding a user to a Keycloak group grants them the corresponding Polaris permissions + on their **next token issuance** (no delay, no operational coupling) +- Removing a user from a group immediately revokes access + +### Group → realm role → Polaris principal role mapping + +| Keycloak group | Composite realm role(s) | Polaris principal role | Effect | +|---|---|---|---| +| `/teehr-read-only` | `teehr-read-only` | `teehr-read-only` | Can list and read tables in the `teehr` namespace | +| `/teehr-read-write` | `teehr-read-write`, `teehr-read-only` | `teehr-read-write`, `teehr-read-only` | Can create, read, write, and drop tables | +| `/iceberg-catalog-admins` | `iceberg-catalog-admin` | `iceberg-catalog-admin` | Full catalog management | + +Realm roles not in this table (e.g. `basic-user`, `jupyter-user`) are ignored by Polaris — +note the *realm role* names are singular/unprefixed even where the *group* name (e.g. +`iceberg-catalog-admins`) isn't; don't confuse the two when tracing a permission issue. + +### Polaris configuration + +Configured in `polaris/manifests/polaris-config.yaml.tpl`: + +```properties +quarkus.oidc.roles.role-claim-path=realm_access/roles +polaris.oidc.principal-roles-mapper.type=default +polaris.oidc.principal-roles-mapper.mappings[0].regex=^iceberg-catalog-admin$ +polaris.oidc.principal-roles-mapper.mappings[0].replacement=PRINCIPAL_ROLE:iceberg-catalog-admin +polaris.oidc.principal-roles-mapper.mappings[1].regex=^teehr-(.+)$ +polaris.oidc.principal-roles-mapper.mappings[1].replacement=PRINCIPAL_ROLE:teehr-$1 +``` + +### Adding a new access tier + +1. Create a Keycloak group named `/teehr-` in `keycloak-bootstrap/manifests/`, with a + composite `realmRoles: ["teehr-"]` mapping — the realm role is what actually reaches + Polaris, so this step is required, not just the group itself +2. Add a namespace policy for the new principal role in `polaris-bootstrap/manifests/acl-config.yaml.tpl` +3. No Polaris config change needed — the `teehr-(.+)` pattern picks it up automatically +4. Add users to the group in Keycloak + +--- + +## Layer 2: Individual/table-level access (principal sync) + +For use cases requiring finer-grained control beyond group defaults: + +- Granting a specific user read access to a specific table (not the whole namespace) +- Temporary elevated access for a single user +- Audit trails tied to a named Polaris principal entity + +### How it works + +The `polaris-sync-principals-script` (run as part of `polaris-bootstrap`) creates a +named Polaris principal for each Keycloak user and assigns principal role bindings +based on their group membership. These bindings are **additive** — they stack on top +of the JWT-based group grants. + +To grant a user access to a specific table: +1. Ensure the user has a synced principal in Polaris (sync script handles this) +2. Create a table-level catalog role with the desired privilege +3. Bind that catalog role to the user's principal role via the Polaris management API + +### Why the sync is optional for basic access + +Since JWT group mapping covers the common case, the sync is only needed when you +require per-principal grants. The sync can be run on-demand or on a schedule — it +does not need to run before users can access the catalog. + +--- + +## Permission model + +### Namespace-level grants (acl-config.yaml.tpl) + +| Principal role | Catalog role | Privileges | +|---|---|---| +| `teehr-read-only` | `teehr_read_only_role` | `NAMESPACE_READ_PROPERTIES`, `TABLE_LIST`, `TABLE_READ_PROPERTIES`, `TABLE_READ_DATA` | +| `teehr-read-write` | `teehr_read_write_role` | All read-only + `NAMESPACE_WRITE_PROPERTIES`, `TABLE_CREATE`, `TABLE_DROP`, `TABLE_WRITE_PROPERTIES`, `TABLE_READ_DATA`, `TABLE_WRITE_DATA` | +| `iceberg-catalog-admin` | `catalog_admin_role` | `CATALOG_MANAGE_CONTENT`, `CATALOG_MANAGE_METADATA` | + +### Storage + +- MinIO (local) / S3 (remote): credentials configured in Polaris catalog `storageConfigInfo` +- `stsUnavailable=true` for local MinIO (no STS credential vending) +- `s3.remote-signing-enabled=false` — clients use their own configured S3 credentials + +--- + +## Authentication paths + +### Direct token (notebooks, API clients) + +``` +User → Keycloak password/refresh grant → JWT (jupyterhub client) + → Spark: spark.sql.catalog.iceberg.token = {jwt} + → Polaris: validates JWT, maps realm_access.roles (from Keycloak group + membership) to principal roles, enforces permissions +``` + +### AuthManager / broker (JupyterHub spawned notebooks) + +``` +User logs in → JupyterHub OAuth → Keycloak issues access_token + refresh_token + → Broker (/auth/polaris-session) stores refresh_token, returns broker_session_token + → TeehrBrokerAuthManager (JAR) holds broker_session_token + → On each Iceberg operation: JAR calls /auth/polaris-token/session + → Broker refreshes token via Keycloak (refresh_token grant) + → Returns refreshed jupyterhub JWT (preserves realm_access.roles claim) + → Polaris: same realm_access.roles-based mapping as direct token path +``` + +Note: The broker does **not** do token exchange — it refreshes the user's token directly +to preserve the `realm_access.roles` claim (which is what Polaris actually reads; it's +derived from the user's Keycloak group membership, per Layer 1 above). Token exchange +with a different audience was found to strip this claim, preventing per-user permission +enforcement. diff --git a/docs/polaris-broker-api-contract.md b/docs/polaris-broker-api-contract.md new file mode 100644 index 0000000..dd90b0c --- /dev/null +++ b/docs/polaris-broker-api-contract.md @@ -0,0 +1,171 @@ +# Polaris Broker API Contract (Prototype v0) + +Last updated: 2026-07-23 + +## Purpose + +Define a minimal broker contract that allows Spark-side Iceberg AuthManager code to acquire and rotate short-lived Polaris bearer tokens without exposing refresh tokens in notebooks. + +## Scope + +This contract is intentionally narrow: + +- one token mint endpoint for interactive notebook Spark sessions +- strict caller/session binding +- short-lived access tokens only +- no refresh token returned to Spark + +## Endpoint + +- Method: `POST` +- Path: `/v1/polaris/token` +- Authn: broker validates caller using cluster-local identity and request signature/session binding +- Content-Type: `application/json` + +Current prototype implementation in this repo: + +- Method: `POST` +- Path: `/auth/polaris-token` +- Service: `teehr-api` +- Caller auth: Keycloak bearer token validated by `teehr-api` + +## Request Body + +```json +{ + "user_id": "user@example.local", + "session_id": "jupyter-6f9f99b5f9-l9z6k", + "realm": "teehr", + "catalog": "iceberg", + "groups": ["iceberg-user", "hydrology-team"], + "requested_ttl_seconds": 600, + "audience": "account" +} +``` + +## Request Field Notes + +- `user_id`: stable user identifier. Prefer immutable subject (`sub`) if available; username is acceptable for prototype. +- `session_id`: Jupyter notebook server identity (pod UID or equivalent) for caller binding and audit. +- `realm`: Polaris realm to target. +- `catalog`: optional catalog hint for policy/audit context. +- `groups`: optional hint for diagnostics only. Broker should derive trusted groups from validated identity when possible. +- `requested_ttl_seconds`: bounded by broker policy (for example 120 to 900). +- `audience`: should be constrained to Polaris data-plane usage. + +## Success Response + +- Status: `200 OK` + +```json +{ + "access_token": "", + "token_type": "Bearer", + "expires_at_epoch_seconds": 1785169492, + "expires_in_seconds": 600, + "issued_for": { + "user_id": "user@example.local", + "session_id": "jupyter-6f9f99b5f9-l9z6k", + "realm": "teehr" + }, + "trace_id": "f8af1e8b3d21469a9adf99c9185d2d10" +} +``` + +## Error Responses + +- `400 Bad Request`: validation failures +- `401 Unauthorized`: caller identity not valid +- `403 Forbidden`: caller/session mismatch or policy denies issuance +- `429 Too Many Requests`: per-user/session throttle exceeded +- `500/502/503`: transient issuer/broker failure + +Error payload shape: + +FastAPI wraps every `HTTPException(detail=...)` payload under a top-level +`"detail"` key by default, and this API has no exception handler that +unwraps it — so error responses are actually: + +```json +{ + "detail": { + "error": "forbidden", + "message": "session is not authorized for requested user_id", + "trace_id": "f8af1e8b3d21469a9adf99c9185d2d10" + } +} +``` + +Callers must read `error`/`message`/`trace_id` from `response.json()["detail"]`, +not from the top level of the response body. + +## Security Requirements + +- Broker must not trust notebook-supplied `groups` blindly. +- Broker must validate caller identity from trusted transport identity (mTLS, workload identity, signed upstream token, or equivalent). +- Broker must bind `user_id` to `session_id` and reject mismatches. +- Broker-issued tokens must be short-lived and audience-restricted. +- Broker credentials used for exchange/mint operations must never be exposed to notebook runtimes. + +## Audit Requirements + +At minimum, log: + +- `trace_id` +- broker caller identity +- `user_id` +- `session_id` +- `realm` +- requested and granted TTL +- outcome (`issued`, `denied`, `error`) +- downstream issuer status/latency + +## Spark AuthManager Expectations + +The Spark-side AuthManager should: + +- request new token on first use +- cache token in-memory +- proactively rotate before expiry (for example 60 seconds early) +- retry once on auth failure when token is near-expiry +- never persist tokens to logs or Spark history + +## Non-goals (v0) + +- multi-tenant broker routing policies +- user-delegated arbitrary scope requests +- long-lived refresh token distribution to Spark +- batch token mint APIs + +## Prototype Test (Current Repo) + +From a Jupyter single-user pod shell: + +```bash +python - <<'PY' +import json +import os +import requests + +token = os.environ["POLARIS_USER_TOKEN"] +realm = os.getenv("POLARIS_DEFAULT_REALM", "teehr") +user_id = os.getenv("JUPYTERHUB_USER", "admin") +session_id = os.getenv("JUPYTERHUB_SERVER_NAME", user_id) + +resp = requests.post( + "http://teehr-api:8000/auth/polaris-token", + headers={"Authorization": f"Bearer {token}"}, + json={ + "user_id": user_id, + "session_id": session_id, + "realm": realm, + "catalog": "iceberg", + "requested_ttl_seconds": 600, + "audience": "account", + }, + timeout=20, +) +print(resp.status_code) +print(json.dumps(resp.json(), indent=2)[:1200]) +PY +``` diff --git a/docs/polaris-identity-propagation-plan.md b/docs/polaris-identity-propagation-plan.md new file mode 100644 index 0000000..36ae2f1 --- /dev/null +++ b/docs/polaris-identity-propagation-plan.md @@ -0,0 +1,897 @@ +# TEEHR Hub Identity Propagation and Fine-Grained Data Authorization Plan + +Last updated: 2026-07-30 + +> **Status: Flow A implemented and tested.** Flow B (Trino with user identity propagation) pending. +> +> **Implemented:** +> - Apache Polaris 1.5.0 replaces `iceberg-rest` as the Iceberg catalog +> - Keycloak OIDC → Polaris JWT group claim mapping (no per-user sync required for group-level access) +> - JupyterHub users authenticated via Keycloak → user JWT → Polaris via AuthManager broker +> - Per-user Polaris permission enforcement verified by integration tests (`garden test`) +> - Three tiers: `teehr-read-only`, `teehr-read-write`, `iceberg-catalog-admin` +> - Service accounts: `trino-polaris` (admin) and `prefect-polaris` (read-write) via Keycloak realm role → Polaris mapping +> +> **Pending:** +> - Flow B: Trino user identity propagation (currently uses `trino-polaris` service account for all queries) +> - Pattern C: S3 credential vending per user (currently shared MinIO/S3 credentials) +> - Per-user table-level grants via optional principal sync +> +> See [`polaris-access-control.md`](./polaris-access-control.md) for the current design. + +--- + +## Goals + +Establish a unified identity and authorization architecture for all TEEHR Hub services that access Iceberg-backed data so that: + +- Keycloak remains the single source of truth for user identity and group membership. +- JupyterHub, FastAPI, and other user-facing entry points propagate end-user identity forward instead of collapsing users into a shared technical principal. +- Apache Polaris becomes the primary enforcement point for fine-grained catalog and table permissions. +- Spark, Trino, and other compute/query services access Iceberg through Polaris in a way that preserves user-level authorization context. +- Existing services continue to work during migration, with clear phases and fallback paths. + +## Principles + +1. **Identity source of truth:** Keycloak owns users, groups, and role/group assignments. +2. **Data authorization source of truth:** Polaris owns fine-grained data authorization for Iceberg catalogs, namespaces, tables, and write operations. +3. **Policy enforcement at the data plane:** JupyterHub and FastAPI should not be the final authority for what data a user can read or write. They should authenticate the user, propagate identity, and rely on downstream systems to enforce. +4. **End-user identity propagation:** Where a user initiates an action, downstream data access should occur with a credential or context representing that user. +5. **Coarse vs fine-grained separation:** Keycloak groups remain coarse platform entitlements; Polaris policies hold fine-grained data access rules. +6. **Hybrid authorization model:** Policies are assigned primarily to groups, but requests should always carry the individual user identity for auditability, traceability, and exception handling. +7. **Least privilege:** Shared Kubernetes, AWS, or service identities must not undermine per-user data authorization. +8. **Short-lived credentials:** Prefer short-lived, revocable credentials/tokens over long-lived shared secrets inside notebook or app runtimes. + +## Target Architecture + +### Flow A: Keycloak → JupyterHub → Polaris → Iceberg/Spark + +1. User authenticates to JupyterHub with Keycloak OIDC. +2. JupyterHub stores auth state and extracts a minimal set of claims/groups needed for spawn-time decisions and downstream identity propagation. +3. Jupyter single-user server receives either: + - a short-lived access token for the logged-in user, or + - a short-lived exchanged token/credential derived from the user identity. +4. Notebook clients (Spark, PyIceberg, direct REST catalog clients) present that per-user credential to Polaris. +5. Polaris evaluates both the individual user principal and the user’s Keycloak-derived groups. +6. Iceberg operations proceed only if Polaris authorizes them. +7. Underlying object storage access is constrained so users cannot bypass Polaris with broad direct bucket permissions. + +### Flow B: Keycloak → FastAPI → Trino → Polaris → Iceberg + +1. User authenticates to FastAPI with a Keycloak JWT. +2. FastAPI validates the JWT and extracts user identity plus coarse claims. +3. For data-plane calls, FastAPI forwards end-user identity to Trino using a supported secure mechanism. +4. Trino uses that end-user context when resolving catalog access through Polaris. +5. Polaris evaluates both the individual user principal and the user’s Keycloak-derived groups. +6. FastAPI remains responsible for app-level capabilities and route access, but not for warehouse data authorization beyond coarse gatekeeping. + +## Identity Model + +### Keycloak responsibilities + +Keycloak should continue to manage: + +- users +- default baseline access +- group membership +- OIDC clients for frontend, JupyterHub, FastAPI, and any service-to-service integrations + +The existing coarse group model is a strong starting point: + +- `basic-user` +- `jupyter-user` +- `jupyter-admin` +- `iceberg-user` +- `key-management-admin` +- `prefect-admin` +- `webapi-admin` + +### Recommended group strategy + +Keep Keycloak groups coarse and human-manageable. Suggested semantics: + +- `basic-user`: baseline authenticated application access +- `jupyter-user`: allowed to access JupyterHub +- `jupyter-admin`: JupyterHub admin rights only +- `iceberg-user`: allowed to use catalog-backed data tools +- optional team/domain groups: e.g. `hydrology-team`, `forecast-team`, `operations-team` + +Avoid expressing every dataset/table permission directly in Keycloak groups. Instead, use Keycloak groups as the default policy-assignment mechanism in Polaris and reserve direct per-user grants for exceptional cases. + +### Hybrid authorization model + +The target authorization model should be hybrid: + +- Every human-originated request carries the individual Keycloak user identity end to end. +- Polaris policy assignment is primarily group-based. +- Individual user grants are exceptions, not the default. +- Non-human workloads use distinct service principals with narrowly scoped rights. +- Audit logs should preserve both the individual principal and the effective groups used for authorization. + +Rule of thumb: + +> Policies are assigned mostly to groups, but requests always carry the individual user identity. + +### Claims to propagate + +At minimum, downstream systems should have access to: + +- subject (`sub`) +- preferred username or stable user identifier +- groups +- optionally realm roles if needed for coarse app behavior +- issuer (`iss`) +- audience/client context as needed for validation + +## Polaris Authorization Model + +### Polaris as the fine-grained authority + +Polaris should become the primary location for: + +- catalog-level permissions +- namespace/schema permissions +- table/view permissions +- read vs write separation +- admin/management operations +- team/project-specific access + +### Mapping strategy + +Prefer these patterns, in order: + +1. group-based policy mapping for team/domain entitlements +2. direct user principal mapping for limited exceptions +3. distinct service principals for non-human automation paths + +Examples: + +- `iceberg-user` grants general eligibility to use catalog-backed services +- `hydrology-team` maps to read access on selected namespaces/tables +- `forecast-team` maps to write access on forecast-derived tables only +- a specific user may receive a temporary direct grant for a narrow exception case +- automation service accounts get narrowly scoped non-human privileges + +### Project namespace model + +Eligible human users should be able to create and manage project namespaces for exploratory, collaborative, and intermediate work. + +Recommended rules: + +- Project namespace creation is allowed only for users with the required coarse entitlement, such as `iceberg-user`. +- Project namespaces live under the `projects` prefix and use a user-provided name: `projects.`. +- Namespace names must satisfy validation and uniqueness rules. +- Project namespaces are created lazily on first use. +- The creating user becomes the initial owner and receives read, write, and manage permissions for that namespace. +- New project namespaces default to `private` visibility. +- The owner may optionally configure the namespace to allow read-only access for all eligible Iceberg users. +- Shared/team/production namespaces remain governed primarily by group-based Polaris policies and separate governance. + +Recommended sharing modes for the initial implementation: + +- `private`: only the owner may access the namespace, except for narrowly scoped admin or maintenance access +- `all-iceberg-users-read`: all eligible Iceberg users may read objects in the namespace, while only the owner may write or manage objects + +This model intentionally uses project namespaces as the initial self-service workspace primitive. A separate `users.*` personal-namespace model is deferred unless later usage demonstrates a clear need for a distinct personal workspace class. + +### Project namespace naming and governance rules + +Recommended v1 naming rules for `projects.`: + +- use lowercase letters, numbers, and hyphens only +- must start with a letter +- should be globally unique under `projects.*` +- names that differ only by case should be treated as the same name +- reserve selected names and prefixes such as `admin`, `system`, `default`, `prod`, `production`, and `shared` +- apply a reasonable maximum length to the provided namespace component, such as 50 characters + +Recommended v1 governance rules: + +- the creating user becomes the initial owner +- the owner receives read, write, and manage permissions for the namespace +- new namespaces default to `private` +- the owner may switch visibility between `private` and `all-iceberg-users-read` +- arbitrary custom ACLs, explicit collaborators, and self-service ownership transfer are out of scope for v1 +- ownership transfer is admin-controlled in the initial implementation +- admins may recover, reassign, archive, or otherwise govern orphaned namespaces when an owner leaves or loses entitlement +- apply an initial per-user limit on the number of project namespaces, such as 10, with admin override if needed + +### Human vs automation separation + +Polaris policy should explicitly separate: + +- human interactive access from JupyterHub and FastAPI +- delegated execution on behalf of a human user +- non-human automation such as Prefect, ingestion jobs, and maintenance workflows + +Non-human automation should use distinct service principals and should not impersonate human users by default. + +## JupyterHub Design + +### JupyterHub responsibilities + +JupyterHub should: + +- authenticate users with Keycloak +- authorize JupyterHub access using coarse groups +- persist auth state securely +- pass minimal downstream identity into notebook runtimes +- optionally shape spawn behavior based on groups + +JupyterHub should not be the final authority for Iceberg permissions. + +### Recommended implementation pattern + +1. Enable and secure `auth_state` persistence. +2. At login or pre-spawn time, read: + - `sub` + - username + - groups + - token expiry metadata + - access token only if needed +3. Pass into notebook pods: + - minimal identity env vars for UX and telemetry + - a short-lived token or exchanged credential for Polaris access +4. Avoid passing refresh tokens into notebook environments unless absolutely necessary. +5. Use spawn hooks to gate profiles/features, not to implement table-level authorization. + +### Notebook runtime behavior + +Notebook runtimes should: + +- authenticate to Polaris as the end user +- use catalog-aware clients for Iceberg access +- avoid direct object-store access patterns that bypass Polaris +- allow creation and management of project namespaces for users with the required entitlement + +### Spark integration + +Spark jobs launched from Jupyter should preserve the originating end-user identity when accessing Polaris. This likely means: + +- Spark driver receives user credential/context from the notebook environment +- Spark catalog configuration uses Polaris endpoints and auth settings +- executor-side access follows the driver’s authenticated catalog interactions or other supported user-context mechanism +- audit context should preserve both the initiating user identity and the effective groups used for authorization + +Exact mechanics depend on the Spark + Iceberg + Polaris auth model you select, but the design goal is unchanged: no shared “all notebooks are the same person” catalog identity. + +### Kubernetes and AWS identity caution + +The current shared `jupyter` service account and shared IRSA role are acceptable for platform operations only if they do not grant blanket data access that bypasses Polaris. + +Recommended direction: + +- keep shared pod identity narrow +- do not rely on shared IRSA for warehouse authorization +- ensure direct S3/object-store permissions are minimized relative to Polaris-mediated access + +## FastAPI Design + +### FastAPI responsibilities + +FastAPI should: + +- validate Keycloak JWTs +- enforce application-level route permissions and coarse feature gates +- propagate end-user identity to Trino/data clients +- not substitute a shared privileged warehouse identity for end-user requests + +### Existing repo alignment + +The API already validates Keycloak JWTs and extracts realm roles. This is a good foundation for: + +- app-level authorization +- request identity extraction +- future downstream identity propagation + +### Forwarding identity to Trino + +The exact Trino integration should be chosen based on supported secure mechanisms, but the plan should require: + +- preserving a stable end-user identity into the Trino session +- preserving effective Keycloak groups or equivalent authorization context where supported +- preventing FastAPI from always querying as a single technical user for end-user traffic +- aligning Trino catalog access with Polaris-enforced permissions + +Candidate approaches to evaluate: + +1. user identity forwarded as authenticated session principal +2. OAuth/OIDC-aware Trino integration if supported by chosen deployment +3. trusted proxy/service pattern only if it still preserves distinguishable end-user principals and auditable enforcement + +## Service-by-Service Policy Split + +### Keycloak + +Owns: + +- authentication +- user lifecycle +- groups/roles +- client registration + +Does not own: + +- fine-grained Iceberg table permissions + +### JupyterHub + +Owns: + +- notebook login authorization +- admin access to JupyterHub +- spawn-time feature gating + +Does not own: + +- final warehouse data authorization + +### FastAPI + +Owns: + +- API authentication +- app feature authorization +- rate limiting / route protection / business rules + +Does not own: + +- final Iceberg table authorization for end-user data access + +### Polaris + +Owns: + +- fine-grained catalog and table authorization +- group-based policy evaluation as the default mechanism +- limited direct user grants for exceptional cases +- project namespace ownership and namespace-level self-service rules +- data-access decisions for Iceberg-aware clients/services + +### Trino / Spark + +Owns: + +- execution under propagated user identity +- honoring Polaris-backed catalog authorization +- preserving auditability of the human initiator where applicable + +### Automation services + +Own: + +- non-human scheduled or background execution under distinct service principals + +Do not own: + +- human-interactive identity or authorization decisions + +## Implementation Checklist + +### Repository and application changes + +#### JupyterHub + +- Locate current JupyterHub auth configuration. +- Confirm Keycloak OIDC integration path. +- Verify whether `auth_state` is enabled and persisted securely. +- Identify where pre-spawn hooks can extract username, `sub`, groups, and token expiry metadata. +- Decide what minimal identity context should be injected into notebook runtimes. +- Decide whether notebook runtimes receive a direct short-lived user token or an exchanged credential. +- Prototype notebook-side access to Polaris as the end user. +- Identify where project namespace create-on-first-use logic should live for Jupyter-driven workflows. + +#### FastAPI + +- Locate JWT validation and auth dependency code. +- Confirm where username, subject, roles, and groups are extracted today. +- Add or refine a canonical request identity object. +- Trace every FastAPI path that triggers Trino or Iceberg-backed access. +- Identify where end-user identity must be forwarded downstream. +- Determine whether project namespace operations will be exposed through API endpoints. +- If project namespace operations are exposed through the API, define behavior for create namespace, get namespace visibility, set namespace visibility, and list owned namespaces. + +#### Trino integration + +- Locate current Trino client or session creation code. +- Identify how user identity is currently represented in Trino sessions. +- Determine where group or authorization context could be forwarded. +- Document whether current behavior uses a shared technical principal. +- Define required code and configuration changes for per-user session context. + +#### Spark and notebook data access + +- Locate Spark catalog configuration used by notebooks. +- Determine how Polaris would be configured as the Iceberg catalog. +- Verify where Spark receives user auth context. +- Determine whether executor behavior preserves user-context semantics. +- Prototype read and write in a user-created project namespace. + +#### Project namespace workflow + +- Define the canonical create-on-first-use workflow. +- Define validation for `projects.`. +- Implement or prototype name validation rules. +- Define the visibility enum for v1: `private` and `all-iceberg-users-read`. +- Define owner permissions. +- Define admin-only ownership transfer handling. +- Define behavior when a namespace already exists. +- Define behavior when the creating user lacks `iceberg-user`. + +#### Audit and observability + +- Identify where to log initiating user identity. +- Identify where to log effective groups. +- Identify where to log namespace creation and visibility changes. +- Define correlation points between Keycloak user, Jupyter session or API request, Trino or Spark execution, and Polaris authorization decisions. + +### Platform and infrastructure changes + +#### Keycloak + +- Confirm the final group model for v1. +- Confirm that groups claims are present in tokens where needed. +- Confirm username stability expectations. +- Confirm whether `preferred_username` is sufficient for namespace naming. +- Confirm whether immutable `sub` should also be logged for ownership and audit. + +#### Polaris + +- Confirm the supported authentication method for end-user principals. +- Confirm how Polaris consumes user and group information. +- Confirm namespace creation APIs or workflow. +- Confirm the grant model for namespace owner read/write/manage, all-iceberg-users read-only, and admin recovery access. +- Confirm whether visibility toggling maps cleanly to grant changes. +- Confirm how to model project namespace ownership operationally. + +#### Object storage, AWS, and IAM + +- Inventory current object-store access paths. +- Identify any direct bucket permissions that bypass Polaris. +- Narrow shared IRSA or service-account access where needed. +- Define operational and admin exceptions. +- Confirm whether project namespace creation requires additional storage-side setup. + +#### Kubernetes and deployment + +- Locate JupyterHub deployment configuration. +- Locate FastAPI deployment configuration. +- Identify the secret and token handling mechanism. +- Confirm how short-lived credentials would be passed and rotated. +- Identify configuration surfaces for Polaris endpoints and auth settings. + +#### Admin and governance operations + +- Define the admin process for namespace reassignment. +- Define the admin process for orphaned namespaces. +- Define the namespace quota override process. +- Define the reserved-name management process. + +### Validation spikes and unknowns + +- Validate whether Polaris can directly authenticate Keycloak-issued user tokens. +- Validate whether Polaris can evaluate group-based policy from those tokens. +- Validate the exact grant model needed for namespace ownership and read-only sharing. +- Validate whether a notebook can authenticate to Polaris as the actual user. +- Validate the safest token propagation pattern for notebooks. +- Validate how long-lived notebook sessions behave when tokens expire. +- Validate whether Spark can access Polaris with preserved user context. +- Validate whether user attribution is maintained only at the driver level or throughout execution. +- Validate whether Trino can preserve end-user identity in the way Polaris needs. +- Validate whether group context can be propagated or reconstructed for Trino-driven requests. +- Validate whether project namespaces can be created lazily without fragile race conditions. +- Validate how duplicate creation attempts should behave. +- Validate how visibility changes will be represented in Polaris grants. +- Validate which current credentials or access paths still allow storage bypass. + +### Recommended implementation order + +1. Validate the Polaris authentication and policy model. +2. Validate the Jupyter-to-Polaris end-user flow. +3. Validate the FastAPI-to-Trino-to-Polaris flow. +4. Inventory current storage bypass paths. +5. Implement the first vertical slice around project namespace creation and default private access. +6. Validate owner read/write access, denied access for a second user, and read-only access after switching visibility to `all-iceberg-users-read`. +7. Add audit logging, admin recovery flows, quotas, and broader Spark/API parity. + +## Migration Phases + +## Token Lifecycle Strategy for Jupyter and Spark + +### Problem statement + +Current behavior relies on a short-lived user access token being injected into a +long-lived notebook runtime. Once the token expires, Spark and direct catalog +operations fail until the user logs out and logs back in. + +### Constraints and observations + +- Interactive notebook UX requires sessions that may outlive an access token. +- We should preserve end-user identity at Polaris for authorization and audit. +- We should avoid exposing long-lived refresh tokens directly in notebook + environments unless there is no practical alternative. +- Spark itself does not provide a complete Keycloak session lifecycle model. + Token refresh behavior must be designed at the client/configuration layer. + +### Approved direction + +#### Short-term (now): Notebook-side token renewal helper + +Implement an explicit token renewal path for Jupyter and Spark sessions that: + +1. detects token expiration proactively using token `exp` metadata +2. acquires a fresh user access token before expiry +3. updates Spark catalog auth configuration in-session +4. retries failed catalog operations once after token renewal + +Short-term implementation notes: + +- Keep renewal logic in a small shared helper used by notebooks and example + scripts. +- Do not require a full logout/login roundtrip for normal token expiry. +- Prefer renewal from a controlled server-side endpoint where possible. +- If direct refresh is used temporarily, limit scope and lifetime and avoid + persisting refresh credentials in notebook files or outputs. + +#### Long-term (target): Token broker / exchange service + +Introduce a dedicated token broker that mints short-lived Polaris-compatible +tokens on behalf of the authenticated Jupyter user. + +Target flow: + +1. user authenticates to JupyterHub with Keycloak +2. notebook runtime requests a short-lived data-plane token from broker +3. broker validates caller identity and exchanges/mints token using secure + server-side credentials +4. notebook and Spark use only short-lived access tokens +5. broker refreshes/exchanges as needed without requiring user relogin + +Broker requirements: + +- preserve individual user identity and effective groups in resulting token +- enforce strict caller binding and audience/scope constraints +- issue short-lived tokens only +- never expose long-lived broker credentials to notebook runtimes +- provide auditable logs that correlate user, request, and issued token metadata + +### Option assessment + +1. Increase Keycloak token TTL only + - Pros: fast operational relief + - Cons: weakens security posture and does not solve lifecycle architecture +2. Notebook helper renewal (approved short-term) + - Pros: immediate UX improvement, compatible with identity goals + - Cons: still transitional without centralized brokering +3. Service principal for interactive Spark + - Pros: simple operational model + - Cons: breaks per-user authorization and audit goals for interactive usage +4. Broker/exchange service (approved target) + - Pros: strongest alignment with security + UX + per-user authorization + - Cons: requires new service and integration work + +### Definition of done + +Short-term done when: + +- notebooks continue through normal token expiry without manual relogin +- Spark catalog operations recover automatically after token rotation +- user identity at Polaris remains user-specific (not collapsed to shared + principal) + +Long-term done when: + +- notebook runtimes no longer require direct refresh-token handling +- broker-issued short-lived tokens are the standard interactive data-plane + credential +- audit logs can correlate Keycloak identity, notebook request, broker issuance, + and Polaris authorization decision + +### Phase 0: Discovery and capability validation + +Validate product capabilities and constraints before committing implementation details: + +- how Polaris authenticates principals and consumes OIDC/user identity +- how Iceberg clients authenticate to Polaris +- how Trino integrates with Polaris and preserves user identity +- how Spark integrates with Polaris and preserves user identity +- how groups/claims can be surfaced to Polaris, directly or indirectly +- how project namespace creation, sharing, and ownership can be represented in Polaris policy +- whether token exchange, service delegation, or direct bearer-token auth is preferred +- what object-store permissions are still required beneath Polaris + +Deliverables: + +- architecture decision record +- supported auth flow matrix for Jupyter, Spark, FastAPI, Trino +- gap list for unsupported assumptions + +### Phase 1: Identity inventory and policy model + +Define the canonical identity and policy model: + +- inventory existing Keycloak groups and roles +- identify coarse platform groups to keep +- define any new team/domain groups +- define how Keycloak groups map into Polaris principals/policies +- define criteria for when direct user grants are allowed +- define naming conventions for users, groups, namespaces, catalogs +- define the project namespace naming, entitlement, and visibility model + +Deliverables: + +- identity map +- group-to-Polaris policy map +- per-user exception policy +- project namespace policy +- example access-control matrix for target state + +### Phase 2: JupyterHub identity propagation foundation + +Implement the JupyterHub foundation for user-context propagation: + +- enable auth state persistence +- add pre-spawn logic to extract minimal claims +- inject minimal identity metadata into notebook sessions +- evaluate secure handling for short-lived user token or exchanged credential +- validate notebook-to-Polaris authentication path +- validate audit visibility of user identity and effective groups +- validate project namespace create/read/write behavior for eligible users + +Deliverables: + +- JupyterHub configuration changes +- secret-handling model +- proof of concept notebook access path +- audit-context validation notes +- project namespace proof of concept + +Token-lifecycle deliverables for this phase: + +- notebook token-renewal helper design +- Spark in-session token update mechanism +- relogin-free expiry recovery validation for representative notebooks + +### Phase 3: Spark + Iceberg + Polaris user-context path + +Implement Spark access through Polaris under end-user identity: + +- configure Spark Iceberg catalog for Polaris +- validate read/write behavior by user/group +- validate create/read/write behavior in user-created project namespaces +- ensure job submissions launched from Jupyter preserve user context +- confirm executor/runtime behavior does not collapse to a shared catalog principal +- confirm how delegated execution is attributed in logs and policy evaluation + +Deliverables: + +- Spark catalog configuration +- end-to-end auth test cases +- operational notes for debugging and token expiry +- delegated execution audit model +- project namespace Spark validation cases + +### Phase 4: FastAPI → Trino user-context propagation + +Implement user-context propagation for API-driven data access: + +- formalize request identity object in FastAPI +- define Trino session principal propagation strategy +- validate per-user access behavior through Polaris +- validate group-based authorization behavior through Polaris +- validate create/read/write behavior for project namespaces where API workflows support it +- keep existing app-level route authorization intact + +Deliverables: + +- FastAPI integration design +- Trino integration configuration +- end-to-end API authz test cases +- request-to-query audit mapping +- project namespace API validation cases + +### Phase 5: Storage hardening / bypass prevention + +Reduce or eliminate bypass paths that would undermine Polaris: + +- audit direct S3/object-store permissions for Jupyter, Spark, Trino, and service accounts +- minimize shared credentials with broad warehouse access +- ensure intended clients access warehouse data through Polaris-mediated paths +- define exceptions explicitly for admin/maintenance automation +- ensure automation principals are separated from human interactive access + +Deliverables: + +- credential inventory +- least-privilege policy changes +- documented exception list +- human-vs-automation access boundary documentation + +### Phase 6: Rollout, observability, and migration cleanup + +Roll out incrementally and verify behavior: + +- pilot with a small set of users/groups +- compare current vs target behavior +- add audit logging and request tracing where possible +- deprecate old shared-identity assumptions +- update docs and developer notebooks/examples +- document the user experience for project namespace creation and visibility configuration + +Deliverables: + +- rollout checklist +- audit/observability plan +- migration completion checklist +- project namespace user guidance + +### Phase 7: Token broker and exchange hardening + +Move from notebook-managed renewal to broker-managed short-lived credentials: + +- implement broker service with strict identity and audience validation +- integrate JupyterHub/notebook clients with broker endpoint +- remove notebook dependence on direct refresh-token handling +- add issuance and exchange audit events with trace correlation +- define fallback behavior when broker is unavailable + +Deliverables: + +- broker service design and implementation +- end-to-end token exchange flow validation for Jupyter + Spark +- security review of broker scopes, TTLs, and credential storage +- migration plan to retire temporary notebook-side renewal paths + +## AuthManager Prototype Path + +Iceberg 1.9+ includes the AuthManager API, which provides a more viable medium-term +path than notebook-managed token refresh for long-lived Spark sessions. + +### Why AuthManager matters here + +- Spark currently relies on Iceberg REST catalog auth behavior that does not recover + reliably with Keycloak token refresh in long-lived sessions. +- Updating notebook variables or Spark conf after catalog initialization is not a + dependable fix once the REST catalog client is already live. +- A custom AuthManager moves token acquisition/rotation into the JVM-side catalog + auth layer where Spark is actually making catalog requests. + +### What is available today + +- Iceberg AuthManager API enablement landed in Iceberg 1.9.0. +- Shared/external AuthManager injection into RESTCatalog was discussed but did not + land in core. +- Therefore, the realistic implementation path is a custom AuthManager class loaded + on the Spark classpath and configured through Iceberg auth properties. + +### Recommended prototype shape + +Build a custom `teehr` AuthManager implementation in Java or Scala that: + +1. accepts a stable user/session identifier from the Spark session context or catalog + properties +2. obtains short-lived Polaris-compatible access tokens from a broker endpoint +3. caches tokens in-memory with proactive refresh ahead of `exp` +4. returns auth headers to Iceberg REST calls without depending on notebook-side + Python token mutation +5. supports recovery by reacquiring tokens independently of the notebook kernel state + +### Strongly preferred token source + +Use a broker or local sidecar endpoint, not direct refresh-token handling inside +Spark catalog properties. + +Preferred flow: + +1. user authenticates to JupyterHub with Keycloak +2. notebook runtime receives stable user/session context +3. Spark AuthManager calls broker with that context +4. broker validates caller/session and returns a short-lived Polaris access token +5. AuthManager refreshes from broker as needed for the life of the Spark session + +### Why broker-backed is preferred + +- avoids storing long-lived refresh credentials in Spark config or executors +- centralizes audit, policy, and failure handling +- preserves per-user authorization semantics at Polaris +- allows independent evolution of Keycloak refresh/exchange logic without pushing + auth complexity into notebooks + +Concrete broker/auth artifacts: + +- broker contract: `docs/polaris-broker-api-contract.md` +- published Spark AuthManager package: `org.rtiamanzi:teehr-iceberg-authmanager` +- manager class in package: `org.teehr.iceberg.auth.TeehrBrokerAuthManager` + +Prototype property contract used by the manager: + +- `rest.auth.teehr.broker.url` +- `rest.auth.teehr.user-id` +- `rest.auth.teehr.session-id` +- `rest.auth.teehr.realm` +- `rest.auth.teehr.catalog` (default `iceberg`) +- `rest.auth.teehr.requested-ttl-seconds` (default `600`) +- `rest.auth.teehr.request-timeout-ms` (default `5000`) +- `rest.auth.teehr.refresh-skew-seconds` (default `60`) + +### Integration points + +- package custom AuthManager in a jar available to Spark driver and executors +- configure Spark Iceberg catalog to use custom auth manager type/class +- pass only minimal user/session context from notebook to Spark +- keep current notebook refresh helper as fallback for direct REST/API calls, but + do not treat it as the primary fix for Spark session longevity + +### Success criteria + +1. Spark interactive session survives normal Keycloak access-token expiry without + full Spark restart +2. per-user Polaris authorization still reflects current Keycloak-derived entitlements +3. no long-lived refresh token is exposed in notebook code or Spark SQL properties +4. audit logs can correlate notebook user, broker issuance, and Polaris access + +### Near-term recommendation + +- keep longer access-token TTL for user experience stability +- keep notebook-side proactive refresh for direct API access +- treat custom AuthManager plus broker as the real fix for Spark session continuity + +Execution checklist: + +1. publish or select a release of `org.rtiamanzi:teehr-iceberg-authmanager` +2. ensure Spark includes that coordinate in `spark.jars.packages` +3. configure `spark.sql.catalog..rest.auth.type` to `org.teehr.iceberg.auth.TeehrBrokerAuthManager` +4. provide the `rest.auth.teehr.*` properties via Spark config +5. validate session behavior across at least one access-token expiration window + +## Open Design Questions + +1. What auth mechanism does the selected Polaris deployment support for end-user principals? +2. Can Polaris directly evaluate Keycloak-issued JWTs, or is an intermediate exchange/delegation layer needed? +3. How are Keycloak groups or equivalent authorization attributes surfaced to Polaris policy evaluation? +4. What naming validation and uniqueness rules should apply to `projects.`? +5. Should project namespace visibility initially support only `private` and `all-iceberg-users-read`, or additional sharing modes? +6. What is the supported user-identity propagation mechanism from Trino to Polaris? +7. What is the supported user-identity propagation mechanism from Spark to Polaris? +8. Do PyIceberg and any direct notebook clients need separate auth handling from Spark? +9. How will token refresh work for long-lived notebook sessions and Spark jobs? +10. What direct object-store permissions are still needed, and how do we prevent them from bypassing Polaris? +11. Which existing services besides Jupyter, FastAPI, Spark, and Trino also need user-context-aware Iceberg access? +12. How should non-human automation be separated from human end-user access? +13. What audit trail is required to correlate Keycloak user, notebook/API request, Trino/Spark execution, effective groups, project namespace ownership, and Polaris decision? +14. What governance process should control direct per-user exceptions? +15. What quotas, lifecycle rules, or cleanup policies should apply to project namespaces? + +## Token Lifecycle Immediate Execution Plan + +1. Build shared notebook token utility with expiry introspection and proactive + renewal threshold. +2. Add Spark catalog token update function and one-time retry wrapper for + authorization failures attributable to token expiry. +3. Wire utility into developer notebooks and example scripts first. +4. Add observability fields: token issue time, expiry, renewal attempts, + renewal outcome, and Spark operation retry outcome. +5. Define broker API contract and security model in parallel. +6. Implement broker, then cut notebooks over from direct renewal to broker + issuance. + +## Immediate Next Steps + +1. Inspect current JupyterHub config and identify where to enable `auth_state` and pre-spawn claim handling. +2. Document current FastAPI → Trino call paths and whether requests already preserve end-user context. +3. Inventory all Iceberg-accessing services in the repo and classify them as human-initiated vs automation. +4. Research and document the chosen Polaris auth capabilities and Trino/Spark integration constraints. +5. Draft the target group-to-policy mapping for current TEEHR personas and datasets. +6. Define the exception policy for direct user grants and the boundary for automation principals. +7. Define the project namespace naming, entitlement, and visibility model. + +## Non-Goals + +This plan does not attempt to: + +- move all fine-grained authorization into Keycloak +- use notebook-side Python logic as the primary enforcement point +- keep broad shared storage credentials as the long-term authorization model +- make direct per-user grants the default authorization strategy +- allow arbitrary self-created shared namespaces without governance +- finalize product-specific config syntax before validating supported auth paths diff --git a/docs/polaris-local-spark-simple-example.md b/docs/polaris-local-spark-simple-example.md new file mode 100644 index 0000000..f3f0e6e --- /dev/null +++ b/docs/polaris-local-spark-simple-example.md @@ -0,0 +1,107 @@ +# Polaris Local Spark Simple Example + +This is a minimal local-development flow for Polaris on KinD with MinIO using Spark. + +## Goal + +- Authenticate as `admin` +- Connect Spark to Polaris REST catalog +- Create a namespace +- Create and query a table + +## Prerequisites + +- Cluster is deployed and healthy: `garden deploy` +- A Jupyter single-user pod is running (for example `jupyter-admin`) +- Local Keycloak users exist (`admin` / `admin` by default) + +## 1. Refresh Polaris bootstrap config + +Run this once after changing Polaris/MinIO/catalog settings: + +```bash +kubectl -n teehr-hub delete job polaris-bootstrap --ignore-not-found=true +kubectl -n teehr-hub apply -f ./polaris-bootstrap/manifests/bootstrap-job.yaml +kubectl -n teehr-hub wait --for=condition=Complete job/polaris-bootstrap --timeout=600s +``` + +## 2. Run the minimal Spark example + +```bash +bash ./scripts/run_jupyter_polaris_spark_example.sh +``` + +Optional: keep created objects instead of dropping them. + +```bash +EXTRA_ARGS="--keep --namespace spark_demo_manual --table demo_table" bash ./scripts/run_jupyter_polaris_spark_example.sh +``` + +## 3. Expected behavior + +The script prints: + +- effective Spark Polaris/MinIO config +- namespace listing +- namespace creation +- table creation, insert, and select + +## 4. Known failure signature and meaning + +If table creation fails with `UnknownHostException` and a host like `warehouse.minio`, then: + +- Spark-to-Polaris auth is working +- namespace operations are working +- Polaris server-side object store write is using virtual-host style DNS +- local MinIO path-style behavior is not being honored for that write path + +Root cause observed in this repo: + +- catalog-level `s3.*` properties were present, but Polaris table-write path required `table-default.s3.*` +- once `table-default.s3.endpoint`, `table-default.s3.path-style-access`, and `table-default.s3.region` were added, Spark table create succeeded + +Check quickly: + +```bash +kubectl -n teehr-hub logs deploy/polaris --since=10m | grep -E 'UnknownHostException|warehouse.minio|Unable to execute HTTP request' +``` + +Verify active catalog settings (from a Jupyter pod): + +```bash +kubectl -n teehr-hub exec jupyter-admin -c notebook -- python -c "import requests; t=requests.post('http://keycloak-service:8080/realms/teehr/protocol/openid-connect/token',data={'grant_type':'password','client_id':'jupyterhub','client_secret':'local-jupyterhub-client-secret','username':'admin','password':'admin','scope':'openid profile email'},timeout=20).json()['access_token']; h={'Authorization':f'Bearer {t}','X-Polaris-Realm':'teehr'}; print(requests.get('http://polaris:8181/api/management/v1/catalogs/teehr',headers=h,timeout=20).json())" +``` + +Look for: + +- `storageConfigInfo.endpoint = http://minio:9000` +- `storageConfigInfo.pathStyleAccess = true` +- catalog properties containing both: + - `s3.path-style-access = true` + - `table-default.s3.path-style-access = true` + - `table-default.s3.endpoint = http://minio:9000` + +## 5. Durable fix in manifests + +The bootstrap job now writes both catalog-level and table-default S3 properties during catalog create/update. + +- `s3.endpoint` and `table-default.s3.endpoint` +- `s3.path-style-access` and `table-default.s3.path-style-access` +- `s3.region` and `table-default.s3.region` +- `s3.remote-signing-enabled` and `table-default.s3.remote-signing-enabled` + +After pulling these changes, rerun bootstrap: + +```bash +kubectl -n teehr-hub delete job polaris-bootstrap --ignore-not-found=true +kubectl -n teehr-hub apply -f ./polaris-bootstrap/manifests/bootstrap-job.yaml +kubectl -n teehr-hub wait --for=condition=Complete job/polaris-bootstrap --timeout=600s +``` + +## 6. Files for this simple flow + +- `examples/developer/polaris_spark_namespace_table_example.py` +- `scripts/run_jupyter_polaris_spark_example.sh` +- `examples/developer/setup_utils.py` +- `polaris-bootstrap/manifests/bootstrap-job.yaml` +- `polaris-bootstrap/manifests/acl-config.yaml.tpl` diff --git a/docs/polaris-migration-plan.md b/docs/polaris-migration-plan.md new file mode 100644 index 0000000..341dfe9 --- /dev/null +++ b/docs/polaris-migration-plan.md @@ -0,0 +1,387 @@ +# Plan: Replace iceberg-rest with Apache Polaris (Keycloak-integrated) + +> **Status: COMPLETED** — Last updated 2026-07-30. +> `iceberg-rest` has been removed and Apache Polaris 1.5.0 is deployed and operational. +> See [`polaris-access-control.md`](./polaris-access-control.md) for the implemented auth design. +> This document is retained as a historical record of the migration plan and rationale. + +--- + +Replace `tabulario/iceberg-rest` with `apache/polaris`. Tight Keycloak integration via JWKS + a `PrincipalRoleMapper` that reads `realm_access.roles` from JWTs, mapping Keycloak roles directly to Polaris principal roles at runtime — no per-user Polaris registration needed. A data-driven bootstrap job creates the full namespace × privilege ACL structure from a ConfigMap. Polaris and Keycloak are the control plane; all other services are the data plane. + +--- + +> **Implementation order for `garden deploy` to work**: Phase 7 (Garden variables) and the secrets additions from Phase 1 must be done first — before any other phase — because Garden resolves `${var.polaris.*}` template variables at render time and pods reference the new secrets at startup. + +## Phase 1: Database — `polaris-pg` + +1. Create `polaris-pg/manifests/polaris-pg.yaml` (static, not `.tpl` — matches `iceberg-pg` which uses `manifestFiles`, no Garden templating needed). Mirror `iceberg-pg/manifests/iceberg-pg.yaml` exactly with these substitutions: + - All `iceberg-pg` → `polaris-pg`, all `iceberg` → `polaris` (names, labels, PVC claim) + - `POSTGRES_DB`/`POSTGRES_USER`/`POSTGRES_PASSWORD` env vars read from `polaris-db-secrets` + - Readiness/liveness `pg_isready` probes use `-U polaris -d polaris` + - PVC storage: 10Gi (same as iceberg-pg) + +2. Create `polaris-pg/garden.yaml` — `type: kubernetes`, `name: polaris-database`, `spec.manifestFiles: [./manifests/polaris-pg.yaml]`, depends on `deploy.secrets` — mirrors `iceberg-pg/garden.yaml` exactly with name substitution. + +3. Add `polaris-db-secrets` to `secrets/secrets.local.yaml` and `secrets/secrets.remote.yaml`. **Secrets are NOT standalone K8s manifests** — the `secrets/garden.yaml` `$forEach` loop creates K8s Secrets automatically from these varfiles. Add: + ```yaml + polaris-db-secrets: + data: + database: polaris + username: polaris + password: polaris123 # local; use strong password in remote + polaris-secrets: + data: + root-credentials: "root:secret123" # POLARIS_ROOT_CREDENTIALS format (:); use strong value in remote + trino-polaris-secrets: + data: + client-secret: local-trino-polaris-client-secret # plain secret — used by keycloak-bootstrap env var + credential: "trino-polaris:local-trino-polaris-client-secret" # full credential string — mounted as Trino credential file + ``` + > **Note**: `trino-polaris-secrets` needs **two keys** because the Keycloak bootstrap job needs the plain secret value (`client-secret`) to set as the Keycloak client secret, while Trino's credential file mount needs the full `trino-polaris:` string (`credential`). Using the `credential` key for the Keycloak env var would inject the wrong value. + +--- + +## Phase 2: Polaris Deployment — `polaris/` + +4. Create `polaris/manifests/polaris-config.yaml.tpl` — ConfigMap with `polaris-server.yml`: + - JDBC persistence → `polaris-pg` + - OIDC issuers as a **list** (multi-realm extensible); JWKS URI internal for `local`, external for `remote` + - **`PrincipalRoleMapper`** pointing at `realm_access.roles` in the JWT — maps Keycloak role names to Polaris principal role names at runtime. This is the critical link; without it user token pass-through grants zero privileges. Exact config key must be verified against the pinned Polaris release. + +5. Create `polaris/manifests/polaris.yaml.tpl` — Deployment + Service + SA: + - Image: `apache/polaris` pinned to a specific release tag + - Ports 8182 (catalog REST API), port 8183 (management API); Service exposes both + - **No IRSA annotation** — Polaris is pure metadata; S3 I/O stays with Trino/Spark + - ServiceAccount metadata must use `${environment.namespace}` — do NOT copy the hardcoded `namespace: teehr-hub` from `iceberg-rest.yaml.tpl` + - Local env gets MinIO credentials (same `${if environment.name == "local"}` pattern as `iceberg-rest.yaml.tpl`) + - `POLARIS_ROOT_CREDENTIALS` from `polaris-secrets` key `root-credentials` + - JDBC credentials from `polaris-db-secrets` (Quarkus datasource env var names must be verified against the pinned release — typically `QUARKUS_DATASOURCE_JDBC_URL`, `QUARKUS_DATASOURCE_USERNAME`, `QUARKUS_DATASOURCE_PASSWORD`) + - **Readiness and liveness probes are required** — the `polaris-bootstrap` exec deploy depends on `deploy.polaris` and will fire immediately when Polaris is marked ready; without probes the Quarkus app may still be starting: + ```yaml + readinessProbe: + httpGet: + path: /q/health/ready + port: 8182 + initialDelaySeconds: 20 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /q/health/live + port: 8182 + initialDelaySeconds: 30 + periodSeconds: 10 + ``` + +6. Create `polaris/garden.yaml` — `type: kubernetes`, `spec.manifestTemplates`, depends on `deploy.secrets`, `deploy.polaris-database`, `deploy.keycloak` + +--- + +## Phase 3: Keycloak Bootstrap Updates + +7. Update `keycloak-bootstrap/manifests/realm-configmap.yaml.tpl` — inside the existing `teehr-realm.json` data key: + - Add to `roles.realm` array: `iceberg-catalog-admin`, `iceberg-namespace-public-read`, `iceberg-namespace-public-write`, `iceberg-namespace-restricted-read`, `iceberg-namespace-restricted-write` + - Add to `groups` array: + - `iceberg-public-readers` (realmRoles: `iceberg-namespace-public-read`) + - `iceberg-public-writers` (realmRoles: `iceberg-namespace-public-write`, `iceberg-namespace-public-read`) + - `iceberg-restricted-readers` (realmRoles: `iceberg-namespace-restricted-read`) + - `iceberg-restricted-writers` (realmRoles: `iceberg-namespace-restricted-write`, `iceberg-namespace-restricted-read`) + - `iceberg-catalog-admins` (realmRoles: `iceberg-catalog-admin`) + - Retain existing `iceberg-user` role and group during transition + - Add to `clients` array — **one** new confidential service account client (Polaris does not need its own Keycloak client; it validates tokens via JWKS only and does not perform token introspection or act as an OAuth2 client itself): + - `trino-polaris` client — `serviceAccountsEnabled: true`, `secret: $(env:TRINO_POLARIS_CLIENT_SECRET)`, add `realm_access` protocol mapper to include roles in access token + - **Note on `realm_access.roles` claim**: Keycloak includes realm roles in access tokens by default, but verify against the running Keycloak version. If the Polaris `PrincipalRoleMapper` needs roles under a custom claim path, add an explicit `oidc-usermodel-realm-role-mapper` protocolMapper to the Polaris-facing clients. + +8. Update `keycloak-bootstrap/manifests/bootstrap-job.yaml` — add one new `env` entry matching the existing pattern: + - `TRINO_POLARIS_CLIENT_SECRET` from `trino-polaris-secrets` key `client-secret` + +--- + +## Phase 4: Polaris Bootstrap Job — `polaris-bootstrap/` + +9. Create `polaris-bootstrap/manifests/acl-config.yaml` — a ConfigMap containing a declarative ACL definition (JSON) that the bootstrap job consumes. This makes the job **data-driven**: adding a new namespace = edit this file + re-run bootstrap job, no code changes: + ```json + { + "realm": "teehr", + "catalog": "teehr", + "catalog_admin_keycloak_role": "iceberg-catalog-admin", + "namespaces": [ + { + "name": "public", + "roles": [ + { + "keycloak_role": "iceberg-namespace-public-read", + "polaris_principal_role": "public_reader", + "polaris_catalog_role": "public_read_role", + "privileges": ["TABLE_READ_DATA", "TABLE_LIST", "NAMESPACE_LIST"] + }, + { + "keycloak_role": "iceberg-namespace-public-write", + "polaris_principal_role": "public_writer", + "polaris_catalog_role": "public_write_role", + "privileges": ["TABLE_WRITE_DATA", "TABLE_READ_DATA", "TABLE_LIST", "NAMESPACE_LIST", "CREATE_TABLE"] + } + ] + }, + { + "name": "restricted", + "roles": [ + { + "keycloak_role": "iceberg-namespace-restricted-read", + "polaris_principal_role": "restricted_reader", + "polaris_catalog_role": "restricted_read_role", + "privileges": ["TABLE_READ_DATA", "TABLE_LIST", "NAMESPACE_LIST"] + }, + { + "keycloak_role": "iceberg-namespace-restricted-write", + "polaris_principal_role": "restricted_writer", + "polaris_catalog_role": "restricted_write_role", + "privileges": ["TABLE_WRITE_DATA", "TABLE_READ_DATA", "TABLE_LIST", "NAMESPACE_LIST", "CREATE_TABLE"] + } + ] + } + ] + } + ``` + +10. Create `polaris-bootstrap/manifests/bootstrap-job.yaml` — reuse a Prefect image (same pattern as `prefect-workflows/manifests/load-secrets.yaml`, which uses `prefecthq/prefect:3.2.0-python3.10`; use whichever tag is current in the codebase — no new image or Dockerfile needed) running an inline Python script (`python -c |`) matching that pattern. The script: + - Accepts `POLARIS_REALM_NAME` env var (default `teehr`); sends `X-Polaris-Realm` header on every Management API call + - Uses stdlib (`json`, `os`) + `requests` (available in the Prefect image) — no custom image build + - Reads ACL config from mounted `acl-config.yaml` ConfigMap + - Creates the `teehr` catalog pointing at the warehouse (S3/MinIO path from env) + - Iterates config to create: namespaces, catalog roles with privilege grants, principal roles, catalog role → principal role assignments + - Creates `iceberg-catalog-admin` principal role with full catalog-level grants + - `ttlSecondsAfterFinished: 100` — matches existing Job pattern in `load-secrets.yaml` + - Required env vars in the Job manifest: + - `POLARIS_MANAGEMENT_URL: http://polaris:8183` — base URL for the management API + - `POLARIS_ROOT_CREDENTIALS` from `polaris-secrets` key `root-credentials` — used to obtain an initial Bearer token from the management API + - `POLARIS_REALM_NAME: teehr` (or from env) + - `CATALOG_WAREHOUSE` from `${var.polaris.catalogWarehouse}` (or hardcoded for local/remote) + +11. Create `polaris-bootstrap/garden.yaml` — mirrors `keycloak-bootstrap/garden.yaml` with **two deploys in one file** (this is required so the ConfigMap exists in Kubernetes before the Job pod tries to mount it): + ```yaml + kind: Deploy + type: kubernetes + name: polaris-acl-config + dependencies: + - deploy.secrets + environments: + - local + - remote + spec: + manifestFiles: + - ./manifests/acl-config.yaml + --- + kind: Deploy + type: exec + name: polaris-bootstrap + dependencies: + - deploy.polaris + - deploy.polaris-acl-config + - deploy.keycloak-bootstrap # logical ordering: Keycloak roles must exist before bootstrap maps them + environments: + - local + - remote + spec: + deployCommand: + - bash + - -c + - >- + kubectl -n "${environment.namespace}" delete job polaris-bootstrap --ignore-not-found=true && + kubectl -n "${environment.namespace}" apply -f ./manifests/bootstrap-job.yaml && + kubectl -n "${environment.namespace}" wait --for=condition=Complete job/polaris-bootstrap --timeout=120s + ``` + +--- + +## Phase 5: Trino Catalog Config Update + +12. Update `trino/garden.yaml` — **two separate Deploy blocks exist (local and remote); both must be updated**. Trino is a **Helm chart deployment** (`type: helm`) with no separate manifests directory — all config goes in Helm values. Changes per block: + + **Catalog properties** (in `catalogs.iceberg` multiline string): + - Replace `iceberg.rest-catalog.uri` value with `${var.polaris.catalogUri}` + - Replace `iceberg.rest-catalog.warehouse` value with `${var.polaris.catalogWarehouse}` + - Add `iceberg.rest-catalog.security=OAUTH2` + - Add `iceberg.rest-catalog.oauth2.server-uri=${var.polaris.oauthServerUri}` + - Add `iceberg.rest-catalog.oauth2.credential-file=/etc/trino/polaris-credential` + - Add `iceberg.rest-catalog.oauth2.scope=openid` + - Add `iceberg.rest-catalog.header.X-Polaris-Realm=teehr` + - Keep `iceberg.catalog.type=rest` (unchanged) + + **Credential file delivery** — Trino catalog properties do not support env var interpolation, so the OAuth2 credential must be a mounted file. The `trino-polaris-secrets` key `credential` stores the full `trino-polaris:` string. Mount it via Helm values: + ```yaml + coordinator: + extraVolumes: + - name: polaris-credential + secret: + secretName: trino-polaris-secrets + items: + - key: credential + path: polaris-credential + extraVolumeMounts: + - name: polaris-credential + mountPath: /etc/trino/polaris-credential + subPath: polaris-credential + readOnly: true + ``` + No init container needed. + + **Per-block constraints:** + - Local block: keep existing MinIO `env` entries; keep `s3.path-style-access` and `s3.endpoint` catalog properties + - Remote block: keep existing `serviceAccount.annotations` IRSA entry + - Both blocks: keep all existing `accessControl` configmap rules unchanged + +--- + +## Phase 6: Spark Session Updates + +13. Update `spark_session_utils.py` — changes are **minimal and additive**, preserving all existing function signatures and behavior: + - Add an optional `oauth2_token: str = None` parameter to `create_spark_session()` — passed through to `_configure_iceberg_catalogs()` + - In `_configure_iceberg_catalogs()`: add OAuth2 conf.set calls at the end of the existing function body: + - If `oauth2_token` provided (JupyterHub user token pass-through): set `rest.auth.type=oauth2` + `rest.auth.oauth2.token=` + - Set `rest.transport.header.X-Polaris-Realm=teehr` + - The existing `update_configs: Dict[str, str]` parameter on `create_spark_session()` remains available as an override escape hatch — no structural change needed + - **No changes** to `_create_spark_base_session`, `_set_spark_cluster_configuration`, `_set_aws_credentials_in_spark`, `_update_configs_and_packages`, `_set_catalog_metadata`, or any other existing functions + +14. Update `teehr/src/teehr/const.py` — add `POLARIS_OAUTH2_SERVER_URI` env var read alongside existing constants. + +15. Update `prefect-workflows/manifests/prefect-deployer-job.yaml` — change `REMOTE_CATALOG_REST_URI` value from `${var.iceberg.catalogUri}` to `${var.polaris.catalogUri}`. + +--- + +## Phase 7: Garden Variables & Wiring + +16. Update `project.garden.yml` — add `polaris` variable group with **local/remote divergence** (same pattern as existing `iceberg.*` divergence): + ```yaml + # local: + polaris: + catalogUri: http://polaris:8182/api/catalog + oauthServerUri: http://keycloak-service:8080/realms/teehr/protocol/openid-connect/token + catalogWarehouse: s3://warehouse/ + catalogType: rest + inCluster: "true" + catalogS3PathStyleAccess: "true" + catalogS3Endpoint: "http://minio:9000" + + # remote: + polaris: + catalogUri: https://polaris.${var.hostname}/api/catalog + oauthServerUri: https://auth.${var.hostname}/realms/teehr/protocol/openid-connect/token + catalogWarehouse: s3://dev-teehr-iceberg-warehouse/ + catalogType: rest + inCluster: "false" + catalogS3PathStyleAccess: "false" + catalogS3Endpoint: "" + ``` + Keep existing `iceberg.*` variables during transition. The `polaris` variable group is a complete superset — once all consumers are migrated, `iceberg.*` can be removed entirely. + +17. Add comment blocks in `project.garden.yml` marking **control plane** (`keycloak*`, `polaris*`, `cert-manager`) vs **data plane** modules to document the intended future cluster boundary. All cross-plane references go through `polaris.*` variable group entries — no hardcoded in-cluster hostnames in data plane configs. + +--- + +## Phase 8: Polaris Ingress — Required + +18. Update `ingress/garden.yaml` — add a new `kind: Deploy` entry `name: polaris-ingress` following the exact pattern of existing entries: `type: kubernetes`, `spec.manifestTemplates: [manifests/polaris.yaml.tpl]`, dependencies on `deploy.cert-manager`, `deploy.letsencrypt`, `deploy.cert`, `deploy.contour`, `deploy.polaris`. + +19. Create `ingress/manifests/polaris.yaml.tpl` — Contour `HTTPProxy` following the exact pattern of existing manifests: + - `fqdn: polaris.${var.hostname}`, TLS `secretName: polaris.${var.hostname}-tls` + - Route `/api/catalog` → `polaris:8182` (catalog REST API) + - Route `/api/management` → `polaris:8183` (management API) + +--- + +## Phase 9: Retire iceberg-rest + +20. Disable `iceberg-rest` Garden deployment after all consumers (Trino, Spark, Prefect workflows) are verified connected to Polaris. Keep `iceberg-pg` until catalog data migration is confirmed complete. + +--- + +## Files — New + +| File | Notes | +|---|---| +| `polaris-pg/garden.yaml` | Mirror `iceberg-pg/garden.yaml`; uses `manifestFiles` | +| `polaris-pg/manifests/polaris-pg.yaml` | Static manifest (no `.tpl`); mirror `iceberg-pg/manifests/iceberg-pg.yaml` | +| `polaris/garden.yaml` | Depends on `polaris-database`, `keycloak`, `secrets` | +| `polaris/manifests/polaris.yaml.tpl` | Deployment + Service + SA; no IRSA annotation | +| `polaris/manifests/polaris-config.yaml.tpl` | ConfigMap with multi-issuer `polaris-server.yml` + `PrincipalRoleMapper` | +| `polaris-bootstrap/garden.yaml` | Two deploys: `type: kubernetes` (acl-config ConfigMap) + `type: exec` (job); both with `environments: [local, remote]` | +| `polaris-bootstrap/manifests/bootstrap-job.yaml` | Reuses `prefecthq/prefect:3.4.24-python3.12`; inline Python; parameterized on `POLARIS_REALM_NAME` | +| `polaris-bootstrap/manifests/acl-config.yaml` | Declarative namespace × privilege ACL ConfigMap | +| `ingress/manifests/polaris.yaml.tpl` | Contour HTTPProxy; routes for ports 8182 and 8183 | + +## Files — Modified + +| File | Change | +|---|---| +| `secrets/secrets.local.yaml` | Add `polaris-db-secrets`, `polaris-secrets`, `trino-polaris-secrets` | +| `secrets/secrets.remote.yaml` | Same three secrets with production-grade values | +| `keycloak-bootstrap/manifests/realm-configmap.yaml.tpl` | Add 5 realm roles, 5 groups, 1 confidential client (`trino-polaris`) | +| `keycloak-bootstrap/manifests/bootstrap-job.yaml` | Add 1 new secret env var (`TRINO_POLARIS_CLIENT_SECRET`) | +| `trino/garden.yaml` | Both local + remote Deploy blocks: OAuth2 catalog auth + credential-file volume mount via Helm values | +| `spark_session_utils.py` | Additive: dual-path OAuth2 + `X-Polaris-Realm` header; no existing signatures changed | +| `teehr/src/teehr/const.py` | Add `POLARIS_OAUTH2_SERVER_URI` | +| `prefect-workflows/manifests/prefect-deployer-job.yaml` | Update `REMOTE_CATALOG_REST_URI` to `${var.polaris.catalogUri}` | +| `project.garden.yml` | Add `polaris` variable group + control/data plane comments | +| `ingress/garden.yaml` | Add `polaris-ingress` Deploy entry | + +--- + +## Verification + +1. `kubectl get pods` — `polaris` and `polaris-pg` both Running +2. `kubectl logs deployment/polaris` — OIDC + `PrincipalRoleMapper` config loaded; no startup errors +3. Fetch `client_credentials` token for `trino-polaris` from Keycloak → `GET https://polaris.${var.hostname}/api/catalog/v1/config` with `X-Polaris-Realm: teehr` → expect 200; confirm `realm_access.roles` present in decoded token +4. `trino --execute "SHOW SCHEMAS IN iceberg"` → teehr schema visible +5. JupyterHub Spark with user token injected → user with no namespace role gets 403 from Polaris +6. `REMOTE_CATALOG_REST_URI` resolves to Polaris +7. **ACL matrix**: + - `iceberg-namespace-public-read` member → read `public` ✓, write `public` ✗, read `restricted` ✗ + - `iceberg-namespace-public-write` member → read+write `public` ✓, `restricted` ✗ + - `iceberg-namespace-restricted-read` member → read `restricted` ✓, write `public` ✗ + - `iceberg-catalog-admin` member → full access to all namespaces ✓ +8. Edit `acl-config.yaml` to add a new namespace, re-run bootstrap job → new namespace ACLs applied; no code changes required +9. Run existing `teehr/tests/` catalog operation tests + +--- + +## Decisions + +- `apache/polaris` image (Apache incubator), pinned to a specific release tag — not `latest` +- `polaris.yaml.tpl` ServiceAccount uses `${environment.namespace}` — not hardcoded `teehr-hub` like `iceberg-rest.yaml.tpl` +- Readiness/liveness probes on `/q/health/ready` and `/q/health/live` (port 8182) required so Garden waits for Polaris to be truly ready before firing `polaris-bootstrap` +- `polaris` Keycloak client removed — Polaris validates tokens via JWKS only and needs no Keycloak service account +- New dedicated `polaris-pg` PostgreSQL instance (not reusing `iceberg-pg`) +- `polaris-pg` uses `manifestFiles` (static, no `.tpl`) — matching `iceberg-pg` pattern exactly +- Secrets via `secrets/secrets.local.yaml` + `secrets/secrets.remote.yaml` varfiles — consistent with existing `$forEach` pattern; no standalone K8s Secret manifests +- Trino OAuth2 credential delivered via mounted credential-file (`iceberg.rest-catalog.oauth2.credential-file`) — avoids env var interpolation limitations in Trino catalog properties; stored as full `trino-polaris:` string, mounted via `subPath`, no init container +- Trino: `client_credentials` — access control enforced at Trino layer; Polaris sees service identity +- Spark in JupyterHub: user token pass-through — Polaris enforces per-user namespace/table ACLs +- Polaris has **no IRSA annotation** — pure metadata service on the control plane +- Control plane: Polaris + Keycloak (future: dedicated cluster); Data plane: all other services +- All cross-plane URLs go through `polaris.*` Garden variable group — no hardcoded in-cluster hostnames in data plane configs +- `polaris-server.yml` uses an issuer allow-list (not single issuer) from day one for multi-realm extensibility +- `polaris-bootstrap` reuses `prefecthq/prefect:3.4.24-python3.12` image (already in codebase via `load-secrets.yaml`) — no new Dockerfile or image build +- `polaris-bootstrap/garden.yaml` uses **two deploys**: `type: kubernetes` (deploys `acl-config.yaml` ConfigMap) + `type: exec` (runs the job) — matches `keycloak-bootstrap` pattern exactly; required because the Job pod mounts the ConfigMap as a volume +- `trino-polaris-secrets` has **two keys**: `client-secret` (plain value, used by `keycloak-bootstrap` env var) and `credential` (full `trino-polaris:` string, mounted as Trino credential file) — the same secret provides both without duplication +- `polaris` variable group is a complete superset of `iceberg` variable group, enabling full future removal of `iceberg.*` after migration +- `polaris-bootstrap` job env vars include `POLARIS_MANAGEMENT_URL` (http://polaris:8183) and `POLARIS_ROOT_CREDENTIALS` from `polaris-secrets` — required for the script to authenticate and call the management API +- `polaris-bootstrap` exec deploy depends on `deploy.keycloak-bootstrap` for logical ordering (Keycloak roles must exist before the bootstrap job maps them) +- Trino `iceberg.rest-catalog.warehouse` updated to `${var.polaris.catalogWarehouse}` alongside the URI change +- `spark_session_utils.py` changes are additive only — new `oauth2_token: str = None` parameter; all existing callers unaffected; `update_configs` escape hatch unchanged +- `X-Polaris-Realm` header explicit in all client configs from day one +- Keycloak role taxonomy: namespace × privilege; group-based assignment; coarse `iceberg-user` retained during transition +- `PrincipalRoleMapper` in `polaris-server.yml` resolves JWT `realm_access.roles` → Polaris principal roles at runtime; no per-user Polaris principal registration needed + +--- + +## Further Considerations + +1. **Polaris image tag**: Pin to a specific release (e.g., `0.9.0`) — `apache/polaris` is under active development and `latest` may break between deployments. + +2. **DB schema init**: Confirm whether Polaris auto-migrates its PostgreSQL schema on first start or requires a separate init job — check release notes for the pinned version before implementing Phase 1. + +3. **Catalog data migration**: Existing tables registered in `iceberg-pg`'s JDBC catalog will not auto-appear in Polaris. The `polaris-bootstrap` job needs a migration step to re-register existing namespaces/tables, or plan for a re-ingest window before retiring `iceberg-rest` in Phase 9. + +4. **OPA for Trino access control (future)**: Trino uses a single `trino-polaris` service identity so Polaris cannot enforce per-user namespace/table ACLs for Trino queries. Open Policy Agent (OPA) — a lightweight Go service on the control plane — can fill this gap. Trino's native OPA system access control plugin receives full query context (user identity, Keycloak groups, target catalog/schema/table) and evaluates Rego policies that mirror the Keycloak role taxonomy. Policy changes hot-reload via ConfigMap without Trino restarts. Would require: new `opa/` Garden module + `access-control.name=opa` in Trino config + policy ConfigMap mirroring the Phase 3 role taxonomy. + diff --git a/iceberg-pg/garden.yaml b/iceberg-pg/garden.yaml index 2618b41..de4c912 100644 --- a/iceberg-pg/garden.yaml +++ b/iceberg-pg/garden.yaml @@ -2,6 +2,7 @@ kind: Deploy type: kubernetes name: iceberg-database description: K8s Deploy Iceberg Catalog Database +disabled: true dependencies: - deploy.secrets environments: diff --git a/iceberg-rest/garden.yaml b/iceberg-rest/garden.yaml index e0e39c3..3d31481 100644 --- a/iceberg-rest/garden.yaml +++ b/iceberg-rest/garden.yaml @@ -2,6 +2,7 @@ kind: Deploy type: kubernetes name: iceberg-rest-deploy description: K8s Deploy Iceberg REST +disabled: true spec: manifestTemplates: - ./manifests/iceberg-rest.yaml.tpl diff --git a/ingress/garden.yaml b/ingress/garden.yaml index 47ef36f..88f68aa 100644 --- a/ingress/garden.yaml +++ b/ingress/garden.yaml @@ -53,15 +53,27 @@ environments: --- kind: Deploy type: kubernetes -name: xpublish-api-ingress +name: polaris-ingress spec: manifestTemplates: - - manifests/xpublish-api.yaml.tpl + - manifests/polaris.yaml.tpl dependencies: - - deploy.xpublish-api + - deploy.polaris environments: - local - - remote + - remote +# --- +# kind: Deploy +# type: kubernetes +# name: xpublish-api-ingress +# spec: +# manifestTemplates: +# - manifests/xpublish-api.yaml.tpl +# dependencies: +# - deploy.xpublish-api +# environments: +# - local +# - remote # --- # kind: Deploy # type: kubernetes diff --git a/ingress/manifests/polaris.yaml.tpl b/ingress/manifests/polaris.yaml.tpl new file mode 100644 index 0000000..98495f2 --- /dev/null +++ b/ingress/manifests/polaris.yaml.tpl @@ -0,0 +1,21 @@ +apiVersion: projectcontour.io/v1 +kind: HTTPProxy +metadata: + name: polaris-httpproxy + namespace: ${environment.namespace} +spec: + virtualhost: + fqdn: polaris.${var.hostname} + tls: + secretName: polaris.${var.hostname}-tls + routes: + - services: + - name: polaris + port: 8181 + conditions: + - prefix: /api/catalog + - services: + - name: polaris + port: 8181 + conditions: + - prefix: /api/management diff --git a/jupyterhub/docker/Dockerfile.jupyter-driver b/jupyterhub-docker/Dockerfile.jupyter-driver similarity index 96% rename from jupyterhub/docker/Dockerfile.jupyter-driver rename to jupyterhub-docker/Dockerfile.jupyter-driver index 8a1fbd2..3775e5d 100644 --- a/jupyterhub/docker/Dockerfile.jupyter-driver +++ b/jupyterhub-docker/Dockerfile.jupyter-driver @@ -72,7 +72,7 @@ RUN ARCH=$(uname -m) && \ if [ "$ARCH" = "aarch64" ]; then \ export GDAL_CONFIG=/usr/bin/gdal-config; \ fi && \ - pip install "git+https://github.com/RTIInternational/teehr.git@${TEEHR_VERSION}" + GIT_LFS_SKIP_SMUDGE=1 pip install --no-cache-dir "git+https://github.com/RTIInternational/teehr.git@${TEEHR_VERSION}" RUN mkdir -p /opt/teehr && chown -R ${NB_USER}:${NB_USER} /opt/teehr diff --git a/jupyterhub/docker/executor-pod-template.yaml b/jupyterhub-docker/executor-pod-template.yaml similarity index 100% rename from jupyterhub/docker/executor-pod-template.yaml rename to jupyterhub-docker/executor-pod-template.yaml diff --git a/jupyterhub/docker/garden.yaml b/jupyterhub-docker/garden.yaml similarity index 100% rename from jupyterhub/docker/garden.yaml rename to jupyterhub-docker/garden.yaml diff --git a/jupyterhub/docker/ipython_kernel_config.py b/jupyterhub-docker/ipython_kernel_config.py similarity index 100% rename from jupyterhub/docker/ipython_kernel_config.py rename to jupyterhub-docker/ipython_kernel_config.py diff --git a/jupyterhub-profiles/profile-list.local.json b/jupyterhub-profiles/profile-list.local.json index e390244..8291ffb 100644 --- a/jupyterhub-profiles/profile-list.local.json +++ b/jupyterhub-profiles/profile-list.local.json @@ -22,6 +22,9 @@ "kubespawner_override": { "node_selector": { "teehr-hub/nodegroup-name": "nb-r5-xlarge" + }, + "environment": { + "TEEHR_PROJECT_ID": "TEEHR" } } }, diff --git a/jupyterhub/garden.yaml b/jupyterhub/garden.yaml index 12d119a..e8c47a4 100644 --- a/jupyterhub/garden.yaml +++ b/jupyterhub/garden.yaml @@ -54,6 +54,7 @@ spec: - openid - profile - email + - offline_access claim_groups_key: groups allowed_groups: - jupyter-user @@ -85,6 +86,11 @@ spec: name: jupyterhub-profile-list key: profile-list.json optional: true + JUPYTERHUB_CRYPT_KEY: + valueFrom: + secretKeyRef: + name: jupyterhub + key: JUPYTERHUB_CRYPT_KEY extraConfig: spark-config: | @@ -120,6 +126,18 @@ spec: raise RuntimeError(f"Profile at index {index} is missing required key: display_name") c.KubeSpawner.profile_list = profile_list + auth-state: | + c.Authenticator.enable_auth_state = True + async def auth_state_hook(spawner, auth_state): + spawner.environment["POLARIS_CLIENT_ID"] = os.environ.get("OAUTH_CLIENT_ID", "jupyterhub") + client_secret = os.environ.get("OAUTH_CLIENT_SECRET") + if client_secret: + spawner.environment["POLARIS_CLIENT_SECRET"] = client_secret + if auth_state and "access_token" in auth_state: + spawner.environment["POLARIS_USER_TOKEN"] = auth_state["access_token"] + if auth_state and "refresh_token" in auth_state: + spawner.environment["POLARIS_REFRESH_TOKEN"] = auth_state["refresh_token"] + c.Spawner.auth_state_hook = auth_state_hook cull-kernels: | c.MappingKernelManager.cull_idle_timeout = 3600 c.MappingKernelManager.cull_connected = True @@ -145,17 +163,31 @@ spec: TEEHR_SPARK_IMAGE: ${actions.build.teehr-spark-executor-image.outputs.deploymentImageId} TEEHR_NAMESPACE: ${environment.namespace} AWS_REGION: ${var.aws.region} + AWS_ACCESS_KEY_ID: + valueFrom: + secretKeyRef: + name: minio-secrets + key: accesskey + AWS_SECRET_ACCESS_KEY: + valueFrom: + secretKeyRef: + name: minio-secrets + key: secretkey TEEHR_DOWNLOAD_API_KEY: valueFrom: secretKeyRef: name: jupyter-user-secrets key: api-key - REMOTE_CATALOG_TYPE: ${var.iceberg.catalogType} - REMOTE_CATALOG_REST_URI: ${var.iceberg.catalogUri} - REMOTE_WAREHOUSE_S3_PATH: ${var.iceberg.catalogWarehouse} - REMOTE_CATALOG_S3_ENDPOINT: ${var.iceberg.catalogS3Endpoint} - REMOTE_CATALOG_S3_PATH_STYLE_ACCESS: ${var.iceberg.catalogS3PathStyleAccess} - IN_CLUSTER: ${var.iceberg.inCluster} + REMOTE_CATALOG_TYPE: ${var.polaris.catalogType} + REMOTE_CATALOG_REST_URI: ${var.polaris.catalogUri} + REMOTE_WAREHOUSE_S3_PATH: ${var.polaris.catalogWarehouse} + REMOTE_CATALOG_S3_ENDPOINT: ${var.polaris.catalogS3Endpoint} + REMOTE_CATALOG_S3_PATH_STYLE_ACCESS: ${var.polaris.catalogS3PathStyleAccess} + REMOTE_WAREHOUSE_IDENTIFIER: ${var.polaris.defaultRealm} + IN_CLUSTER: ${var.polaris.inCluster} + POLARIS_DEFAULT_REALM: ${var.polaris.defaultRealm} + POLARIS_OAUTH2_SERVER_URI: ${var.polaris.oauthServerUri} + POLARIS_USE_AUTHMANAGER: "true" extraFiles: jupyter_config: mountPath: /etc/jupyter/jupyter_notebook_config.py @@ -241,6 +273,7 @@ spec: - openid - profile - email + - offline_access claim_groups_key: groups allowed_groups: - jupyter-user @@ -272,6 +305,11 @@ spec: name: jupyterhub-profile-list key: profile-list.json optional: true + JUPYTERHUB_CRYPT_KEY: + valueFrom: + secretKeyRef: + name: jupyterhub + key: JUPYTERHUB_CRYPT_KEY extraConfig: spark-config: | # Allow users to create Spark sessions @@ -306,6 +344,18 @@ spec: raise RuntimeError(f"Profile at index {index} is missing required key: display_name") c.KubeSpawner.profile_list = profile_list + auth-state: | + c.Authenticator.enable_auth_state = True + async def auth_state_hook(spawner, auth_state): + spawner.environment["POLARIS_CLIENT_ID"] = os.environ.get("OAUTH_CLIENT_ID", "jupyterhub") + client_secret = os.environ.get("OAUTH_CLIENT_SECRET") + if client_secret: + spawner.environment["POLARIS_CLIENT_SECRET"] = client_secret + if auth_state and "access_token" in auth_state: + spawner.environment["POLARIS_USER_TOKEN"] = auth_state["access_token"] + if auth_state and "refresh_token" in auth_state: + spawner.environment["POLARIS_REFRESH_TOKEN"] = auth_state["refresh_token"] + c.Spawner.auth_state_hook = auth_state_hook cull-kernels: | c.MappingKernelManager.cull_idle_timeout = 3600 c.MappingKernelManager.cull_connected = True @@ -332,12 +382,17 @@ spec: TEEHR_SPARK_IMAGE: ${actions.build.teehr-spark-executor-image.outputs.deploymentImageId} TEEHR_NAMESPACE: ${environment.namespace} AWS_REGION: ${var.aws.region} - REMOTE_CATALOG_TYPE: ${var.iceberg.catalogType} - REMOTE_CATALOG_REST_URI: ${var.iceberg.catalogUri} - REMOTE_WAREHOUSE_S3_PATH: ${var.iceberg.catalogWarehouse} - REMOTE_CATALOG_S3_ENDPOINT: ${var.iceberg.catalogS3Endpoint} - REMOTE_CATALOG_S3_PATH_STYLE_ACCESS: ${var.iceberg.catalogS3PathStyleAccess} - IN_CLUSTER: ${var.iceberg.inCluster} + REMOTE_CATALOG_TYPE: ${var.polaris.catalogType} + REMOTE_CATALOG_REST_URI: ${var.polaris.catalogUri} + REMOTE_WAREHOUSE_S3_PATH: ${var.polaris.defaultRealm} + REMOTE_CATALOG_S3_ENDPOINT: ${var.polaris.catalogS3Endpoint} + REMOTE_CATALOG_S3_PATH_STYLE_ACCESS: ${var.polaris.catalogS3PathStyleAccess} + REMOTE_WAREHOUSE_IDENTIFIER: ${var.polaris.defaultRealm} + IN_CLUSTER: ${var.polaris.inCluster} + POLARIS_DEFAULT_REALM: ${var.polaris.defaultRealm} + POLARIS_OAUTH2_SERVER_URI: ${var.polaris.oauthServerUri} + POLARIS_USE_AUTHMANAGER: "true" + POLARIS_USE_STS: "true" TRINO_HOST: ${var.trino.host} TRINO_PORT: ${var.trino.port} TRINO_CATALOG: ${var.trino.catalog} diff --git a/jupyterhub/manifests/jupyter-serviceaccount.yaml.tpl b/jupyterhub/manifests/jupyter-serviceaccount.yaml.tpl index 034b8c5..28b1cfc 100644 --- a/jupyterhub/manifests/jupyter-serviceaccount.yaml.tpl +++ b/jupyterhub/manifests/jupyter-serviceaccount.yaml.tpl @@ -3,10 +3,6 @@ kind: ServiceAccount metadata: name: jupyter namespace: ${environment.namespace} - ${if environment.name == "remote"} - annotations: - eks.amazonaws.com/role-arn: ${var.irsa.jupyterRoleArn} - ${endif} labels: app: jupyterhub component: jupyter \ No newline at end of file diff --git a/keycloak-bootstrap/manifests/bootstrap-job.yaml b/keycloak-bootstrap/manifests/bootstrap-job.yaml index 8fd8522..2c529b1 100644 --- a/keycloak-bootstrap/manifests/bootstrap-job.yaml +++ b/keycloak-bootstrap/manifests/bootstrap-job.yaml @@ -43,6 +43,16 @@ spec: secretKeyRef: name: teehr-api-secrets key: client-secret + - name: TRINO_POLARIS_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: trino-polaris-secrets + key: client-secret + - name: PREFECT_POLARIS_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: prefect-polaris-secrets + key: client-secret - name: SMTP_HOST valueFrom: configMapKeyRef: diff --git a/keycloak-bootstrap/manifests/local-users-configmap.yaml.tpl b/keycloak-bootstrap/manifests/local-users-configmap.yaml.tpl index eb7f634..7a7b735 100644 --- a/keycloak-bootstrap/manifests/local-users-configmap.yaml.tpl +++ b/keycloak-bootstrap/manifests/local-users-configmap.yaml.tpl @@ -25,7 +25,9 @@ data: ], "groups": [ "/basic-user", - "/iceberg-user", + "/teehr-read-only", + "/teehr-read-write", + "/iceberg-catalog-admins", "/jupyter-admin", "/key-management-admin", "/prefect-admin", @@ -48,6 +50,27 @@ data: ], "groups": [ "/basic-user", + "/teehr-read-only", + "/jupyter-user" + ] + }, + { + "username": "poweruser", + "enabled": true, + "email": "poweruser@example.local", + "emailVerified": true, + "firstName": "Local", + "lastName": "PowerUser", + "credentials": [ + { + "type": "password", + "value": "poweruser", + "temporary": false + } + ], + "groups": [ + "/basic-user", + "/teehr-read-write", "/jupyter-user" ] } diff --git a/keycloak-bootstrap/manifests/realm-configmap.yaml.tpl b/keycloak-bootstrap/manifests/realm-configmap.yaml.tpl index 6e70ed0..0f2effd 100644 --- a/keycloak-bootstrap/manifests/realm-configmap.yaml.tpl +++ b/keycloak-bootstrap/manifests/realm-configmap.yaml.tpl @@ -9,6 +9,7 @@ data: { "realm": "teehr", "enabled": true, + "accessTokenLifespan": 300, "loginTheme": "teehr", "registrationAllowed": true, "loginWithEmailAllowed": true, @@ -33,7 +34,9 @@ data: { "name": "admin" }, { "name": "basic-user" }, { "name": "jupyter-user" }, - { "name": "iceberg-user" } + { "name": "teehr-read-only" }, + { "name": "teehr-read-write" }, + { "name": "iceberg-catalog-admin" }, ] }, "groups": [ @@ -50,8 +53,12 @@ data: "realmRoles": ["jupyter-user"] }, { - "name": "iceberg-user", - "realmRoles": ["iceberg-user"] + "name": "teehr-read-only", + "realmRoles": ["teehr-read-only"] + }, + { + "name": "teehr-read-write", + "realmRoles": ["teehr-read-write", "teehr-read-only"] }, { "name": "key-management-admin", @@ -73,10 +80,15 @@ data: "view-realm" ] } + }, + { + "name": "iceberg-catalog-admins", + "realmRoles": ["iceberg-catalog-admin"] } ], "defaultGroups": [ - "/basic-user" + "/basic-user", + "/teehr-read-only" ], "clients": [ { @@ -102,6 +114,9 @@ data: "protocol": "openid-connect", "publicClient": false, "serviceAccountsEnabled": true, + "attributes": { + "standard.token.exchange.enabled": "true" + }, "secret": "$(env:TEEHR_API_CLIENT_SECRET)" }, { @@ -126,6 +141,31 @@ data: "userinfo.token.claim": "true", "claim.name": "groups" } + }, + { + "name": "realm-roles", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-realm-role-mapper", + "consentRequired": false, + "config": { + "multivalued": "true", + "id.token.claim": "true", + "access.token.claim": "true", + "userinfo.token.claim": "true", + "claim.name": "realm_access.roles", + "jsonType.label": "String" + } + }, + { + "name": "audience-teehr-api", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-mapper", + "consentRequired": false, + "config": { + "included.client.audience": "teehr-api", + "id.token.claim": "false", + "access.token.claim": "true" + } } ], "redirectUris": [ @@ -165,6 +205,68 @@ data: "webOrigins": [ "https://prefect.${var.hostname}" ] + }, + { + "clientId": "trino-polaris", + "enabled": true, + "protocol": "openid-connect", + "publicClient": false, + "serviceAccountsEnabled": true, + "secret": "$(env:TRINO_POLARIS_CLIENT_SECRET)", + "protocolMappers": [ + { + "name": "realm-roles", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-realm-role-mapper", + "consentRequired": false, + "config": { + "multivalued": "true", + "id.token.claim": "true", + "access.token.claim": "true", + "userinfo.token.claim": "true", + "claim.name": "realm_access.roles", + "jsonType.label": "String" + } + } + ] + }, + { + "clientId": "prefect-polaris", + "enabled": true, + "protocol": "openid-connect", + "publicClient": false, + "serviceAccountsEnabled": true, + "secret": "$(env:PREFECT_POLARIS_CLIENT_SECRET)", + "protocolMappers": [ + { + "name": "realm-roles", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-realm-role-mapper", + "consentRequired": false, + "config": { + "multivalued": "true", + "id.token.claim": "true", + "access.token.claim": "true", + "userinfo.token.claim": "true", + "claim.name": "realm_access.roles", + "jsonType.label": "String" + } + } + ] + } + ], + "users": [ + { + "username": "service-account-trino-polaris", + "enabled": true, + "serviceAccountClientId": "trino-polaris", + "realmRoles": ["iceberg-catalog-admin"] + }, + { + "username": "service-account-prefect-polaris", + "enabled": true, + "serviceAccountClientId": "prefect-polaris", + "realmRoles": ["teehr-read-write"] } ] } diff --git a/polaris-bootstrap/garden.yaml b/polaris-bootstrap/garden.yaml new file mode 100644 index 0000000..96c1149 --- /dev/null +++ b/polaris-bootstrap/garden.yaml @@ -0,0 +1,47 @@ +kind: Deploy +type: kubernetes +name: polaris-acl-config +description: K8s Deploy Polaris ACL ConfigMap and shared sync script +dependencies: + - deploy.secrets +environments: + - local + - remote +spec: + manifestTemplates: + - ./manifests/acl-config.yaml.tpl + - ./manifests/polaris-sync-principals-script.yaml +--- +kind: Deploy +type: kubernetes +name: polaris-principal-sync +description: CronJob that syncs enabled Keycloak users to Polaris principals every 5 minutes +dependencies: + - deploy.secrets + - deploy.polaris-bootstrap +environments: + - local + - remote +spec: + manifestTemplates: + - ./manifests/polaris-principal-sync-cronjob.yaml.tpl +--- +kind: Deploy +type: exec +name: polaris-bootstrap +description: Run Polaris bootstrap job to create catalog, namespaces, and ACL grants +dependencies: + - deploy.polaris + - deploy.polaris-acl-config + - deploy.keycloak-bootstrap +environments: + - local + - remote +spec: + deployCommand: + - bash + - -c + - >- + kubectl -n "${environment.namespace}" delete job polaris-bootstrap --ignore-not-found=true && + kubectl -n "${environment.namespace}" apply -f ./manifests/bootstrap-job.yaml && + kubectl -n "${environment.namespace}" wait --for=condition=Complete job/polaris-bootstrap --timeout=600s diff --git a/polaris-bootstrap/manifests/acl-config.yaml.tpl b/polaris-bootstrap/manifests/acl-config.yaml.tpl new file mode 100644 index 0000000..7452d32 --- /dev/null +++ b/polaris-bootstrap/manifests/acl-config.yaml.tpl @@ -0,0 +1,76 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: polaris-acl-config +data: + acl-config.json: | + { + "realms": [ + { + "realm": "${var.polaris.defaultRealm}", + "catalog": "${var.polaris.defaultRealm}", + "warehouse": "${var.polaris.catalogWarehouse}", + "storage_type": "S3", + "s3_endpoint": "${var.polaris.catalogS3Endpoint}", + "path_style_access": "${var.polaris.catalogS3PathStyleAccess}", + "s3_region": "${var.polaris.catalogS3Region}", + "sts_unavailable": ${var.polaris.storageStsUnavailable}, + "role_arn": "${var.polaris.catalogRoleArn}", + "allowed_locations": [ + "${var.polaris.catalogWarehouse}" + ], + "admin": { + "principal_role": "iceberg-catalog-admin", + "catalog_role": "catalog_admin_role", + "privileges": [ + "CATALOG_MANAGE_CONTENT", + "CATALOG_MANAGE_METADATA" + ] + }, + "namespace_policies": [ + { + "namespace": "teehr", + "roles": [ + { + "principal_role": "teehr-read-only", + "catalog_role": "teehr_read_only_role", + "grants": [ + { + "type": "namespace", + "privileges": [ + "NAMESPACE_READ_PROPERTIES", + "TABLE_LIST", + "TABLE_READ_PROPERTIES", + "TABLE_READ_DATA" + ] + } + ] + }, + { + "principal_role": "teehr-read-write", + "catalog_role": "teehr_read_write_role", + "grants": [ + { + "type": "namespace", + "privileges": [ + "NAMESPACE_READ_PROPERTIES", + "NAMESPACE_WRITE_PROPERTIES", + "TABLE_CREATE", + "TABLE_DROP", + "TABLE_LIST", + "TABLE_READ_PROPERTIES", + "TABLE_WRITE_PROPERTIES", + "TABLE_READ_DATA", + "TABLE_WRITE_DATA" + ] + } + ] + } + ] + } + ], + "table_policies": [], + "principals": [] + } + ] + } diff --git a/polaris-bootstrap/manifests/bootstrap-job.yaml b/polaris-bootstrap/manifests/bootstrap-job.yaml new file mode 100644 index 0000000..c202ea8 --- /dev/null +++ b/polaris-bootstrap/manifests/bootstrap-job.yaml @@ -0,0 +1,401 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: polaris-bootstrap + labels: + app: polaris-bootstrap +spec: + ttlSecondsAfterFinished: 100 + template: + metadata: + labels: + app: polaris-bootstrap + spec: + restartPolicy: Never + containers: + - name: polaris-bootstrap + image: prefecthq/prefect:3.2.0-python3.10 + env: + - name: POLARIS_MANAGEMENT_URL + value: "http://polaris:8181" + - name: POLARIS_ROOT_CREDENTIALS + valueFrom: + secretKeyRef: + name: polaris-secrets + key: root-credentials + - name: KEYCLOAK_URL + value: "http://keycloak-service.teehr-hub.svc.cluster.local:8080" + - name: KEYCLOAK_ADMIN_USER + valueFrom: + secretKeyRef: + name: keycloak-admin-secrets + key: username + - name: KEYCLOAK_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: keycloak-admin-secrets + key: password + command: + - python + - -c + - | + import json + import os + import subprocess + import sys + + import requests + + MGMT_URL = os.environ.get("POLARIS_MANAGEMENT_URL", "http://polaris:8181") + ROOT_CREDS = os.environ["POLARIS_ROOT_CREDENTIALS"] + username, password = ROOT_CREDS.split(":", 1) + + SUCCESS_CODES = (200, 201, 204, 409) + + def parse_bool(value): + if isinstance(value, bool): + return value + if value is None: + return None + return str(value).strip().lower() in ("1", "true", "t", "yes", "y", "on") + + def is_duplicate_error(resp): + return resp.status_code == 500 and "duplicate key" in resp.text.lower() + + def realm_headers(realm, token=None, content_type="application/json"): + headers = { + "X-Polaris-Realm": realm, + } + if token: + headers["Authorization"] = f"Bearer {token}" + if content_type: + headers["Content-Type"] = content_type + return headers + + def request_token(realm): + token_resp = requests.post( + f"{MGMT_URL}/api/catalog/v1/oauth/tokens", + headers=realm_headers(realm, content_type="application/x-www-form-urlencoded"), + data={ + "grant_type": "client_credentials", + "client_id": username, + "client_secret": password, + "scope": "PRINCIPAL_ROLE:ALL", + "realm": realm, + }, + ) + token_resp.raise_for_status() + return token_resp.json()["access_token"] + + def mgmt(method, realm, token, path, **kwargs): + url = f"{MGMT_URL}/api/management/v1{path}" + resp = getattr(requests, method)(url, headers=realm_headers(realm, token=token), **kwargs) + + if resp.status_code in SUCCESS_CODES or is_duplicate_error(resp): + return resp + + print(f"ERROR {resp.status_code}: {resp.text}", file=sys.stderr) + resp.raise_for_status() + return resp + + def ensure_namespace(realm, token, catalog_name, namespace): + resp = requests.post( + f"{MGMT_URL}/api/catalog/v1/{catalog_name}/namespaces", + headers=realm_headers(realm, token=token), + json={"namespace": [namespace]}, + ) + if resp.status_code in SUCCESS_CODES or is_duplicate_error(resp): + return + resp.raise_for_status() + + def ensure_catalog(realm, token, catalog_name, properties, storage_config_info): + get_resp = requests.get( + f"{MGMT_URL}/api/management/v1/catalogs/{catalog_name}", + headers=realm_headers(realm, token=token, content_type=None), + ) + + if get_resp.status_code == 404: + mgmt( + "post", + realm, + token, + "/catalogs", + json={ + "name": catalog_name, + "type": "INTERNAL", + "properties": properties, + "storageConfigInfo": storage_config_info, + }, + ) + return + + get_resp.raise_for_status() + current = get_resp.json() + current_entity_version = current.get("entityVersion") + if current_entity_version is None: + raise ValueError(f"Catalog '{catalog_name}' missing entityVersion") + + update_payload = { + "currentEntityVersion": current_entity_version, + "properties": properties, + "storageConfigInfo": storage_config_info, + } + + update_resp = requests.put( + f"{MGMT_URL}/api/management/v1/catalogs/{catalog_name}", + headers=realm_headers(realm, token=token), + json=update_payload, + ) + + if update_resp.status_code in SUCCESS_CODES: + return + + # Retry once on optimistic-concurrency conflict. + if update_resp.status_code == 409: + refreshed = requests.get( + f"{MGMT_URL}/api/management/v1/catalogs/{catalog_name}", + headers=realm_headers(realm, token=token, content_type=None), + ) + refreshed.raise_for_status() + update_payload["currentEntityVersion"] = refreshed.json()["entityVersion"] + update_resp = requests.put( + f"{MGMT_URL}/api/management/v1/catalogs/{catalog_name}", + headers=realm_headers(realm, token=token), + json=update_payload, + ) + if update_resp.status_code in SUCCESS_CODES: + return + + print(f"ERROR {update_resp.status_code}: {update_resp.text}", file=sys.stderr) + update_resp.raise_for_status() + + def grant_to_catalog_role(realm, token, catalog_name, catalog_role, grant, fallback_namespace=None): + payload = { + "type": grant["type"], + "privilege": grant["privilege"], + } + + if grant["type"] in ("namespace", "table"): + namespace = grant.get("namespace") or fallback_namespace + if not namespace: + raise ValueError("Namespace/table grant missing namespace") + payload["namespace"] = [namespace] + + if grant["type"] == "table": + table_name = grant.get("table") + if not table_name: + raise ValueError("Table grant missing table") + payload["table"] = table_name + + mgmt( + "put", + realm, + token, + f"/catalogs/{catalog_name}/catalog-roles/{catalog_role}/grants", + json=payload, + ) + + def create_role_bindings(realm, token, catalog_name, role_bindings, namespace=None): + for role_cfg in role_bindings: + principal_role = role_cfg["principal_role"] + catalog_role = role_cfg["catalog_role"] + + mgmt("post", realm, token, "/principal-roles", json={"name": principal_role}) + mgmt("post", realm, token, f"/catalogs/{catalog_name}/catalog-roles", json={"name": catalog_role}) + + for grant_cfg in role_cfg.get("grants", []): + for privilege in grant_cfg.get("privileges", []): + grant_to_catalog_role( + realm, + token, + catalog_name, + catalog_role, + { + "type": grant_cfg["type"], + "namespace": grant_cfg.get("namespace"), + "table": grant_cfg.get("table"), + "privilege": privilege, + }, + fallback_namespace=namespace, + ) + + mgmt( + "put", + realm, + token, + f"/principal-roles/{principal_role}/catalog-roles/{catalog_name}", + json={"name": catalog_role}, + ) + + with open("/config/acl-config.json", encoding="utf-8") as f: + config = json.load(f) + + realms_cfg = config.get("realms", []) + if not realms_cfg: + raise ValueError("acl-config.json must define a non-empty 'realms' list") + + for realm_cfg in realms_cfg: + realm = realm_cfg["realm"] + catalog_name = realm_cfg["catalog"] + warehouse = realm_cfg["warehouse"] + storage_type = realm_cfg.get("storage_type", "S3") + s3_endpoint = realm_cfg.get("s3_endpoint") + path_style_access = realm_cfg.get("path_style_access") + s3_region = realm_cfg.get("s3_region") + sts_unavailable = realm_cfg.get("sts_unavailable") + role_arn = realm_cfg.get("role_arn") + allowed_locations = realm_cfg.get("allowed_locations") or [warehouse] + + print(f"[polaris-bootstrap] realm={realm} catalog={catalog_name}") + token = request_token(realm) + + storage_config_info = { + "storageType": storage_type, + "allowedLocations": allowed_locations, + } + catalog_properties = { + "default-base-location": warehouse, + } + if s3_endpoint: + storage_config_info["endpoint"] = s3_endpoint + catalog_properties["s3.endpoint"] = s3_endpoint + catalog_properties["table-default.s3.endpoint"] = s3_endpoint + if path_style_access is not None: + path_style_access_bool = parse_bool(path_style_access) + storage_config_info["pathStyleAccess"] = path_style_access_bool + catalog_properties["s3.path-style-access"] = str( + path_style_access_bool + ).lower() + catalog_properties["table-default.s3.path-style-access"] = str( + path_style_access_bool + ).lower() + if s3_region: + storage_config_info["region"] = s3_region + catalog_properties["s3.region"] = s3_region + catalog_properties["table-default.s3.region"] = s3_region + sts_unavailable_bool = False + if sts_unavailable is not None: + sts_unavailable_bool = parse_bool(sts_unavailable) + storage_config_info["stsUnavailable"] = sts_unavailable_bool + if role_arn: + storage_config_info["roleArn"] = role_arn + + ensure_catalog( + realm, + token, + catalog_name, + catalog_properties, + storage_config_info, + ) + + admin_cfg = realm_cfg.get("admin") + if admin_cfg: + create_role_bindings( + realm, + token, + catalog_name, + [ + { + "principal_role": admin_cfg["principal_role"], + "catalog_role": admin_cfg.get("catalog_role", "catalog_admin_role"), + "grants": [ + { + "type": "catalog", + "privileges": admin_cfg.get( + "privileges", + ["CATALOG_MANAGE_CONTENT", "CATALOG_MANAGE_METADATA"], + ), + } + ], + } + ], + ) + + namespace_set = set() + for ns_policy in realm_cfg.get("namespace_policies", []): + namespace_set.add(ns_policy["namespace"]) + for table_policy in realm_cfg.get("table_policies", []): + namespace_set.add(table_policy["namespace"]) + + for namespace in sorted(namespace_set): + ensure_namespace(realm, token, catalog_name, namespace) + + for ns_policy in realm_cfg.get("namespace_policies", []): + create_role_bindings( + realm, + token, + catalog_name, + ns_policy.get("roles", []), + namespace=ns_policy["namespace"], + ) + + for table_policy in realm_cfg.get("table_policies", []): + table_namespace = table_policy["namespace"] + table_name = table_policy["table"] + normalized_roles = [] + + for role_cfg in table_policy.get("roles", []): + normalized_roles.append( + { + "principal_role": role_cfg["principal_role"], + "catalog_role": role_cfg["catalog_role"], + "grants": [ + { + "type": "table", + "namespace": table_namespace, + "table": table_name, + "privileges": grant_cfg.get("privileges", []), + } + for grant_cfg in role_cfg.get("grants", []) + ], + } + ) + + create_role_bindings( + realm, + token, + catalog_name, + normalized_roles, + namespace=table_namespace, + ) + + # Sync Keycloak users to Polaris principals via shared script. + # Role assignment is omitted — the OIDC role mapper handles it at auth time. + subprocess.run( + ["python", "/scripts/sync_principals.py"], + check=True, + env={**os.environ, "POLARIS_REALM": realm}, + ) + + # Service account principals (static list from acl-config). + # These authenticate via client credentials (not OIDC user tokens), + # so their principal role must be assigned explicitly here. + for principal_cfg in realm_cfg.get("principals", []): + principal_name = principal_cfg["name"] + principal_role = principal_cfg["principal_role"] + print(f"[polaris-bootstrap] Ensuring service principal: {principal_name}") + mgmt("post", realm, token, "/principals", + json={"name": principal_name, "type": "USER", "properties": {}}) + mgmt("put", realm, token, + f"/principals/{principal_name}/principal-roles", + json={"name": principal_role}) + + print("[polaris-bootstrap] Bootstrap complete") + volumeMounts: + - name: acl-config + mountPath: /config/acl-config.json + subPath: acl-config.json + readOnly: true + - name: sync-script + mountPath: /scripts + readOnly: true + volumes: + - name: acl-config + configMap: + name: polaris-acl-config + items: + - key: acl-config.json + path: acl-config.json + - name: sync-script + configMap: + name: polaris-sync-principals-script diff --git a/polaris-bootstrap/manifests/polaris-principal-sync-cronjob.yaml.tpl b/polaris-bootstrap/manifests/polaris-principal-sync-cronjob.yaml.tpl new file mode 100644 index 0000000..32451b9 --- /dev/null +++ b/polaris-bootstrap/manifests/polaris-principal-sync-cronjob.yaml.tpl @@ -0,0 +1,50 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: polaris-principal-sync +spec: + schedule: "*/5 * * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + template: + spec: + restartPolicy: Never + containers: + - name: polaris-principal-sync + image: prefecthq/prefect:3.2.0-python3.10 + env: + - name: POLARIS_MANAGEMENT_URL + value: "http://polaris:8181" + - name: POLARIS_ROOT_CREDENTIALS + valueFrom: + secretKeyRef: + name: polaris-secrets + key: root-credentials + - name: POLARIS_REALM + value: "${var.polaris.defaultRealm}" + - name: KEYCLOAK_URL + value: "http://keycloak-service.teehr-hub.svc.cluster.local:8080" + - name: KEYCLOAK_ADMIN_USER + valueFrom: + secretKeyRef: + name: keycloak-admin-secrets + key: username + - name: KEYCLOAK_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: keycloak-admin-secrets + key: password + command: + - python + - /scripts/sync_principals.py + volumeMounts: + - name: sync-script + mountPath: /scripts + readOnly: true + volumes: + - name: sync-script + configMap: + name: polaris-sync-principals-script diff --git a/polaris-bootstrap/manifests/polaris-sync-principals-script.yaml b/polaris-bootstrap/manifests/polaris-sync-principals-script.yaml new file mode 100644 index 0000000..d4a628d --- /dev/null +++ b/polaris-bootstrap/manifests/polaris-sync-principals-script.yaml @@ -0,0 +1,193 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: polaris-sync-principals-script +data: + sync_principals.py: | + """ + Syncs Keycloak users into Polaris as named principal entities and grants + principal role bindings matching their effective (composite) realm roles -- + the same realm_access.roles claim Polaris's own JWT-based principal-roles-mapper + reads (polaris/manifests/polaris-config.yaml.tpl), so named-principal bindings + can't silently diverge from what group-based JWT mapping would have granted. + + NOTE ON DESIGN: When a named Polaris principal exists for a user, Polaris uses + that principal's explicit role bindings rather than the JWT group claim mapper. + Therefore this script must sync ALL intended PrincipalRoles — teehr-*, and + iceberg-catalog-admin — not just teehr-* roles. + + This script is required for group-level access to work correctly once named + principals exist. It is also used for table-level grants and individual + permission overrides beyond group-level defaults. + + Required environment variables: + POLARIS_MANAGEMENT_URL e.g. http://polaris:8181 + POLARIS_ROOT_CREDENTIALS e.g. root:secret (used only to obtain a management token) + POLARIS_REALM e.g. teehr + KEYCLOAK_URL e.g. http://keycloak-service:8080 + KEYCLOAK_ADMIN_USER Keycloak admin username + KEYCLOAK_ADMIN_PASSWORD Keycloak admin password + """ + import os + import sys + import requests + + MGMT_URL = os.environ["POLARIS_MANAGEMENT_URL"] + REALM = os.environ["POLARIS_REALM"] + ROOT_CREDS = os.environ["POLARIS_ROOT_CREDENTIALS"] + KC_URL = os.environ["KEYCLOAK_URL"] + KC_ADMIN_USER = os.environ["KEYCLOAK_ADMIN_USER"] + KC_ADMIN_PASSWORD = os.environ["KEYCLOAK_ADMIN_PASSWORD"] + _root_user, _root_password = ROOT_CREDS.split(":", 1) + + + def get_polaris_token(): + resp = requests.post( + f"{MGMT_URL}/api/catalog/v1/oauth/tokens", + headers={"X-Polaris-Realm": REALM, "Content-Type": "application/x-www-form-urlencoded"}, + data={ + "grant_type": "client_credentials", + "client_id": _root_user, + "client_secret": _root_password, + "scope": "PRINCIPAL_ROLE:ALL", + }, + ) + resp.raise_for_status() + return resp.json()["access_token"] + + + def get_keycloak_token(): + resp = requests.post( + f"{KC_URL}/realms/master/protocol/openid-connect/token", + data={ + "grant_type": "password", + "client_id": "admin-cli", + "username": KC_ADMIN_USER, + "password": KC_ADMIN_PASSWORD, + }, + ) + resp.raise_for_status() + return resp.json()["access_token"] + + + def iter_keycloak_users(kc_token): + first = 0 + max_per_page = 100 + while True: + resp = requests.get( + f"{KC_URL}/admin/realms/{REALM}/users", + headers={"Authorization": f"Bearer {kc_token}"}, + params={"first": first, "max": max_per_page, "enabled": "true"}, + ) + resp.raise_for_status() + page = resp.json() + if not page: + break + yield from page + if len(page) < max_per_page: + break + first += max_per_page + + + def get_keycloak_realm_roles(kc_token, user_id): + resp = requests.get( + f"{KC_URL}/admin/realms/{REALM}/users/{user_id}/role-mappings/realm/composite", + headers={"Authorization": f"Bearer {kc_token}"}, + ) + resp.raise_for_status() + return [role.get("name") for role in resp.json() if role.get("name")] + + + def ensure_principal(polaris_token, principal_name): + resp = requests.post( + f"{MGMT_URL}/api/management/v1/principals", + headers={ + "X-Polaris-Realm": REALM, + "Authorization": f"Bearer {polaris_token}", + "Content-Type": "application/json", + }, + json={"name": principal_name, "type": "USER", "properties": {}}, + ) + if resp.status_code in (200, 201, 204, 409): + return + print(f"ERROR creating principal '{principal_name}': {resp.status_code} {resp.text}", file=sys.stderr) + resp.raise_for_status() + + + def ensure_principal_role(polaris_token, principal_role): + resp = requests.post( + f"{MGMT_URL}/api/management/v1/principal-roles", + headers={ + "X-Polaris-Realm": REALM, + "Authorization": f"Bearer {polaris_token}", + "Content-Type": "application/json", + }, + json={"name": principal_role}, + ) + if resp.status_code in (200, 201, 204, 409): + return + print(f"ERROR creating principal role '{principal_role}': {resp.status_code} {resp.text}", file=sys.stderr) + resp.raise_for_status() + + + def _is_duplicate_grant_error(resp): + if resp.status_code != 500: + return False + body = (resp.text or "").lower() + return ( + "duplicate key value violates unique constraint" in body + or "grant_records_pkey" in body + or "sql-state '23505'" in body + or "already exists" in body + ) + + + def ensure_principal_role_binding(polaris_token, principal_name, principal_role): + ensure_principal_role(polaris_token, principal_role) + resp = requests.put( + f"{MGMT_URL}/api/management/v1/principals/{principal_name}/principal-roles", + headers={ + "X-Polaris-Realm": REALM, + "Authorization": f"Bearer {polaris_token}", + "Content-Type": "application/json", + }, + json={"name": principal_role}, + ) + if resp.status_code in (200, 201, 204, 409) or _is_duplicate_grant_error(resp): + return + print( + f"ERROR binding principal role '{principal_role}' to principal '{principal_name}': " + f"{resp.status_code} {resp.text}", + file=sys.stderr, + ) + resp.raise_for_status() + + + polaris_token = get_polaris_token() + kc_token = get_keycloak_token() + + synced = 0 + granted = 0 + for user in iter_keycloak_users(kc_token): + name = user.get("username") + user_id = user.get("id") + if name: + ensure_principal(polaris_token, name) + if user_id: + # Grant principal roles from the user's effective (composite) realm + # roles -- this is exactly what ends up in the JWT's realm_access.roles + # claim, i.e. the same input Polaris's own principal-roles-mapper regex + # (polaris/manifests/polaris-config.yaml.tpl) uses for JWT-based mapping. + # Matching that regex here (rather than deriving from Keycloak group + # names/paths separately) keeps named-principal bindings from silently + # diverging from what group-based JWT mapping would have granted. + for role_name in get_keycloak_realm_roles(kc_token, user_id): + if role_name.startswith("teehr-") or role_name == "iceberg-catalog-admin": + ensure_principal_role_binding(polaris_token, name, role_name) + granted += 1 + synced += 1 + + print( + f"[polaris-principal-sync] Synced {synced} principals and ensured {granted} " + f"principal-role grants from Keycloak realm '{REALM}'" + ) diff --git a/polaris-pg/garden.yaml b/polaris-pg/garden.yaml new file mode 100644 index 0000000..b1d7783 --- /dev/null +++ b/polaris-pg/garden.yaml @@ -0,0 +1,12 @@ +kind: Deploy +type: kubernetes +name: polaris-database +description: K8s Deploy Polaris Catalog Database +dependencies: + - deploy.secrets +environments: + - local + - remote +spec: + manifestFiles: + - ./manifests/polaris-pg.yaml diff --git a/polaris-pg/manifests/polaris-pg.yaml b/polaris-pg/manifests/polaris-pg.yaml new file mode 100644 index 0000000..09d1f4d --- /dev/null +++ b/polaris-pg/manifests/polaris-pg.yaml @@ -0,0 +1,102 @@ +apiVersion: v1 +kind: Service +metadata: + name: polaris-pg +spec: + type: ClusterIP + ports: + - port: 5432 + targetPort: 5432 + selector: + app: polaris-pg + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: polaris-pg +spec: + replicas: 1 + selector: + matchLabels: + app: polaris-pg + template: + metadata: + labels: + app: polaris-pg + spec: + # nodeSelector: + # teehr-hub/nodegroup-name: core-a + containers: + - name: postgres + image: postgres:15 + env: + - name: POSTGRES_DB + valueFrom: + secretKeyRef: + name: polaris-db-secrets + key: database + - name: POSTGRES_USER + valueFrom: + secretKeyRef: + name: polaris-db-secrets + key: username + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: polaris-db-secrets + key: password + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + ports: + - containerPort: 5432 + readinessProbe: + exec: + command: + - pg_isready + - -U + - polaris + - -d + - polaris + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + livenessProbe: + exec: + command: + - pg_isready + - -U + - polaris + - -d + - polaris + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + volumeMounts: + - name: pgdata + mountPath: /var/lib/postgresql/data + volumes: + - name: pgdata + persistentVolumeClaim: + claimName: polaris-pg-data + +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: polaris-pg-data +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi diff --git a/polaris/garden.yaml b/polaris/garden.yaml new file mode 100644 index 0000000..3159fe0 --- /dev/null +++ b/polaris/garden.yaml @@ -0,0 +1,15 @@ +kind: Deploy +type: kubernetes +name: polaris +description: K8s Deploy Apache Polaris Iceberg Catalog +dependencies: + - deploy.secrets + - deploy.polaris-database + - deploy.keycloak +environments: + - local + - remote +spec: + manifestTemplates: + - ./manifests/polaris-config.yaml.tpl + - ./manifests/polaris.yaml.tpl diff --git a/polaris/manifests/polaris-config.yaml.tpl b/polaris/manifests/polaris-config.yaml.tpl new file mode 100644 index 0000000..66b970f --- /dev/null +++ b/polaris/manifests/polaris-config.yaml.tpl @@ -0,0 +1,65 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: polaris-config +data: + application.properties: | + # Persistence: PostgreSQL Configuration + polaris.persistence.type=relational-jdbc + polaris.persistence.relational.jdbc.database-type=postgresql + + quarkus.datasource.db-kind=postgresql + quarkus.datasource.jdbc.url=jdbc:postgresql://polaris-pg:5432/polaris + quarkus.datasource.username=polaris + + polaris.persistence.relational.jdbc.max-retries=5 + polaris.persistence.relational.jdbc.initial-delay-in-ms=100 + polaris.persistence.relational.jdbc.max-duration-in-ms=5000 + + # Authentication Context Configuration + # This realm is configured to use both the internal and external authentication. + # It accepts tokens issued by both Polaris and Keycloak. + polaris.authentication.type=mixed + # These are global. You can also set per realm like: + # polaris.authentication.realm1.type=external + polaris.oidc.principal-mapper.name-claim-path=preferred_username + + # Quarkus OIDC — tenant-enabled=true is required; without it Quarkus disables the + # Default tenant and rejects all Bearer tokens with 401 regardless of other config. + quarkus.oidc.tenant-enabled=true + quarkus.oidc.application-type=service + quarkus.oidc.client-id=jupyterhub + # auth-server-url + relative jwks-path enables local JWT validation without discovery. + # Without auth-server-url, Quarkus falls back to userinfo introspection which fails for service accounts. + quarkus.oidc.auth-server-url=${var.polaris.oidcIssuerUri} + quarkus.oidc.discovery-enabled=false + quarkus.oidc.jwks-path=/protocol/openid-connect/certs + quarkus.tls.trust-all=true + quarkus.oidc.connection-delay=PT10S + quarkus.oidc.connection-retry-count=5 + quarkus.oidc.token.audience=account + quarkus.oidc.token.issuer=any + + # Access control: map Keycloak realm roles to Polaris PrincipalRoles. + # Uses realm_access/roles (not groups) to cover both: + # - human users: realm roles propagated from Keycloak group membership + # - service accounts: realm roles assigned directly (trino-polaris, prefect-polaris) + # + # Patterns: + # iceberg-catalog-admin → PRINCIPAL_ROLE:iceberg-catalog-admin + # teehr- → PRINCIPAL_ROLE:teehr- + # + # Individual/table-level grants: use the polaris-sync-principals script. + # Named principal role bindings from the sync take precedence over JWT mapping. + quarkus.oidc.roles.role-claim-path=realm_access/roles + polaris.oidc.principal-roles-mapper.type=default + polaris.oidc.principal-roles-mapper.mappings[0].regex=^iceberg-catalog-admin$ + polaris.oidc.principal-roles-mapper.mappings[0].replacement=PRINCIPAL_ROLE:iceberg-catalog-admin + polaris.oidc.principal-roles-mapper.mappings[1].regex=^teehr-(.+)$ + polaris.oidc.principal-roles-mapper.mappings[1].replacement=PRINCIPAL_ROLE:teehr-$1 + + # Storage Properties Integration + polaris.features."SUPPORTED_CATALOG_STORAGE_TYPES"=["S3","GCS","AZURE","FILE"] + polaris.features."ALLOW_INSECURE_STORAGE_TYPES"=true + polaris.features."SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION"=${var.polaris.skipCredentialSubscopingIndirection} + polaris.readiness.ignore-severe-issues=true diff --git a/polaris/manifests/polaris.yaml.tpl b/polaris/manifests/polaris.yaml.tpl new file mode 100644 index 0000000..bddb240 --- /dev/null +++ b/polaris/manifests/polaris.yaml.tpl @@ -0,0 +1,198 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: polaris + namespace: ${environment.namespace} + ${if environment.name == "remote"} + annotations: + eks.amazonaws.com/role-arn: ${var.irsa.polarisRoleArn} + ${endif} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: polaris + name: polaris +spec: + replicas: 1 + selector: + matchLabels: + app: polaris + template: + metadata: + labels: + app: polaris + spec: + serviceAccountName: polaris + initContainers: + - name: schema-bootstrap + image: apache/polaris-admin-tool:1.5.0 + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + - | + JAR=/deployments/polaris-admin-tool.jar + echo "Found jar: $JAR" + OUTPUT=$(java $JAVA_OPTS -jar "$JAR" bootstrap \ + --realm="$POLARIS_BOOTSTRAP_REALM" \ + -c "$POLARIS_BOOTSTRAP_REALM,$ROOT_USERNAME,$ROOT_PASSWORD" \ + -p 2>&1) + EXIT_CODE=$? + echo "$OUTPUT" + if [ $EXIT_CODE -eq 0 ]; then + echo "Bootstrap succeeded." + exit 0 + fi + if echo "$OUTPUT" | grep -q "already been bootstrapped"; then + echo "Metastore already bootstrapped — skipping." + exit 0 + fi + echo "Bootstrap failed with unexpected error (exit code $EXIT_CODE)." + exit $EXIT_CODE + env: + # Core persistence assignment + - name: POLARIS_PERSISTENCE_TYPE + value: relational-jdbc + - name: POLARIS_PERSISTENCE_AUTO_BOOTSTRAP_TYPES + value: relational-jdbc + - name: POLARIS_REALM_CONTEXT_REALMS + value: ${var.polaris.realmsCsv} + - name: POLARIS_BOOTSTRAP_REALM + value: ${var.polaris.defaultRealm} + - name: ROOT_USERNAME + valueFrom: + secretKeyRef: + name: polaris-secrets + key: root-username + - name: ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: polaris-secrets + key: root-password + + # Explicit lowercase system translations to force Agroal activation + - name: quarkus_datasource_db-kind + value: postgresql + - name: quarkus_datasource_jdbc_url + value: jdbc:postgresql://polaris-pg:5432/polaris + - name: quarkus_datasource_username + value: polaris + + # Fetch secret credentials cleanly + - name: quarkus_datasource_password + valueFrom: + secretKeyRef: + name: polaris-db-secrets + key: password + volumeMounts: + - name: polaris-config + mountPath: /deployments/config/application.properties + subPath: application.properties + readOnly: true + + containers: + - name: polaris + image: apache/polaris:1.5.0 + imagePullPolicy: IfNotPresent + env: + - name: POLARIS_REALM_CONTEXT_REALMS + value: ${var.polaris.realmsCsv} + - name: POLARIS_PERSISTENCE_TYPE + value: relational-jdbc + - name: QUARKUS_DATASOURCE_JDBC_URL + value: jdbc:postgresql://polaris-pg:5432/polaris + - name: QUARKUS_DATASOURCE_USERNAME + valueFrom: + secretKeyRef: + name: polaris-db-secrets + key: username + - name: QUARKUS_DATASOURCE_PASSWORD + valueFrom: + secretKeyRef: + name: polaris-db-secrets + key: password + ${if environment.name == "local"} + - name: QUARKUS_OIDC_AUTH_SERVER_URL + value: ${var.polaris.oidcIssuerUri} + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: minio-secrets + key: accesskey + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: minio-secrets + key: secretkey + - name: AWS_S3_ENDPOINT + value: ${var.polaris.catalogS3Endpoint} + - name: AWS_S3_PATH_STYLE_ACCESS + value: "${var.polaris.catalogS3PathStyleAccess}" + ${endif} + ${if environment.name != "local"} + - name: QUARKUS_OIDC_AUTH_SERVER_URL + value: ${var.polaris.oidcIssuerUri} + ${endif} + - name: AWS_REGION + value: us-east-2 + ports: + - name: api + containerPort: 8181 + protocol: TCP + - name: management + containerPort: 8182 + protocol: TCP + readinessProbe: + httpGet: + path: /q/health/ready + port: 8182 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + livenessProbe: + httpGet: + path: /q/health/live + port: 8182 + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + resources: + requests: + cpu: 200m + memory: 512Mi + limits: + cpu: 1000m + memory: 1Gi + volumeMounts: + - name: polaris-config + mountPath: /deployments/config/application.properties + subPath: application.properties + readOnly: true + volumes: + - name: polaris-config + configMap: + name: polaris-config + +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: polaris + name: polaris +spec: + ports: + - name: api + protocol: TCP + port: 8181 + targetPort: 8181 + - name: management + protocol: TCP + port: 8182 + targetPort: 8182 + selector: + app: polaris diff --git a/example.project.garden.yml b/project.garden.yml.template similarity index 60% rename from example.project.garden.yml rename to project.garden.yml.template index ac7e564..342a9b0 100644 --- a/example.project.garden.yml +++ b/project.garden.yml.template @@ -13,9 +13,9 @@ variables: ssl: "false" aws: region: us-east-2 - # remoteCluster: - # contextArn: "" - # ecrRegistry: "" + remoteCluster: + contextArn: "" + ecrRegistry: "" environments: - name: local @@ -23,15 +23,30 @@ environments: variables: hostname: teehr.local.app.garden certificateIssuerName: letsencrypt-prod - devTeehrVersion: 267f8a75034132aefe84749af10ab1562a6ac169 + devTeehrVersion: 27f0c3dd1c56db78ce4d2284b8f3caa966bebbe1 + previousTeehrVersion: place-holder stableTeehrVersion: place-holder # Needed for sync to work for some reason. - iceberg: + # iceberg: + # inCluster: "true" + # catalogS3PathStyleAccess: "true" + # catalogS3Endpoint: "http://minio:9000" + # catalogType: rest + # catalogUri: http://iceberg-rest:8181 + # catalogWarehouse: s3://warehouse/ + polaris: inCluster: "true" catalogS3PathStyleAccess: "true" catalogS3Endpoint: "http://minio:9000" + catalogS3Region: ${var.aws.region} catalogType: rest - catalogUri: http://iceberg-rest:8181 + catalogUri: http://polaris:8181/api/catalog catalogWarehouse: s3://warehouse/ + skipCredentialSubscopingIndirection: "true" + storageStsUnavailable: "true" + defaultRealm: teehr + realmsCsv: teehr + oauthServerUri: http://keycloak-service:8080/realms/teehr/protocol/openid-connect/token + oidcIssuerUri: http://keycloak-service:8080/realms/teehr trino: host: trino port: "8080" diff --git a/scripts/run_jupyter_polaris_spark_example.sh b/scripts/run_jupyter_polaris_spark_example.sh new file mode 100755 index 0000000..4bf345d --- /dev/null +++ b/scripts/run_jupyter_polaris_spark_example.sh @@ -0,0 +1,134 @@ +#!/usr/bin/env bash +set -euo pipefail + +NAMESPACE="${NAMESPACE:-teehr-hub}" +POD_NAME="${POD_NAME:-}" +CONTAINER_NAME="${CONTAINER_NAME:-notebook}" +SCRIPT_PATH="${SCRIPT_PATH:-examples/developer/polaris_spark_namespace_table_example.py}" +SETUP_UTILS_PATH="${SETUP_UTILS_PATH:-examples/developer/setup_utils.py}" +EXTRA_ARGS="${EXTRA_ARGS:-}" +POLARIS_TEST_USERNAME="${POLARIS_TEST_USERNAME:-admin}" +POLARIS_TEST_PASSWORD="${POLARIS_TEST_PASSWORD:-admin}" +POLARIS_OAUTH_CLIENT_ID="${POLARIS_OAUTH_CLIENT_ID:-jupyterhub}" +DEBUG_POLARIS="${DEBUG_POLARIS:-0}" +POLARIS_DOCTOR="${POLARIS_DOCTOR:-0}" + +if [[ -z "$POD_NAME" ]]; then + POD_NAME="$(kubectl -n "$NAMESPACE" get pods -o name | sed 's#^pod/##' | grep -E '^jupyter-' | head -n1 || true)" +fi + +if [[ -z "$POD_NAME" ]]; then + echo "No Jupyter single-user pod found in namespace '$NAMESPACE'." >&2 + echo "Start a notebook server first, or set POD_NAME explicitly." >&2 + exit 1 +fi + +if [[ -z "${POLARIS_OAUTH_CLIENT_SECRET:-}" ]]; then + POLARIS_OAUTH_CLIENT_SECRET="$(kubectl -n "$NAMESPACE" get secret jupyterhub -o jsonpath='{.data.OAUTH_CLIENT_SECRET}' | base64 --decode)" +fi + +if [[ ! -f "$SCRIPT_PATH" ]]; then + echo "Script not found: $SCRIPT_PATH" >&2 + exit 1 +fi + +if [[ ! -f "$SETUP_UTILS_PATH" ]]; then + echo "Setup utils not found: $SETUP_UTILS_PATH" >&2 + exit 1 +fi + +echo "Running $SCRIPT_PATH in pod $POD_NAME (namespace: $NAMESPACE, container: $CONTAINER_NAME)" + +start_epoch="$(date +%s)" + +diagnostics_dir="" +run_log="" +if [[ "$POLARIS_DOCTOR" == "1" ]]; then + diagnostics_dir="${TMPDIR:-/tmp}/polaris-doctor-$(date +%Y%m%d-%H%M%S)-$$" + mkdir -p "$diagnostics_dir" + run_log="$diagnostics_dir/run.log" + echo "[doctor] diagnostics dir: $diagnostics_dir" +fi + +tmpdir="${TMPDIR:-/tmp}/polaris-spark-example.$$" +mkdir -p "$tmpdir" +cp "$SCRIPT_PATH" "$tmpdir/polaris_spark_namespace_table_example.py" +cp "$SETUP_UTILS_PATH" "$tmpdir/setup_utils.py" + +set +e +if [[ "$POLARIS_DOCTOR" == "1" ]]; then + ( + tar -C "$tmpdir" -cf - polaris_spark_namespace_table_example.py setup_utils.py | \ + kubectl -n "$NAMESPACE" exec -i "$POD_NAME" -c "$CONTAINER_NAME" -- sh -lc \ + "mkdir -p /tmp/polaris-spark-example && cd /tmp/polaris-spark-example && tar -xf - && export PYTHONPATH=/tmp/polaris-spark-example:\$PYTHONPATH POLARIS_TEST_USERNAME='$POLARIS_TEST_USERNAME' POLARIS_TEST_PASSWORD='$POLARIS_TEST_PASSWORD' POLARIS_OAUTH_CLIENT_ID='$POLARIS_OAUTH_CLIENT_ID' POLARIS_OAUTH_CLIENT_SECRET='$POLARIS_OAUTH_CLIENT_SECRET'; python /tmp/polaris-spark-example/polaris_spark_namespace_table_example.py $EXTRA_ARGS" + ) 2>&1 | tee "$run_log" + cmd_exit_code=${PIPESTATUS[0]} +else + tar -C "$tmpdir" -cf - polaris_spark_namespace_table_example.py setup_utils.py | \ + kubectl -n "$NAMESPACE" exec -i "$POD_NAME" -c "$CONTAINER_NAME" -- sh -lc \ + "mkdir -p /tmp/polaris-spark-example && cd /tmp/polaris-spark-example && tar -xf - && export PYTHONPATH=/tmp/polaris-spark-example:\$PYTHONPATH POLARIS_TEST_USERNAME='$POLARIS_TEST_USERNAME' POLARIS_TEST_PASSWORD='$POLARIS_TEST_PASSWORD' POLARIS_OAUTH_CLIENT_ID='$POLARIS_OAUTH_CLIENT_ID' POLARIS_OAUTH_CLIENT_SECRET='$POLARIS_OAUTH_CLIENT_SECRET'; python /tmp/polaris-spark-example/polaris_spark_namespace_table_example.py $EXTRA_ARGS" + cmd_exit_code=$? +fi +set -e + +if [[ "$POLARIS_DOCTOR" == "1" ]]; then + { + echo "exit_code=$cmd_exit_code" + echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + echo "namespace=$NAMESPACE" + echo "pod_name=$POD_NAME" + echo "container_name=$CONTAINER_NAME" + echo "script_path=$SCRIPT_PATH" + } > "$diagnostics_dir/context.txt" +fi + +if [[ "$DEBUG_POLARIS" == "1" ]]; then + now_epoch="$(date +%s)" + since_seconds="$((now_epoch - start_epoch + 15))" + if (( since_seconds < 30 )); then + since_seconds=30 + fi + echo + echo "[debug] Polaris logs for the last ${since_seconds}s" + kubectl -n "$NAMESPACE" logs deploy/polaris --since="${since_seconds}s" | \ + grep -E "(POST /api/catalog/v1/oauth/tokens|GET /api/catalog/v1/config|HTTP/1.1\" 401|HTTP/1.1\" 200|principal=|roles=|Some principal roles were not found)" || true + + if [[ "$POLARIS_DOCTOR" == "1" ]]; then + kubectl -n "$NAMESPACE" logs deploy/polaris --since="${since_seconds}s" > "$diagnostics_dir/polaris.log" || true + grep -E "(POST /api/catalog/v1/oauth/tokens|/api/catalog/v1/config|/api/catalog/v1/.*/namespaces|HTTP/1.1\" 401|HTTP/1.1\" 403|HTTP/1.1\" 500|Some principal roles were not found|UnknownHostException|warehouse.minio)" "$diagnostics_dir/polaris.log" > "$diagnostics_dir/polaris-summary.log" || true + fi +fi + +if [[ "$POLARIS_DOCTOR" == "1" ]]; then + diagnosis="unknown" + if [[ "$cmd_exit_code" -eq 0 ]]; then + diagnosis="success" + elif grep -qi "NotAuthorizedException\|HTTP Error 401\|401 Unauthorized" "$run_log" 2>/dev/null; then + diagnosis="authz_or_realm_mismatch" + elif grep -qi "UnknownHostException\|warehouse\.minio\|NoSuchBucket\|AccessDenied" "$run_log" 2>/dev/null; then + diagnosis="storage_or_warehouse_misconfig" + elif grep -qi "Failed to write to grant records\|grant_records_pkey\|duplicate key value" "$run_log" 2>/dev/null; then + diagnosis="principal_role_grant_idempotency" + fi + + { + echo "diagnosis=$diagnosis" + if [[ "$diagnosis" == "authz_or_realm_mismatch" ]]; then + echo "hint=Verify token issuer/realm and Polaris principal-role grants for this principal" + elif [[ "$diagnosis" == "storage_or_warehouse_misconfig" ]]; then + echo "hint=Verify Polaris catalog warehouse and MinIO endpoint/path-style settings" + elif [[ "$diagnosis" == "principal_role_grant_idempotency" ]]; then + echo "hint=Principal sync is re-granting existing role; ensure duplicate grant is treated as success" + fi + } > "$diagnostics_dir/diagnosis.txt" + + echo + echo "[doctor] diagnosis: $diagnosis" + echo "[doctor] bundle: $diagnostics_dir" + if [[ -f "$diagnostics_dir/polaris-summary.log" ]]; then + echo "[doctor] summary:" + tail -n 60 "$diagnostics_dir/polaris-summary.log" || true + fi +fi + +exit "$cmd_exit_code" diff --git a/scripts/test_jupyter_broker_token.sh b/scripts/test_jupyter_broker_token.sh new file mode 100755 index 0000000..8670b96 --- /dev/null +++ b/scripts/test_jupyter_broker_token.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +set -euo pipefail + +NAMESPACE="${NAMESPACE:-teehr-hub}" +POD_NAME="${POD_NAME:-}" +CONTAINER_NAME="${CONTAINER_NAME:-notebook}" +BROKER_URL="${BROKER_URL:-http://teehr-api:8000/auth/polaris-token}" +REQUESTED_TTL_SECONDS="${REQUESTED_TTL_SECONDS:-600}" +AUDIENCE="${AUDIENCE:-account}" + +if [[ -z "$POD_NAME" ]]; then + POD_NAME="$(kubectl -n "$NAMESPACE" get pods -o name | sed 's#^pod/##' | grep -E '^jupyter-' | head -n1 || true)" +fi + +if [[ -z "$POD_NAME" ]]; then + echo "No Jupyter single-user pod found in namespace '$NAMESPACE'." >&2 + exit 1 +fi + +echo "Testing broker token endpoint from pod $POD_NAME" + +kubectl -n "$NAMESPACE" exec -i "$POD_NAME" -c "$CONTAINER_NAME" -- env \ + BROKER_URL="$BROKER_URL" \ + REQUESTED_TTL_SECONDS="$REQUESTED_TTL_SECONDS" \ + AUDIENCE="$AUDIENCE" \ + python - <<'PY' +import json +import os +import requests + +def get_fresh_subject_token() -> str: + token_endpoint = os.getenv( + "POLARIS_OAUTH2_SERVER_URI", + "http://keycloak-service:8080/realms/teehr/protocol/openid-connect/token", + ) + client_id = os.getenv("POLARIS_CLIENT_ID", "jupyterhub") + client_secret = os.getenv("POLARIS_CLIENT_SECRET", "") + + refresh_token = os.getenv("POLARIS_REFRESH_TOKEN", "") + if refresh_token: + payload = { + "grant_type": "refresh_token", + "client_id": client_id, + "refresh_token": refresh_token, + } + if client_secret: + payload["client_secret"] = client_secret + resp = requests.post(token_endpoint, data=payload, timeout=20) + if resp.status_code < 400: + token = resp.json().get("access_token") + if token: + return token + + token = os.getenv("POLARIS_USER_TOKEN", "") + if token: + return token + + raise RuntimeError("Unable to obtain a usable subject token from pod environment") + + +token = get_fresh_subject_token() + +realm = os.getenv("POLARIS_DEFAULT_REALM", "teehr") +user_id = os.getenv("JUPYTERHUB_USER", "admin") +session_id = (os.getenv("JUPYTERHUB_SERVER_NAME") or "").strip() or user_id + +response = requests.post( + os.environ["BROKER_URL"], + headers={"Authorization": f"Bearer {token}"}, + json={ + "user_id": user_id, + "session_id": session_id, + "realm": realm, + "catalog": "iceberg", + "requested_ttl_seconds": int(os.environ["REQUESTED_TTL_SECONDS"]), + "audience": os.environ["AUDIENCE"], + }, + timeout=20, +) + +print("status:", response.status_code) +try: + payload = response.json() +except ValueError: + print(response.text[:1500]) + raise + +if response.status_code >= 400: + print(json.dumps(payload, indent=2)[:2000]) + raise RuntimeError("broker token call failed") + +print("trace_id:", payload.get("trace_id")) +print("token_type:", payload.get("token_type")) +print("expires_in_seconds:", payload.get("expires_in_seconds")) +print("issued_for:", payload.get("issued_for")) +print("access_token_prefix:", str(payload.get("access_token", ""))[:24]) +PY diff --git a/secrets/secrets.local.yaml b/secrets/secrets.local.yaml index 5e806e8..8994283 100644 --- a/secrets/secrets.local.yaml +++ b/secrets/secrets.local.yaml @@ -3,6 +3,7 @@ secrets: data: OAUTH_CLIENT_ID: jupyterhub OAUTH_CLIENT_SECRET: local-jupyterhub-client-secret + JUPYTERHUB_CRYPT_KEY: 55b6f7f4ead8f8fec6fe024c2ecdaff76cb024916876f50d668437e6f8d0e051 prefect-db-secrets: data: database: prefect @@ -19,6 +20,11 @@ secrets: database: keycloak username: keycloak password: keycloak123 + teehr-api-db-secrets: + data: + database: teehr_api + username: keycloak + password: keycloak123 keycloak-admin-secrets: data: username: admin @@ -37,10 +43,33 @@ secrets: teehr-api-secrets: data: client-secret: local-teehr-api-client-secret + broker-secrets: + data: + session-signing-secret: local-broker-session-signing-secret + refresh-token-encryption-secret: local-broker-refresh-token-encryption-secret minio-secrets: data: accesskey: minioadmin secretkey: minioadmin123 prefect-workflow-secrets: data: - api-usgs-pat: CHANGE_ME_API_USGS_PAT \ No newline at end of file + api-usgs-pat: CHANGE_ME_API_USGS_PAT + polaris-db-secrets: + data: + database: polaris + username: polaris + password: polaris123 + polaris-secrets: + data: + root-credentials: "root:secret123" + bootstrap-credentials: "teehr,root,secret123" + root-username: "root" + root-password: "secret123" + trino-polaris-secrets: + data: + client-secret: local-trino-polaris-client-secret + credential: "trino-polaris:local-trino-polaris-client-secret" + prefect-polaris-secrets: + data: + client-secret: local-prefect-polaris-client-secret + credential: "prefect-polaris:local-prefect-polaris-client-secret" \ No newline at end of file diff --git a/spark/docker/Dockerfile.spark-executor b/spark/docker/Dockerfile.spark-executor index 2eaf5b2..ae30b79 100644 --- a/spark/docker/Dockerfile.spark-executor +++ b/spark/docker/Dockerfile.spark-executor @@ -59,11 +59,11 @@ RUN ARCH=$(uname -m) && \ echo "rasterio==1.3.11" > /tmp/constraints.txt && \ python3.12 -m pip install --no-cache-dir --ignore-installed "setuptools<81" "wheel" "cython<3.1" "numpy<2" && \ python3.12 -m pip install --no-cache-dir --no-build-isolation "rasterio==1.3.11" && \ - python3.12 -m pip install --no-cache-dir "git+https://github.com/RTIInternational/teehr.git@${TEEHR_VERSION}" --constraint /tmp/constraints.txt && \ + GIT_LFS_SKIP_SMUDGE=1 python3.12 -m pip install --no-cache-dir "git+https://github.com/RTIInternational/teehr.git@${TEEHR_VERSION}" --constraint /tmp/constraints.txt && \ rm /tmp/constraints.txt; \ else \ echo "Installing TEEHR normally for x86_64..." && \ - python3.12 -m pip install --no-cache-dir "git+https://github.com/RTIInternational/teehr.git@${TEEHR_VERSION}"; \ + GIT_LFS_SKIP_SMUDGE=1 python3.12 -m pip install --no-cache-dir "git+https://github.com/RTIInternational/teehr.git@${TEEHR_VERSION}"; \ fi # change spark UID and GID to 1000 diff --git a/spark/manifests/spark-roles.yaml.tpl b/spark/manifests/spark-roles.yaml.tpl index 00826b1..6477cb7 100644 --- a/spark/manifests/spark-roles.yaml.tpl +++ b/spark/manifests/spark-roles.yaml.tpl @@ -3,10 +3,6 @@ kind: ServiceAccount metadata: name: spark namespace: ${environment.namespace} - ${if environment.name == "remote"} - annotations: - eks.amazonaws.com/role-arn: ${var.irsa.sparkRoleArn} - ${endif} --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role diff --git a/tests/Dockerfile b/tests/Dockerfile new file mode 100644 index 0000000..c3ddc90 --- /dev/null +++ b/tests/Dockerfile @@ -0,0 +1,18 @@ +ARG BASE_IMAGE=python:3.11-slim +FROM ${BASE_IMAGE} + +WORKDIR /app + +# Create tests directory +RUN mkdir -p /app/tests + +# Copy test scripts +COPY keycloak_users_test.py /app/tests/ +COPY polaris_oidc_test.py /app/tests/ +COPY polaris_namespace_test.py /app/tests/ +COPY polaris_roles_test.py /app/tests/ +COPY polaris_auth_token_test.py /app/tests/ +COPY polaris_permissions_config_test.py /app/tests/ +COPY spark_permission_test.py /app/tests/ +COPY spark_authmanager_test.py /app/tests/ +COPY spark_authmanager_executor_test.py /app/tests/ diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..b7526dc --- /dev/null +++ b/tests/README.md @@ -0,0 +1,130 @@ +# Polaris Integration Tests + +This directory contains **Python-based integration tests** for the Polaris + Keycloak + Spark ecosystem running in KinD. + +## Test Files + +- `keycloak_users_test.py` — Validates user provisioning in Keycloak +- `polaris_oidc_test.py` — Validates Polaris OIDC token acceptance +- `polaris_namespace_test.py` — Validates namespace provisioning +- `polaris_roles_test.py` — Validates role-based access control setup +- `spark_session_auth_test.py` — Validates Spark session creation and write permissions + +## What These Tests Do + +These are **end-to-end integration tests** that validate: + +1. **Keycloak User Provisioning** (`polaris-keycloak-users`) + - Admin, user, and poweruser accounts exist + - Users have correct group membership + - Users can authenticate to Keycloak + +2. **Polaris OIDC Integration** (`polaris-oidc-token-validation`) + - Polaris accepts Keycloak-issued tokens + - Token validation succeeds with correct realm header + - JWT claims are properly mapped + +3. **Namespace Provisioning** (`polaris-namespace-list`) + - The `iceberg` catalog exists + - The `teehr` namespace exists in the catalog + - Root credentials can list namespaces + +4. **Role-Based Access Control** (`polaris-role-acl-validation`) + - Principal roles (`teehr-read-only`, `teehr-read-write`, `iceberg-catalog-admin`) exist + - Catalog roles (`teehr_read_only_role`, `teehr_read_write_role`, `catalog_admin_role`) exist + - Roles are properly bound to namespaces and privileges + +5. **Spark Session Authentication** (`spark-session-auth`) + - Poweruser can create Spark sessions with Keycloak credentials + - Read-only user can create Spark sessions with Keycloak credentials + - Poweruser can list tables in `iceberg.teehr` namespace + - Read-only user can list tables in `iceberg.teehr` namespace + - Poweruser **can create tables** in `iceberg.teehr` namespace + - Read-only user **cannot create tables** (write denied) + +## Running the Tests + +### Run all tests through Garden: +```bash +garden test +``` + +### Run a specific test through Garden: +```bash +garden test polaris-keycloak-users +garden test polaris-oidc-token-validation +garden test spark-session-auth +``` + +### Run tests locally (for development): +```bash +# Each test can be run directly from the command line +python3 tests/keycloak_users_test.py +python3 tests/polaris_oidc_test.py +python3 tests/polaris_namespace_test.py +python3 tests/polaris_roles_test.py +python3 tests/spark_session_auth_test.py +``` + +### Run tests with verbose output: +```bash +garden test --verbose +``` + +## Test Execution Order + +Garden automatically resolves dependencies. The test order is: + +1. `polaris-keycloak-users` (runs after `deploy.keycloak-bootstrap`) +2. `polaris-oidc-token-validation` (runs after `deploy.polaris-bootstrap`) +3. `polaris-namespace-list` (runs after `deploy.polaris-bootstrap`) +4. `polaris-role-acl-validation` (runs after `deploy.polaris-bootstrap`) +5. `spark-session-auth` (runs after `deploy.polaris-bootstrap` and `deploy.spark`) + +## Future Tests + +### Spark Integration Tests (to implement) +- Spark session creation with Keycloak credentials +- Table creation in iceberg.teehr namespace as read-write user +- Read-only access validation (write attempt should fail) +- Namespace isolation across roles + +### Trino Integration Tests (to implement) +- Trino catalog configuration validation +- Query access with different user roles +- Namespace-level permission enforcement + +### End-to-End Flow (to implement) +- Full user journey: Keycloak login → JupyterHub → Spark session → Polaris access +- Data pipeline execution with role-based filtering + +## Extending These Tests + +To add a new test, add a new `kind: Test` block to `garden.yaml`: + +```yaml +--- +kind: Test +name: my-new-test +dependencies: + - deploy.some-service +timeout: 60 +spec: + image: some-container-image + command: + - /bin/sh + - -c + - | + # your test logic here + exit 0 # success + exit 1 # failure +``` + +### Tips for Writing Tests + +- Use `set -e` to fail fast on errors +- Print progress with `echo "[test] ..."` for clarity +- Use `grep` and pipe to `/dev/null` for silent checks +- Use `curl -w "\n%{http_code}"` to capture HTTP status separately +- Resolve service names via internal DNS (e.g., `keycloak-service:8080`) +- Use environment secrets from Garden where available diff --git a/tests/garden.yaml b/tests/garden.yaml new file mode 100644 index 0000000..4b89a70 --- /dev/null +++ b/tests/garden.yaml @@ -0,0 +1,178 @@ +kind: Build +type: container +name: test-scripts +dependencies: + - build.teehr-jupyter-driver-image-edge +environments: + - local +spec: + dockerfile: Dockerfile + localId: test-scripts + buildArgs: + BASE_IMAGE: ${actions.build.teehr-jupyter-driver-image-edge.outputs.deploymentImageName}:${actions.build.teehr-jupyter-driver-image-edge.version} + +--- +kind: Test +name: polaris-keycloak-users +dependencies: + - deploy.keycloak-local-users-bootstrap + - build.test-scripts +timeout: 180 +type: container +spec: + image: ${actions.build.test-scripts.outputs.deploymentImageName}:${actions.build.test-scripts.version} + command: + - python3 + - /app/tests/keycloak_users_test.py + +--- +kind: Test +name: polaris-oidc-token-validation +dependencies: + - deploy.polaris-bootstrap + - build.test-scripts +timeout: 180 +type: container +spec: + image: ${actions.build.test-scripts.outputs.deploymentImageName}:${actions.build.test-scripts.version} + command: + - python3 + - /app/tests/polaris_oidc_test.py + +--- +kind: Test +name: polaris-namespace-list +dependencies: + - deploy.polaris-bootstrap + - build.test-scripts +timeout: 180 +type: container +spec: + image: ${actions.build.test-scripts.outputs.deploymentImageName}:${actions.build.test-scripts.version} + command: + - python3 + - /app/tests/polaris_namespace_test.py + +--- +kind: Test +name: polaris-role-acl-validation +dependencies: + - deploy.polaris-bootstrap + - build.test-scripts +timeout: 180 +type: container +spec: + image: ${actions.build.test-scripts.outputs.deploymentImageName}:${actions.build.test-scripts.version} + command: + - python3 + - /app/tests/polaris_roles_test.py + +--- +kind: Test +name: polaris-auth-token +dependencies: + - deploy.polaris-bootstrap + - deploy.keycloak-local-users-bootstrap + - build.test-scripts +timeout: 240 +type: container +spec: + image: ${actions.build.test-scripts.outputs.deploymentImageName}:${actions.build.test-scripts.version} + command: + - python3 + - /app/tests/polaris_auth_token_test.py + +--- +kind: Test +name: polaris-permissions-config +dependencies: + - deploy.polaris-bootstrap + - deploy.keycloak-local-users-bootstrap + - build.test-scripts +timeout: 180 +type: container +spec: + image: ${actions.build.test-scripts.outputs.deploymentImageName}:${actions.build.test-scripts.version} + command: + - python3 + - /app/tests/polaris_permissions_config_test.py + +--- +kind: Test +name: spark-permission-enforcement +dependencies: + - deploy.polaris-bootstrap + - deploy.keycloak-local-users-bootstrap + - build.test-scripts +timeout: 300 +type: container +spec: + image: ${actions.build.test-scripts.outputs.deploymentImageName}:${actions.build.test-scripts.version} + command: + - python3 + - /app/tests/spark_permission_test.py + memory: + min: 2048 + max: 3072 + cpu: + min: 1000 + max: 2000 + +--- +kind: Test +name: spark-authmanager-enforcement +dependencies: + - deploy.polaris-bootstrap + - deploy.keycloak-local-users-bootstrap + - deploy.teehr-api + - build.test-scripts +timeout: 600 +type: container +spec: + image: ${actions.build.test-scripts.outputs.deploymentImageName}:${actions.build.test-scripts.version} + command: + - python3 + - /app/tests/spark_authmanager_test.py + memory: + min: 2048 + max: 3072 + cpu: + min: 1000 + max: 2000 + +--- +kind: Test +name: spark-authmanager-executor-enforcement +description: >- + Cluster-mode AuthManager test -- confirms Spark executors (not just the + driver) get the Polaris auth env vars they need and can participate in a + real distributed Iceberg write/read, unlike spark-authmanager-enforcement + above which only exercises local (driver-only) mode. +dependencies: + - deploy.polaris-bootstrap + - deploy.keycloak-local-users-bootstrap + - deploy.teehr-api + - build.test-scripts + - build.teehr-spark-executor-image +timeout: 600 +type: kubernetes-pod +spec: + podSpec: + serviceAccountName: jupyter + restartPolicy: Never + containers: + - name: spark-authmanager-executor-test + image: ${actions.build.test-scripts.outputs.deploymentImageName}:${actions.build.test-scripts.version} + command: + - python3 + - /app/tests/spark_authmanager_executor_test.py + env: + - name: TEEHR_SPARK_IMAGE + value: ${actions.build.teehr-spark-executor-image.outputs.deploymentImageId} + resources: + requests: + memory: 2Gi + cpu: "1" + limits: + memory: 3Gi + cpu: "2" diff --git a/tests/keycloak_users_test.py b/tests/keycloak_users_test.py new file mode 100644 index 0000000..2aa5e32 --- /dev/null +++ b/tests/keycloak_users_test.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +""" +Integration test: Keycloak user provisioning + +Validates: +- Admin, user, and poweruser accounts exist in Keycloak +- Users have correct group membership +""" + +import sys +import requests +import time + +KEYCLOAK_URL = "http://keycloak-service:8080" +REALM = "teehr" +ADMIN_USERNAME = "admin" +ADMIN_PASSWORD = "admin123" + +EXPECTED_USERS = ["admin", "user", "poweruser"] + +# Retry configuration +MAX_RETRIES = 10 +RETRY_DELAY = 2 # seconds + + +def get_admin_token(): + """Get admin token to query Keycloak""" + for attempt in range(MAX_RETRIES): + try: + token_url = f"{KEYCLOAK_URL}/realms/master/protocol/openid-connect/token" + payload = { + "grant_type": "password", + "client_id": "admin-cli", + "username": ADMIN_USERNAME, + "password": ADMIN_PASSWORD + } + + response = requests.post(token_url, data=payload, timeout=5) + if response.status_code == 200: + return response.json()["access_token"] + elif attempt < MAX_RETRIES - 1: + print(f" Attempt {attempt + 1}/{MAX_RETRIES}: Retrying Keycloak connection...") + time.sleep(RETRY_DELAY) + else: + raise Exception(f"Failed to get admin token: {response.text}") + except requests.exceptions.RequestException as e: + if attempt < MAX_RETRIES - 1: + print(f" Attempt {attempt + 1}/{MAX_RETRIES}: Keycloak not ready, retrying...") + time.sleep(RETRY_DELAY) + else: + raise Exception(f"Failed to connect to Keycloak: {e}") + + +def check_user_exists(admin_token, username): + """Check if a user exists in Keycloak""" + url = f"{KEYCLOAK_URL}/admin/realms/{REALM}/users?username={username}&exact=true" + headers = {"Authorization": f"Bearer {admin_token}"} + + response = requests.get(url, headers=headers) + if response.status_code != 200: + raise Exception(f"Failed to query users: {response.text}") + + users = response.json() + if len(users) == 0: + print(f" DEBUG: No users found for '{username}'. Response: {users}") + return False + + if users[0]["username"] != username: + print(f" DEBUG: Expected '{username}' but got '{users[0]['username']}'") + return False + + return True + + +def main(): + """Run user provisioning validation""" + print("[test] Validating Keycloak users...") + + try: + admin_token = get_admin_token() + print(" ✓ Got admin token") + + for username in EXPECTED_USERS: + print(f" Checking user: {username}") + + if check_user_exists(admin_token, username): + print(f" ✓ User {username} exists") + else: + print(f" ✗ ERROR: User {username} not found") + return 1 + + print("[test] All Keycloak users validated successfully") + return 0 + + except Exception as e: + print(f"[test] ERROR: {e}") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/polaris_auth_token_test.py b/tests/polaris_auth_token_test.py new file mode 100644 index 0000000..4e13c91 --- /dev/null +++ b/tests/polaris_auth_token_test.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 +""" +Integration test: Spark session authentication with Polaris + Keycloak + +Validates: +- Keycloak token acquisition for test users (poweruser, user) +- Polaris accepts Keycloak JWT tokens with correct realm and warehouse +- Different users can obtain tokens and authenticate with Polaris +""" + +import sys +import json +import requests +import time +import base64 + +# Configuration +KEYCLOAK_URL = "http://keycloak-service:8080" +POLARIS_URL = "http://polaris:8181" +REALM = "teehr" +CATALOG = "teehr" +NAMESPACE = "teehr" + +TEST_USERS = { + "poweruser": {"password": "poweruser", "should_write": True}, + "user": {"password": "user", "should_write": False} +} + +# Retry configuration +MAX_RETRIES = 10 +RETRY_DELAY = 2 # seconds + + +def get_keycloak_token(username, password): + """Get a JWT token from Keycloak for a user""" + for attempt in range(MAX_RETRIES): + try: + token_url = f"{KEYCLOAK_URL}/realms/{REALM}/protocol/openid-connect/token" + payload = { + "grant_type": "password", + "client_id": "jupyterhub", + "client_secret": "local-jupyterhub-client-secret", + "username": username, + "password": password, + "scope": "openid" + } + + response = requests.post(token_url, data=payload, timeout=5) + if response.status_code == 200: + return response.json()["access_token"] + elif response.status_code == 401 and attempt < MAX_RETRIES - 1: + print(f" Attempt {attempt + 1}/{MAX_RETRIES}: Keycloak not ready, retrying...") + time.sleep(RETRY_DELAY) + else: + raise Exception(f"Failed to get token for {username}: {response.text}") + except requests.exceptions.RequestException as e: + if attempt < MAX_RETRIES - 1: + print(f" Attempt {attempt + 1}/{MAX_RETRIES}: Connection failed, retrying...") + time.sleep(RETRY_DELAY) + else: + raise Exception(f"Failed to get token for {username}: {e}") + + +def validate_token_with_polaris(token): + """Validate that Polaris accepts the token""" + for attempt in range(MAX_RETRIES): + try: + headers = { + "Authorization": f"Bearer {token}", + "X-Polaris-Realm": REALM + } + response = requests.get( + f"{POLARIS_URL}/api/catalog/v1/config", + headers=headers, + params={"warehouse": CATALOG}, + timeout=5 + ) + if response.status_code == 200: + return True + elif response.status_code >= 500 and attempt < MAX_RETRIES - 1: + print(f" Attempt {attempt + 1}/{MAX_RETRIES}: Polaris error, retrying...") + time.sleep(RETRY_DELAY) + else: + print(f" DEBUG: HTTP {response.status_code}: {response.text[:300]}") + return False + except requests.exceptions.RequestException as e: + if attempt < MAX_RETRIES - 1: + print(f" Attempt {attempt + 1}/{MAX_RETRIES}: Connection failed, retrying...") + time.sleep(RETRY_DELAY) + else: + return False + return False + + +def check_token_roles(token, username): + """Verify expected roles in JWT token""" + try: + parts = token.split(".") + # Add padding if needed + payload_part = parts[1] + padding = 4 - len(payload_part) % 4 + if padding != 4: + payload_part += "=" * padding + + payload = json.loads(base64.urlsafe_b64decode(payload_part)) + roles = payload.get("realm_access", {}).get("roles", []) + + # Check if user has expected roles + if username == "poweruser": + if "teehr-read-write" in roles: + print(f" ✓ User has teehr-read-write role") + return True + else: + print(f" WARNING: User missing teehr-read-write role (has: {roles})") + return False + elif username == "user": + if "teehr-read-only" in roles: + print(f" ✓ User has teehr-read-only role") + return True + else: + print(f" WARNING: User missing teehr-read-only role (has: {roles})") + return False + except Exception as e: + print(f" WARNING: Could not decode token roles: {e}") + return False + + +def main(): + """Run Spark session authentication tests""" + print("[test] Starting Spark session authentication tests...") + print() + + all_passed = True + + for username, config in TEST_USERS.items(): + print(f"Testing user: {username}") + + try: + # Step 1: Get Keycloak token + print(" Getting Keycloak token...") + token = get_keycloak_token(username, config["password"]) + print(" ✓ Token obtained") + + # Step 2: Validate token with Polaris + print(" Validating token with Polaris...") + if validate_token_with_polaris(token): + print(" ✓ Polaris accepted token") + else: + print(" ✗ ERROR: Polaris rejected token") + all_passed = False + continue + + # Step 3: Verify role in token + print(" Checking token roles...") + check_token_roles(token, username) + + except Exception as e: + print(f" ✗ ERROR: {e}") + all_passed = False + + print() + + print("[test] Spark session auth tests", "PASSED" if all_passed else "FAILED") + + return 0 if all_passed else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/polaris_namespace_test.py b/tests/polaris_namespace_test.py new file mode 100644 index 0000000..c6c3da5 --- /dev/null +++ b/tests/polaris_namespace_test.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +""" +Integration test: Polaris namespace provisioning + +Validates: +- The iceberg catalog exists in Polaris +- The teehr namespace exists in the iceberg catalog +- Root credentials can be used to query Polaris +""" + +import sys +import requests +import time + +POLARIS_URL = "http://polaris:8181" +REALM = "teehr" +ROOT_CLIENT_ID = "root" +ROOT_CLIENT_SECRET = "secret123" + +# Retry configuration +MAX_RETRIES = 10 +RETRY_DELAY = 2 # seconds + + +def get_root_token(): + """Get root token using client credentials""" + for attempt in range(MAX_RETRIES): + try: + token_url = f"{POLARIS_URL}/api/catalog/v1/oauth/tokens" + payload = { + "grant_type": "client_credentials", + "client_id": ROOT_CLIENT_ID, + "client_secret": ROOT_CLIENT_SECRET, + "scope": "PRINCIPAL_ROLE:ALL" + } + headers = { + "X-Polaris-Realm": REALM, + "Content-Type": "application/x-www-form-urlencoded" + } + + response = requests.post(token_url, data=payload, headers=headers, timeout=5) + if response.status_code == 200: + return response.json()["access_token"] + elif attempt < MAX_RETRIES - 1: + print(f" Attempt {attempt + 1}/{MAX_RETRIES}: Polaris not ready, retrying...") + time.sleep(RETRY_DELAY) + else: + raise Exception(f"Failed to get root token: {response.text}") + except requests.exceptions.RequestException as e: + if attempt < MAX_RETRIES - 1: + print(f" Attempt {attempt + 1}/{MAX_RETRIES}: Polaris not reachable, retrying...") + time.sleep(RETRY_DELAY) + else: + raise Exception(f"Failed to get root token: {e}") + + +def list_namespaces(root_token): + """List namespaces in the teehr catalog""" + headers = { + "Authorization": f"Bearer {root_token}", + "X-Polaris-Realm": REALM + } + + response = requests.get( + f"{POLARIS_URL}/api/catalog/v1/teehr/namespaces", + headers=headers + ) + + if response.status_code != 200: + raise Exception(f"Failed to list namespaces: {response.text}") + + return response.json() + + +def main(): + """Run namespace provisioning validation""" + print("[test] Validating teehr namespace exists...") + + try: + # Step 1: Get root token + print(" Getting root credentials token...") + root_token = get_root_token() + print(" ✓ Root token obtained") + + # Step 2: List namespaces + print(" Listing namespaces in iceberg catalog...") + namespaces_response = list_namespaces(root_token) + print(" ✓ Namespace list retrieved") + + # Step 3: Check if teehr namespace exists + # Response is a list of namespace objects directly + if isinstance(namespaces_response, list): + namespaces = namespaces_response + else: + namespaces = namespaces_response.get("namespaces", []) + + found_teehr = False + namespace_names = [] + for ns in namespaces: + # namespace can be a list or a dict + namespace_path = ns if isinstance(ns, (list, tuple)) else ns.get("namespace", []) + if namespace_path: + namespace_names.append(".".join(namespace_path) if isinstance(namespace_path, (list, tuple)) else str(namespace_path)) + if isinstance(namespace_path, (list, tuple)) and len(namespace_path) > 0 and namespace_path[0] == "teehr": + found_teehr = True + elif isinstance(namespace_path, str) and namespace_path == "teehr": + found_teehr = True + + if found_teehr: + print(" ✓ teehr namespace found") + else: + print(f" WARNING: teehr namespace not found") + if namespace_names: + print(f" Available namespaces: {namespace_names}") + else: + print(f" Response type: {type(namespaces_response)}, content: {str(namespaces_response)[:200]}") + + print("[test] Namespace validation successful") + return 0 + + except Exception as e: + print(f"[test] ERROR: {e}") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/polaris_oidc_test.py b/tests/polaris_oidc_test.py new file mode 100644 index 0000000..d9d1e68 --- /dev/null +++ b/tests/polaris_oidc_test.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +""" +Integration test: Polaris OIDC token validation + +Validates: +- Keycloak can issue tokens to users +- Polaris accepts and validates Keycloak-issued JWT tokens +- JWT claims are properly mapped +""" + +import sys +import requests +import time + +KEYCLOAK_URL = "http://keycloak-service:8080" +POLARIS_URL = "http://polaris:8181" +REALM = "teehr" +TEST_USERNAME = "user" +TEST_PASSWORD = "user" + +# Retry configuration +MAX_RETRIES = 10 +RETRY_DELAY = 2 # seconds + + +def get_user_token(username, password): + """Get a JWT token from Keycloak for a user""" + for attempt in range(MAX_RETRIES): + try: + token_url = f"{KEYCLOAK_URL}/realms/{REALM}/protocol/openid-connect/token" + payload = { + "grant_type": "password", + "client_id": "jupyterhub", + "client_secret": "local-jupyterhub-client-secret", + "username": username, + "password": password, + "scope": "openid" + } + + response = requests.post(token_url, data=payload, timeout=5) + if response.status_code == 200: + return response.json()["access_token"] + elif response.status_code == 401 and attempt < MAX_RETRIES - 1: + print(f" Attempt {attempt + 1}/{MAX_RETRIES}: Keycloak not ready, retrying...") + time.sleep(RETRY_DELAY) + else: + raise Exception(f"Failed to get token for {username}: {response.text}") + except requests.exceptions.RequestException as e: + if attempt < MAX_RETRIES - 1: + print(f" Attempt {attempt + 1}/{MAX_RETRIES}: Connection failed, retrying...") + time.sleep(RETRY_DELAY) + else: + raise Exception(f"Failed to get token for {username}: {e}") + + +def validate_token_with_polaris(token): + """Test that Polaris accepts and validates the token""" + headers = { + "Authorization": f"Bearer {token}", + "X-Polaris-Realm": REALM + } + + response = requests.get( + f"{POLARIS_URL}/api/catalog/v1/config", + headers=headers, + params={"warehouse": REALM} + ) + + if response.status_code != 200: + print(f" DEBUG: HTTP {response.status_code}: {response.text[:300]}") + + return response.status_code == 200 + + +def main(): + """Run Polaris OIDC validation tests""" + print("[test] Validating Polaris OIDC token acceptance...") + + try: + # Step 1: Get user token from Keycloak + print(" Getting user token from Keycloak...") + token = get_user_token(TEST_USERNAME, TEST_PASSWORD) + print(" ✓ Token obtained") + + # Step 2: Test Polaris accepts the token + print(" Testing Polaris accepts the token...") + if validate_token_with_polaris(token): + print(" ✓ Polaris accepted token") + else: + print(" ✗ ERROR: Polaris rejected token") + return 1 + + print("[test] Polaris OIDC validation successful") + return 0 + + except Exception as e: + print(f"[test] ERROR: {e}") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/polaris_permissions_config_test.py b/tests/polaris_permissions_config_test.py new file mode 100644 index 0000000..87847f0 --- /dev/null +++ b/tests/polaris_permissions_config_test.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +""" +Integration test: Polaris permission configuration verification + +Validates: +- Catalog roles and principal roles are correctly created +- Principal roles have the expected permissions granted on the teehr namespace +- Permissions are configured for read-only and read-write access patterns +""" + +import sys +import json +import requests +import time + +# Configuration +POLARIS_URL = "http://polaris:8181" +REALM = "teehr" +CATALOG = "teehr" +NAMESPACE = "teehr" + +# Root credentials for admin access +ROOT_CLIENT_ID = "root" +ROOT_CLIENT_SECRET = "secret123" + +# Retry configuration +MAX_RETRIES = 10 +RETRY_DELAY = 2 # seconds + + +def get_root_token(): + """Get root token using client credentials""" + for attempt in range(MAX_RETRIES): + try: + token_url = f"{POLARIS_URL}/api/catalog/v1/oauth/tokens" + payload = { + "grant_type": "client_credentials", + "client_id": ROOT_CLIENT_ID, + "client_secret": ROOT_CLIENT_SECRET, + "scope": "PRINCIPAL_ROLE:ALL" + } + headers = { + "X-Polaris-Realm": REALM, + "Content-Type": "application/x-www-form-urlencoded" + } + + response = requests.post(token_url, data=payload, headers=headers, timeout=5) + if response.status_code == 200: + return response.json()["access_token"] + elif attempt < MAX_RETRIES - 1: + print(f" Attempt {attempt + 1}/{MAX_RETRIES}: Polaris not ready, retrying...") + time.sleep(RETRY_DELAY) + else: + raise Exception(f"Failed to get root token: {response.text}") + except requests.exceptions.RequestException as e: + if attempt < MAX_RETRIES - 1: + print(f" Attempt {attempt + 1}/{MAX_RETRIES}: Polaris not reachable, retrying...") + time.sleep(RETRY_DELAY) + else: + raise Exception(f"Failed to get root token: {e}") + + +def get_principal_role_grants(root_token, principal_role): + """Get all grants for a principal role""" + headers = { + "Authorization": f"Bearer {root_token}", + "X-Polaris-Realm": REALM + } + + response = requests.get( + f"{POLARIS_URL}/api/management/v1/principal-roles/{principal_role}", + headers=headers + ) + + status = response.status_code + if status != 200: + return None + + return response.json() + + +def get_catalog_role_grants(root_token, catalog_role): + """Get all grants for a catalog role""" + headers = { + "Authorization": f"Bearer {root_token}", + "X-Polaris-Realm": REALM + } + + response = requests.get( + f"{POLARIS_URL}/api/management/v1/catalogs/{CATALOG}/catalog-roles/{catalog_role}/grants", + headers=headers + ) + + status = response.status_code + if status != 200: + return None + + return response.json() + + +def main(): + """Verify Polaris permission configuration""" + print("[test] Verifying Polaris permission configuration...\n") + + all_passed = True + + try: + # Step 1: Get root token + print(" Getting root credentials token...") + root_token = get_root_token() + print(" ✓ Root token obtained\n") + + # Step 2: Verify teehr-read-only role has correct permissions + print(" Checking teehr-read-only role permissions...") + read_only_role_info = get_principal_role_grants(root_token, "teehr-read-only") + if read_only_role_info: + print(" ✓ teehr-read-only principal role exists") + else: + print(" ✗ ERROR: teehr-read-only principal role not found") + all_passed = False + + # Step 3: Verify teehr-read-write role has correct permissions + print(" Checking teehr-read-write role permissions...") + read_write_role_info = get_principal_role_grants(root_token, "teehr-read-write") + if read_write_role_info: + print(" ✓ teehr-read-write principal role exists") + else: + print(" ✗ ERROR: teehr-read-write principal role not found") + all_passed = False + + # Step 4: Verify catalog roles have the expected grants + print(" Checking teehr_read_only_role grants...") + read_only_catalog_grants = get_catalog_role_grants(root_token, "teehr_read_only_role") + if read_only_catalog_grants: + print(" ✓ teehr_read_only_role catalog role exists") + grants = read_only_catalog_grants.get("grants", []) + privileges = [g.get("privilege") for g in grants] + has_read_props = any("NAMESPACE_READ_PROPERTIES" in p for p in privileges) + has_table_list = any("TABLE_LIST" in p for p in privileges) + has_table_read = any("TABLE_READ_DATA" in p for p in privileges) + if has_read_props: + print(" ✓ NAMESPACE_READ_PROPERTIES permission is granted") + else: + print(" ✗ NAMESPACE_READ_PROPERTIES missing") + all_passed = False + if has_table_list: + print(" ✓ TABLE_LIST permission is granted") + else: + print(" ✗ TABLE_LIST missing") + all_passed = False + if has_table_read: + print(" ✓ TABLE_READ_DATA permission is granted") + else: + print(" ✗ TABLE_READ_DATA missing") + all_passed = False + else: + print(" ✗ ERROR: teehr_read_only_role not found") + all_passed = False + + print(" Checking teehr_read_write_role grants...") + read_write_catalog_grants = get_catalog_role_grants(root_token, "teehr_read_write_role") + if read_write_catalog_grants: + print(" ✓ teehr_read_write_role catalog role exists") + grants = read_write_catalog_grants.get("grants", []) + privileges = [g.get("privilege") for g in grants] + has_create = any("TABLE_CREATE" in p for p in privileges) + has_write = any("TABLE_WRITE_DATA" in p for p in privileges) + has_read = any("TABLE_READ_DATA" in p for p in privileges) + if has_create: + print(" ✓ TABLE_CREATE permission is granted") + else: + print(" ✗ TABLE_CREATE missing") + all_passed = False + if has_write: + print(" ✓ TABLE_WRITE_DATA permission is granted") + else: + print(" ✗ TABLE_WRITE_DATA missing") + all_passed = False + if has_read: + print(" ✓ TABLE_READ_DATA permission is granted") + else: + print(" ✗ TABLE_READ_DATA missing") + all_passed = False + else: + print(" ✗ ERROR: teehr_read_write_role not found") + all_passed = False + + print() + if all_passed: + print("[test] Polaris permission configuration verification PASSED") + return 0 + else: + print("[test] Polaris permission configuration verification FAILED") + return 1 + + except Exception as e: + print(f"[test] ERROR: {e}") + import traceback + traceback.print_exc() + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/polaris_roles_test.py b/tests/polaris_roles_test.py new file mode 100644 index 0000000..f41b254 --- /dev/null +++ b/tests/polaris_roles_test.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +""" +Integration test: Polaris role-based ACL validation + +Validates: +- Principal roles (teehr-read-only, teehr-read-write, iceberg-catalog-admin) exist +- Catalog roles (teehr_read_only_role, teehr_read_write_role, catalog_admin_role) exist +- Roles are properly bound to permissions +""" + +import sys +import requests +import time + +POLARIS_URL = "http://polaris:8181" +REALM = "teehr" +ROOT_CLIENT_ID = "root" +ROOT_CLIENT_SECRET = "secret123" + +EXPECTED_CATALOG_ROLES = [ + "teehr_read_only_role", + "teehr_read_write_role", + "catalog_admin_role" +] + +EXPECTED_PRINCIPAL_ROLES = [ + "teehr-read-only", + "teehr-read-write", + "iceberg-catalog-admin" +] + +# Retry configuration +MAX_RETRIES = 10 +RETRY_DELAY = 2 # seconds + + +def get_root_token(): + """Get root token using client credentials""" + for attempt in range(MAX_RETRIES): + try: + token_url = f"{POLARIS_URL}/api/catalog/v1/oauth/tokens" + payload = { + "grant_type": "client_credentials", + "client_id": ROOT_CLIENT_ID, + "client_secret": ROOT_CLIENT_SECRET, + "scope": "PRINCIPAL_ROLE:ALL" + } + headers = { + "X-Polaris-Realm": REALM, + "Content-Type": "application/x-www-form-urlencoded" + } + + response = requests.post(token_url, data=payload, headers=headers, timeout=5) + if response.status_code == 200: + return response.json()["access_token"] + elif attempt < MAX_RETRIES - 1: + print(f" Attempt {attempt + 1}/{MAX_RETRIES}: Polaris not ready, retrying...") + time.sleep(RETRY_DELAY) + else: + raise Exception(f"Failed to get root token: {response.text}") + except requests.exceptions.RequestException as e: + if attempt < MAX_RETRIES - 1: + print(f" Attempt {attempt + 1}/{MAX_RETRIES}: Polaris not reachable, retrying...") + time.sleep(RETRY_DELAY) + else: + raise Exception(f"Failed to get root token: {e}") + + +def check_catalog_role(root_token, role_name): + """Check if a catalog role exists""" + headers = { + "Authorization": f"Bearer {root_token}", + "X-Polaris-Realm": REALM + } + + response = requests.get( + f"{POLARIS_URL}/api/management/v1/catalogs/teehr/catalog-roles/{role_name}", + headers=headers + ) + + return response.status_code == 200 + + +def check_principal_role(root_token, role_name): + """Check if a principal role exists""" + headers = { + "Authorization": f"Bearer {root_token}", + "X-Polaris-Realm": REALM + } + + response = requests.get( + f"{POLARIS_URL}/api/management/v1/principal-roles/{role_name}", + headers=headers + ) + + return response.status_code == 200 + + +def main(): + """Run role-based ACL validation""" + print("[test] Validating Polaris role-based ACLs...") + + try: + root_token = get_root_token() + + # Check catalog roles + print(" Checking catalog roles...") + all_passed = True + for role in EXPECTED_CATALOG_ROLES: + if check_catalog_role(root_token, role): + print(f" ✓ Catalog role {role} exists") + else: + print(f" WARNING: Catalog role {role} not found") + all_passed = False + + # Check principal roles + print(" Checking principal roles...") + for role in EXPECTED_PRINCIPAL_ROLES: + if check_principal_role(root_token, role): + print(f" ✓ Principal role {role} exists") + else: + print(f" WARNING: Principal role {role} not found") + all_passed = False + + print("[test] Role-based ACL validation complete") + return 0 if all_passed else 1 + + except Exception as e: + print(f"[test] ERROR: {e}") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/spark_authmanager_executor_test.py b/tests/spark_authmanager_executor_test.py new file mode 100644 index 0000000..6650e02 --- /dev/null +++ b/tests/spark_authmanager_executor_test.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python3 +""" +Integration test: Spark cluster-mode executor auth via AuthManager + +Validates that Spark EXECUTORS (not just the driver) can participate in +real distributed Iceberg catalog operations when POLARIS_USE_AUTHMANAGER +is enabled with start_spark_cluster=True, and that the Polaris auth env +vars AuthManager needs are actually propagated to executor pods. + +TEST_USERNAME - Keycloak username (default: admin) +TEST_PASSWORD - Keycloak password (default: admin) +""" + +import sys +import os +import socket +import time +import gc + +import requests +from pyspark.sql import Row +from pyspark.sql import functions as F + +# Set up environment for Polaris/Spark before importing PySpark +os.environ.setdefault("POLARIS_DEFAULT_REALM", "teehr") +# create_spark_session() (unlike create_minio_spark_session(), which we +# deliberately don't use here) defaults remote_warehouse_dir to "" rather +# than the realm name when this isn't set, which Polaris's REST catalog +# rejects with "Please specify a warehouse" on CREATE TABLE. +os.environ.setdefault("REMOTE_WAREHOUSE_IDENTIFIER", "teehr") +os.environ.setdefault("REMOTE_CATALOG_REST_URI", "http://polaris:8181/api/catalog") +os.environ.setdefault("REMOTE_CATALOG_S3_ENDPOINT", "http://minio:9000") +os.environ.setdefault("REMOTE_CATALOG_S3_PATH_STYLE_ACCESS", "true") +os.environ.setdefault("AWS_ACCESS_KEY_ID", "minioadmin") +os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "minioadmin123") +os.environ.setdefault("AWS_REGION", "us-east-2") +# Broker and OAuth endpoints +os.environ.setdefault("POLARIS_BROKER_URL", "http://teehr-api:8000/auth/polaris-token") +os.environ.setdefault( + "POLARIS_OAUTH2_TOKEN_ENDPOINT", + "http://keycloak-service:8080/realms/teehr/protocol/openid-connect/token", +) +os.environ.setdefault("POLARIS_CLIENT_ID", "jupyterhub") +os.environ.setdefault("POLARIS_CLIENT_SECRET", "local-jupyterhub-client-secret") +# JVM heap +os.environ["JAVA_TOOL_OPTIONS"] = "-Xmx1g" +os.environ.setdefault("SPARK_LOCAL_IP", "127.0.0.1") + +sys.path.insert(0, "/opt/teehr") + +KEYCLOAK_URL = "http://keycloak-service:8080" +REALM = "teehr" +CATALOG = "iceberg" +NAMESPACE = "teehr" + +MAX_RETRIES = 10 +RETRY_DELAY = 2 + +TEST_USERNAME = os.getenv("TEST_USERNAME", "admin") +TEST_PASSWORD = os.getenv("TEST_PASSWORD", "admin") + +EXPECTED_EXECUTOR_ENV_KEYS = [ + "POLARIS_DEFAULT_REALM", + "POLARIS_BROKER_SESSION_TOKEN", +] + + +def get_keycloak_tokens(username: str, password: str) -> tuple: + """Get access_token and refresh_token from Keycloak via password grant.""" + for attempt in range(MAX_RETRIES): + try: + token_url = f"{KEYCLOAK_URL}/realms/{REALM}/protocol/openid-connect/token" + payload = { + "grant_type": "password", + "client_id": "jupyterhub", + "client_secret": "local-jupyterhub-client-secret", + "username": username, + "password": password, + "scope": "openid", + } + response = requests.post(token_url, data=payload, timeout=10) + if response.status_code == 200: + data = response.json() + return data["access_token"], data.get("refresh_token", "") + elif attempt < MAX_RETRIES - 1: + print(f" Attempt {attempt + 1}/{MAX_RETRIES}: Keycloak not ready, retrying...") + time.sleep(RETRY_DELAY) + else: + raise Exception(f"Failed to get token for {username}: {response.text}") + except requests.exceptions.RequestException as e: + if attempt < MAX_RETRIES - 1: + print(f" Attempt {attempt + 1}/{MAX_RETRIES}: Connection failed, retrying...") + time.sleep(RETRY_DELAY) + else: + raise Exception(f"Failed to get token for {username}: {e}") + + raise Exception(f"Failed to get token for {username} after {MAX_RETRIES} attempts") + + +def probe_executor_env(spark, expected_env_keys, partitions=4): + """Confirm expected env vars are present on executors. + + Returns one row per partition attempt with only booleans + executor + identity. Never returns secret values. + """ + keys = list(expected_env_keys) + + def _probe_partition(it): + import os + # Force execution of partition iterator so Spark doesn't prune the task. + _ = list(it) + result = { + "executor_host": socket.gethostname(), + "pid_present": os.getpid() > 0, + } + for k in keys: + result[f"has_{k}"] = bool(os.environ.get(k)) + yield Row(**result) + + rdd = spark.sparkContext.parallelize(range(partitions), partitions) + return rdd.mapPartitions(_probe_partition).collect() + + +def main(): + print(f"[test] Starting Spark cluster-mode executor AuthManager test for user: {TEST_USERNAME}") + + all_passed = True + spark = None + try: + print(" Getting Keycloak tokens...") + access_token, refresh_token = get_keycloak_tokens(TEST_USERNAME, TEST_PASSWORD) + if not refresh_token: + print(" ✗ ERROR: No refresh_token returned - required for AuthManager broker session") + return 1 + print(" ✓ Tokens obtained (access + refresh)") + + os.environ["POLARIS_USER_TOKEN"] = access_token + os.environ["POLARIS_REFRESH_TOKEN"] = refresh_token + os.environ["JUPYTERHUB_USER"] = TEST_USERNAME + + print(" Creating cluster-mode Spark session via AuthManager...") + from teehr.evaluation.spark_session_utils import create_spark_session + spark = create_spark_session( + update_configs={ + "spark.kubernetes.executor.node.selector.teehr-hub/nodegroup-name": "spark-r5-4xlarge", + }, + start_spark_cluster=True, + use_authmanager=True, + force_recreate_session=True, + executor_instances=1, + executor_cores=1, + executor_memory="1g", + ) + print(" ✓ Spark session created") + + print(" Probing executor environment for propagated Polaris auth vars...") + rows = probe_executor_env(spark, EXPECTED_EXECUTOR_ENV_KEYS, partitions=4) + if not rows: + print(" ✗ ERROR: no executor probe results returned") + all_passed = False + for row in rows: + print(f" {row.asDict()}") + for key in EXPECTED_EXECUTOR_ENV_KEYS: + if not row[f"has_{key}"]: + print(f" ✗ ERROR: executor {row['executor_host']} missing {key}") + all_passed = False + + # Real distributed write + read through the executors just probed, + # exercising the same Iceberg/Polaris auth path end to end. This + # table is created fresh and dropped at the end -- it never reads + # from or depends on any pre-existing warehouse table/data. + table = f"executor_authmanager_test_{int(time.time())}" + full_table = f"{CATALOG}.{NAMESPACE}.{table}" + + n = 200_000 + parts = 4 + print(f" Building {n}-row distributed dataset across {parts} partitions...") + df = ( + spark.range(0, n) + .repartition(parts) + .withColumn("grp", (F.col("id") % 17).cast("int")) + .withColumn("payload", F.concat(F.lit("v-"), F.col("id").cast("string"))) + ) + input_count = df.count() + print(f" input_count: {input_count}") + if input_count != n: + print(f" ✗ ERROR: expected input_count={n}, got {input_count}") + all_passed = False + + print(f" Writing distributed table {full_table}...") + df.writeTo(full_table).using("iceberg").create() + + read_df = spark.read.table(full_table).repartition(parts) + table_count = read_df.count() + print(f" table_count: {table_count}") + if table_count != n: + print(f" ✗ ERROR: expected table_count={n}, got {table_count}") + all_passed = False + + group_count = read_df.groupBy("grp").count().count() + if group_count != 17: + print(f" ✗ ERROR: expected 17 groups, got {group_count}") + all_passed = False + + spark.sql(f"DROP TABLE {full_table}") + print(f" dropped: {full_table}") + + except Exception as e: + print(f" ✗ ERROR: {type(e).__name__}: {str(e)[:300]}") + all_passed = False + finally: + if spark: + try: + spark.stop() + except Exception: + pass + gc.collect() + + result = "PASSED" if all_passed else "FAILED" + print(f"\n[test] Spark cluster-mode executor AuthManager test: {result}") + return 0 if all_passed else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/spark_authmanager_test.py b/tests/spark_authmanager_test.py new file mode 100644 index 0000000..0bfbed0 --- /dev/null +++ b/tests/spark_authmanager_test.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python3 +""" +Integration test: Spark session permission enforcement via AuthManager + +When run without TEST_USERNAME set, orchestrates per-user subprocesses so each +user gets a fresh JVM (matching JupyterHub's per-user-pod model and avoiding +static state leakage in TeehrBrokerAuthManager across spark.stop() calls). + +Single-user mode (invoked by orchestrator or directly): + TEST_USERNAME - Keycloak username (default: admin) + TEST_PASSWORD - Keycloak password (default: admin) + TEST_EXPECTED_READ - "true"/"false" (default: true) + TEST_EXPECTED_WRITE - "true"/"false" (default: true) +""" + +import sys +import os +import subprocess +import requests +import time +import gc + +# --- Orchestrator mode: spawn one subprocess per user for clean JVM isolation --- +TEST_USERS = { + "admin": {"password": "admin", "expected_read": True, "expected_write": True}, + "poweruser": {"password": "poweruser", "expected_read": True, "expected_write": True}, + "user": {"password": "user", "expected_read": True, "expected_write": False}, +} + +if "TEST_USERNAME" not in os.environ: + all_passed = True + for username, config in TEST_USERS.items(): + print(f"\n{'='*60}") + print(f"Running AuthManager test for: {username}") + print('='*60) + env = os.environ.copy() + env["TEST_USERNAME"] = username + env["TEST_PASSWORD"] = config["password"] + env["TEST_EXPECTED_READ"] = str(config["expected_read"]).lower() + env["TEST_EXPECTED_WRITE"] = str(config["expected_write"]).lower() + result = subprocess.run( + [sys.executable, __file__], + env=env, + timeout=300, + ) + if result.returncode != 0: + all_passed = False + + print(f"\n[test] Spark AuthManager enforcement tests {'PASSED' if all_passed else 'FAILED'}") + sys.exit(0 if all_passed else 1) + +# --- Single-user mode: actual test logic, called by orchestrator subprocess --- + +# Set up environment before importing PySpark +os.environ.setdefault("POLARIS_DEFAULT_REALM", "teehr") +# create_spark_session() defaults remote_warehouse_dir to "" (not the realm +# name) when this isn't set, which Polaris's REST catalog rejects with +# "Please specify a warehouse" on any catalog read/write. +os.environ.setdefault("REMOTE_WAREHOUSE_IDENTIFIER", "teehr") +os.environ.setdefault("REMOTE_CATALOG_REST_URI", "http://polaris:8181/api/catalog") +os.environ.setdefault("REMOTE_WAREHOUSE_S3_PATH", "s3://warehouse/") +os.environ.setdefault("REMOTE_CATALOG_S3_ENDPOINT", "http://minio:9000") +os.environ.setdefault("REMOTE_CATALOG_S3_PATH_STYLE_ACCESS", "true") +os.environ.setdefault("AWS_ACCESS_KEY_ID", "minioadmin") +os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "minioadmin123") +os.environ.setdefault("AWS_REGION", "us-east-2") +# Broker and OAuth endpoints +os.environ.setdefault("POLARIS_BROKER_URL", "http://teehr-api:8000/auth/polaris-token") +os.environ.setdefault( + "POLARIS_OAUTH2_TOKEN_ENDPOINT", + "http://keycloak-service:8080/realms/teehr/protocol/openid-connect/token", +) +# Use jupyterhub client (same as the password grant below) +os.environ.setdefault("POLARIS_CLIENT_ID", "jupyterhub") +os.environ.setdefault("POLARIS_CLIENT_SECRET", "local-jupyterhub-client-secret") +# JVM heap +os.environ["JAVA_TOOL_OPTIONS"] = "-Xmx1g" +os.environ.setdefault("SPARK_LOCAL_IP", "127.0.0.1") + +sys.path.insert(0, "/opt/teehr") + +# Configuration +KEYCLOAK_URL = "http://keycloak-service:8080" +REALM = "teehr" +CATALOG = "iceberg" +NAMESPACE = "teehr" + +MAX_RETRIES = 10 +RETRY_DELAY = 2 + +# Single-user mode — configured via env vars to match JupyterHub's per-user-pod model +USERNAME = os.getenv("TEST_USERNAME", "admin") +PASSWORD = os.getenv("TEST_PASSWORD", "admin") +EXPECTED_READ = os.getenv("TEST_EXPECTED_READ", "true").lower() == "true" +EXPECTED_WRITE = os.getenv("TEST_EXPECTED_WRITE", "true").lower() == "true" + + +def get_keycloak_tokens(username: str, password: str) -> tuple: + """Get access_token and refresh_token from Keycloak via password grant.""" + for attempt in range(MAX_RETRIES): + try: + token_url = f"{KEYCLOAK_URL}/realms/{REALM}/protocol/openid-connect/token" + payload = { + "grant_type": "password", + "client_id": "jupyterhub", + "client_secret": "local-jupyterhub-client-secret", + "username": username, + "password": password, + "scope": "openid", + } + response = requests.post(token_url, data=payload, timeout=10) + if response.status_code == 200: + data = response.json() + return data["access_token"], data.get("refresh_token", "") + elif attempt < MAX_RETRIES - 1: + print(f" Attempt {attempt + 1}/{MAX_RETRIES}: Keycloak not ready, retrying...") + time.sleep(RETRY_DELAY) + else: + raise Exception(f"Failed to get token for {username}: {response.text}") + except requests.exceptions.RequestException as e: + if attempt < MAX_RETRIES - 1: + print(f" Attempt {attempt + 1}/{MAX_RETRIES}: Connection failed, retrying...") + time.sleep(RETRY_DELAY) + else: + raise Exception(f"Failed to get token for {username}: {e}") + + raise Exception(f"Failed to get token for {username} after {MAX_RETRIES} attempts") + + +def test_spark_read_access(spark, username: str) -> bool: + """Test read access by listing tables in namespace.""" + try: + print(f" Testing READ access...") + tables = spark.sql(f"SHOW TABLES IN {CATALOG}.{NAMESPACE}").collect() + print(f" ✓ {username} can READ from {NAMESPACE} namespace (found {len(tables)} tables)") + return True + except Exception as e: + error_msg = str(e).lower() + if any(w in error_msg for w in ("permission", "forbidden", "denied", "403")): + print(f" ✓ {username} correctly denied READ access") + else: + print(f" ✗ ERROR during READ test: {type(e).__name__}: {str(e)[:100]}") + return False + + +def test_spark_write_access(spark, username: str, should_write: bool = True) -> bool: + """Test write access via CREATE TABLE + INSERT. Returns True if write succeeded.""" + table_name = f"test_authmanager_{username}_{int(time.time() * 1000)}" + full_table_name = f"{CATALOG}.{NAMESPACE}.{table_name}" + + try: + print(f" Testing WRITE access (CREATE TABLE + INSERT)...") + spark.sql(f""" + CREATE TABLE {full_table_name} ( + id INT, + name STRING + ) + USING iceberg + """) + spark.sql(f"INSERT INTO {full_table_name} VALUES (1, 'test')") + + try: + spark.sql(f"DROP TABLE {full_table_name}") + except Exception: + pass + + if should_write: + print(f" ✓ {username} successfully created table and inserted data") + else: + print(f" ✗ ERROR: {username} should NOT be able to write but succeeded!") + return True + + except Exception as e: + try: + spark.sql(f"DROP TABLE IF EXISTS {full_table_name}") + except Exception: + pass + + error_msg = str(e).lower() + is_permission_error = any(w in error_msg for w in ( + "permission", "forbidden", "denied", "403", "not authorized", "unauthorized", "access" + )) + + if should_write: + print(f" ✗ ERROR: {username} should be able to write but got: {type(e).__name__}") + print(f" {str(e)[:150]}") + elif is_permission_error: + print(f" ✓ {username} correctly denied WRITE access (permission error)") + else: + print(f" ✓ {username} denied WRITE access: {type(e).__name__}") + return False + + +def main(): + print(f"[test] Starting Spark AuthManager test for user: {USERNAME}") + print(f" Expected: read={EXPECTED_READ}, write={EXPECTED_WRITE}\n") + + all_passed = True + spark = None + try: + print(" Getting Keycloak tokens...") + access_token, refresh_token = get_keycloak_tokens(USERNAME, PASSWORD) + if not refresh_token: + print(" ✗ ERROR: No refresh_token returned - required for AuthManager broker session") + return 1 + print(" ✓ Tokens obtained (access + refresh)") + + os.environ["POLARIS_USER_TOKEN"] = access_token + os.environ["POLARIS_REFRESH_TOKEN"] = refresh_token + os.environ["JUPYTERHUB_USER"] = USERNAME + + print(" Creating Spark session via AuthManager...") + try: + from teehr.evaluation.spark_session_utils import create_spark_session + spark = create_spark_session( + use_authmanager=True, + force_recreate_session=True, + ) + print(" ✓ Spark session created via AuthManager") + except ImportError: + print(" ✗ ERROR: spark_session_utils not available") + return 1 + + can_read = test_spark_read_access(spark, USERNAME) + if can_read != EXPECTED_READ: + print(f" ✗ ERROR: Expected read={EXPECTED_READ} but got {can_read}") + all_passed = False + + can_write = test_spark_write_access(spark, USERNAME, should_write=EXPECTED_WRITE) + if can_write != EXPECTED_WRITE: + print(f" ✗ ERROR: Expected write={EXPECTED_WRITE} but got {can_write}") + all_passed = False + + except Exception as e: + print(f" ✗ ERROR: {type(e).__name__}: {str(e)[:200]}") + all_passed = False + finally: + if spark: + try: + spark.stop() + except Exception: + pass + gc.collect() + + result = "PASSED" if all_passed else "FAILED" + print(f"\n[test] Spark AuthManager test for {USERNAME}: {result}") + return 0 if all_passed else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/spark_iceberg_permissions_test.py b/tests/spark_iceberg_permissions_test.py new file mode 100644 index 0000000..a691c06 --- /dev/null +++ b/tests/spark_iceberg_permissions_test.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +""" +Integration test: Polaris permission configuration verification + +Validates: +- Catalog roles and principal roles are correctly created +- Principal roles have the expected permissions granted on the teehr namespace +- Permissions are configured for read-only and read-write access patterns +""" + +import sys +import json +import requests +import time + +# Configuration +POLARIS_URL = "http://polaris:8181" +REALM = "teehr" +CATALOG = "teehr" +NAMESPACE = "teehr" + +# Root credentials for admin access +ROOT_CLIENT_ID = "root" +ROOT_CLIENT_SECRET = "secret123" + +# Retry configuration +MAX_RETRIES = 10 +RETRY_DELAY = 2 # seconds + + +def get_root_token(): + """Get root token using client credentials""" + for attempt in range(MAX_RETRIES): + try: + token_url = f"{POLARIS_URL}/api/catalog/v1/oauth/tokens" + payload = { + "grant_type": "client_credentials", + "client_id": ROOT_CLIENT_ID, + "client_secret": ROOT_CLIENT_SECRET, + "scope": "PRINCIPAL_ROLE:ALL" + } + headers = { + "X-Polaris-Realm": REALM, + "Content-Type": "application/x-www-form-urlencoded" + } + + response = requests.post(token_url, data=payload, headers=headers, timeout=5) + if response.status_code == 200: + return response.json()["access_token"] + elif attempt < MAX_RETRIES - 1: + print(f" Attempt {attempt + 1}/{MAX_RETRIES}: Polaris not ready, retrying...") + time.sleep(RETRY_DELAY) + else: + raise Exception(f"Failed to get root token: {response.text}") + except requests.exceptions.RequestException as e: + if attempt < MAX_RETRIES - 1: + print(f" Attempt {attempt + 1}/{MAX_RETRIES}: Polaris not reachable, retrying...") + time.sleep(RETRY_DELAY) + else: + raise Exception(f"Failed to get root token: {e}") + + +def get_principal_role_grants(root_token, principal_role): + """Get all grants for a principal role""" + headers = { + "Authorization": f"Bearer {root_token}", + "X-Polaris-Realm": REALM + } + + response = requests.get( + f"{POLARIS_URL}/api/management/v1/principal-roles/{principal_role}", + headers=headers + ) + + status = response.status_code + if status != 200: + return None + + return response.json() + + +def get_catalog_role_grants(root_token, catalog_role): + """Get all grants for a catalog role""" + headers = { + "Authorization": f"Bearer {root_token}", + "X-Polaris-Realm": REALM + } + + response = requests.get( + f"{POLARIS_URL}/api/management/v1/catalogs/{CATALOG}/catalog-roles/{catalog_role}/grants", + headers=headers + ) + + status = response.status_code + if status != 200: + return None + + return response.json() + + +def main(): + """Verify Polaris permission configuration""" + print("[test] Verifying Polaris permission configuration...\n") + + all_passed = True + + try: + # Step 1: Get root token + print(" Getting root credentials token...") + root_token = get_root_token() + print(" ✓ Root token obtained\n") + + # Step 2: Verify teehr-read-only role has correct permissions + print(" Checking teehr-read-only role permissions...") + read_only_role_info = get_principal_role_grants(root_token, "teehr-read-only") + if read_only_role_info: + print(" ✓ teehr-read-only principal role exists") + else: + print(" ✗ ERROR: teehr-read-only principal role not found") + all_passed = False + + # Step 3: Verify teehr-read-write role has correct permissions + print(" Checking teehr-read-write role permissions...") + read_write_role_info = get_principal_role_grants(root_token, "teehr-read-write") + if read_write_role_info: + print(" ✓ teehr-read-write principal role exists") + else: + print(" ✗ ERROR: teehr-read-write principal role not found") + all_passed = False + + # Step 4: Verify catalog roles have the expected grants + print(" Checking teehr_read_only_role grants...") + read_only_catalog_grants = get_catalog_role_grants(root_token, "teehr_read_only_role") + if read_only_catalog_grants: + print(" ✓ teehr_read_only_role catalog role exists") + grants = read_only_catalog_grants.get("grants", []) + if any("READ_PROPERTIES" in g.get("privilege", "") for g in grants): + print(" ✓ READ_PROPERTIES permission is granted") + else: + print(" ℹ Available grants:", [g.get("privilege") for g in grants]) + else: + print(" ✗ ERROR: teehr_read_only_role not found") + all_passed = False + + print(" Checking teehr_read_write_role grants...") + read_write_catalog_grants = get_catalog_role_grants(root_token, "teehr_read_write_role") + if read_write_catalog_grants: + print(" ✓ teehr_read_write_role catalog role exists") + grants = read_write_catalog_grants.get("grants", []) + has_create = any("CREATE" in g.get("privilege", "") for g in grants) + has_write = any("WRITE" in g.get("privilege", "") for g in grants) + if has_create: + print(" ✓ TABLE_CREATE permission is granted") + if has_write: + print(" ✓ WRITE permission is granted") + if not (has_create or has_write): + print(" ℹ Available grants:", [g.get("privilege") for g in grants]) + else: + print(" ✗ ERROR: teehr_read_write_role not found") + all_passed = False + + print() + if all_passed: + print("[test] Polaris permission configuration verification PASSED") + return 0 + else: + print("[test] Polaris permission configuration verification FAILED") + return 1 + + except Exception as e: + print(f"[test] ERROR: {e}") + import traceback + traceback.print_exc() + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/spark_permission_test.py b/tests/spark_permission_test.py new file mode 100644 index 0000000..1ba51e7 --- /dev/null +++ b/tests/spark_permission_test.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 +""" +Integration test: Spark session permission enforcement with Polaris + +Validates real permission enforcement through Spark operations: +- admin: Can create tables, insert, read +- poweruser (teehr-read-write): Can create tables, insert, read +- user (teehr-read-only): Can read tables, but NOT create or insert +""" + +import sys +import os +import requests +import time +import gc +from typing import Optional + +# Set up environment for Polaris/Spark before importing PySpark +os.environ.setdefault("POLARIS_DEFAULT_REALM", "teehr") +# create_spark_session() defaults remote_warehouse_dir to "" (not the realm +# name) when this isn't set, which Polaris's REST catalog rejects with +# "Please specify a warehouse" on any catalog read/write. +os.environ.setdefault("REMOTE_WAREHOUSE_IDENTIFIER", "teehr") +os.environ.setdefault("REMOTE_CATALOG_REST_URI", "http://polaris:8181/api/catalog") +os.environ.setdefault("REMOTE_WAREHOUSE_S3_PATH", "s3://warehouse/") +os.environ.setdefault("REMOTE_CATALOG_S3_ENDPOINT", "http://minio:9000") +os.environ.setdefault("REMOTE_CATALOG_S3_PATH_STYLE_ACCESS", "true") +os.environ.setdefault("AWS_ACCESS_KEY_ID", "minioadmin") +os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "minioadmin123") +os.environ.setdefault("AWS_REGION", "us-east-2") +# Set JVM heap size BEFORE PySpark initializes the JVM - must use JAVA_TOOL_OPTIONS +# spark.driver.memory config is ignored if JVM heap is already too small +os.environ["JAVA_TOOL_OPTIONS"] = "-Xmx1g" +os.environ.setdefault("SPARK_LOCAL_IP", "127.0.0.1") + +# Set up path to import spark_session_utils from /opt/teehr (copied in Dockerfile) +sys.path.insert(0, "/opt/teehr") + +# Configuration +KEYCLOAK_URL = "http://keycloak-service:8080" +REALM = "teehr" +CATALOG = "iceberg" # Spark catalog name (spark.sql.catalog.) +NAMESPACE = "teehr" # Polaris/Iceberg namespace name + +# Retry configuration +MAX_RETRIES = 10 +RETRY_DELAY = 2 # seconds + +TEST_USERS = { + "admin": { + "password": "admin", + "expected_read": True, + "expected_write": True + }, + "poweruser": { + "password": "poweruser", + "expected_read": True, + "expected_write": True + }, + "user": { + "password": "user", + "expected_read": True, + "expected_write": False + } +} + + +def get_keycloak_token(username: str, password: str) -> str: + """Get a JWT token from Keycloak for a user""" + for attempt in range(MAX_RETRIES): + try: + token_url = f"{KEYCLOAK_URL}/realms/{REALM}/protocol/openid-connect/token" + payload = { + "grant_type": "password", + "client_id": "jupyterhub", + "client_secret": "local-jupyterhub-client-secret", + "username": username, + "password": password, + "scope": "openid" + } + + response = requests.post(token_url, data=payload, timeout=5) + if response.status_code == 200: + return response.json()["access_token"] + elif response.status_code == 401 and attempt < MAX_RETRIES - 1: + print(f" Attempt {attempt + 1}/{MAX_RETRIES}: Keycloak not ready, retrying...") + time.sleep(RETRY_DELAY) + else: + raise Exception(f"Failed to get token for {username}: {response.text}") + except requests.exceptions.RequestException as e: + if attempt < MAX_RETRIES - 1: + print(f" Attempt {attempt + 1}/{MAX_RETRIES}: Connection failed, retrying...") + time.sleep(RETRY_DELAY) + else: + raise Exception(f"Failed to get token for {username}: {e}") + + raise Exception(f"Failed to get token for {username} after {MAX_RETRIES} attempts") + + +def test_spark_read_access(spark, username: str, catalog: str = "iceberg", namespace: str = "teehr") -> bool: + """Test read access by listing tables in namespace""" + try: + print(f" Testing READ access...") + # List tables using fully-qualified catalog.namespace reference + tables = spark.sql(f"SHOW TABLES IN {catalog}.{namespace}").collect() + print(f" ✓ {username} can READ from {namespace} namespace (found {len(tables)} tables)") + return True + except Exception as e: + error_msg = str(e).lower() + if "permission" in error_msg or "forbidden" in error_msg or "denied" in error_msg or "403" in error_msg: + print(f" ✓ {username} correctly denied READ access") + return False + else: + print(f" ✗ ERROR during READ test: {type(e).__name__}: {str(e)[:100]}") + return False + + +def test_spark_write_access(spark, username: str, catalog: str = "iceberg", namespace: str = "teehr", should_write: bool = True) -> bool: + """Test write access by creating a table and inserting data. Returns True if write succeeded, False if denied/failed.""" + table_name = f"test_table_{username}_{int(time.time() * 1000)}" + full_table_name = f"{catalog}.{namespace}.{table_name}" + + try: + print(f" Testing WRITE access (CREATE TABLE + INSERT)...") + spark.sql(f""" + CREATE TABLE {full_table_name} ( + id INT, + name STRING + ) + USING iceberg + """) + spark.sql(f"INSERT INTO {full_table_name} VALUES (1, 'test')") + + # Clean up + try: + spark.sql(f"DROP TABLE {full_table_name}") + except Exception: + pass + + if should_write: + print(f" ✓ {username} successfully created table and inserted data") + else: + print(f" ✗ ERROR: {username} should NOT be able to write but succeeded!") + return True + + except Exception as e: + # Always attempt cleanup even if write failed partway through + try: + spark.sql(f"DROP TABLE IF EXISTS {full_table_name}") + except Exception: + pass + + error_msg = str(e).lower() + is_permission_error = any(w in error_msg for w in ("permission", "forbidden", "denied", "403", "not authorized", "unauthorized", "access")) + + if should_write: + print(f" ✗ ERROR: {username} should be able to write but got: {type(e).__name__}") + print(f" {str(e)[:150]}") + elif is_permission_error: + print(f" ✓ {username} correctly denied WRITE access (permission error)") + else: + print(f" ✓ {username} denied WRITE access: {type(e).__name__}") + return False + + +def main(): + """Run Spark permission enforcement tests""" + print("[test] Starting Spark permission enforcement tests...\n") + + all_passed = True + + for username, config in TEST_USERS.items(): + print(f"Testing user: {username}") + print(f" Expected: read={config['expected_read']}, write={config['expected_write']}") + + spark = None + try: + # Step 1: Get Keycloak token + print(" Getting Keycloak token...") + token = get_keycloak_token(username, config["password"]) + print(" ✓ Token obtained") + + # Step 2: Create Spark session with Polaris catalog + print(" Creating Spark session with Polaris catalog...") + try: + from teehr.evaluation.spark_session_utils import create_spark_session + spark = create_spark_session( + polaris_token=token, + ) + print(" ✓ Spark session created") + except ImportError: + # If spark_session_utils is not available, skip this user + print(" ✗ ERROR: spark_session_utils not available") + all_passed = False + continue + + # Step 3: Test read access + can_read = test_spark_read_access(spark, username, catalog=CATALOG, namespace=NAMESPACE) + if can_read != config["expected_read"]: + print(f" ✗ ERROR: Expected read={config['expected_read']} but got {can_read}") + all_passed = False + + # Step 4: Test write access + can_write = test_spark_write_access(spark, username, catalog=CATALOG, namespace=NAMESPACE, should_write=config["expected_write"]) + if can_write != config["expected_write"]: + print(f" ✗ ERROR: Expected write={config['expected_write']} but got {can_write}") + all_passed = False + + print() + + except Exception as e: + print(f" ✗ ERROR: {type(e).__name__}: {str(e)[:200]}\n") + all_passed = False + finally: + # Cleanup: stop Spark session and force garbage collection + if spark: + try: + spark.stop() + except: + pass + # Force garbage collection to free memory between sessions + gc.collect() + time.sleep(0.5) + + print("[test] Spark permission enforcement tests", "PASSED" if all_passed else "FAILED") + + return 0 if all_passed else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_executor_auth.py b/tests/test_executor_auth.py new file mode 100644 index 0000000..c15984a --- /dev/null +++ b/tests/test_executor_auth.py @@ -0,0 +1,92 @@ +from pyspark.sql import Row +from pyspark.sql import functions as F +from teehr import RemoteReadWriteEvaluation +from teehr.evaluation.spark_session_utils import create_spark_session +import time + +spark = create_spark_session( + update_configs={"spark.kubernetes.executor.node.selector.teehr-hub/nodegroup-name": "spark-r5-4xlarge"}, + start_spark_cluster=True, + use_authmanager=True, + executor_instances=1, + executor_cores=1, + executor_memory="1g" +) + +ev = RemoteReadWriteEvaluation(spark=spark) + +print(ev.list_tables()) + +print(ev.configurations.to_sdf().show()) + +def probe_executor_env(spark, expected_env_keys, partitions=8): + """ + Returns one row per partition attempt with only booleans + executor identity. + Never returns secret values. + """ + keys = list(expected_env_keys) + + def _probe_partition(it): + import os + import socket + # Force execution of partition iterator so Spark doesn't prune the task. + _ = list(it) + result = { + "executor_host": socket.gethostname(), + "pid_present": os.getpid() > 0, + } + for k in keys: + result[f"has_{k}"] = bool(os.environ.get(k)) + yield Row(**result) + + # Use enough partitions to spread across executors + rdd = spark.sparkContext.parallelize(range(partitions), partitions) + rows = rdd.mapPartitions(_probe_partition).collect() + return rows + +expected = [ + "POLARIS_DEFAULT_REALM", + "POLARIS_BROKER_SESSION_TOKEN" +] +rows = probe_executor_env(spark, expected, partitions=12) +for r in rows: + print(r.asDict()) + + catalog = "iceberg" +namespace = "teehr" +table = f"executor_probe_{int(time.time())}" +full_table = f"{catalog}.{namespace}.{table}" + +# 1) Build distributed data (force executor work via repartition) +n = 200_000 +parts = 12 +df = ( + spark.range(0, n) + .repartition(parts) + .withColumn("grp", (F.col("id") % 17).cast("int")) + .withColumn("payload", F.concat(F.lit("v-"), F.col("id").cast("string"))) +) + +# Optional: materialize first to ensure tasks run +print("input_count:", df.count()) + +# 2) Real distributed WRITE to Polaris/Iceberg +df.writeTo(full_table).using("iceberg").create() + +# 3) Real distributed READ from Polaris/Iceberg +read_df = spark.read.table(full_table).repartition(parts) +print("table_count:", read_df.count()) + +# 4) A distributed aggregate to exercise more executor paths +agg = ( + read_df.groupBy("grp") + .count() + .orderBy("grp") +) +agg.show(20, truncate=False) + +# 5) Cleanup +spark.sql(f"DROP TABLE {full_table}") +print("dropped:", full_table) + +spark.stop() \ No newline at end of file diff --git a/trino/garden.yaml b/trino/garden.yaml index 46be5d4..dcc8e64 100644 --- a/trino/garden.yaml +++ b/trino/garden.yaml @@ -12,7 +12,7 @@ spec: name: trino repo: https://trinodb.github.io/charts version: 1.41.0 - + values: server: workers: 1 @@ -26,13 +26,17 @@ spec: catalogs: iceberg: |- connector.name=iceberg - iceberg.catalog.type=${var.iceberg.catalogType} - iceberg.rest-catalog.uri=${var.iceberg.catalogUri} - iceberg.rest-catalog.warehouse=${var.iceberg.catalogWarehouse} + iceberg.catalog.type=${var.polaris.catalogType} + iceberg.rest-catalog.uri=${var.polaris.catalogUri} + iceberg.rest-catalog.warehouse=${var.polaris.defaultRealm} + iceberg.rest-catalog.security=OAUTH2 + iceberg.rest-catalog.oauth2.server-uri=${var.polaris.oauthServerUri} + iceberg.rest-catalog.oauth2.credential=$${ENV:POLARIS_CREDENTIAL} + iceberg.rest-catalog.oauth2.scope=openid # S3 Configuration fs.native-s3.enabled=true - s3.path-style-access=${var.iceberg.catalogS3PathStyleAccess} - s3.endpoint=${var.iceberg.catalogS3Endpoint} + s3.path-style-access=${var.polaris.catalogS3PathStyleAccess} + s3.endpoint=${var.polaris.catalogS3Endpoint} s3.region=${var.aws.region} accessControl: type: configmap @@ -61,6 +65,11 @@ spec: secretKeyRef: name: minio-secrets key: secretkey + - name: POLARIS_CREDENTIAL + valueFrom: + secretKeyRef: + name: trino-polaris-secrets + key: credential --- # Remote environment deployment kind: Deploy @@ -76,7 +85,7 @@ spec: name: trino repo: https://trinodb.github.io/charts version: 1.41.0 - + values: server: workers: "${environment.name == 'local' ? 1 : 2}" @@ -106,10 +115,18 @@ spec: catalogs: iceberg: |- connector.name=iceberg - iceberg.catalog.type=${var.iceberg.catalogType} - iceberg.rest-catalog.uri=${var.iceberg.catalogUri} - iceberg.rest-catalog.warehouse=${var.iceberg.catalogWarehouse} + iceberg.catalog.type=${var.polaris.catalogType} + iceberg.rest-catalog.uri=${var.polaris.catalogUri} + iceberg.rest-catalog.warehouse=${var.polaris.defaultRealm} + iceberg.rest-catalog.security=OAUTH2 + iceberg.rest-catalog.oauth2.server-uri=${var.polaris.oauthServerUri} + iceberg.rest-catalog.oauth2.credential=$${ENV:POLARIS_CREDENTIAL} + iceberg.rest-catalog.oauth2.scope=openid # S3 Configuration + # Trino holds no AWS identity of its own for the warehouse bucket; + # it relies entirely on Polaris to vend scoped, short-lived S3 + # credentials via the REST catalog protocol. + iceberg.rest-catalog.vended-credentials-enabled=true fs.native-s3.enabled=true s3.region=${var.aws.region} accessControl: @@ -128,8 +145,13 @@ spec: ] } + env: + - name: POLARIS_CREDENTIAL + valueFrom: + secretKeyRef: + name: trino-polaris-secrets + key: credential + serviceAccount: create: true - name: trino - annotations: - eks.amazonaws.com/role-arn: ${var.irsa.trinoRoleArn} \ No newline at end of file + name: trino \ No newline at end of file