Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -318,9 +318,13 @@ spec:
variables.toEndpoints.all(selector, selector.?matchExpressions.orValue([]).all(item,
!item.?key.orValue('').matches(variables.broadSelectorKey) ||
item.?operator.orValue('') in ['In', 'Exists']))
# An egress rule that is exactly {} names no destination and no port, so
# Cilium allows nothing through it; it only switches the endpoint to egress
# default deny (#3501). A rule with only toPorts still allows every peer on
# those ports and stays refused below.
- name: egressRulesNameDestinations
expression: >-
variables.egress.all(rule,
variables.egress.all(rule, size(rule) == 0 ||
['toEndpoints', 'toCIDR', 'toCIDRSet', 'toEntities', 'toNodes', 'toGroups', 'toServices', 'toFQDNs']
.exists(field, field in rule && rule[field] != null && size(rule[field]) > 0))
- name: egressOmitsNodesAndGroups
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,13 +12,15 @@
# hostname on the listener.
#
# Coverage: every namespace bound to the tenant-edit ClusterRole gets a validation.
# Two namespaces reconcile tenant-controlled manifests as a tenant
# ServiceAccount today — wedding-app and ascoachingogvaner — and each has its own
# allow-list below, because tenant hostnames are not derivable from the namespace
# name (wedding-app serves wedding.*, ascoachingogvaner serves an apex customer
# Three namespaces reconcile tenant-controlled routes today — wedding-app,
# ascoachingogvaner and business-site — and each has its own allow-list below,
# because tenant hostnames are not derivable from the namespace name
# (wedding-app serves wedding.*, ascoachingogvaner serves an apex customer
# domain). Each list is that tenant's own declared set from its
# deploy/httproute.yaml, covering the production hostname and the *.platform.lan
# name used on local/dev clusters where the tenant declares one.
# name used on local/dev clusters where the tenant declares one. business-site
# declares its host in deploy/webapp.yaml and has one public staging name only;
# its production hostname stays on GitHub Pages until a separate cutover.
#
# Fail-closed for anything NOT listed: the final validation denies every HTTPRoute in a
# ksail tenant namespace that has no allow-list above, so a newly onboarded
Expand Down Expand Up @@ -112,11 +114,21 @@ spec:
ascoachingogvaner.dk, www.ascoachingogvaner.dk or
ascoachingogvaner.platform.lan. Omitting hostnames claims every hostname
on the shared platform Gateway listener.
# The KRO controller renders this route from the tenant-owned WebApp, so
# the allow-list applies to it exactly as to a tenant-applied route.
- expression: >-
object.metadata.namespace != 'business-site' ||
(size(variables.hostnames) > 0 &&
variables.hostnames.all(h, h in ['business-site.platform.devantler.tech']))
message: >-
business-site HTTPRoutes must set spec.hostnames and may only use
business-site.platform.devantler.tech. Omitting hostnames claims every
hostname on the shared platform Gateway listener.
# Catch-all: any ksail tenant namespace without an allow-list above. This is
# what makes onboarding fail loudly instead of silently unrestricted, the
# exact gap that once left this policy covering only wedding-app while other
# tenants shared the same wildcard Gateway.
- expression: "object.metadata.namespace in ['wedding-app', 'ascoachingogvaner']"
- expression: "object.metadata.namespace in ['wedding-app', 'ascoachingogvaner', 'business-site']"
message: >-
This tenant namespace has no approved hostnames. Add an allow-list
validation for it to the restrict-tenant-route-hostnames ValidatingPolicy
Expand Down
10 changes: 6 additions & 4 deletions k8s/providers/hetzner/apps/business-site/tenant.yaml
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
# Source-only staging: the signed manifest can be fetched and verified, but no
# tenant application, route or private identity configuration is reconciled.
# Public-only staging: the tenant reconciles its signed v3.0.0 package — the
# WebApp at business-site.platform.devantler.tech, its network policy, budget
# and disclosure input — while the platform-owned staging hold still denies
# gateway traffic. Portal identity, payments and devantler.tech stay off.
apiVersion: kro.run/v1alpha1
kind: Tenant
metadata:
Expand All @@ -9,8 +11,8 @@ spec:
name: business-site
externalDns: false
sops: false
webApp: false
webApp: true
source:
digest: sha256:7ec5b206635db783d8fd18789605a771cc772946f65339d3f0a2b4e08a14ebc6
publisherRevision: d8b5d0e9429cebf12907e0d4cb448433d3108e4b
reconcileApplication: false
reconcileApplication: true
6 changes: 4 additions & 2 deletions scripts/canonical-tenant-source.lib.sh
Original file line number Diff line number Diff line change
Expand Up @@ -111,10 +111,12 @@ canonical_tenant_projection() (
select(.apiVersion==$record.tenantIdentity.apiVersion and .kind==$record.tenantIdentity.kind and
.metadata.name==$record.tenantIdentity.name and .metadata.namespace==$record.tenantIdentity.namespace and
(.spec|keys)==["externalDns","name","sops","source","webApp"] and .spec.name==$record.consumer and
.spec.webApp==false and .spec.externalDns==false and .spec.sops==false and
.spec.externalDns==false and .spec.sops==false and
(.spec.source|keys)==["digest","publisherRevision","reconcileApplication"] and
.spec.source.digest==$record.sourceIdentity.digest and .spec.source.publisherRevision==$pin and
.spec.source.reconcileApplication==false) |
# Source-only bootstrap, or the public staging application with its own
# WebApp. A half-enabled instance is neither reviewed shape.
(.spec.webApp|type)=="boolean" and .spec.webApp==.spec.source.reconcileApplication) |
{apiVersion:"source.toolkit.fluxcd.io/v1",kind:"OCIRepository",
metadata:{namespace:$record.sourceIdentity.namespace,name:$record.sourceIdentity.name},
spec:{url:$record.sourceIdentity.artifactURL,ref:{digest:$record.sourceIdentity.digest},
Expand Down
2 changes: 1 addition & 1 deletion scripts/rgd-template-static-scan-baseline.tsv
Original file line number Diff line number Diff line change
Expand Up @@ -42,4 +42,4 @@
1 k8s/bases/infrastructure/resource-graph-definitions/webapp/resource-graph-definition.yaml.resources/service.yaml KSV-0040 LOW CAUSE-SHA256:79c2597bf128e3ca530288250548c211e5e43a66a10d3c88748c6db4ac565920
1 k8s/providers/docker/apps/tenant-ascoachingogvaner.yaml INSTANCE-CONTENT SHA256 7e65481a62deb24c89dca2e5f83969e308e7604e770d97edb15e86ecc948f22c
1 k8s/providers/docker/apps/web-app-wedding-app.yaml INSTANCE-CONTENT SHA256 3d42027c7e549014cad1e717876a6eb72e0a6641f3bf181f7a0e1a70d94306d6
1 k8s/providers/hetzner/apps/business-site/tenant.yaml INSTANCE-CONTENT SHA256 63fd7c3a83133a8debdb3d0c2cd0ade8c7cedf2ced5bb3646fd07608823545e1
1 k8s/providers/hetzner/apps/business-site/tenant.yaml INSTANCE-CONTENT SHA256 f41cf288531722e872f273a55d0ed3d8827c642f49d37341f9e7f4a88fd75d45
9 changes: 5 additions & 4 deletions scripts/tests/test-business-staging-hold.sh
Original file line number Diff line number Diff line change
Expand Up @@ -71,14 +71,15 @@ for change in \
exit 1
fi
done
# Installing the hold is deliberately independent of enabling the application.
# The public staging application reconciles behind the hold; the hold, not the
# Tenant, decides whether gateway traffic reaches it.
jq -e '
[.[] | select(.kind == "Tenant" and .metadata.name == "business-site")] |
length == 1 and .[0].metadata.namespace == "flux-system" and
.[0].spec.webApp == false and .[0].spec.externalDns == false and
.[0].spec.source.reconcileApplication == false
.[0].spec.webApp == true and .[0].spec.externalDns == false and
.[0].spec.source.reconcileApplication == true
' "$scratch/production.json" >/dev/null
kubectl kustomize "$root/k8s/providers/docker/apps" >"$scratch/local.yaml"
yq -o=json -I=0 '.' "$scratch/local.yaml" |
jq -es 'all(.[]; .metadata.name != "business-site-staging-hold" and .metadata.name != "business-site-customer-identity-hold")' >/dev/null
printf 'PASS: production-only staging and identity holds; 16 unsafe mutations refused; application remains disabled\n'
printf 'PASS: production-only staging and identity holds; 16 unsafe mutations refused; application reconciles behind the hold\n'
21 changes: 20 additions & 1 deletion scripts/tests/test-canonical-source-bootstrap.sh
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
#!/usr/bin/env bash
# Exercise the real raw discovery path with one fully qualified source-only binding.
# Exercise the real raw discovery path with one fully qualified binding: the
# source-only bootstrap shape and the enabled public staging shape.
set -euo pipefail
test_repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
work="$(mktemp -d)"
Expand Down Expand Up @@ -99,6 +100,24 @@ for mutation in '.spec.source.reconcileApplication=true' '.spec.webApp=true' '.s
test ! -s "$work/rows"
controls=$((controls + 1))
done
# The enabled public staging shape turns both switches on together and projects
# the same reviewed source; each switch alone was refused above.
yq '.spec.webApp=true | .spec.source.reconcileApplication=true' "$work/good-tenant.yaml" >"$tree/$source_path"
if ! discover >"$work/rows" 2>"$work/error"; then
echo 'enabled staging Tenant was omitted or refused by actual discovery' >&2
cat "$work/error" >&2
exit 1
fi
test "$(wc -l <"$work/rows" | tr -d ' ')" = 1
for mutation in '.spec.webApp="true"' '.spec.externalDns=true' '.spec.sops=true'; do
yq ".spec.webApp=true | .spec.source.reconcileApplication=true | $mutation" "$work/good-tenant.yaml" >"$tree/$source_path"
if discover >"$work/rows" 2>"$work/error"; then
echo "changed enabled instance admitted: $mutation" >&2
exit 1
fi
test ! -s "$work/rows"
controls=$((controls + 1))
done
cp "$work/good-tenant.yaml" "$tree/$source_path"
cp "$tree/$source_path" "$tree/$(dirname "$source_path")/clone.yaml"
if discover >"$work/rows" 2>"$work/error"; then
Expand Down
6 changes: 4 additions & 2 deletions scripts/tests/test-canonical-source-upgrade.sh
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ kubectl() {
jq '.bindings[0].publication.workflowRunAttempt=2' "$binding_file" >"$upgrade_test_work/raced.json"
mv "$upgrade_test_work/raced.json" "$binding_file"
;;
tenant) yq -i '.spec.webApp=true' "$tree/$tenant_path" ;;
tenant) yq -i '.spec.externalDns=true' "$tree/$tenant_path" ;;
constructor) yq -i '.spec.schema.group="unapproved.example"' "$tree/k8s/bases/infrastructure/resource-graph-definitions/tenant/resource-graph-definition.yaml" ;;
output)
mkdir "$upgrade_test_work/candidate"
Expand Down Expand Up @@ -144,7 +144,9 @@ jq -e --arg digest "$next_digest" --argjson old "$old_record" '
.previousBinding==$old and .previousObservation.generation==2 and
(.previousObservation.uidSha256|test("^[0-9a-f]{64}$"))' "$candidate_record" >/dev/null
test "$(yq '.spec.source.digest' "$upgrade_test_work/candidate/$tenant_path")" = "$next_digest"
test "$(yq '.spec.webApp or .spec.externalDns or .spec.sops or .spec.source.reconcileApplication' "$upgrade_test_work/candidate/$tenant_path")" = false
# A digest upgrade keeps every reviewed switch exactly as it was.
switches='[.spec.webApp, .spec.externalDns, .spec.sops, .spec.source.reconcileApplication]'
test "$(yq -o=json -I=0 "$switches" "$upgrade_test_work/candidate/$tenant_path")" = "$(yq -o=json -I=0 "$switches" "$upgrade_test_work/original-tenant.yaml")"
if prepare >"$upgrade_test_work/result" 2>&1; then
echo 'existing candidate directory overwritten' >&2
exit 1
Expand Down
11 changes: 6 additions & 5 deletions scripts/tests/test-restrict-tenant-network-policies.sh
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ expect_summary() {
# used to be refused now admits. The unmanaged namespace control is an explicit
# match-condition skip; a namespaceSelector previously omitted its result.
apply "${tests}/resources.yaml" "${tests}/values.yaml" "${tests}/user-info.yaml"
expect_summary "pass: 18, fail: 80, warn: 0, error: 0, skip: 1" \
expect_summary "pass: 19, fail: 80, warn: 0, error: 0, skip: 1" \
"tenant CiliumNetworkPolicy boundary"

# 2. Carve-out: kyverno's background controller owns the generated floor. These
Expand All @@ -57,10 +57,11 @@ expect_summary "pass: 0, fail: 0, warn: 0, error: 0, skip: 3" \
# Besides the reserved name, allow-dns reaches CoreDNS by naming kube-system
# through the namespace label and allow-cnpg-operator names cnpg-system, the
# cross-namespace selector a tenant is refused (#3399, #3655). default-deny's
# one empty rule per direction names no source or destination, which is refused
# because a sourceless rule WITH toPorts allows every peer on that port. Without
# toPorts, as here, Cilium allows nothing (#3501), so the refusal is conservative,
# not a hole. The generated floor is exempt only through its applying identity.
# empty ingress rule names no source, which is refused because a sourceless rule
# WITH toPorts allows every peer on that port. Without toPorts, as here, Cilium
# allows nothing (#3501), so the refusal is conservative, not a hole; its empty
# egress rule is admitted for the same reason. The generated floor is exempt
# only through its applying identity.
apply "${tests}/kyverno-author/resources.yaml" \
"${tests}/kyverno-author/values.yaml" "${tests}/user-info.yaml"
expect_summary "pass: 0, fail: 3, warn: 0, error: 0, skip: 0" \
Expand Down
4 changes: 2 additions & 2 deletions scripts/tests/test-tenant-controller-namespace-coverage.sh
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@ while IFS=$'\t' read -r family passed denied skipped; do
done <<'CASES'
restrict-tenant-secret-stores 5 7 0
restrict-tenant-issuer-refs 4 3 0
restrict-tenant-route-hostnames 3 13 0
restrict-tenant-network-policies 18 80 1
restrict-tenant-route-hostnames 4 17 0
restrict-tenant-network-policies 19 80 1
restrict-tenant-image-repositories 7 11 0
CASES
[[ "$failures" == 0 ]] || exit 1
Expand Down
2 changes: 1 addition & 1 deletion scripts/validate-eks-ci-role-policy/approved-surface.txt
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ iam.aws.m.upbound.io/v1beta1|Role|aws|eks-ci c40205b3f48d193e8be1a39abda738865b9

kro.run/v1alpha1|ResourceGraphDefinition||tenant.kro.run 72f3f972cecdf3ca2e315deb4224bd28c1df1244fc7973a88eb7f4fc03ac1d05

kro.run/v1alpha1|Tenant|flux-system|business-site 5854dbac8e7f4ff52f5d858ee7e673e82cd758f0c770c3203b9409c8dc15aba2
kro.run/v1alpha1|Tenant|flux-system|business-site 47380ca8c8781c2de6b17226e44ac7cef030e13a89c8b41dae9ddf459304b3a4

kustomize.toolkit.fluxcd.io/v1|Kustomization|ascoachingogvaner|ascoachingogvaner ac00ca29c04de8d2c30a045d583570536a97fc305a1b06c4300da018ecdfb864

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,21 +9,22 @@ import (
"gopkg.in/yaml.v3"
)

// Approve only the exact source-only typed instance, never its future app or
// capabilities. The full ordinary validator must reject each widened intent.
func TestBusinessSourceOnlyAuthorization(t *testing.T) {
// Approve only the exact public staging instance: its own application and
// WebApp, with no DNS writer, SOPS or other source. The full ordinary
// validator must reject every other intent, including a return to source-only.
func TestBusinessStagingAuthorization(t *testing.T) {
role, boundary, rendered := repositoryInputs(t)
if err := validateAuthorization(role, boundary, rendered); err != nil {
t.Fatalf("reviewed source-only registration refused: %v", err)
t.Fatalf("reviewed staging registration refused: %v", err)
}
for _, tc := range []struct {
name string
remove bool
mutate func(map[string]any)
}{
{name: "source removed", remove: true},
{name: "application reconciliation", mutate: func(spec map[string]any) { spec["source"].(map[string]any)["reconcileApplication"] = true }},
{name: "webapp capability", mutate: func(spec map[string]any) { spec["webApp"] = true }},
{name: "no application reconciliation", mutate: func(spec map[string]any) { spec["source"].(map[string]any)["reconcileApplication"] = false }},
{name: "no webapp capability", mutate: func(spec map[string]any) { spec["webApp"] = false }},
{name: "external DNS", mutate: func(spec map[string]any) { spec["externalDns"] = true }},
{name: "SOPS", mutate: func(spec map[string]any) { spec["sops"] = true }},
{name: "different publisher", mutate: func(spec map[string]any) {
Expand Down Expand Up @@ -58,15 +59,15 @@ func TestBusinessSourceOnlyAuthorization(t *testing.T) {
t.Fatal(err)
}
if selected != 1 {
t.Fatalf("selected %d source-only registrations, want one", selected)
t.Fatalf("selected %d staging registrations, want one", selected)
}
var mismatch *surfaceMismatchError
if err := validateAuthorization(role, boundary, changed.Bytes()); !errors.As(err, &mismatch) {
t.Fatalf("changed source-only authorization error = %v, want ledger mismatch", err)
t.Fatalf("changed staging authorization error = %v, want ledger mismatch", err)
}
for _, line := range mismatch.delta {
if !strings.Contains(line, "kro.run/v1alpha1|Tenant|flux-system|business-site ") {
t.Fatalf("source-only mutation changed an unrelated authorization: %s", line)
t.Fatalf("staging mutation changed an unrelated authorization: %s", line)
}
}
})
Expand Down
1 change: 1 addition & 0 deletions tests/restrict-tenant-network-policies/kyverno-test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,7 @@ results:
- tenant-ns/allow-web-egress-to-kube-apiserver-selector
- tenant-ns/allow-web-ingress-from-documentation-cidr
- tenant-ns/allow-web-ingress-from-gateway
- tenant-ns/allow-web-ingress-from-gateway-egress-default-deny
- tenant-ns/allow-web-ingress-from-ingress-selector
- policy: restrict-tenant-network-policies
kind: CiliumNetworkPolicy
Expand Down
21 changes: 21 additions & 0 deletions tests/restrict-tenant-network-policies/resources.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1650,3 +1650,24 @@ spec:
- k8sService:
serviceName: db
namespace: other-tenant
---
# ALLOW case: the business-site package's own policy. Its one egress rule is
# exactly {}, which allows no destination; it only selects egress default deny.
apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
name: allow-web-ingress-from-gateway-egress-default-deny
namespace: tenant-ns
spec:
endpointSelector:
matchLabels:
app: web
ingress:
- fromEntities:
- ingress
toPorts:
- ports:
- port: "8080"
protocol: TCP
egress:
- {}
14 changes: 14 additions & 0 deletions tests/restrict-tenant-route-hostnames/kyverno-test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,20 @@ results:
- ascoachingogvaner/approved-plus-wildcard
kind: HTTPRoute
result: fail
# --- business-site: one public staging hostname ---
- policy: restrict-tenant-route-hostnames
resources:
- business-site/approved-staging-hostname
kind: HTTPRoute
result: pass
- policy: restrict-tenant-route-hostnames
resources:
- business-site/production-hostname
- business-site/steals-wedding-hostname
- business-site/no-hostnames
- business-site/wildcard-covering-approved
kind: HTTPRoute
result: fail
# --- fail-closed catch-all ---
- policy: restrict-tenant-route-hostnames
resources:
Expand Down
66 changes: 66 additions & 0 deletions tests/restrict-tenant-route-hostnames/resources.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -252,3 +252,69 @@ spec:
- backendRefs:
- name: ascoachingogvaner
port: 80
---
# business-site — the KRO-rendered route from its tenant-owned WebApp. Only the
# public staging hostname is approved.
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: approved-staging-hostname
namespace: business-site
spec:
hostnames:
- business-site.platform.devantler.tech
rules:
- backendRefs:
- name: business-site
port: 80
---
# The production hostname stays on GitHub Pages until a separate cutover.
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: production-hostname
namespace: business-site
spec:
hostnames:
- devantler.tech
rules:
- backendRefs:
- name: business-site
port: 80
---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: steals-wedding-hostname
namespace: business-site
spec:
hostnames:
- wedding.platform.devantler.tech
rules:
- backendRefs:
- name: business-site
port: 80
---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: no-hostnames
namespace: business-site
spec:
rules:
- backendRefs:
- name: business-site
port: 80
---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: wildcard-covering-approved
namespace: business-site
spec:
hostnames:
- "*.platform.devantler.tech"
rules:
- backendRefs:
- name: business-site
port: 80
3 changes: 3 additions & 0 deletions tests/restrict-tenant-route-hostnames/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -17,3 +17,6 @@ namespaceSelector:
app.kubernetes.io/managed-by: ksail
- name: platform-ns
labels: {}
- name: business-site
labels:
app.kubernetes.io/managed-by: kro
Loading