From 418569970094be3d22ff38f481a0bff298146952 Mon Sep 17 00:00:00 2001 From: Fernando Escolar Date: Wed, 16 Sep 2026 16:57:29 +0200 Subject: [PATCH 01/18] feat: add guardrail policy foundation with regex evaluation - add GuardrailPolicy CRD in v1alpha1 and v1beta1 - support dual-version schema and runtime config wiring - compile regex-based guardrails into runtime matcher state - enforce request/response guardrails in extproc before upstream or client response - add guardrail regression tests for runtime evaluation Signed-off-by: Fernando Escolar --- .../typed/api/v1alpha1/api_client.go | 5 + .../api/v1alpha1/fake/fake_api_client.go | 4 + .../api/v1alpha1/fake/fake_guardrailpolicy.go | 41 ++ .../typed/api/v1alpha1/generated_expansion.go | 2 + .../typed/api/v1alpha1/guardrailpolicy.go | 59 ++ .../api/v1alpha1/guardrailpolicy.go | 91 +++ .../api/v1alpha1/interface.go | 7 + .../informers/externalversions/generic.go | 2 + .../api/v1alpha1/expansion_generated.go | 8 + .../listers/api/v1alpha1/guardrailpolicy.go | 59 ++ api/v1alpha1/guardrail_policy.go | 114 ++++ api/v1alpha1/registry.go | 3 + api/v1alpha1/status.go | 9 + api/v1alpha1/zz_generated.deepcopy.go | 144 ++++- .../versioned/typed/api/v1beta1/api_client.go | 5 + .../typed/api/v1beta1/fake/fake_api_client.go | 4 + .../api/v1beta1/fake/fake_guardrailpolicy.go | 41 ++ .../typed/api/v1beta1/generated_expansion.go | 2 + .../typed/api/v1beta1/guardrailpolicy.go | 59 ++ .../api/v1beta1/guardrailpolicy.go | 91 +++ .../externalversions/api/v1beta1/interface.go | 7 + .../informers/externalversions/generic.go | 2 + .../api/v1beta1/expansion_generated.go | 8 + .../listers/api/v1beta1/guardrailpolicy.go | 59 ++ api/v1beta1/guardrail_policy.go | 114 ++++ api/v1beta1/registry.go | 3 + api/v1beta1/status.go | 9 + api/v1beta1/zz_generated.deepcopy.go | 139 ++++- internal/extproc/guardrails.go | 48 ++ internal/extproc/guardrails_test.go | 65 ++ internal/extproc/processor_impl.go | 11 + internal/filterapi/filterconfig.go | 42 ++ internal/filterapi/runtime.go | 38 ++ internal/filterapi/runtime_test.go | 20 + ...teway.envoyproxy.io_guardrailpolicies.yaml | 410 +++++++++++++ site/docs/api/api.mdx | 560 ++++++++++++++++++ 36 files changed, 2278 insertions(+), 7 deletions(-) create mode 100644 api/v1alpha1/client/clientset/versioned/typed/api/v1alpha1/fake/fake_guardrailpolicy.go create mode 100644 api/v1alpha1/client/clientset/versioned/typed/api/v1alpha1/guardrailpolicy.go create mode 100644 api/v1alpha1/client/informers/externalversions/api/v1alpha1/guardrailpolicy.go create mode 100644 api/v1alpha1/client/listers/api/v1alpha1/guardrailpolicy.go create mode 100644 api/v1alpha1/guardrail_policy.go create mode 100644 api/v1beta1/client/clientset/versioned/typed/api/v1beta1/fake/fake_guardrailpolicy.go create mode 100644 api/v1beta1/client/clientset/versioned/typed/api/v1beta1/guardrailpolicy.go create mode 100644 api/v1beta1/client/informers/externalversions/api/v1beta1/guardrailpolicy.go create mode 100644 api/v1beta1/client/listers/api/v1beta1/guardrailpolicy.go create mode 100644 api/v1beta1/guardrail_policy.go create mode 100644 internal/extproc/guardrails.go create mode 100644 internal/extproc/guardrails_test.go create mode 100644 manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_guardrailpolicies.yaml diff --git a/api/v1alpha1/client/clientset/versioned/typed/api/v1alpha1/api_client.go b/api/v1alpha1/client/clientset/versioned/typed/api/v1alpha1/api_client.go index 5ee81af3c9..1f0d60a5ed 100644 --- a/api/v1alpha1/client/clientset/versioned/typed/api/v1alpha1/api_client.go +++ b/api/v1alpha1/client/clientset/versioned/typed/api/v1alpha1/api_client.go @@ -21,6 +21,7 @@ type AigatewayV1alpha1Interface interface { AIServiceBackendsGetter BackendSecurityPoliciesGetter GatewayConfigsGetter + GuardrailPoliciesGetter MCPRoutesGetter QuotaPoliciesGetter } @@ -46,6 +47,10 @@ func (c *AigatewayV1alpha1Client) GatewayConfigs(namespace string) GatewayConfig return newGatewayConfigs(c, namespace) } +func (c *AigatewayV1alpha1Client) GuardrailPolicies(namespace string) GuardrailPolicyInterface { + return newGuardrailPolicies(c, namespace) +} + func (c *AigatewayV1alpha1Client) MCPRoutes(namespace string) MCPRouteInterface { return newMCPRoutes(c, namespace) } diff --git a/api/v1alpha1/client/clientset/versioned/typed/api/v1alpha1/fake/fake_api_client.go b/api/v1alpha1/client/clientset/versioned/typed/api/v1alpha1/fake/fake_api_client.go index 5e14621080..b17694ba5d 100644 --- a/api/v1alpha1/client/clientset/versioned/typed/api/v1alpha1/fake/fake_api_client.go +++ b/api/v1alpha1/client/clientset/versioned/typed/api/v1alpha1/fake/fake_api_client.go @@ -33,6 +33,10 @@ func (c *FakeAigatewayV1alpha1) GatewayConfigs(namespace string) v1alpha1.Gatewa return newFakeGatewayConfigs(c, namespace) } +func (c *FakeAigatewayV1alpha1) GuardrailPolicies(namespace string) v1alpha1.GuardrailPolicyInterface { + return newFakeGuardrailPolicies(c, namespace) +} + func (c *FakeAigatewayV1alpha1) MCPRoutes(namespace string) v1alpha1.MCPRouteInterface { return newFakeMCPRoutes(c, namespace) } diff --git a/api/v1alpha1/client/clientset/versioned/typed/api/v1alpha1/fake/fake_guardrailpolicy.go b/api/v1alpha1/client/clientset/versioned/typed/api/v1alpha1/fake/fake_guardrailpolicy.go new file mode 100644 index 0000000000..78f1960461 --- /dev/null +++ b/api/v1alpha1/client/clientset/versioned/typed/api/v1alpha1/fake/fake_guardrailpolicy.go @@ -0,0 +1,41 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + v1alpha1 "github.com/envoyproxy/ai-gateway/api/v1alpha1" + apiv1alpha1 "github.com/envoyproxy/ai-gateway/api/v1alpha1/client/clientset/versioned/typed/api/v1alpha1" + gentype "k8s.io/client-go/gentype" +) + +// fakeGuardrailPolicies implements GuardrailPolicyInterface +type fakeGuardrailPolicies struct { + *gentype.FakeClientWithList[*v1alpha1.GuardrailPolicy, *v1alpha1.GuardrailPolicyList] + Fake *FakeAigatewayV1alpha1 +} + +func newFakeGuardrailPolicies(fake *FakeAigatewayV1alpha1, namespace string) apiv1alpha1.GuardrailPolicyInterface { + return &fakeGuardrailPolicies{ + gentype.NewFakeClientWithList[*v1alpha1.GuardrailPolicy, *v1alpha1.GuardrailPolicyList]( + fake.Fake, + namespace, + v1alpha1.SchemeGroupVersion.WithResource("guardrailpolicies"), + v1alpha1.SchemeGroupVersion.WithKind("GuardrailPolicy"), + func() *v1alpha1.GuardrailPolicy { return &v1alpha1.GuardrailPolicy{} }, + func() *v1alpha1.GuardrailPolicyList { return &v1alpha1.GuardrailPolicyList{} }, + func(dst, src *v1alpha1.GuardrailPolicyList) { dst.ListMeta = src.ListMeta }, + func(list *v1alpha1.GuardrailPolicyList) []*v1alpha1.GuardrailPolicy { + return gentype.ToPointerSlice(list.Items) + }, + func(list *v1alpha1.GuardrailPolicyList, items []*v1alpha1.GuardrailPolicy) { + list.Items = gentype.FromPointerSlice(items) + }, + ), + fake, + } +} diff --git a/api/v1alpha1/client/clientset/versioned/typed/api/v1alpha1/generated_expansion.go b/api/v1alpha1/client/clientset/versioned/typed/api/v1alpha1/generated_expansion.go index 3b0636bead..1c6e837fdf 100644 --- a/api/v1alpha1/client/clientset/versioned/typed/api/v1alpha1/generated_expansion.go +++ b/api/v1alpha1/client/clientset/versioned/typed/api/v1alpha1/generated_expansion.go @@ -15,6 +15,8 @@ type BackendSecurityPolicyExpansion interface{} type GatewayConfigExpansion interface{} +type GuardrailPolicyExpansion interface{} + type MCPRouteExpansion interface{} type QuotaPolicyExpansion interface{} diff --git a/api/v1alpha1/client/clientset/versioned/typed/api/v1alpha1/guardrailpolicy.go b/api/v1alpha1/client/clientset/versioned/typed/api/v1alpha1/guardrailpolicy.go new file mode 100644 index 0000000000..55bffbf124 --- /dev/null +++ b/api/v1alpha1/client/clientset/versioned/typed/api/v1alpha1/guardrailpolicy.go @@ -0,0 +1,59 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + context "context" + + apiv1alpha1 "github.com/envoyproxy/ai-gateway/api/v1alpha1" + scheme "github.com/envoyproxy/ai-gateway/api/v1alpha1/client/clientset/versioned/scheme" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + gentype "k8s.io/client-go/gentype" +) + +// GuardrailPoliciesGetter has a method to return a GuardrailPolicyInterface. +// A group's client should implement this interface. +type GuardrailPoliciesGetter interface { + GuardrailPolicies(namespace string) GuardrailPolicyInterface +} + +// GuardrailPolicyInterface has methods to work with GuardrailPolicy resources. +type GuardrailPolicyInterface interface { + Create(ctx context.Context, guardrailPolicy *apiv1alpha1.GuardrailPolicy, opts v1.CreateOptions) (*apiv1alpha1.GuardrailPolicy, error) + Update(ctx context.Context, guardrailPolicy *apiv1alpha1.GuardrailPolicy, opts v1.UpdateOptions) (*apiv1alpha1.GuardrailPolicy, error) + // Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). + UpdateStatus(ctx context.Context, guardrailPolicy *apiv1alpha1.GuardrailPolicy, opts v1.UpdateOptions) (*apiv1alpha1.GuardrailPolicy, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*apiv1alpha1.GuardrailPolicy, error) + List(ctx context.Context, opts v1.ListOptions) (*apiv1alpha1.GuardrailPolicyList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *apiv1alpha1.GuardrailPolicy, err error) + GuardrailPolicyExpansion +} + +// guardrailPolicies implements GuardrailPolicyInterface +type guardrailPolicies struct { + *gentype.ClientWithList[*apiv1alpha1.GuardrailPolicy, *apiv1alpha1.GuardrailPolicyList] +} + +// newGuardrailPolicies returns a GuardrailPolicies +func newGuardrailPolicies(c *AigatewayV1alpha1Client, namespace string) *guardrailPolicies { + return &guardrailPolicies{ + gentype.NewClientWithList[*apiv1alpha1.GuardrailPolicy, *apiv1alpha1.GuardrailPolicyList]( + "guardrailpolicies", + c.RESTClient(), + scheme.ParameterCodec, + namespace, + func() *apiv1alpha1.GuardrailPolicy { return &apiv1alpha1.GuardrailPolicy{} }, + func() *apiv1alpha1.GuardrailPolicyList { return &apiv1alpha1.GuardrailPolicyList{} }, + ), + } +} diff --git a/api/v1alpha1/client/informers/externalversions/api/v1alpha1/guardrailpolicy.go b/api/v1alpha1/client/informers/externalversions/api/v1alpha1/guardrailpolicy.go new file mode 100644 index 0000000000..8996cbdc5e --- /dev/null +++ b/api/v1alpha1/client/informers/externalversions/api/v1alpha1/guardrailpolicy.go @@ -0,0 +1,91 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +// Code generated by informer-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + context "context" + time "time" + + aigatewayapiv1alpha1 "github.com/envoyproxy/ai-gateway/api/v1alpha1" + versioned "github.com/envoyproxy/ai-gateway/api/v1alpha1/client/clientset/versioned" + internalinterfaces "github.com/envoyproxy/ai-gateway/api/v1alpha1/client/informers/externalversions/internalinterfaces" + apiv1alpha1 "github.com/envoyproxy/ai-gateway/api/v1alpha1/client/listers/api/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// GuardrailPolicyInformer provides access to a shared informer and lister for +// GuardrailPolicies. +type GuardrailPolicyInformer interface { + Informer() cache.SharedIndexInformer + Lister() apiv1alpha1.GuardrailPolicyLister +} + +type guardrailPolicyInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc + namespace string +} + +// NewGuardrailPolicyInformer constructs a new informer for GuardrailPolicy type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewGuardrailPolicyInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredGuardrailPolicyInformer(client, namespace, resyncPeriod, indexers, nil) +} + +// NewFilteredGuardrailPolicyInformer constructs a new informer for GuardrailPolicy type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredGuardrailPolicyInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + cache.ToListWatcherWithWatchListSemantics(&cache.ListWatch{ + ListFunc: func(options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.AigatewayV1alpha1().GuardrailPolicies(namespace).List(context.Background(), options) + }, + WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.AigatewayV1alpha1().GuardrailPolicies(namespace).Watch(context.Background(), options) + }, + ListWithContextFunc: func(ctx context.Context, options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.AigatewayV1alpha1().GuardrailPolicies(namespace).List(ctx, options) + }, + WatchFuncWithContext: func(ctx context.Context, options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.AigatewayV1alpha1().GuardrailPolicies(namespace).Watch(ctx, options) + }, + }, client), + &aigatewayapiv1alpha1.GuardrailPolicy{}, + resyncPeriod, + indexers, + ) +} + +func (f *guardrailPolicyInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredGuardrailPolicyInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *guardrailPolicyInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&aigatewayapiv1alpha1.GuardrailPolicy{}, f.defaultInformer) +} + +func (f *guardrailPolicyInformer) Lister() apiv1alpha1.GuardrailPolicyLister { + return apiv1alpha1.NewGuardrailPolicyLister(f.Informer().GetIndexer()) +} diff --git a/api/v1alpha1/client/informers/externalversions/api/v1alpha1/interface.go b/api/v1alpha1/client/informers/externalversions/api/v1alpha1/interface.go index a208a0db37..ee83a90986 100644 --- a/api/v1alpha1/client/informers/externalversions/api/v1alpha1/interface.go +++ b/api/v1alpha1/client/informers/externalversions/api/v1alpha1/interface.go @@ -21,6 +21,8 @@ type Interface interface { BackendSecurityPolicies() BackendSecurityPolicyInformer // GatewayConfigs returns a GatewayConfigInformer. GatewayConfigs() GatewayConfigInformer + // GuardrailPolicies returns a GuardrailPolicyInformer. + GuardrailPolicies() GuardrailPolicyInformer // MCPRoutes returns a MCPRouteInformer. MCPRoutes() MCPRouteInformer // QuotaPolicies returns a QuotaPolicyInformer. @@ -58,6 +60,11 @@ func (v *version) GatewayConfigs() GatewayConfigInformer { return &gatewayConfigInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} } +// GuardrailPolicies returns a GuardrailPolicyInformer. +func (v *version) GuardrailPolicies() GuardrailPolicyInformer { + return &guardrailPolicyInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} +} + // MCPRoutes returns a MCPRouteInformer. func (v *version) MCPRoutes() MCPRouteInformer { return &mCPRouteInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} diff --git a/api/v1alpha1/client/informers/externalversions/generic.go b/api/v1alpha1/client/informers/externalversions/generic.go index 171d834867..09c8c134a9 100644 --- a/api/v1alpha1/client/informers/externalversions/generic.go +++ b/api/v1alpha1/client/informers/externalversions/generic.go @@ -50,6 +50,8 @@ func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource return &genericInformer{resource: resource.GroupResource(), informer: f.Aigateway().V1alpha1().BackendSecurityPolicies().Informer()}, nil case v1alpha1.SchemeGroupVersion.WithResource("gatewayconfigs"): return &genericInformer{resource: resource.GroupResource(), informer: f.Aigateway().V1alpha1().GatewayConfigs().Informer()}, nil + case v1alpha1.SchemeGroupVersion.WithResource("guardrailpolicies"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Aigateway().V1alpha1().GuardrailPolicies().Informer()}, nil case v1alpha1.SchemeGroupVersion.WithResource("mcproutes"): return &genericInformer{resource: resource.GroupResource(), informer: f.Aigateway().V1alpha1().MCPRoutes().Informer()}, nil case v1alpha1.SchemeGroupVersion.WithResource("quotapolicies"): diff --git a/api/v1alpha1/client/listers/api/v1alpha1/expansion_generated.go b/api/v1alpha1/client/listers/api/v1alpha1/expansion_generated.go index ed502a96bb..ad81ab7c21 100644 --- a/api/v1alpha1/client/listers/api/v1alpha1/expansion_generated.go +++ b/api/v1alpha1/client/listers/api/v1alpha1/expansion_generated.go @@ -39,6 +39,14 @@ type GatewayConfigListerExpansion interface{} // GatewayConfigNamespaceLister. type GatewayConfigNamespaceListerExpansion interface{} +// GuardrailPolicyListerExpansion allows custom methods to be added to +// GuardrailPolicyLister. +type GuardrailPolicyListerExpansion interface{} + +// GuardrailPolicyNamespaceListerExpansion allows custom methods to be added to +// GuardrailPolicyNamespaceLister. +type GuardrailPolicyNamespaceListerExpansion interface{} + // MCPRouteListerExpansion allows custom methods to be added to // MCPRouteLister. type MCPRouteListerExpansion interface{} diff --git a/api/v1alpha1/client/listers/api/v1alpha1/guardrailpolicy.go b/api/v1alpha1/client/listers/api/v1alpha1/guardrailpolicy.go new file mode 100644 index 0000000000..b1ff880e1f --- /dev/null +++ b/api/v1alpha1/client/listers/api/v1alpha1/guardrailpolicy.go @@ -0,0 +1,59 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +// Code generated by lister-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + apiv1alpha1 "github.com/envoyproxy/ai-gateway/api/v1alpha1" + labels "k8s.io/apimachinery/pkg/labels" + listers "k8s.io/client-go/listers" + cache "k8s.io/client-go/tools/cache" +) + +// GuardrailPolicyLister helps list GuardrailPolicies. +// All objects returned here must be treated as read-only. +type GuardrailPolicyLister interface { + // List lists all GuardrailPolicies in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*apiv1alpha1.GuardrailPolicy, err error) + // GuardrailPolicies returns an object that can list and get GuardrailPolicies. + GuardrailPolicies(namespace string) GuardrailPolicyNamespaceLister + GuardrailPolicyListerExpansion +} + +// guardrailPolicyLister implements the GuardrailPolicyLister interface. +type guardrailPolicyLister struct { + listers.ResourceIndexer[*apiv1alpha1.GuardrailPolicy] +} + +// NewGuardrailPolicyLister returns a new GuardrailPolicyLister. +func NewGuardrailPolicyLister(indexer cache.Indexer) GuardrailPolicyLister { + return &guardrailPolicyLister{listers.New[*apiv1alpha1.GuardrailPolicy](indexer, apiv1alpha1.Resource("guardrailpolicy"))} +} + +// GuardrailPolicies returns an object that can list and get GuardrailPolicies. +func (s *guardrailPolicyLister) GuardrailPolicies(namespace string) GuardrailPolicyNamespaceLister { + return guardrailPolicyNamespaceLister{listers.NewNamespaced[*apiv1alpha1.GuardrailPolicy](s.ResourceIndexer, namespace)} +} + +// GuardrailPolicyNamespaceLister helps list and get GuardrailPolicies. +// All objects returned here must be treated as read-only. +type GuardrailPolicyNamespaceLister interface { + // List lists all GuardrailPolicies in the indexer for a given namespace. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*apiv1alpha1.GuardrailPolicy, err error) + // Get retrieves the GuardrailPolicy from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. + Get(name string) (*apiv1alpha1.GuardrailPolicy, error) + GuardrailPolicyNamespaceListerExpansion +} + +// guardrailPolicyNamespaceLister implements the GuardrailPolicyNamespaceLister +// interface. +type guardrailPolicyNamespaceLister struct { + listers.ResourceIndexer[*apiv1alpha1.GuardrailPolicy] +} diff --git a/api/v1alpha1/guardrail_policy.go b/api/v1alpha1/guardrail_policy.go new file mode 100644 index 0000000000..bc30b97a68 --- /dev/null +++ b/api/v1alpha1/guardrail_policy.go @@ -0,0 +1,114 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" +) + +// GuardrailPolicy evaluates content safety checks for request and response payloads. +// +// +genclient +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.conditions[-1:].type` +// +kubebuilder:metadata:labels="gateway.networking.k8s.io/policy=direct" +// +kubebuilder:deprecatedversion:warning="aigateway.envoyproxy.io/v1alpha1 is deprecated; use aigateway.envoyproxy.io/v1beta1 instead" +type GuardrailPolicy struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + Spec GuardrailPolicySpec `json:"spec,omitempty"` + // Status defines the status details of the GuardrailPolicy. + Status GuardrailPolicyStatus `json:"status,omitempty"` +} + +// GuardrailPolicySpec contains the configured checks attached to an AIServiceBackend. +type GuardrailPolicySpec struct { + // TargetRefs are the names of the AIServiceBackend resources this GuardrailPolicy is attached to. + // + // +optional + // +kubebuilder:validation:MaxItems=16 + // +kubebuilder:validation:XValidation:rule="self.all(ref, ref.group == 'aigateway.envoyproxy.io' && ref.kind == 'AIServiceBackend')", message="targetRefs must reference AIServiceBackend resources" + TargetRefs []gwapiv1a2.LocalPolicyTargetReference `json:"targetRefs,omitempty"` + // Rules are executed in order and can evaluate request or response payloads. + // + // +optional + // +kubebuilder:validation:MaxItems=32 + Rules []GuardrailRule `json:"rules,omitempty"` +} + +// GuardrailRule defines one content-safety check to apply to a request or response. +type GuardrailRule struct { + // Name is a stable identifier for the rule. + // + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + Name string `json:"name"` + // Phase determines whether the rule runs against the request or the response payload. + // + // +kubebuilder:validation:Enum=Request;Response + Phase GuardrailPhase `json:"phase"` + // Provider configures how the rule is evaluated. + Provider GuardrailProvider `json:"provider"` +} + +// GuardrailPhase determines when a guardrail runs. +type GuardrailPhase string + +const ( + GuardrailPhaseRequest GuardrailPhase = "Request" + GuardrailPhaseResponse GuardrailPhase = "Response" +) + +// GuardrailProvider describes the implementation used to evaluate a rule. +type GuardrailProvider struct { + // Type identifies the guardrail implementation. + // + // +kubebuilder:validation:Enum=Regex;Presidio;Bedrock;AzureContentSafety + Type GuardrailProviderType `json:"type"` + // Pattern is used for deterministic regex-based evaluations. + // + // +optional + Pattern string `json:"pattern,omitempty"` + // Action is the action taken when the rule is matched. + // + // +optional + // +kubebuilder:default=Block + Action GuardrailAction `json:"action,omitempty"` + // Message is returned to the caller when the rule blocks a request or response. + // + // +optional + Message string `json:"message,omitempty"` +} + +// GuardrailProviderType is the guardrail implementation. +type GuardrailProviderType string + +const ( + GuardrailProviderTypeRegex GuardrailProviderType = "Regex" + GuardrailProviderTypePresidio GuardrailProviderType = "Presidio" + GuardrailProviderTypeBedrockGuardrails GuardrailProviderType = "Bedrock" + GuardrailProviderTypeAzureContentSafety GuardrailProviderType = "AzureContentSafety" +) + +// GuardrailAction defines the safeguard action. +type GuardrailAction string + +const ( + GuardrailActionBlock GuardrailAction = "Block" +) + +// GuardrailPolicyList contains a list of GuardrailPolicy resources. +// +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:object:root=true +type GuardrailPolicyList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []GuardrailPolicy `json:"items"` +} diff --git a/api/v1alpha1/registry.go b/api/v1alpha1/registry.go index 92bfd4e2cb..92f175cd12 100644 --- a/api/v1alpha1/registry.go +++ b/api/v1alpha1/registry.go @@ -19,6 +19,7 @@ func init() { SchemeBuilder.Register(&MCPRoute{}, &MCPRouteList{}) SchemeBuilder.Register(&GatewayConfig{}, &GatewayConfigList{}) SchemeBuilder.Register(&QuotaPolicy{}, &QuotaPolicyList{}) + SchemeBuilder.Register(&GuardrailPolicy{}, &GuardrailPolicyList{}) } const GroupName = "aigateway.envoyproxy.io" @@ -54,6 +55,8 @@ func AddKnownTypes(scheme *runtime.Scheme) error { &GatewayConfigList{}, &QuotaPolicy{}, &QuotaPolicyList{}, + &GuardrailPolicy{}, + &GuardrailPolicyList{}, ) metav1.AddToGroupVersion(scheme, SchemeGroupVersion) return nil diff --git a/api/v1alpha1/status.go b/api/v1alpha1/status.go index 4874d1b9b8..f6f5edd52b 100644 --- a/api/v1alpha1/status.go +++ b/api/v1alpha1/status.go @@ -52,6 +52,15 @@ type MCPRouteStatus struct { Conditions []metav1.Condition `json:"conditions,omitempty"` } +// GuardrailPolicyStatus contains the conditions by the reconciliation result. +type GuardrailPolicyStatus struct { + // Conditions is the list of conditions by the reconciliation result. + // Currently, at most one condition is set. + // + // Known .status.conditions.type are: "Accepted", "NotAccepted". + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + // QuotaPolicyStatus contains the conditions by the reconciliation result. type QuotaPolicyStatus struct { // Conditions is the list of conditions by the reconciliation result. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 3c44727050..e43e5d825e 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -1,8 +1,3 @@ -// Copyright Envoy AI Gateway Authors -// SPDX-License-Identifier: Apache-2.0 -// The full text of the Apache license is available in the LICENSE file at -// the root of the repo. - //go:build !ignore_autogenerated // Code generated by controller-gen. DO NOT EDIT. @@ -13,7 +8,7 @@ import ( apiv1alpha1 "github.com/envoyproxy/gateway/api/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "sigs.k8s.io/gateway-api/apis/v1" + v1 "sigs.k8s.io/gateway-api/apis/v1" "sigs.k8s.io/gateway-api/apis/v1alpha2" ) @@ -942,6 +937,143 @@ func (in *GatewayConfigStatus) DeepCopy() *GatewayConfigStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GuardrailPolicy) DeepCopyInto(out *GuardrailPolicy) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GuardrailPolicy. +func (in *GuardrailPolicy) DeepCopy() *GuardrailPolicy { + if in == nil { + return nil + } + out := new(GuardrailPolicy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *GuardrailPolicy) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GuardrailPolicyList) DeepCopyInto(out *GuardrailPolicyList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]GuardrailPolicy, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GuardrailPolicyList. +func (in *GuardrailPolicyList) DeepCopy() *GuardrailPolicyList { + if in == nil { + return nil + } + out := new(GuardrailPolicyList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *GuardrailPolicyList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GuardrailPolicySpec) DeepCopyInto(out *GuardrailPolicySpec) { + *out = *in + if in.TargetRefs != nil { + in, out := &in.TargetRefs, &out.TargetRefs + *out = make([]v1alpha2.LocalPolicyTargetReference, len(*in)) + copy(*out, *in) + } + if in.Rules != nil { + in, out := &in.Rules, &out.Rules + *out = make([]GuardrailRule, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GuardrailPolicySpec. +func (in *GuardrailPolicySpec) DeepCopy() *GuardrailPolicySpec { + if in == nil { + return nil + } + out := new(GuardrailPolicySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GuardrailPolicyStatus) DeepCopyInto(out *GuardrailPolicyStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]metav1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GuardrailPolicyStatus. +func (in *GuardrailPolicyStatus) DeepCopy() *GuardrailPolicyStatus { + if in == nil { + return nil + } + out := new(GuardrailPolicyStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GuardrailProvider) DeepCopyInto(out *GuardrailProvider) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GuardrailProvider. +func (in *GuardrailProvider) DeepCopy() *GuardrailProvider { + if in == nil { + return nil + } + out := new(GuardrailProvider) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GuardrailRule) DeepCopyInto(out *GuardrailRule) { + *out = *in + out.Provider = in.Provider +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GuardrailRule. +func (in *GuardrailRule) DeepCopy() *GuardrailRule { + if in == nil { + return nil + } + out := new(GuardrailRule) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HTTPBodyField) DeepCopyInto(out *HTTPBodyField) { *out = *in diff --git a/api/v1beta1/client/clientset/versioned/typed/api/v1beta1/api_client.go b/api/v1beta1/client/clientset/versioned/typed/api/v1beta1/api_client.go index 6b6cc75409..21ef334fc4 100644 --- a/api/v1beta1/client/clientset/versioned/typed/api/v1beta1/api_client.go +++ b/api/v1beta1/client/clientset/versioned/typed/api/v1beta1/api_client.go @@ -21,6 +21,7 @@ type AigatewayV1beta1Interface interface { AIServiceBackendsGetter BackendSecurityPoliciesGetter GatewayConfigsGetter + GuardrailPoliciesGetter MCPRoutesGetter } @@ -45,6 +46,10 @@ func (c *AigatewayV1beta1Client) GatewayConfigs(namespace string) GatewayConfigI return newGatewayConfigs(c, namespace) } +func (c *AigatewayV1beta1Client) GuardrailPolicies(namespace string) GuardrailPolicyInterface { + return newGuardrailPolicies(c, namespace) +} + func (c *AigatewayV1beta1Client) MCPRoutes(namespace string) MCPRouteInterface { return newMCPRoutes(c, namespace) } diff --git a/api/v1beta1/client/clientset/versioned/typed/api/v1beta1/fake/fake_api_client.go b/api/v1beta1/client/clientset/versioned/typed/api/v1beta1/fake/fake_api_client.go index d05148a148..6deb1795df 100644 --- a/api/v1beta1/client/clientset/versioned/typed/api/v1beta1/fake/fake_api_client.go +++ b/api/v1beta1/client/clientset/versioned/typed/api/v1beta1/fake/fake_api_client.go @@ -33,6 +33,10 @@ func (c *FakeAigatewayV1beta1) GatewayConfigs(namespace string) v1beta1.GatewayC return newFakeGatewayConfigs(c, namespace) } +func (c *FakeAigatewayV1beta1) GuardrailPolicies(namespace string) v1beta1.GuardrailPolicyInterface { + return newFakeGuardrailPolicies(c, namespace) +} + func (c *FakeAigatewayV1beta1) MCPRoutes(namespace string) v1beta1.MCPRouteInterface { return newFakeMCPRoutes(c, namespace) } diff --git a/api/v1beta1/client/clientset/versioned/typed/api/v1beta1/fake/fake_guardrailpolicy.go b/api/v1beta1/client/clientset/versioned/typed/api/v1beta1/fake/fake_guardrailpolicy.go new file mode 100644 index 0000000000..e7ea5555df --- /dev/null +++ b/api/v1beta1/client/clientset/versioned/typed/api/v1beta1/fake/fake_guardrailpolicy.go @@ -0,0 +1,41 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + v1beta1 "github.com/envoyproxy/ai-gateway/api/v1beta1" + apiv1beta1 "github.com/envoyproxy/ai-gateway/api/v1beta1/client/clientset/versioned/typed/api/v1beta1" + gentype "k8s.io/client-go/gentype" +) + +// fakeGuardrailPolicies implements GuardrailPolicyInterface +type fakeGuardrailPolicies struct { + *gentype.FakeClientWithList[*v1beta1.GuardrailPolicy, *v1beta1.GuardrailPolicyList] + Fake *FakeAigatewayV1beta1 +} + +func newFakeGuardrailPolicies(fake *FakeAigatewayV1beta1, namespace string) apiv1beta1.GuardrailPolicyInterface { + return &fakeGuardrailPolicies{ + gentype.NewFakeClientWithList[*v1beta1.GuardrailPolicy, *v1beta1.GuardrailPolicyList]( + fake.Fake, + namespace, + v1beta1.SchemeGroupVersion.WithResource("guardrailpolicies"), + v1beta1.SchemeGroupVersion.WithKind("GuardrailPolicy"), + func() *v1beta1.GuardrailPolicy { return &v1beta1.GuardrailPolicy{} }, + func() *v1beta1.GuardrailPolicyList { return &v1beta1.GuardrailPolicyList{} }, + func(dst, src *v1beta1.GuardrailPolicyList) { dst.ListMeta = src.ListMeta }, + func(list *v1beta1.GuardrailPolicyList) []*v1beta1.GuardrailPolicy { + return gentype.ToPointerSlice(list.Items) + }, + func(list *v1beta1.GuardrailPolicyList, items []*v1beta1.GuardrailPolicy) { + list.Items = gentype.FromPointerSlice(items) + }, + ), + fake, + } +} diff --git a/api/v1beta1/client/clientset/versioned/typed/api/v1beta1/generated_expansion.go b/api/v1beta1/client/clientset/versioned/typed/api/v1beta1/generated_expansion.go index 57e2bb7675..9ec2c16cdc 100644 --- a/api/v1beta1/client/clientset/versioned/typed/api/v1beta1/generated_expansion.go +++ b/api/v1beta1/client/clientset/versioned/typed/api/v1beta1/generated_expansion.go @@ -15,4 +15,6 @@ type BackendSecurityPolicyExpansion interface{} type GatewayConfigExpansion interface{} +type GuardrailPolicyExpansion interface{} + type MCPRouteExpansion interface{} diff --git a/api/v1beta1/client/clientset/versioned/typed/api/v1beta1/guardrailpolicy.go b/api/v1beta1/client/clientset/versioned/typed/api/v1beta1/guardrailpolicy.go new file mode 100644 index 0000000000..b82deb93e2 --- /dev/null +++ b/api/v1beta1/client/clientset/versioned/typed/api/v1beta1/guardrailpolicy.go @@ -0,0 +1,59 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +// Code generated by client-gen. DO NOT EDIT. + +package v1beta1 + +import ( + context "context" + + apiv1beta1 "github.com/envoyproxy/ai-gateway/api/v1beta1" + scheme "github.com/envoyproxy/ai-gateway/api/v1beta1/client/clientset/versioned/scheme" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + gentype "k8s.io/client-go/gentype" +) + +// GuardrailPoliciesGetter has a method to return a GuardrailPolicyInterface. +// A group's client should implement this interface. +type GuardrailPoliciesGetter interface { + GuardrailPolicies(namespace string) GuardrailPolicyInterface +} + +// GuardrailPolicyInterface has methods to work with GuardrailPolicy resources. +type GuardrailPolicyInterface interface { + Create(ctx context.Context, guardrailPolicy *apiv1beta1.GuardrailPolicy, opts v1.CreateOptions) (*apiv1beta1.GuardrailPolicy, error) + Update(ctx context.Context, guardrailPolicy *apiv1beta1.GuardrailPolicy, opts v1.UpdateOptions) (*apiv1beta1.GuardrailPolicy, error) + // Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). + UpdateStatus(ctx context.Context, guardrailPolicy *apiv1beta1.GuardrailPolicy, opts v1.UpdateOptions) (*apiv1beta1.GuardrailPolicy, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*apiv1beta1.GuardrailPolicy, error) + List(ctx context.Context, opts v1.ListOptions) (*apiv1beta1.GuardrailPolicyList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *apiv1beta1.GuardrailPolicy, err error) + GuardrailPolicyExpansion +} + +// guardrailPolicies implements GuardrailPolicyInterface +type guardrailPolicies struct { + *gentype.ClientWithList[*apiv1beta1.GuardrailPolicy, *apiv1beta1.GuardrailPolicyList] +} + +// newGuardrailPolicies returns a GuardrailPolicies +func newGuardrailPolicies(c *AigatewayV1beta1Client, namespace string) *guardrailPolicies { + return &guardrailPolicies{ + gentype.NewClientWithList[*apiv1beta1.GuardrailPolicy, *apiv1beta1.GuardrailPolicyList]( + "guardrailpolicies", + c.RESTClient(), + scheme.ParameterCodec, + namespace, + func() *apiv1beta1.GuardrailPolicy { return &apiv1beta1.GuardrailPolicy{} }, + func() *apiv1beta1.GuardrailPolicyList { return &apiv1beta1.GuardrailPolicyList{} }, + ), + } +} diff --git a/api/v1beta1/client/informers/externalversions/api/v1beta1/guardrailpolicy.go b/api/v1beta1/client/informers/externalversions/api/v1beta1/guardrailpolicy.go new file mode 100644 index 0000000000..c3bbe20eb2 --- /dev/null +++ b/api/v1beta1/client/informers/externalversions/api/v1beta1/guardrailpolicy.go @@ -0,0 +1,91 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +// Code generated by informer-gen. DO NOT EDIT. + +package v1beta1 + +import ( + context "context" + time "time" + + aigatewayapiv1beta1 "github.com/envoyproxy/ai-gateway/api/v1beta1" + versioned "github.com/envoyproxy/ai-gateway/api/v1beta1/client/clientset/versioned" + internalinterfaces "github.com/envoyproxy/ai-gateway/api/v1beta1/client/informers/externalversions/internalinterfaces" + apiv1beta1 "github.com/envoyproxy/ai-gateway/api/v1beta1/client/listers/api/v1beta1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// GuardrailPolicyInformer provides access to a shared informer and lister for +// GuardrailPolicies. +type GuardrailPolicyInformer interface { + Informer() cache.SharedIndexInformer + Lister() apiv1beta1.GuardrailPolicyLister +} + +type guardrailPolicyInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc + namespace string +} + +// NewGuardrailPolicyInformer constructs a new informer for GuardrailPolicy type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewGuardrailPolicyInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredGuardrailPolicyInformer(client, namespace, resyncPeriod, indexers, nil) +} + +// NewFilteredGuardrailPolicyInformer constructs a new informer for GuardrailPolicy type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredGuardrailPolicyInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + cache.ToListWatcherWithWatchListSemantics(&cache.ListWatch{ + ListFunc: func(options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.AigatewayV1beta1().GuardrailPolicies(namespace).List(context.Background(), options) + }, + WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.AigatewayV1beta1().GuardrailPolicies(namespace).Watch(context.Background(), options) + }, + ListWithContextFunc: func(ctx context.Context, options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.AigatewayV1beta1().GuardrailPolicies(namespace).List(ctx, options) + }, + WatchFuncWithContext: func(ctx context.Context, options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.AigatewayV1beta1().GuardrailPolicies(namespace).Watch(ctx, options) + }, + }, client), + &aigatewayapiv1beta1.GuardrailPolicy{}, + resyncPeriod, + indexers, + ) +} + +func (f *guardrailPolicyInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredGuardrailPolicyInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *guardrailPolicyInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&aigatewayapiv1beta1.GuardrailPolicy{}, f.defaultInformer) +} + +func (f *guardrailPolicyInformer) Lister() apiv1beta1.GuardrailPolicyLister { + return apiv1beta1.NewGuardrailPolicyLister(f.Informer().GetIndexer()) +} diff --git a/api/v1beta1/client/informers/externalversions/api/v1beta1/interface.go b/api/v1beta1/client/informers/externalversions/api/v1beta1/interface.go index 587fca5078..3c87eb2bc5 100644 --- a/api/v1beta1/client/informers/externalversions/api/v1beta1/interface.go +++ b/api/v1beta1/client/informers/externalversions/api/v1beta1/interface.go @@ -21,6 +21,8 @@ type Interface interface { BackendSecurityPolicies() BackendSecurityPolicyInformer // GatewayConfigs returns a GatewayConfigInformer. GatewayConfigs() GatewayConfigInformer + // GuardrailPolicies returns a GuardrailPolicyInformer. + GuardrailPolicies() GuardrailPolicyInformer // MCPRoutes returns a MCPRouteInformer. MCPRoutes() MCPRouteInformer } @@ -56,6 +58,11 @@ func (v *version) GatewayConfigs() GatewayConfigInformer { return &gatewayConfigInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} } +// GuardrailPolicies returns a GuardrailPolicyInformer. +func (v *version) GuardrailPolicies() GuardrailPolicyInformer { + return &guardrailPolicyInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} +} + // MCPRoutes returns a MCPRouteInformer. func (v *version) MCPRoutes() MCPRouteInformer { return &mCPRouteInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} diff --git a/api/v1beta1/client/informers/externalversions/generic.go b/api/v1beta1/client/informers/externalversions/generic.go index 2393d2db0b..623b1c6c42 100644 --- a/api/v1beta1/client/informers/externalversions/generic.go +++ b/api/v1beta1/client/informers/externalversions/generic.go @@ -50,6 +50,8 @@ func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource return &genericInformer{resource: resource.GroupResource(), informer: f.Aigateway().V1beta1().BackendSecurityPolicies().Informer()}, nil case v1beta1.SchemeGroupVersion.WithResource("gatewayconfigs"): return &genericInformer{resource: resource.GroupResource(), informer: f.Aigateway().V1beta1().GatewayConfigs().Informer()}, nil + case v1beta1.SchemeGroupVersion.WithResource("guardrailpolicies"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Aigateway().V1beta1().GuardrailPolicies().Informer()}, nil case v1beta1.SchemeGroupVersion.WithResource("mcproutes"): return &genericInformer{resource: resource.GroupResource(), informer: f.Aigateway().V1beta1().MCPRoutes().Informer()}, nil diff --git a/api/v1beta1/client/listers/api/v1beta1/expansion_generated.go b/api/v1beta1/client/listers/api/v1beta1/expansion_generated.go index 7b135a7511..c3a85eac96 100644 --- a/api/v1beta1/client/listers/api/v1beta1/expansion_generated.go +++ b/api/v1beta1/client/listers/api/v1beta1/expansion_generated.go @@ -39,6 +39,14 @@ type GatewayConfigListerExpansion interface{} // GatewayConfigNamespaceLister. type GatewayConfigNamespaceListerExpansion interface{} +// GuardrailPolicyListerExpansion allows custom methods to be added to +// GuardrailPolicyLister. +type GuardrailPolicyListerExpansion interface{} + +// GuardrailPolicyNamespaceListerExpansion allows custom methods to be added to +// GuardrailPolicyNamespaceLister. +type GuardrailPolicyNamespaceListerExpansion interface{} + // MCPRouteListerExpansion allows custom methods to be added to // MCPRouteLister. type MCPRouteListerExpansion interface{} diff --git a/api/v1beta1/client/listers/api/v1beta1/guardrailpolicy.go b/api/v1beta1/client/listers/api/v1beta1/guardrailpolicy.go new file mode 100644 index 0000000000..ed3f50c682 --- /dev/null +++ b/api/v1beta1/client/listers/api/v1beta1/guardrailpolicy.go @@ -0,0 +1,59 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +// Code generated by lister-gen. DO NOT EDIT. + +package v1beta1 + +import ( + apiv1beta1 "github.com/envoyproxy/ai-gateway/api/v1beta1" + labels "k8s.io/apimachinery/pkg/labels" + listers "k8s.io/client-go/listers" + cache "k8s.io/client-go/tools/cache" +) + +// GuardrailPolicyLister helps list GuardrailPolicies. +// All objects returned here must be treated as read-only. +type GuardrailPolicyLister interface { + // List lists all GuardrailPolicies in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*apiv1beta1.GuardrailPolicy, err error) + // GuardrailPolicies returns an object that can list and get GuardrailPolicies. + GuardrailPolicies(namespace string) GuardrailPolicyNamespaceLister + GuardrailPolicyListerExpansion +} + +// guardrailPolicyLister implements the GuardrailPolicyLister interface. +type guardrailPolicyLister struct { + listers.ResourceIndexer[*apiv1beta1.GuardrailPolicy] +} + +// NewGuardrailPolicyLister returns a new GuardrailPolicyLister. +func NewGuardrailPolicyLister(indexer cache.Indexer) GuardrailPolicyLister { + return &guardrailPolicyLister{listers.New[*apiv1beta1.GuardrailPolicy](indexer, apiv1beta1.Resource("guardrailpolicy"))} +} + +// GuardrailPolicies returns an object that can list and get GuardrailPolicies. +func (s *guardrailPolicyLister) GuardrailPolicies(namespace string) GuardrailPolicyNamespaceLister { + return guardrailPolicyNamespaceLister{listers.NewNamespaced[*apiv1beta1.GuardrailPolicy](s.ResourceIndexer, namespace)} +} + +// GuardrailPolicyNamespaceLister helps list and get GuardrailPolicies. +// All objects returned here must be treated as read-only. +type GuardrailPolicyNamespaceLister interface { + // List lists all GuardrailPolicies in the indexer for a given namespace. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*apiv1beta1.GuardrailPolicy, err error) + // Get retrieves the GuardrailPolicy from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. + Get(name string) (*apiv1beta1.GuardrailPolicy, error) + GuardrailPolicyNamespaceListerExpansion +} + +// guardrailPolicyNamespaceLister implements the GuardrailPolicyNamespaceLister +// interface. +type guardrailPolicyNamespaceLister struct { + listers.ResourceIndexer[*apiv1beta1.GuardrailPolicy] +} diff --git a/api/v1beta1/guardrail_policy.go b/api/v1beta1/guardrail_policy.go new file mode 100644 index 0000000000..32d8fb1cc0 --- /dev/null +++ b/api/v1beta1/guardrail_policy.go @@ -0,0 +1,114 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package v1beta1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" +) + +// GuardrailPolicy evaluates content safety checks for request and response payloads. +// +// +genclient +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.conditions[-1:].type` +// +kubebuilder:metadata:labels="gateway.networking.k8s.io/policy=direct" +// +kubebuilder:storageversion +type GuardrailPolicy struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + Spec GuardrailPolicySpec `json:"spec,omitempty"` + // Status defines the status details of the GuardrailPolicy. + Status GuardrailPolicyStatus `json:"status,omitempty"` +} + +// GuardrailPolicySpec contains the configured checks attached to an AIServiceBackend. +type GuardrailPolicySpec struct { + // TargetRefs are the names of the AIServiceBackend resources this GuardrailPolicy is attached to. + // + // +optional + // +kubebuilder:validation:MaxItems=16 + // +kubebuilder:validation:XValidation:rule="self.all(ref, ref.group == 'aigateway.envoyproxy.io' && ref.kind == 'AIServiceBackend')", message="targetRefs must reference AIServiceBackend resources" + TargetRefs []gwapiv1a2.LocalPolicyTargetReference `json:"targetRefs,omitempty"` + // Rules are executed in order and can evaluate request or response payloads. + // + // +optional + // +kubebuilder:validation:MaxItems=32 + Rules []GuardrailRule `json:"rules,omitempty"` +} + +// GuardrailRule defines one content-safety check to apply to a request or response. +type GuardrailRule struct { + // Name is a stable identifier for the rule. + // + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + Name string `json:"name"` + // Phase determines whether the rule runs against the request or the response payload. + // + // +kubebuilder:validation:Enum=Request;Response + Phase GuardrailPhase `json:"phase"` + // Provider configures how the rule is evaluated. + Provider GuardrailProvider `json:"provider"` +} + +// GuardrailPhase determines when a guardrail runs. +type GuardrailPhase string + +const ( + GuardrailPhaseRequest GuardrailPhase = "Request" + GuardrailPhaseResponse GuardrailPhase = "Response" +) + +// GuardrailProvider describes the implementation used to evaluate a rule. +type GuardrailProvider struct { + // Type identifies the guardrail implementation. + // + // +kubebuilder:validation:Enum=Regex;Presidio;Bedrock;AzureContentSafety + Type GuardrailProviderType `json:"type"` + // Pattern is used for deterministic regex-based evaluations. + // + // +optional + Pattern string `json:"pattern,omitempty"` + // Action is the action taken when the rule is matched. + // + // +optional + // +kubebuilder:default=Block + Action GuardrailAction `json:"action,omitempty"` + // Message is returned to the caller when the rule blocks a request or response. + // + // +optional + Message string `json:"message,omitempty"` +} + +// GuardrailProviderType is the guardrail implementation. +type GuardrailProviderType string + +const ( + GuardrailProviderTypeRegex GuardrailProviderType = "Regex" + GuardrailProviderTypePresidio GuardrailProviderType = "Presidio" + GuardrailProviderTypeBedrockGuardrails GuardrailProviderType = "Bedrock" + GuardrailProviderTypeAzureContentSafety GuardrailProviderType = "AzureContentSafety" +) + +// GuardrailAction defines the safeguard action. +type GuardrailAction string + +const ( + GuardrailActionBlock GuardrailAction = "Block" +) + +// GuardrailPolicyList contains a list of GuardrailPolicy resources. +// +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:object:root=true +type GuardrailPolicyList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []GuardrailPolicy `json:"items"` +} diff --git a/api/v1beta1/registry.go b/api/v1beta1/registry.go index 01c2e169b6..d38dd9b9cd 100644 --- a/api/v1beta1/registry.go +++ b/api/v1beta1/registry.go @@ -18,6 +18,7 @@ func init() { SchemeBuilder.Register(&BackendSecurityPolicy{}, &BackendSecurityPolicyList{}) SchemeBuilder.Register(&MCPRoute{}, &MCPRouteList{}) SchemeBuilder.Register(&GatewayConfig{}, &GatewayConfigList{}) + SchemeBuilder.Register(&GuardrailPolicy{}, &GuardrailPolicyList{}) } const GroupName = "aigateway.envoyproxy.io" @@ -51,6 +52,8 @@ func AddKnownTypes(scheme *runtime.Scheme) error { &MCPRouteList{}, &GatewayConfig{}, &GatewayConfigList{}, + &GuardrailPolicy{}, + &GuardrailPolicyList{}, ) metav1.AddToGroupVersion(scheme, SchemeGroupVersion) return nil diff --git a/api/v1beta1/status.go b/api/v1beta1/status.go index 09dafe7376..162e6fecee 100644 --- a/api/v1beta1/status.go +++ b/api/v1beta1/status.go @@ -51,3 +51,12 @@ type MCPRouteStatus struct { // Known .status.conditions.type are: "Accepted", "NotAccepted". Conditions []metav1.Condition `json:"conditions,omitempty"` } + +// GuardrailPolicyStatus contains the conditions by the reconciliation result. +type GuardrailPolicyStatus struct { + // Conditions is the list of conditions by the reconciliation result. + // Currently, at most one condition is set. + // + // Known .status.conditions.type are: "Accepted", "NotAccepted". + Conditions []metav1.Condition `json:"conditions,omitempty"` +} diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 10823978cf..7c4b204789 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -13,7 +13,7 @@ import ( "github.com/envoyproxy/gateway/api/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "sigs.k8s.io/gateway-api/apis/v1" + v1 "sigs.k8s.io/gateway-api/apis/v1" "sigs.k8s.io/gateway-api/apis/v1alpha2" ) @@ -1024,6 +1024,143 @@ func (in *GatewayConfigStatus) DeepCopy() *GatewayConfigStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GuardrailPolicy) DeepCopyInto(out *GuardrailPolicy) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GuardrailPolicy. +func (in *GuardrailPolicy) DeepCopy() *GuardrailPolicy { + if in == nil { + return nil + } + out := new(GuardrailPolicy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *GuardrailPolicy) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GuardrailPolicyList) DeepCopyInto(out *GuardrailPolicyList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]GuardrailPolicy, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GuardrailPolicyList. +func (in *GuardrailPolicyList) DeepCopy() *GuardrailPolicyList { + if in == nil { + return nil + } + out := new(GuardrailPolicyList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *GuardrailPolicyList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GuardrailPolicySpec) DeepCopyInto(out *GuardrailPolicySpec) { + *out = *in + if in.TargetRefs != nil { + in, out := &in.TargetRefs, &out.TargetRefs + *out = make([]v1alpha2.LocalPolicyTargetReference, len(*in)) + copy(*out, *in) + } + if in.Rules != nil { + in, out := &in.Rules, &out.Rules + *out = make([]GuardrailRule, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GuardrailPolicySpec. +func (in *GuardrailPolicySpec) DeepCopy() *GuardrailPolicySpec { + if in == nil { + return nil + } + out := new(GuardrailPolicySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GuardrailPolicyStatus) DeepCopyInto(out *GuardrailPolicyStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]metav1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GuardrailPolicyStatus. +func (in *GuardrailPolicyStatus) DeepCopy() *GuardrailPolicyStatus { + if in == nil { + return nil + } + out := new(GuardrailPolicyStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GuardrailProvider) DeepCopyInto(out *GuardrailProvider) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GuardrailProvider. +func (in *GuardrailProvider) DeepCopy() *GuardrailProvider { + if in == nil { + return nil + } + out := new(GuardrailProvider) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GuardrailRule) DeepCopyInto(out *GuardrailRule) { + *out = *in + out.Provider = in.Provider +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GuardrailRule. +func (in *GuardrailRule) DeepCopy() *GuardrailRule { + if in == nil { + return nil + } + out := new(GuardrailRule) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HTTPBodyField) DeepCopyInto(out *HTTPBodyField) { *out = *in diff --git a/internal/extproc/guardrails.go b/internal/extproc/guardrails.go new file mode 100644 index 0000000000..4274da6a5f --- /dev/null +++ b/internal/extproc/guardrails.go @@ -0,0 +1,48 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package extproc + +import ( + "fmt" + + "github.com/envoyproxy/ai-gateway/internal/filterapi" +) + +// guardrailViolation describes a trigger that should block the request/response. +type guardrailViolation struct { + Name string + Message string +} + +func evaluateGuardrailsForPhase(guardrails []filterapi.RuntimeGuardrail, phase filterapi.GuardrailPhase, body []byte) (*guardrailViolation, error) { + for i := range guardrails { + g := &guardrails[i] + if g.Phase != phase { + continue + } + if g.Provider.Type == filterapi.GuardrailProviderTypeRegex { + if g.Matcher == nil { + return nil, fmt.Errorf("guardrail %q uses regex provider without a compiled matcher", g.Name) + } + if g.Matcher.Match(body) { + msg := g.Provider.Message + if msg == "" { + msg = fmt.Sprintf("request blocked by guardrail %q", g.Name) + } + return &guardrailViolation{Name: g.Name, Message: msg}, nil + } + } + } + return nil, nil +} + +func evaluateRequestGuardrails(guardrails []filterapi.RuntimeGuardrail, body []byte) (*guardrailViolation, error) { + return evaluateGuardrailsForPhase(guardrails, filterapi.GuardrailPhaseRequest, body) +} + +func evaluateResponseGuardrails(guardrails []filterapi.RuntimeGuardrail, body []byte) (*guardrailViolation, error) { + return evaluateGuardrailsForPhase(guardrails, filterapi.GuardrailPhaseResponse, body) +} diff --git a/internal/extproc/guardrails_test.go b/internal/extproc/guardrails_test.go new file mode 100644 index 0000000000..26fa0dc57a --- /dev/null +++ b/internal/extproc/guardrails_test.go @@ -0,0 +1,65 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package extproc + +import ( + "regexp" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/envoyproxy/ai-gateway/internal/filterapi" +) + +func TestEvaluateGuardrailsForPhase(t *testing.T) { + t.Run("request guardrail matches and returns violation", func(t *testing.T) { + guardrails := []filterapi.RuntimeGuardrail{{ + Name: "deny-pii", + Phase: filterapi.GuardrailPhaseRequest, + Provider: filterapi.GuardrailProvider{ + Type: filterapi.GuardrailProviderTypeRegex, + Message: "PII detected in request", + }, + Matcher: regexp.MustCompile(`\bSSN\b`), + }} + + violation, err := evaluateRequestGuardrails(guardrails, []byte("customer SSN is present")) + require.NoError(t, err) + require.NotNil(t, violation) + require.Equal(t, "deny-pii", violation.Name) + require.Equal(t, "PII detected in request", violation.Message) + }) + + t.Run("response guardrail ignores different phase", func(t *testing.T) { + guardrails := []filterapi.RuntimeGuardrail{{ + Name: "block-sensitive-response", + Phase: filterapi.GuardrailPhaseResponse, + Provider: filterapi.GuardrailProvider{ + Type: filterapi.GuardrailProviderTypeRegex, + }, + Matcher: regexp.MustCompile(`forbidden`), + }} + + violation, err := evaluateRequestGuardrails(guardrails, []byte("forbidden")) + require.NoError(t, err) + require.Nil(t, violation) + }) + + t.Run("regex guardrail without compiled matcher returns error on matching phase", func(t *testing.T) { + guardrails := []filterapi.RuntimeGuardrail{{ + Name: "missing-matcher", + Phase: filterapi.GuardrailPhaseRequest, + Provider: filterapi.GuardrailProvider{ + Type: filterapi.GuardrailProviderTypeRegex, + }, + }} + + violation, err := evaluateRequestGuardrails(guardrails, []byte("forbidden")) + require.Error(t, err) + require.Nil(t, violation) + require.Contains(t, err.Error(), "uses regex provider without a compiled matcher") + }) +} diff --git a/internal/extproc/processor_impl.go b/internal/extproc/processor_impl.go index 5bcdb2d269..090a8937da 100644 --- a/internal/extproc/processor_impl.go +++ b/internal/extproc/processor_impl.go @@ -252,6 +252,11 @@ func (r *routerProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) ProcessRequest } return nil, fmt.Errorf("failed to parse request body: %w", err) } + if violation, err := evaluateRequestGuardrails(r.config.Guardrails, rawBody.Body); err != nil { + return nil, fmt.Errorf("failed to evaluate request guardrails: %w", err) + } else if violation != nil { + return createUserFacingErrorResponse(400, "BadRequest", violation.Message), nil + } // Use the request-scoped logger from context if available, otherwise fall back to processor logger logger := loggerFromContext(ctx) @@ -618,6 +623,12 @@ func (u *upstreamProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) ProcessRespo // Translator reports the latest cumulative token usage which we use to override existing costs. u.costs.Override(tokenUsage) + if violation, err := evaluateResponseGuardrails(u.parent.config.Guardrails, body.Body); err != nil { + return nil, fmt.Errorf("failed to evaluate response guardrails: %w", err) + } else if violation != nil { + return u.respondLocally(ctx, 400, "BadRequest", violation.Message), nil + } + // Set the response model for metrics u.metrics.SetResponseModel(responseModel) diff --git a/internal/filterapi/filterconfig.go b/internal/filterapi/filterconfig.go index 7c6f8ecae2..698ff87588 100644 --- a/internal/filterapi/filterconfig.go +++ b/internal/filterapi/filterconfig.go @@ -52,8 +52,50 @@ type Config struct { UnscopedModels []Model `json:"unscopedModels,omitempty"` // MCPConfig is the configuration for the MCPRoute implementations. MCPConfig *MCPConfig `json:"mcpConfig,omitempty"` + // Guardrails are content-safety checks evaluated before the request is forwarded upstream or before the response is returned to the client. + Guardrails []Guardrail `json:"guardrails,omitempty"` } +// Guardrail specifies a content-safety rule to evaluate. +type Guardrail struct { + Name string `json:"name"` + Phase GuardrailPhase `json:"phase"` + Provider GuardrailProvider `json:"provider"` +} + +// GuardrailPhase determines when the rule is evaluated. +type GuardrailPhase string + +const ( + GuardrailPhaseRequest GuardrailPhase = "Request" + GuardrailPhaseResponse GuardrailPhase = "Response" +) + +// GuardrailProvider describes the implementation used to evaluate a guardrail. +type GuardrailProvider struct { + Type GuardrailProviderType `json:"type"` + Pattern string `json:"pattern,omitempty"` + Action GuardrailAction `json:"action,omitempty"` + Message string `json:"message,omitempty"` +} + +// GuardrailProviderType identifies a guardrail implementation. +type GuardrailProviderType string + +const ( + GuardrailProviderTypeRegex GuardrailProviderType = "Regex" + GuardrailProviderTypePresidio GuardrailProviderType = "Presidio" + GuardrailProviderTypeBedrockGuardrails GuardrailProviderType = "Bedrock" + GuardrailProviderTypeAzureContentSafety GuardrailProviderType = "AzureContentSafety" +) + +// GuardrailAction is the action taken when the rule matches. +type GuardrailAction string + +const ( + GuardrailActionBlock GuardrailAction = "Block" +) + // Model corresponds to the OpenAI model object in the OpenAI-compatible APIs // and is used to populate the "/models" endpoint in OpenAI-compatible APIs. type Model struct { diff --git a/internal/filterapi/runtime.go b/internal/filterapi/runtime.go index 3885fb71f9..c08b5ac12d 100644 --- a/internal/filterapi/runtime.go +++ b/internal/filterapi/runtime.go @@ -8,6 +8,7 @@ package filterapi import ( "context" "fmt" + "regexp" "github.com/google/cel-go/cel" @@ -45,6 +46,8 @@ type RuntimeConfig struct { UnscopedModels []Model // Backends is the map of backends by name. Backends map[string]*RuntimeBackend + // Guardrails is the list of compiled runtime guardrails. + Guardrails []RuntimeGuardrail } // RuntimeBackend is a filter backend with its auth handler that is derived from the filterapi.Backend configuration. @@ -69,6 +72,14 @@ type RuntimeRequestCost struct { CELProg cel.Program } +// RuntimeGuardrail is a compiled guardrail rule for runtime evaluation. +type RuntimeGuardrail struct { + Name string + Phase GuardrailPhase + Provider GuardrailProvider + Matcher *regexp.Regexp +} + // NewRuntimeConfig creates a new runtime filter configuration from the given filterapi.Config and a function to create backend auth handlers. func NewRuntimeConfig(ctx context.Context, config *Config, fn NewBackendAuthHandlerFunc) (*RuntimeConfig, error) { backends := make(map[string]*RuntimeBackend, len(config.Backends)) @@ -123,6 +134,32 @@ func NewRuntimeConfig(ctx context.Context, config *Config, fn NewBackendAuthHand costs = append(costs, RuntimeRequestCost{LLMRequestCost: c, CELProg: prog}) } + guardrails := make([]RuntimeGuardrail, 0, len(config.Guardrails)) + for i := range config.Guardrails { + g := &config.Guardrails[i] + if g.Provider.Type == GuardrailProviderTypeRegex { + if g.Provider.Pattern == "" { + return nil, fmt.Errorf("guardrail %q uses regex provider without a pattern", g.Name) + } + re, err := regexp.Compile(g.Provider.Pattern) + if err != nil { + return nil, fmt.Errorf("guardrail %q has an invalid regex pattern: %w", g.Name, err) + } + guardrails = append(guardrails, RuntimeGuardrail{ + Name: g.Name, + Phase: g.Phase, + Provider: g.Provider, + Matcher: re, + }) + continue + } + guardrails = append(guardrails, RuntimeGuardrail{ + Name: g.Name, + Phase: g.Phase, + Provider: g.Provider, + }) + } + return &RuntimeConfig{ UUID: config.UUID, Backends: backends, @@ -131,5 +168,6 @@ func NewRuntimeConfig(ctx context.Context, config *Config, fn NewBackendAuthHand DeclaredModels: config.Models, ModelsByHost: config.ModelsByHost, UnscopedModels: config.UnscopedModels, + Guardrails: guardrails, }, nil } diff --git a/internal/filterapi/runtime_test.go b/internal/filterapi/runtime_test.go index 1d58be6fd2..9e1758c2d6 100644 --- a/internal/filterapi/runtime_test.go +++ b/internal/filterapi/runtime_test.go @@ -118,6 +118,26 @@ func TestServer_LoadConfig(t *testing.T) { require.Contains(t, err.Error(), "cannot create CEL program for cost") }) + t.Run("guardrail regex config compiles", func(t *testing.T) { + config := &Config{ + Guardrails: []Guardrail{{ + Name: "deny-pii", + Phase: GuardrailPhaseRequest, + Provider: GuardrailProvider{ + Type: GuardrailProviderTypeRegex, + Pattern: `\bSSN\b`, + }, + }}, + } + rc, err := NewRuntimeConfig(t.Context(), config, func(_ context.Context, _ *BackendAuth) (BackendAuthHandler, error) { + return nil, nil + }) + require.NoError(t, err) + require.Len(t, rc.Guardrails, 1) + require.NotNil(t, rc.Guardrails[0].Matcher) + require.True(t, rc.Guardrails[0].Matcher.MatchString("SSN")) + }) + t.Run("error - route cost with empty RouteName", func(t *testing.T) { config := &Config{ LLMRequestCosts: []LLMRequestCost{ diff --git a/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_guardrailpolicies.yaml b/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_guardrailpolicies.yaml new file mode 100644 index 0000000000..261f2cea08 --- /dev/null +++ b/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_guardrailpolicies.yaml @@ -0,0 +1,410 @@ +# Copyright Envoy AI Gateway Authors +# SPDX-License-Identifier: Apache-2.0 +# The full text of the Apache license is available in the LICENSE file at +# the root of the repo. + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.0 + labels: + gateway.networking.k8s.io/policy: direct + name: guardrailpolicies.aigateway.envoyproxy.io +spec: + group: aigateway.envoyproxy.io + names: + kind: GuardrailPolicy + listKind: GuardrailPolicyList + plural: guardrailpolicies + singular: guardrailpolicy + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.conditions[-1:].type + name: Status + type: string + deprecated: true + deprecationWarning: aigateway.envoyproxy.io/v1alpha1 is deprecated; use aigateway.envoyproxy.io/v1beta1 + instead + name: v1alpha1 + schema: + openAPIV3Schema: + description: GuardrailPolicy evaluates content safety checks for request and + response payloads. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: GuardrailPolicySpec contains the configured checks attached + to an AIServiceBackend. + properties: + rules: + description: Rules are executed in order and can evaluate request + or response payloads. + items: + description: GuardrailRule defines one content-safety check to apply + to a request or response. + properties: + name: + description: Name is a stable identifier for the rule. + minLength: 1 + type: string + phase: + description: Phase determines whether the rule runs against + the request or the response payload. + enum: + - Request + - Response + type: string + provider: + description: Provider configures how the rule is evaluated. + properties: + action: + default: Block + description: Action is the action taken when the rule is + matched. + type: string + message: + description: Message is returned to the caller when the + rule blocks a request or response. + type: string + pattern: + description: Pattern is used for deterministic regex-based + evaluations. + type: string + type: + description: Type identifies the guardrail implementation. + enum: + - Regex + - Presidio + - Bedrock + - AzureContentSafety + type: string + required: + - type + type: object + required: + - name + - phase + - provider + type: object + maxItems: 32 + type: array + targetRefs: + description: TargetRefs are the names of the AIServiceBackend resources + this GuardrailPolicy is attached to. + items: + description: |- + LocalPolicyTargetReference identifies an API object to apply a direct or + inherited policy to. This should be used as part of Policy resources + that can target Gateway API resources. For more information on how this + policy attachment model works, and a sample Policy resource, refer to + the policy attachment documentation for Gateway API. + properties: + group: + description: Group is the group of the target resource. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + description: Kind is kind of the target resource. + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the target resource. + maxLength: 253 + minLength: 1 + type: string + required: + - group + - kind + - name + type: object + maxItems: 16 + type: array + x-kubernetes-validations: + - message: targetRefs must reference AIServiceBackend resources + rule: self.all(ref, ref.group == 'aigateway.envoyproxy.io' && ref.kind + == 'AIServiceBackend') + type: object + status: + description: Status defines the status details of the GuardrailPolicy. + properties: + conditions: + description: |- + Conditions is the list of conditions by the reconciliation result. + Currently, at most one condition is set. + + Known .status.conditions.type are: "Accepted", "NotAccepted". + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - jsonPath: .status.conditions[-1:].type + name: Status + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: GuardrailPolicy evaluates content safety checks for request and + response payloads. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: GuardrailPolicySpec contains the configured checks attached + to an AIServiceBackend. + properties: + rules: + description: Rules are executed in order and can evaluate request + or response payloads. + items: + description: GuardrailRule defines one content-safety check to apply + to a request or response. + properties: + name: + description: Name is a stable identifier for the rule. + minLength: 1 + type: string + phase: + description: Phase determines whether the rule runs against + the request or the response payload. + enum: + - Request + - Response + type: string + provider: + description: Provider configures how the rule is evaluated. + properties: + action: + default: Block + description: Action is the action taken when the rule is + matched. + type: string + message: + description: Message is returned to the caller when the + rule blocks a request or response. + type: string + pattern: + description: Pattern is used for deterministic regex-based + evaluations. + type: string + type: + description: Type identifies the guardrail implementation. + enum: + - Regex + - Presidio + - Bedrock + - AzureContentSafety + type: string + required: + - type + type: object + required: + - name + - phase + - provider + type: object + maxItems: 32 + type: array + targetRefs: + description: TargetRefs are the names of the AIServiceBackend resources + this GuardrailPolicy is attached to. + items: + description: |- + LocalPolicyTargetReference identifies an API object to apply a direct or + inherited policy to. This should be used as part of Policy resources + that can target Gateway API resources. For more information on how this + policy attachment model works, and a sample Policy resource, refer to + the policy attachment documentation for Gateway API. + properties: + group: + description: Group is the group of the target resource. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + description: Kind is kind of the target resource. + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the target resource. + maxLength: 253 + minLength: 1 + type: string + required: + - group + - kind + - name + type: object + maxItems: 16 + type: array + x-kubernetes-validations: + - message: targetRefs must reference AIServiceBackend resources + rule: self.all(ref, ref.group == 'aigateway.envoyproxy.io' && ref.kind + == 'AIServiceBackend') + type: object + status: + description: Status defines the status details of the GuardrailPolicy. + properties: + conditions: + description: |- + Conditions is the list of conditions by the reconciliation result. + Currently, at most one condition is set. + + Known .status.conditions.type are: "Accepted", "NotAccepted". + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/site/docs/api/api.mdx b/site/docs/api/api.mdx index 85f7896439..172061f0ab 100644 --- a/site/docs/api/api.mdx +++ b/site/docs/api/api.mdx @@ -27,6 +27,8 @@ API group. - [BackendSecurityPolicyList](#github-com-envoyproxy-ai-gateway-api-v1alpha1-backendsecuritypolicylist) - [GatewayConfig](#github-com-envoyproxy-ai-gateway-api-v1alpha1-gatewayconfig) - [GatewayConfigList](#github-com-envoyproxy-ai-gateway-api-v1alpha1-gatewayconfiglist) +- [GuardrailPolicy](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailpolicy) +- [GuardrailPolicyList](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailpolicylist) - [MCPRoute](#github-com-envoyproxy-ai-gateway-api-v1alpha1-mcproute) - [MCPRouteList](#github-com-envoyproxy-ai-gateway-api-v1alpha1-mcproutelist) - [QuotaPolicy](#github-com-envoyproxy-ai-gateway-api-v1alpha1-quotapolicy) @@ -391,6 +393,85 @@ GatewayConfigList contains a list of GatewayConfig. /> +#### GuardrailPolicy + + + +**Appears in:** +- [GuardrailPolicyList](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailpolicylist) + +GuardrailPolicy evaluates content safety checks for request and response payloads. + +##### Fields + + + + + + + + +#### GuardrailPolicyList + + + + +GuardrailPolicyList contains a list of GuardrailPolicy resources. + +##### Fields + + + + + + + + #### MCPRoute @@ -588,6 +669,13 @@ QuotaPolicyList contains a list of QuotaPolicy - [GatewayConfigForwardProxy](#github-com-envoyproxy-ai-gateway-api-v1alpha1-gatewayconfigforwardproxy) - [GatewayConfigSpec](#github-com-envoyproxy-ai-gateway-api-v1alpha1-gatewayconfigspec) - [GatewayConfigStatus](#github-com-envoyproxy-ai-gateway-api-v1alpha1-gatewayconfigstatus) +- [GuardrailAction](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailaction) +- [GuardrailPhase](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailphase) +- [GuardrailPolicySpec](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailpolicyspec) +- [GuardrailPolicyStatus](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailpolicystatus) +- [GuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailprovider) +- [GuardrailProviderType](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailprovidertype) +- [GuardrailRule](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailrule) - [HTTPBodyField](#github-com-envoyproxy-ai-gateway-api-v1alpha1-httpbodyfield) - [HTTPBodyMutation](#github-com-envoyproxy-ai-gateway-api-v1alpha1-httpbodymutation) - [HTTPHeaderMutation](#github-com-envoyproxy-ai-gateway-api-v1alpha1-httpheadermutation) @@ -1580,6 +1668,198 @@ GatewayConfigStatus defines the observed state of GatewayConfig. /> +#### GuardrailAction + +**Underlying type:** string + +**Appears in:** +- [GuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailprovider) + +GuardrailAction defines the safeguard action. + + + +##### Possible Values + + +#### GuardrailPhase + +**Underlying type:** string + +**Appears in:** +- [GuardrailRule](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailrule) + +GuardrailPhase determines when a guardrail runs. + + + +##### Possible Values + + +#### GuardrailPolicySpec + + + +**Appears in:** +- [GuardrailPolicy](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailpolicy) + +GuardrailPolicySpec contains the configured checks attached to an AIServiceBackend. + +##### Fields + + + + + + +#### GuardrailPolicyStatus + + + +**Appears in:** +- [GuardrailPolicy](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailpolicy) + +GuardrailPolicyStatus contains the conditions by the reconciliation result. + +##### Fields + + + + + + +#### GuardrailProvider + + + +**Appears in:** +- [GuardrailRule](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailrule) + +GuardrailProvider describes the implementation used to evaluate a rule. + +##### Fields + + + + + + +#### GuardrailProviderType + +**Underlying type:** string + +**Appears in:** +- [GuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailprovider) + +GuardrailProviderType is the guardrail implementation. + + + +##### Possible Values + + +#### GuardrailRule + + + +**Appears in:** +- [GuardrailPolicySpec](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailpolicyspec) + +GuardrailRule defines one content-safety check to apply to a request or response. + +##### Fields + + + + + + #### HTTPBodyField @@ -2676,6 +2956,8 @@ API group. This is the beta version of the API, preferred over v1alpha1 for new - [BackendSecurityPolicyList](#github-com-envoyproxy-ai-gateway-api-v1beta1-backendsecuritypolicylist) - [GatewayConfig](#github-com-envoyproxy-ai-gateway-api-v1beta1-gatewayconfig) - [GatewayConfigList](#github-com-envoyproxy-ai-gateway-api-v1beta1-gatewayconfiglist) +- [GuardrailPolicy](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailpolicy) +- [GuardrailPolicyList](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailpolicylist) - [MCPRoute](#github-com-envoyproxy-ai-gateway-api-v1beta1-mcproute) - [MCPRouteList](#github-com-envoyproxy-ai-gateway-api-v1beta1-mcproutelist) @@ -3038,6 +3320,85 @@ GatewayConfigList contains a list of GatewayConfig. /> +#### GuardrailPolicy + + + +**Appears in:** +- [GuardrailPolicyList](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailpolicylist) + +GuardrailPolicy evaluates content safety checks for request and response payloads. + +##### Fields + + + + + + + + +#### GuardrailPolicyList + + + + +GuardrailPolicyList contains a list of GuardrailPolicy resources. + +##### Fields + + + + + + + + #### MCPRoute @@ -3156,6 +3517,13 @@ MCPRouteList contains a list of MCPRoute. - [GatewayConfigForwardProxy](#github-com-envoyproxy-ai-gateway-api-v1beta1-gatewayconfigforwardproxy) - [GatewayConfigSpec](#github-com-envoyproxy-ai-gateway-api-v1beta1-gatewayconfigspec) - [GatewayConfigStatus](#github-com-envoyproxy-ai-gateway-api-v1beta1-gatewayconfigstatus) +- [GuardrailAction](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailaction) +- [GuardrailPhase](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailphase) +- [GuardrailPolicySpec](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailpolicyspec) +- [GuardrailPolicyStatus](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailpolicystatus) +- [GuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailprovider) +- [GuardrailProviderType](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailprovidertype) +- [GuardrailRule](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailrule) - [HTTPBodyField](#github-com-envoyproxy-ai-gateway-api-v1beta1-httpbodyfield) - [HTTPBodyMutation](#github-com-envoyproxy-ai-gateway-api-v1beta1-httpbodymutation) - [HTTPHeaderMutation](#github-com-envoyproxy-ai-gateway-api-v1beta1-httpheadermutation) @@ -4253,6 +4621,198 @@ GatewayConfigStatus defines the observed state of GatewayConfig. /> +#### GuardrailAction + +**Underlying type:** string + +**Appears in:** +- [GuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailprovider) + +GuardrailAction defines the safeguard action. + + + +##### Possible Values + + +#### GuardrailPhase + +**Underlying type:** string + +**Appears in:** +- [GuardrailRule](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailrule) + +GuardrailPhase determines when a guardrail runs. + + + +##### Possible Values + + +#### GuardrailPolicySpec + + + +**Appears in:** +- [GuardrailPolicy](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailpolicy) + +GuardrailPolicySpec contains the configured checks attached to an AIServiceBackend. + +##### Fields + + + + + + +#### GuardrailPolicyStatus + + + +**Appears in:** +- [GuardrailPolicy](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailpolicy) + +GuardrailPolicyStatus contains the conditions by the reconciliation result. + +##### Fields + + + + + + +#### GuardrailProvider + + + +**Appears in:** +- [GuardrailRule](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailrule) + +GuardrailProvider describes the implementation used to evaluate a rule. + +##### Fields + + + + + + +#### GuardrailProviderType + +**Underlying type:** string + +**Appears in:** +- [GuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailprovider) + +GuardrailProviderType is the guardrail implementation. + + + +##### Possible Values + + +#### GuardrailRule + + + +**Appears in:** +- [GuardrailPolicySpec](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailpolicyspec) + +GuardrailRule defines one content-safety check to apply to a request or response. + +##### Fields + + + + + + #### HTTPBodyField From 13c23e1097b7795e81b483e9a49c599228d5b2c3 Mon Sep 17 00:00:00 2001 From: Fernando Escolar Date: Wed, 16 Sep 2026 17:13:38 +0200 Subject: [PATCH 02/18] add GuardrailPolicy controller with reconciliation logic Signed-off-by: Fernando Escolar --- internal/controller/controller.go | 25 ++++ internal/controller/guardrail_policy.go | 147 ++++++++++++++++++++++++ 2 files changed, 172 insertions(+) create mode 100644 internal/controller/guardrail_policy.go diff --git a/internal/controller/controller.go b/internal/controller/controller.go index 70ba755ed4..07eb6c7121 100644 --- a/internal/controller/controller.go +++ b/internal/controller/controller.go @@ -258,6 +258,13 @@ func StartControllers(ctx context.Context, mgr manager.Manager, config *rest.Con } } + guardrailPolicyC := NewGuardrailPolicyController(c, kube, logger.WithName("guardrail-policy"), aiGatewayRouteEventChan) + if err = TypedControllerBuilderForCRD(mgr, &aigv1b1.GuardrailPolicy{}). + Watches(&aigv1b1.AIServiceBackend{}, handler.EnqueueRequestsFromMapFunc(guardrailPolicyC.BackendToGuardrailPolicy)). + Complete(guardrailPolicyC); err != nil { + return fmt.Errorf("failed to create controller for GuardrailPolicy: %w", err) + } + // ReferenceGrant controller for cross-namespace access validation referenceGrantC := NewReferenceGrantController(c, logger.WithName("reference-grant"), aiGatewayRouteEventChan) if err = TypedControllerBuilderForCRD(mgr, &gwapiv1b1.ReferenceGrant{}). @@ -312,6 +319,9 @@ const ( // k8sClientIndexAIServiceBackendToTargetingQuotaPolicy is the index name that maps from an AIServiceBackend // to the QuotaPolicy whose targetRefs contains the AIServiceBackend. k8sClientIndexAIServiceBackendToTargetingQuotaPolicy = "AIServiceBackendToTargetingQuotaPolicy" + // k8sClientIndexAIServiceBackendToTargetingGuardrailPolicy is the index name that maps from an AIServiceBackend + // to the GuardrailPolicy whose targetRefs contains the AIServiceBackend. + k8sClientIndexAIServiceBackendToTargetingGuardrailPolicy = "AIServiceBackendToTargetingGuardrailPolicy" // k8sClientIndexGatewayToGatewayConfig maps from a GatewayConfig name to Gateways referencing it. k8sClientIndexGatewayToGatewayConfig = "GatewayToGatewayConfig" @@ -361,6 +371,12 @@ func ApplyIndexing(ctx context.Context, indexer func(ctx context.Context, obj cl return fmt.Errorf("failed to index field for QuotaPolicy targetRefs: %w", err) } + err = indexer(ctx, &aigv1b1.GuardrailPolicy{}, + k8sClientIndexAIServiceBackendToTargetingGuardrailPolicy, guardrailPolicyTargetRefsIndexFunc) + if err != nil { + return fmt.Errorf("failed to index field for GuardrailPolicy targetRefs: %w", err) + } + err = indexer(ctx, &gwapiv1.Gateway{}, k8sClientIndexGatewayToGatewayConfig, gatewayToGatewayConfigIndexFunc) if err != nil { @@ -523,6 +539,15 @@ func quotaPolicyTargetRefsIndexFunc(o client.Object) []string { return ret } +func guardrailPolicyTargetRefsIndexFunc(o client.Object) []string { + guardrailPolicy := o.(*aigv1b1.GuardrailPolicy) + var ret []string + for _, targetRef := range guardrailPolicy.Spec.TargetRefs { + ret = append(ret, fmt.Sprintf("%s.%s", targetRef.Name, guardrailPolicy.Namespace)) + } + return ret +} + func getSecretNameAndNamespace(secretRef *gwapiv1.SecretObjectReference, namespace string) string { if secretRef.Namespace != nil { return fmt.Sprintf("%s.%s", secretRef.Name, *secretRef.Namespace) diff --git a/internal/controller/guardrail_policy.go b/internal/controller/guardrail_policy.go new file mode 100644 index 0000000000..376369e373 --- /dev/null +++ b/internal/controller/guardrail_policy.go @@ -0,0 +1,147 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/util/retry" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + aigv1b1 "github.com/envoyproxy/ai-gateway/api/v1beta1" +) + +// GuardrailPolicyController implements [reconcile.TypedReconciler] for [aigv1b1.GuardrailPolicy]. +type GuardrailPolicyController struct { + client client.Client + kube kubernetes.Interface + logger logr.Logger + aiGatewayRouteChan chan event.GenericEvent +} + +// NewGuardrailPolicyController creates a new reconciler for GuardrailPolicy resources. +func NewGuardrailPolicyController(client client.Client, kube kubernetes.Interface, logger logr.Logger, aiGatewayRouteChan chan event.GenericEvent) *GuardrailPolicyController { + return &GuardrailPolicyController{ + client: client, + kube: kube, + logger: logger, + aiGatewayRouteChan: aiGatewayRouteChan, + } +} + +// Reconcile implements [reconcile.TypedReconciler] for [aigv1b1.GuardrailPolicy]. +func (c *GuardrailPolicyController) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { + var policy aigv1b1.GuardrailPolicy + if err := c.client.Get(ctx, req.NamespacedName, &policy); err != nil { + if client.IgnoreNotFound(err) == nil { + c.logger.Info("Deleting GuardrailPolicy", + "namespace", req.Namespace, "name", req.Name) + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + } + + c.logger.Info("Reconciling GuardrailPolicy", "namespace", req.Namespace, "name", req.Name) + if handleFinalizer(ctx, c.client, c.logger, &policy, nil) { + return ctrl.Result{}, nil + } + if len(policy.Spec.TargetRefs) == 0 { + c.updateGuardrailPolicyStatus(ctx, &policy, aigv1b1.ConditionTypeAccepted, "GuardrailPolicy reconciled successfully") + return ctrl.Result{}, nil + } + if err := c.syncGuardrailPolicy(ctx, &policy); err != nil { + c.logger.Error(err, "failed to sync GuardrailPolicy") + c.updateGuardrailPolicyStatus(ctx, &policy, aigv1b1.ConditionTypeNotAccepted, err.Error()) + return ctrl.Result{}, err + } + + c.updateGuardrailPolicyStatus(ctx, &policy, aigv1b1.ConditionTypeAccepted, "GuardrailPolicy reconciled successfully") + c.notifyAIGatewayRoutesForGuardrailPolicy(ctx, &policy) + return ctrl.Result{}, nil +} + +func (c *GuardrailPolicyController) syncGuardrailPolicy(ctx context.Context, policy *aigv1b1.GuardrailPolicy) error { + if len(policy.Spec.TargetRefs) == 0 { + return nil + } + + for _, ref := range policy.Spec.TargetRefs { + var backend aigv1b1.AIServiceBackend + key := client.ObjectKey{Namespace: policy.Namespace, Name: string(ref.Name)} + if err := c.client.Get(ctx, key, &backend); err != nil { + if apierrors.IsNotFound(err) { + c.logger.Info("AIServiceBackend not found, skipping guardrail policy target", + "namespace", key.Namespace, "name", key.Name, + "guardrailPolicy", policy.Name) + continue + } + return fmt.Errorf("failed to get AIServiceBackend %s: %w", key, err) + } + } + + return nil +} + +// BackendToGuardrailPolicy maps AIServiceBackend changes to GuardrailPolicy reconcile requests. +func (c *GuardrailPolicyController) BackendToGuardrailPolicy(ctx context.Context, obj client.Object) []reconcile.Request { + var policies aigv1b1.GuardrailPolicyList + key := fmt.Sprintf("%s.%s", obj.GetName(), obj.GetNamespace()) + if err := c.client.List(ctx, &policies, + client.MatchingFields{k8sClientIndexAIServiceBackendToTargetingGuardrailPolicy: key}); err != nil { + c.logger.Error(err, "failed to list GuardrailPolicies for backend", "backend", key) + return nil + } + + var requests []reconcile.Request + for i := range policies.Items { + policy := &policies.Items[i] + requests = append(requests, reconcile.Request{ + NamespacedName: client.ObjectKeyFromObject(policy), + }) + } + return requests +} + +func (c *GuardrailPolicyController) notifyAIGatewayRoutesForGuardrailPolicy(ctx context.Context, policy *aigv1b1.GuardrailPolicy) { + for _, ref := range policy.Spec.TargetRefs { + key := fmt.Sprintf("%s.%s", ref.Name, policy.Namespace) + var routes aigv1b1.AIGatewayRouteList + if err := c.client.List(ctx, &routes, + client.MatchingFields{k8sClientIndexBackendToReferencingAIGatewayRoute: key}); err != nil { + c.logger.Error(err, "failed to list AIGatewayRoutes for guardrail policy", "backend", key) + continue + } + for i := range routes.Items { + route := &routes.Items[i] + c.logger.Info("notifying AIGatewayRoute of GuardrailPolicy change", + "route", route.Name, "namespace", route.Namespace, + "guardrailPolicy", policy.Name) + c.aiGatewayRouteChan <- event.GenericEvent{Object: route} + } + } +} + +func (c *GuardrailPolicyController) updateGuardrailPolicyStatus(ctx context.Context, policy *aigv1b1.GuardrailPolicy, conditionType string, message string) { + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + if err := c.client.Get(ctx, client.ObjectKey{Name: policy.Name, Namespace: policy.Namespace}, policy); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return err + } + policy.Status.Conditions = newConditions(conditionType, message) + return c.client.Status().Update(ctx, policy) + }) + if err != nil { + c.logger.Error(err, "failed to update GuardrailPolicy status", + "namespace", policy.Namespace, "name", policy.Name) + } +} \ No newline at end of file From a81ac8322bb13bfa066c0791619c3881030c8718 Mon Sep 17 00:00:00 2001 From: Fernando Escolar Date: Wed, 16 Sep 2026 17:24:59 +0200 Subject: [PATCH 03/18] feat: implement guardrail metrics and evaluation logic Signed-off-by: Fernando Escolar --- cmd/extproc/mainlib/main.go | 27 +++++----- internal/controller/guardrail_policy.go | 4 +- internal/extproc/guardrails.go | 9 ++++ internal/extproc/guardrails_test.go | 60 ++++++++++++++++++++++ internal/extproc/processor_impl.go | 30 ++++++++++- internal/extproc/processor_impl_test.go | 4 +- internal/metrics/guardrail_metrics.go | 55 ++++++++++++++++++++ internal/metrics/guardrail_metrics_test.go | 31 +++++++++++ 8 files changed, 203 insertions(+), 17 deletions(-) create mode 100644 internal/metrics/guardrail_metrics.go create mode 100644 internal/metrics/guardrail_metrics_test.go diff --git a/cmd/extproc/mainlib/main.go b/cmd/extproc/mainlib/main.go index df9ec510ad..73a272f706 100644 --- a/cmd/extproc/mainlib/main.go +++ b/cmd/extproc/mainlib/main.go @@ -308,6 +308,7 @@ func Main(ctx context.Context, args []string, stderr io.Writer) (err error) { responsesInputTokensMetricsFactory := metrics.NewMetricsFactory(meter, metricsRequestHeaderAttributes, metrics.GenAIOperationResponsesInputTokens) countTokensMetricsFactory := metrics.NewMetricsFactory(meter, metricsRequestHeaderAttributes, metrics.GenAIOperationCountTokens) mcpMetrics := metrics.NewMCP(meter, metricsRequestHeaderAttributes) + guardrailMetrics := metrics.NewGuardrailMetrics(meter) extproc.LogRequestHeaderAttributes = logRequestHeaderAttributes @@ -316,34 +317,34 @@ func Main(ctx context.Context, args []string, stderr io.Writer) (err error) { return fmt.Errorf("failed to create external processor server: %w", err) } server.Register(path.Join(flags.rootPrefix, endpointPrefixes.OpenAI, "/v1/chat/completions"), extproc.NewFactory( - chatCompletionMetricsFactory, tracing.ChatCompletionTracer(), endpointspec.ChatCompletionsEndpointSpec{})) + chatCompletionMetricsFactory, guardrailMetrics, tracing.ChatCompletionTracer(), endpointspec.ChatCompletionsEndpointSpec{})) server.Register(path.Join(flags.rootPrefix, endpointPrefixes.OpenAI, "/v1/completions"), extproc.NewFactory( - completionMetricsFactory, tracing.CompletionTracer(), endpointspec.CompletionsEndpointSpec{})) + completionMetricsFactory, guardrailMetrics, tracing.CompletionTracer(), endpointspec.CompletionsEndpointSpec{})) server.Register(path.Join(flags.rootPrefix, endpointPrefixes.OpenAI, "/v1/embeddings"), extproc.NewFactory( - embeddingsMetricsFactory, tracing.EmbeddingsTracer(), endpointspec.EmbeddingsEndpointSpec{})) + embeddingsMetricsFactory, guardrailMetrics, tracing.EmbeddingsTracer(), endpointspec.EmbeddingsEndpointSpec{})) server.Register(path.Join(flags.rootPrefix, endpointPrefixes.OpenAI, "/v1/responses"), extproc.NewFactory( - responsesMetricsFactory, tracing.ResponsesTracer(), endpointspec.ResponsesEndpointSpec{})) + responsesMetricsFactory, guardrailMetrics, tracing.ResponsesTracer(), endpointspec.ResponsesEndpointSpec{})) server.Register(path.Join(flags.rootPrefix, endpointPrefixes.OpenAI, "/v1/responses/input_tokens"), extproc.NewFactory( - responsesInputTokensMetricsFactory, tracing.ResponsesInputTokensTracer(), endpointspec.ResponsesInputTokensEndpointSpec{})) + responsesInputTokensMetricsFactory, guardrailMetrics, tracing.ResponsesInputTokensTracer(), endpointspec.ResponsesInputTokensEndpointSpec{})) server.Register(path.Join(flags.rootPrefix, endpointPrefixes.OpenAI, "/v1/audio/speech"), extproc.NewFactory( - speechMetricsFactory, tracing.SpeechTracer(), endpointspec.SpeechEndpointSpec{})) + speechMetricsFactory, guardrailMetrics, tracing.SpeechTracer(), endpointspec.SpeechEndpointSpec{})) server.Register(path.Join(flags.rootPrefix, endpointPrefixes.OpenAI, "/v1/audio/transcriptions"), extproc.NewFactory( - transcriptionMetricsFactory, tracing.TranscriptionTracer(), endpointspec.TranscriptionEndpointSpec{})) + transcriptionMetricsFactory, guardrailMetrics, tracing.TranscriptionTracer(), endpointspec.TranscriptionEndpointSpec{})) server.Register(path.Join(flags.rootPrefix, endpointPrefixes.OpenAI, "/v1/audio/translations"), extproc.NewFactory( - translationMetricsFactory, tracing.TranslationTracer(), endpointspec.TranslationEndpointSpec{})) + translationMetricsFactory, guardrailMetrics, tracing.TranslationTracer(), endpointspec.TranslationEndpointSpec{})) server.Register(path.Join(flags.rootPrefix, endpointPrefixes.OpenAI, "/v1/images/generations"), extproc.NewFactory( - imageGenerationMetricsFactory, tracing.ImageGenerationTracer(), endpointspec.ImageGenerationEndpointSpec{})) + imageGenerationMetricsFactory, guardrailMetrics, tracing.ImageGenerationTracer(), endpointspec.ImageGenerationEndpointSpec{})) server.Register(path.Join(flags.rootPrefix, endpointPrefixes.Cohere, "/v2/rerank"), extproc.NewFactory( - rerankMetricsFactory, tracing.RerankTracer(), endpointspec.RerankEndpointSpec{})) + rerankMetricsFactory, guardrailMetrics, tracing.RerankTracer(), endpointspec.RerankEndpointSpec{})) server.Register(path.Join(flags.rootPrefix, endpointPrefixes.OpenAI, "/v1/models"), extproc.NewModelsProcessor) server.Register(path.Join(flags.rootPrefix, endpointPrefixes.Anthropic, "/v1/models"), extproc.NewAnthropicModelsProcessor) server.Register(path.Join(flags.rootPrefix, endpointPrefixes.Anthropic, "/v1/messages"), extproc.NewFactory( - messagesMetricsFactory, tracing.MessageTracer(), endpointspec.MessagesEndpointSpec{})) + messagesMetricsFactory, guardrailMetrics, tracing.MessageTracer(), endpointspec.MessagesEndpointSpec{})) // Use /tokenize to be consistent with vLLM: https://github.com/vllm-project/vllm/blob/344b50d5258d7cf3f136416e1dbcd9b5ee99bb00/vllm/entrypoints/serve/tokenize/api_router.py#L37 server.Register(path.Join(flags.rootPrefix, endpointPrefixes.OpenAI, "/tokenize"), extproc.NewFactory( - tokenizeMetricsFactory, tracing.TokenizeTracer(), endpointspec.TokenizeEndpointSpec{})) + tokenizeMetricsFactory, guardrailMetrics, tracing.TokenizeTracer(), endpointspec.TokenizeEndpointSpec{})) server.Register(path.Join(flags.rootPrefix, endpointPrefixes.Anthropic, "/v1/messages/count_tokens"), extproc.NewFactory( - countTokensMetricsFactory, tracing.CountTokensTracer(), endpointspec.MessagesCountTokensEndpointSpec{})) + countTokensMetricsFactory, guardrailMetrics, tracing.CountTokensTracer(), endpointspec.MessagesCountTokensEndpointSpec{})) // Create and register gRPC server with ExternalProcessorServer (the service Envoy calls). if err = filterapi.StartConfigBundleWatcher(ctx, flags.configBundlePath, server, l, time.Second*5); err != nil { diff --git a/internal/controller/guardrail_policy.go b/internal/controller/guardrail_policy.go index 376369e373..08ef5f3fed 100644 --- a/internal/controller/guardrail_policy.go +++ b/internal/controller/guardrail_policy.go @@ -1,5 +1,7 @@ // Copyright Envoy AI Gateway Authors // SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. package controller @@ -144,4 +146,4 @@ func (c *GuardrailPolicyController) updateGuardrailPolicyStatus(ctx context.Cont c.logger.Error(err, "failed to update GuardrailPolicy status", "namespace", policy.Namespace, "name", policy.Name) } -} \ No newline at end of file +} diff --git a/internal/extproc/guardrails.go b/internal/extproc/guardrails.go index 4274da6a5f..0f0e1f625b 100644 --- a/internal/extproc/guardrails.go +++ b/internal/extproc/guardrails.go @@ -17,6 +17,15 @@ type guardrailViolation struct { Message string } +func guardrailsConfiguredForPhase(guardrails []filterapi.RuntimeGuardrail, phase filterapi.GuardrailPhase) bool { + for i := range guardrails { + if guardrails[i].Phase == phase { + return true + } + } + return false +} + func evaluateGuardrailsForPhase(guardrails []filterapi.RuntimeGuardrail, phase filterapi.GuardrailPhase, body []byte) (*guardrailViolation, error) { for i := range guardrails { g := &guardrails[i] diff --git a/internal/extproc/guardrails_test.go b/internal/extproc/guardrails_test.go index 26fa0dc57a..7c28b7542f 100644 --- a/internal/extproc/guardrails_test.go +++ b/internal/extproc/guardrails_test.go @@ -6,14 +6,32 @@ package extproc import ( + "context" + "log/slog" "regexp" "testing" + extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" "github.com/stretchr/testify/require" + "github.com/envoyproxy/ai-gateway/internal/endpointspec" "github.com/envoyproxy/ai-gateway/internal/filterapi" + "github.com/envoyproxy/ai-gateway/internal/metrics" + "github.com/envoyproxy/ai-gateway/internal/tracing/tracingapi" ) +type recordingGuardrailMetrics struct { + phase string + result metrics.GuardrailResult + count int +} + +func (m *recordingGuardrailMetrics) RecordEvaluation(_ context.Context, phase string, result metrics.GuardrailResult) { + m.phase = phase + m.result = result + m.count++ +} + func TestEvaluateGuardrailsForPhase(t *testing.T) { t.Run("request guardrail matches and returns violation", func(t *testing.T) { guardrails := []filterapi.RuntimeGuardrail{{ @@ -63,3 +81,45 @@ func TestEvaluateGuardrailsForPhase(t *testing.T) { require.Contains(t, err.Error(), "uses regex provider without a compiled matcher") }) } + +func TestRequestGuardrailBlockRecordsMetric(t *testing.T) { + recorder := &recordingGuardrailMetrics{} + config := &filterapi.RuntimeConfig{ + Guardrails: []filterapi.RuntimeGuardrail{{ + Name: "deny-pii", + Phase: filterapi.GuardrailPhaseRequest, + Provider: filterapi.GuardrailProvider{ + Type: filterapi.GuardrailProviderTypeRegex, + }, + Matcher: regexp.MustCompile(`SSN`), + }}, + } + factory := NewFactory(nil, recorder, tracingapi.NoopChatCompletionTracer{}, endpointspec.ChatCompletionsEndpointSpec{}) + processor, err := factory(config, map[string]string{ + "content-type": "application/json", + ":path": "/v1/chat/completions", + }, slog.Default(), false, false) + require.NoError(t, err) + + response, err := processor.ProcessRequestBody(t.Context(), &extprocv3.HttpBody{ + Body: []byte(`{"model":"test","messages":[{"role":"user","content":"customer SSN"}]}`), + }) + require.NoError(t, err) + require.NotNil(t, response.GetImmediateResponse()) + require.Equal(t, 1, recorder.count) + require.Equal(t, string(filterapi.GuardrailPhaseRequest), recorder.phase) + require.Equal(t, metrics.GuardrailResultBlocked, recorder.result) +} + +func TestRecordGuardrailEvaluationIgnoresUnconfiguredPhase(t *testing.T) { + recorder := &recordingGuardrailMetrics{} + processor := &chatCompletionProcessorRouterFilter{ + config: &filterapi.RuntimeConfig{Guardrails: []filterapi.RuntimeGuardrail{{ + Phase: filterapi.GuardrailPhaseResponse, + }}}, + guardrailMetrics: recorder, + } + + processor.recordGuardrailEvaluation(t.Context(), filterapi.GuardrailPhaseRequest, metrics.GuardrailResultAllowed) + require.Zero(t, recorder.count) +} diff --git a/internal/extproc/processor_impl.go b/internal/extproc/processor_impl.go index 090a8937da..720c3a9a60 100644 --- a/internal/extproc/processor_impl.go +++ b/internal/extproc/processor_impl.go @@ -58,13 +58,14 @@ var LogRequestHeaderAttributes map[string]string // * ProcessorFactory: A factory function to create processors based on the configuration. func NewFactory[ReqT any, RespT any, RespChunkT any, EndpointSpecT endpointspec.Spec[ReqT, RespT, RespChunkT]]( f metrics.Factory, + guardrailMetrics metrics.GuardrailMetrics, tracer tracingapi.RequestTracer[ReqT, RespT, RespChunkT], _ EndpointSpecT, // This is a type marker to bind EndpointSpecT without specifying ReqT, RespT, RespChunkT explicitly. ) ProcessorFactory { return func(config *filterapi.RuntimeConfig, requestHeaders map[string]string, logger *slog.Logger, isUpstreamFilter bool, enableRedaction bool) (Processor, error) { logger = logger.With("isUpstreamFilter", fmt.Sprintf("%v", isUpstreamFilter)) if !isUpstreamFilter { - return newRouterProcessor[ReqT, RespT, RespChunkT, EndpointSpecT](config, requestHeaders, logger, tracer, enableRedaction), nil + return newRouterProcessor[ReqT, RespT, RespChunkT, EndpointSpecT](config, requestHeaders, logger, guardrailMetrics, tracer, enableRedaction), nil } return newUpstreamProcessor[ReqT, RespT, RespChunkT, EndpointSpecT](requestHeaders, f.NewMetrics(), logger), nil } @@ -96,6 +97,8 @@ type ( forceBodyMutation bool // tracer is the tracer used for requests. tracer tracingapi.RequestTracer[ReqT, RespT, RespChunkT] + // guardrailMetrics records guardrail evaluation outcomes. + guardrailMetrics metrics.GuardrailMetrics // span is the tracing span for this request, created in ProcessRequestBody. span tracingapi.Span[RespT, RespChunkT] // upstreamFilterCount is the number of upstream filters that have been processed. @@ -139,6 +142,7 @@ func newRouterProcessor[ReqT, RespT, RespChunkT any, EndpointSpecT endpointspec. config *filterapi.RuntimeConfig, requestHeaders map[string]string, logger *slog.Logger, + guardrailMetrics metrics.GuardrailMetrics, tracer tracingapi.RequestTracer[ReqT, RespT, RespChunkT], enableRedaction bool, ) *routerProcessor[ReqT, RespT, RespChunkT, EndpointSpecT] { @@ -147,6 +151,7 @@ func newRouterProcessor[ReqT, RespT, RespChunkT any, EndpointSpecT endpointspec. config: config, requestHeaders: requestHeaders, logger: logger, + guardrailMetrics: guardrailMetrics, tracer: tracer, forceBodyMutation: false, debugLogEnabled: debugLogEnabled, @@ -154,6 +159,17 @@ func newRouterProcessor[ReqT, RespT, RespChunkT any, EndpointSpecT endpointspec. } } +func (r *routerProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) recordGuardrailEvaluation( + ctx context.Context, + phase filterapi.GuardrailPhase, + result metrics.GuardrailResult, +) { + if r.guardrailMetrics == nil || !guardrailsConfiguredForPhase(r.config.Guardrails, phase) { + return + } + r.guardrailMetrics.RecordEvaluation(ctx, string(phase), result) +} + func newUpstreamProcessor[ReqT, RespT, RespChunkT any, EndpointSpecT endpointspec.Spec[ReqT, RespT, RespChunkT]]( reqHeader map[string]string, metrics metrics.Metrics, logger *slog.Logger, @@ -253,10 +269,16 @@ func (r *routerProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) ProcessRequest return nil, fmt.Errorf("failed to parse request body: %w", err) } if violation, err := evaluateRequestGuardrails(r.config.Guardrails, rawBody.Body); err != nil { + r.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultError) return nil, fmt.Errorf("failed to evaluate request guardrails: %w", err) } else if violation != nil { + r.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultBlocked) + r.logger.Warn("request blocked by guardrail", + slog.String("guardrail.name", violation.Name), + slog.String("guardrail.phase", string(filterapi.GuardrailPhaseRequest))) return createUserFacingErrorResponse(400, "BadRequest", violation.Message), nil } + r.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultAllowed) // Use the request-scoped logger from context if available, otherwise fall back to processor logger logger := loggerFromContext(ctx) @@ -624,10 +646,16 @@ func (u *upstreamProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) ProcessRespo u.costs.Override(tokenUsage) if violation, err := evaluateResponseGuardrails(u.parent.config.Guardrails, body.Body); err != nil { + u.parent.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseResponse, metrics.GuardrailResultError) return nil, fmt.Errorf("failed to evaluate response guardrails: %w", err) } else if violation != nil { + u.parent.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseResponse, metrics.GuardrailResultBlocked) + u.logger.Warn("response blocked by guardrail", + slog.String("guardrail.name", violation.Name), + slog.String("guardrail.phase", string(filterapi.GuardrailPhaseResponse))) return u.respondLocally(ctx, 400, "BadRequest", violation.Message), nil } + u.parent.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseResponse, metrics.GuardrailResultAllowed) // Set the response model for metrics u.metrics.SetResponseModel(responseModel) diff --git a/internal/extproc/processor_impl_test.go b/internal/extproc/processor_impl_test.go index 4482ca98e6..faa3e3cd93 100644 --- a/internal/extproc/processor_impl_test.go +++ b/internal/extproc/processor_impl_test.go @@ -47,7 +47,7 @@ func TestNewFactory(t *testing.T) { t.Run("router", func(t *testing.T) { t.Parallel() - factory := NewFactory(nil, tracingapi.NoopChatCompletionTracer{}, endpointspec.ChatCompletionsEndpointSpec{}) + factory := NewFactory(nil, nil, tracingapi.NoopChatCompletionTracer{}, endpointspec.ChatCompletionsEndpointSpec{}) proc, err := factory(cfg, headers, slog.Default(), false, false) require.NoError(t, err) require.IsType(t, &chatCompletionProcessorRouterFilter{}, proc) @@ -62,7 +62,7 @@ func TestNewFactory(t *testing.T) { t.Run("upstream", func(t *testing.T) { t.Parallel() - factory := NewFactory(&mockMetricsFactory{}, tracingapi.NoopChatCompletionTracer{}, endpointspec.ChatCompletionsEndpointSpec{}) + factory := NewFactory(&mockMetricsFactory{}, nil, tracingapi.NoopChatCompletionTracer{}, endpointspec.ChatCompletionsEndpointSpec{}) proc, err := factory(cfg, headers, slog.Default(), true, false) require.NoError(t, err) require.IsType(t, &chatCompletionProcessorUpstreamFilter{}, proc) diff --git a/internal/metrics/guardrail_metrics.go b/internal/metrics/guardrail_metrics.go new file mode 100644 index 0000000000..95bfe6170d --- /dev/null +++ b/internal/metrics/guardrail_metrics.go @@ -0,0 +1,55 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package metrics + +import ( + "context" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" +) + +const ( + guardrailEvaluationCount = "aigateway.guardrail.evaluation.count" + guardrailAttributePhase = "aigateway.guardrail.phase" + guardrailAttributeResult = "aigateway.guardrail.result" +) + +// GuardrailResult is the outcome of evaluating a payload against guardrails. +type GuardrailResult string + +const ( + GuardrailResultAllowed GuardrailResult = "allowed" + GuardrailResultBlocked GuardrailResult = "blocked" + GuardrailResultError GuardrailResult = "error" +) + +// GuardrailMetrics records guardrail evaluation outcomes. +type GuardrailMetrics interface { + RecordEvaluation(ctx context.Context, phase string, result GuardrailResult) +} + +type guardrailMetrics struct { + evaluationCount metric.Float64Counter +} + +// NewGuardrailMetrics creates guardrail metrics backed by the provided meter. +func NewGuardrailMetrics(meter metric.Meter) GuardrailMetrics { + return &guardrailMetrics{ + evaluationCount: mustRegisterCounter( + meter, + guardrailEvaluationCount, + metric.WithDescription("Total number of payloads evaluated by guardrails"), + ), + } +} + +func (m *guardrailMetrics) RecordEvaluation(ctx context.Context, phase string, result GuardrailResult) { + m.evaluationCount.Add(ctx, 1, metric.WithAttributes( + attribute.String(guardrailAttributePhase, phase), + attribute.String(guardrailAttributeResult, string(result)), + )) +} diff --git a/internal/metrics/guardrail_metrics_test.go b/internal/metrics/guardrail_metrics_test.go new file mode 100644 index 0000000000..d78dcac11b --- /dev/null +++ b/internal/metrics/guardrail_metrics_test.go @@ -0,0 +1,31 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package metrics + +import ( + "testing" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/sdk/metric" + + "github.com/envoyproxy/ai-gateway/internal/testing/testotel" +) + +func TestGuardrailMetricsRecordEvaluation(t *testing.T) { + reader := metric.NewManualReader() + meter := metric.NewMeterProvider(metric.WithReader(reader)).Meter("test") + guardrails := NewGuardrailMetrics(meter) + + guardrails.RecordEvaluation(t.Context(), "Request", GuardrailResultBlocked) + guardrails.RecordEvaluation(t.Context(), "Request", GuardrailResultBlocked) + + value := testotel.GetCounterValue(t, reader, guardrailEvaluationCount, attribute.NewSet( + attribute.String(guardrailAttributePhase, "Request"), + attribute.String(guardrailAttributeResult, string(GuardrailResultBlocked)), + )) + require.Equal(t, float64(2), value) +} From 6953c2ae81d60bcacb1d6a3b0a862b6866c94bb6 Mon Sep 17 00:00:00 2001 From: Fernando Escolar Date: Wed, 16 Sep 2026 19:09:08 +0200 Subject: [PATCH 04/18] feat: add support for guardrail policies with external providers - Implemented support for regex guardrails and external providers including Presidio, AWS Bedrock, and Azure AI Content Safety. - Added failure behavior configuration for external providers. - Enhanced observability with structured logging for guardrail evaluations. - Created test cases for various guardrail policy scenarios including validation of rules and provider configurations. - Added documentation for content guardrails and examples for usage. Signed-off-by: Fernando Escolar --- api/v1alpha1/guardrail_policy.go | 91 +++- api/v1alpha1/zz_generated.deepcopy.go | 103 +++- api/v1beta1/guardrail_policy.go | 91 +++- api/v1beta1/zz_generated.deepcopy.go | 98 +++- examples/README.md | 4 + examples/guardrails/README.md | 24 + examples/guardrails/providers.yaml | 58 +++ examples/guardrails/regex.yaml | 29 ++ internal/controller/controller.go | 39 ++ internal/controller/gateway.go | 173 +++++++ internal/controller/guardrail_policy.go | 119 ++++- internal/controller/guardrail_policy_test.go | 325 +++++++++++++ internal/extproc/guardrails.go | 72 ++- internal/extproc/guardrails_test.go | 50 +- internal/extproc/processor_impl.go | 100 +++- internal/extproc/processor_impl_test.go | 17 + internal/extproc/server.go | 3 +- internal/filterapi/filterconfig.go | 48 +- internal/filterapi/runtime.go | 40 +- internal/filterapi/runtime_test.go | 19 + internal/guardrails/adapters.go | 76 +++ internal/guardrails/adapters_live_test.go | 76 +++ internal/guardrails/adapters_test.go | 122 +++++ internal/guardrails/azure.go | 62 +++ internal/guardrails/bedrock.go | 127 +++++ internal/guardrails/presidio.go | 55 +++ .../guardrails/presidio_container_test.go | 59 +++ internal/tracing/span.go | 10 + internal/tracing/span_test.go | 16 + internal/tracing/tracingapi/api.go | 4 + ...teway.envoyproxy.io_guardrailpolicies.yaml | 454 ++++++++++++++++++ site/docs/api/api.mdx | 348 +++++++++++++- site/docs/capabilities/security/guardrails.md | 137 ++++++ tests/crdcel/main_test.go | 31 ++ .../testdata/guardrailpolicies/basic.yaml | 47 ++ .../duplicate_rule_names.yaml | 26 + .../guardrailpolicies/invalid_action.yaml | 22 + .../guardrailpolicies/invalid_endpoint.yaml | 22 + .../guardrailpolicies/provider_mismatch.yaml | 24 + .../regex_missing_pattern.yaml | 20 + tests/data-plane/testupstream_test.go | 54 +++ 41 files changed, 3219 insertions(+), 76 deletions(-) create mode 100644 examples/guardrails/README.md create mode 100644 examples/guardrails/providers.yaml create mode 100644 examples/guardrails/regex.yaml create mode 100644 internal/controller/guardrail_policy_test.go create mode 100644 internal/guardrails/adapters.go create mode 100644 internal/guardrails/adapters_live_test.go create mode 100644 internal/guardrails/adapters_test.go create mode 100644 internal/guardrails/azure.go create mode 100644 internal/guardrails/bedrock.go create mode 100644 internal/guardrails/presidio.go create mode 100644 internal/guardrails/presidio_container_test.go create mode 100644 site/docs/capabilities/security/guardrails.md create mode 100644 tests/crdcel/testdata/guardrailpolicies/basic.yaml create mode 100644 tests/crdcel/testdata/guardrailpolicies/duplicate_rule_names.yaml create mode 100644 tests/crdcel/testdata/guardrailpolicies/invalid_action.yaml create mode 100644 tests/crdcel/testdata/guardrailpolicies/invalid_endpoint.yaml create mode 100644 tests/crdcel/testdata/guardrailpolicies/provider_mismatch.yaml create mode 100644 tests/crdcel/testdata/guardrailpolicies/regex_missing_pattern.yaml diff --git a/api/v1alpha1/guardrail_policy.go b/api/v1alpha1/guardrail_policy.go index bc30b97a68..a5f86ca7c2 100644 --- a/api/v1alpha1/guardrail_policy.go +++ b/api/v1alpha1/guardrail_policy.go @@ -7,6 +7,7 @@ package v1alpha1 import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" ) @@ -31,14 +32,15 @@ type GuardrailPolicy struct { type GuardrailPolicySpec struct { // TargetRefs are the names of the AIServiceBackend resources this GuardrailPolicy is attached to. // - // +optional + // +kubebuilder:validation:MinItems=1 // +kubebuilder:validation:MaxItems=16 // +kubebuilder:validation:XValidation:rule="self.all(ref, ref.group == 'aigateway.envoyproxy.io' && ref.kind == 'AIServiceBackend')", message="targetRefs must reference AIServiceBackend resources" TargetRefs []gwapiv1a2.LocalPolicyTargetReference `json:"targetRefs,omitempty"` // Rules are executed in order and can evaluate request or response payloads. // - // +optional + // +kubebuilder:validation:MinItems=1 // +kubebuilder:validation:MaxItems=32 + // +kubebuilder:validation:XValidation:rule="self.all(rule, self.exists_one(other, other.name == rule.name))",message="rule name must be unique within the policy" Rules []GuardrailRule `json:"rules,omitempty"` } @@ -48,6 +50,7 @@ type GuardrailRule struct { // // +kubebuilder:validation:Required // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=63 Name string `json:"name"` // Phase determines whether the rule runs against the request or the response payload. // @@ -66,6 +69,10 @@ const ( ) // GuardrailProvider describes the implementation used to evaluate a rule. +// +kubebuilder:validation:XValidation:rule="self.type != 'Regex' || (has(self.pattern) && self.pattern != ” && !has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety))",message="Regex requires pattern and no external provider configuration" +// +kubebuilder:validation:XValidation:rule="self.type != 'Presidio' || (!has(self.pattern) && has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety))",message="Presidio requires only presidio provider configuration" +// +kubebuilder:validation:XValidation:rule="self.type != 'Bedrock' || (!has(self.pattern) && has(self.bedrock) && !has(self.presidio) && !has(self.azureContentSafety))",message="Bedrock requires only bedrock provider configuration" +// +kubebuilder:validation:XValidation:rule="self.type != 'AzureContentSafety' || (!has(self.pattern) && has(self.azureContentSafety) && !has(self.presidio) && !has(self.bedrock))",message="AzureContentSafety requires only azureContentSafety provider configuration" type GuardrailProvider struct { // Type identifies the guardrail implementation. // @@ -79,11 +86,83 @@ type GuardrailProvider struct { // // +optional // +kubebuilder:default=Block + // +kubebuilder:validation:Enum=Block Action GuardrailAction `json:"action,omitempty"` // Message is returned to the caller when the rule blocks a request or response. // // +optional Message string `json:"message,omitempty"` + // Presidio configures the Presidio analyzer provider. + // + // +optional + Presidio *PresidioGuardrailProvider `json:"presidio,omitempty"` + // Bedrock configures AWS Bedrock Guardrails. + // + // +optional + Bedrock *BedrockGuardrailProvider `json:"bedrock,omitempty"` + // AzureContentSafety configures Azure AI Content Safety. + // + // +optional + AzureContentSafety *AzureContentSafetyGuardrailProvider `json:"azureContentSafety,omitempty"` + // TimeoutSeconds limits each external provider evaluation. + // +optional + // +kubebuilder:default=10 + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=60 + TimeoutSeconds *int32 `json:"timeoutSeconds,omitempty"` + // FailureMode determines whether provider errors block or allow the request. + // +optional + // +kubebuilder:default=FailClosed + // +kubebuilder:validation:Enum=FailClosed;FailOpen + FailureMode GuardrailFailureMode `json:"failureMode,omitempty"` +} + +// PresidioGuardrailProvider configures calls to a Presidio analyzer service. +type PresidioGuardrailProvider struct { + // +kubebuilder:validation:Format=uri + Endpoint string `json:"endpoint"` + // +optional + // +kubebuilder:default=en + Language string `json:"language,omitempty"` + // +optional + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Maximum=100 + ScoreThresholdPercent *int32 `json:"scoreThresholdPercent,omitempty"` + // APIKeySecretRef optionally references a Secret whose apiKey entry is sent as a Bearer token. + // +optional + APIKeySecretRef *gwapiv1.SecretObjectReference `json:"apiKeySecretRef,omitempty"` +} + +// BedrockGuardrailProvider configures calls to the AWS Bedrock ApplyGuardrail API. +type BedrockGuardrailProvider struct { + // +kubebuilder:validation:MinLength=1 + Region string `json:"region"` + // +kubebuilder:validation:MinLength=1 + GuardrailIdentifier string `json:"guardrailIdentifier"` + // +kubebuilder:validation:MinLength=1 + GuardrailVersion string `json:"guardrailVersion"` + // Endpoint overrides the Bedrock runtime endpoint, primarily for private endpoints and testing. + // +optional + // +kubebuilder:validation:Format=uri + Endpoint string `json:"endpoint,omitempty"` + // CredentialsSecretRef optionally references a Secret whose credentials entry contains an AWS shared credentials file. + // +optional + CredentialsSecretRef *gwapiv1.SecretObjectReference `json:"credentialsSecretRef,omitempty"` +} + +// AzureContentSafetyGuardrailProvider configures calls to Azure AI Content Safety. +type AzureContentSafetyGuardrailProvider struct { + // +kubebuilder:validation:Format=uri + Endpoint string `json:"endpoint"` + // +optional + // +kubebuilder:default="2024-09-01" + APIVersion string `json:"apiVersion,omitempty"` + // +optional + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Maximum=7 + SeverityThreshold *int32 `json:"severityThreshold,omitempty"` + // APIKeySecretRef references a Secret whose apiKey entry is sent to Azure. + APIKeySecretRef *gwapiv1.SecretObjectReference `json:"apiKeySecretRef"` } // GuardrailProviderType is the guardrail implementation. @@ -103,6 +182,14 @@ const ( GuardrailActionBlock GuardrailAction = "Block" ) +// GuardrailFailureMode defines behavior when an external provider cannot evaluate content. +type GuardrailFailureMode string + +const ( + GuardrailFailureModeFailClosed GuardrailFailureMode = "FailClosed" + GuardrailFailureModeFailOpen GuardrailFailureMode = "FailOpen" +) + // GuardrailPolicyList contains a list of GuardrailPolicy resources. // // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index e43e5d825e..8375e211a0 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -1,3 +1,8 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + //go:build !ignore_autogenerated // Code generated by controller-gen. DO NOT EDIT. @@ -8,7 +13,7 @@ import ( apiv1alpha1 "github.com/envoyproxy/gateway/api/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - v1 "sigs.k8s.io/gateway-api/apis/v1" + "sigs.k8s.io/gateway-api/apis/v1" "sigs.k8s.io/gateway-api/apis/v1alpha2" ) @@ -403,6 +408,31 @@ func (in *AWSOIDCExchangeToken) DeepCopy() *AWSOIDCExchangeToken { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureContentSafetyGuardrailProvider) DeepCopyInto(out *AzureContentSafetyGuardrailProvider) { + *out = *in + if in.SeverityThreshold != nil { + in, out := &in.SeverityThreshold, &out.SeverityThreshold + *out = new(int32) + **out = **in + } + if in.APIKeySecretRef != nil { + in, out := &in.APIKeySecretRef, &out.APIKeySecretRef + *out = new(v1.SecretObjectReference) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureContentSafetyGuardrailProvider. +func (in *AzureContentSafetyGuardrailProvider) DeepCopy() *AzureContentSafetyGuardrailProvider { + if in == nil { + return nil + } + out := new(AzureContentSafetyGuardrailProvider) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AzureOIDCExchangeToken) DeepCopyInto(out *AzureOIDCExchangeToken) { *out = *in @@ -701,6 +731,26 @@ func (in *BackendSecurityPolicyStatus) DeepCopy() *BackendSecurityPolicyStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BedrockGuardrailProvider) DeepCopyInto(out *BedrockGuardrailProvider) { + *out = *in + if in.CredentialsSecretRef != nil { + in, out := &in.CredentialsSecretRef, &out.CredentialsSecretRef + *out = new(v1.SecretObjectReference) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BedrockGuardrailProvider. +func (in *BedrockGuardrailProvider) DeepCopy() *BedrockGuardrailProvider { + if in == nil { + return nil + } + out := new(BedrockGuardrailProvider) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GCPCredentialsFile) DeepCopyInto(out *GCPCredentialsFile) { *out = *in @@ -1007,7 +1057,9 @@ func (in *GuardrailPolicySpec) DeepCopyInto(out *GuardrailPolicySpec) { if in.Rules != nil { in, out := &in.Rules, &out.Rules *out = make([]GuardrailRule, len(*in)) - copy(*out, *in) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } } } @@ -1046,6 +1098,26 @@ func (in *GuardrailPolicyStatus) DeepCopy() *GuardrailPolicyStatus { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GuardrailProvider) DeepCopyInto(out *GuardrailProvider) { *out = *in + if in.Presidio != nil { + in, out := &in.Presidio, &out.Presidio + *out = new(PresidioGuardrailProvider) + (*in).DeepCopyInto(*out) + } + if in.Bedrock != nil { + in, out := &in.Bedrock, &out.Bedrock + *out = new(BedrockGuardrailProvider) + (*in).DeepCopyInto(*out) + } + if in.AzureContentSafety != nil { + in, out := &in.AzureContentSafety, &out.AzureContentSafety + *out = new(AzureContentSafetyGuardrailProvider) + (*in).DeepCopyInto(*out) + } + if in.TimeoutSeconds != nil { + in, out := &in.TimeoutSeconds, &out.TimeoutSeconds + *out = new(int32) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GuardrailProvider. @@ -1061,7 +1133,7 @@ func (in *GuardrailProvider) DeepCopy() *GuardrailProvider { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GuardrailRule) DeepCopyInto(out *GuardrailRule) { *out = *in - out.Provider = in.Provider + in.Provider.DeepCopyInto(&out.Provider) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GuardrailRule. @@ -1763,6 +1835,31 @@ func (in *PerModelQuota) DeepCopy() *PerModelQuota { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PresidioGuardrailProvider) DeepCopyInto(out *PresidioGuardrailProvider) { + *out = *in + if in.ScoreThresholdPercent != nil { + in, out := &in.ScoreThresholdPercent, &out.ScoreThresholdPercent + *out = new(int32) + **out = **in + } + if in.APIKeySecretRef != nil { + in, out := &in.APIKeySecretRef, &out.APIKeySecretRef + *out = new(v1.SecretObjectReference) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PresidioGuardrailProvider. +func (in *PresidioGuardrailProvider) DeepCopy() *PresidioGuardrailProvider { + if in == nil { + return nil + } + out := new(PresidioGuardrailProvider) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ProtectedResourceMetadata) DeepCopyInto(out *ProtectedResourceMetadata) { *out = *in diff --git a/api/v1beta1/guardrail_policy.go b/api/v1beta1/guardrail_policy.go index 32d8fb1cc0..33f4af0cae 100644 --- a/api/v1beta1/guardrail_policy.go +++ b/api/v1beta1/guardrail_policy.go @@ -7,6 +7,7 @@ package v1beta1 import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" ) @@ -31,14 +32,15 @@ type GuardrailPolicy struct { type GuardrailPolicySpec struct { // TargetRefs are the names of the AIServiceBackend resources this GuardrailPolicy is attached to. // - // +optional + // +kubebuilder:validation:MinItems=1 // +kubebuilder:validation:MaxItems=16 // +kubebuilder:validation:XValidation:rule="self.all(ref, ref.group == 'aigateway.envoyproxy.io' && ref.kind == 'AIServiceBackend')", message="targetRefs must reference AIServiceBackend resources" TargetRefs []gwapiv1a2.LocalPolicyTargetReference `json:"targetRefs,omitempty"` // Rules are executed in order and can evaluate request or response payloads. // - // +optional + // +kubebuilder:validation:MinItems=1 // +kubebuilder:validation:MaxItems=32 + // +kubebuilder:validation:XValidation:rule="self.all(rule, self.exists_one(other, other.name == rule.name))",message="rule name must be unique within the policy" Rules []GuardrailRule `json:"rules,omitempty"` } @@ -48,6 +50,7 @@ type GuardrailRule struct { // // +kubebuilder:validation:Required // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=63 Name string `json:"name"` // Phase determines whether the rule runs against the request or the response payload. // @@ -66,6 +69,10 @@ const ( ) // GuardrailProvider describes the implementation used to evaluate a rule. +// +kubebuilder:validation:XValidation:rule="self.type != 'Regex' || (has(self.pattern) && self.pattern != ” && !has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety))",message="Regex requires pattern and no external provider configuration" +// +kubebuilder:validation:XValidation:rule="self.type != 'Presidio' || (!has(self.pattern) && has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety))",message="Presidio requires only presidio provider configuration" +// +kubebuilder:validation:XValidation:rule="self.type != 'Bedrock' || (!has(self.pattern) && has(self.bedrock) && !has(self.presidio) && !has(self.azureContentSafety))",message="Bedrock requires only bedrock provider configuration" +// +kubebuilder:validation:XValidation:rule="self.type != 'AzureContentSafety' || (!has(self.pattern) && has(self.azureContentSafety) && !has(self.presidio) && !has(self.bedrock))",message="AzureContentSafety requires only azureContentSafety provider configuration" type GuardrailProvider struct { // Type identifies the guardrail implementation. // @@ -79,11 +86,83 @@ type GuardrailProvider struct { // // +optional // +kubebuilder:default=Block + // +kubebuilder:validation:Enum=Block Action GuardrailAction `json:"action,omitempty"` // Message is returned to the caller when the rule blocks a request or response. // // +optional Message string `json:"message,omitempty"` + // Presidio configures the Presidio analyzer provider. + // + // +optional + Presidio *PresidioGuardrailProvider `json:"presidio,omitempty"` + // Bedrock configures AWS Bedrock Guardrails. + // + // +optional + Bedrock *BedrockGuardrailProvider `json:"bedrock,omitempty"` + // AzureContentSafety configures Azure AI Content Safety. + // + // +optional + AzureContentSafety *AzureContentSafetyGuardrailProvider `json:"azureContentSafety,omitempty"` + // TimeoutSeconds limits each external provider evaluation. + // +optional + // +kubebuilder:default=10 + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=60 + TimeoutSeconds *int32 `json:"timeoutSeconds,omitempty"` + // FailureMode determines whether provider errors block or allow the request. + // +optional + // +kubebuilder:default=FailClosed + // +kubebuilder:validation:Enum=FailClosed;FailOpen + FailureMode GuardrailFailureMode `json:"failureMode,omitempty"` +} + +// PresidioGuardrailProvider configures calls to a Presidio analyzer service. +type PresidioGuardrailProvider struct { + // +kubebuilder:validation:Format=uri + Endpoint string `json:"endpoint"` + // +optional + // +kubebuilder:default=en + Language string `json:"language,omitempty"` + // +optional + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Maximum=100 + ScoreThresholdPercent *int32 `json:"scoreThresholdPercent,omitempty"` + // APIKeySecretRef optionally references a Secret whose apiKey entry is sent as a Bearer token. + // +optional + APIKeySecretRef *gwapiv1.SecretObjectReference `json:"apiKeySecretRef,omitempty"` +} + +// BedrockGuardrailProvider configures calls to the AWS Bedrock ApplyGuardrail API. +type BedrockGuardrailProvider struct { + // +kubebuilder:validation:MinLength=1 + Region string `json:"region"` + // +kubebuilder:validation:MinLength=1 + GuardrailIdentifier string `json:"guardrailIdentifier"` + // +kubebuilder:validation:MinLength=1 + GuardrailVersion string `json:"guardrailVersion"` + // Endpoint overrides the Bedrock runtime endpoint, primarily for private endpoints and testing. + // +optional + // +kubebuilder:validation:Format=uri + Endpoint string `json:"endpoint,omitempty"` + // CredentialsSecretRef optionally references a Secret whose credentials entry contains an AWS shared credentials file. + // +optional + CredentialsSecretRef *gwapiv1.SecretObjectReference `json:"credentialsSecretRef,omitempty"` +} + +// AzureContentSafetyGuardrailProvider configures calls to Azure AI Content Safety. +type AzureContentSafetyGuardrailProvider struct { + // +kubebuilder:validation:Format=uri + Endpoint string `json:"endpoint"` + // +optional + // +kubebuilder:default="2024-09-01" + APIVersion string `json:"apiVersion,omitempty"` + // +optional + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Maximum=7 + SeverityThreshold *int32 `json:"severityThreshold,omitempty"` + // APIKeySecretRef references a Secret whose apiKey entry is sent to Azure. + APIKeySecretRef *gwapiv1.SecretObjectReference `json:"apiKeySecretRef"` } // GuardrailProviderType is the guardrail implementation. @@ -103,6 +182,14 @@ const ( GuardrailActionBlock GuardrailAction = "Block" ) +// GuardrailFailureMode defines behavior when an external provider cannot evaluate content. +type GuardrailFailureMode string + +const ( + GuardrailFailureModeFailClosed GuardrailFailureMode = "FailClosed" + GuardrailFailureModeFailOpen GuardrailFailureMode = "FailOpen" +) + // GuardrailPolicyList contains a list of GuardrailPolicy resources. // // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 7c4b204789..0732566768 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -13,7 +13,7 @@ import ( "github.com/envoyproxy/gateway/api/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - v1 "sigs.k8s.io/gateway-api/apis/v1" + "sigs.k8s.io/gateway-api/apis/v1" "sigs.k8s.io/gateway-api/apis/v1alpha2" ) @@ -415,6 +415,31 @@ func (in *AWSOIDCExchangeToken) DeepCopy() *AWSOIDCExchangeToken { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureContentSafetyGuardrailProvider) DeepCopyInto(out *AzureContentSafetyGuardrailProvider) { + *out = *in + if in.SeverityThreshold != nil { + in, out := &in.SeverityThreshold, &out.SeverityThreshold + *out = new(int32) + **out = **in + } + if in.APIKeySecretRef != nil { + in, out := &in.APIKeySecretRef, &out.APIKeySecretRef + *out = new(v1.SecretObjectReference) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureContentSafetyGuardrailProvider. +func (in *AzureContentSafetyGuardrailProvider) DeepCopy() *AzureContentSafetyGuardrailProvider { + if in == nil { + return nil + } + out := new(AzureContentSafetyGuardrailProvider) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AzureOIDCExchangeToken) DeepCopyInto(out *AzureOIDCExchangeToken) { *out = *in @@ -743,6 +768,26 @@ func (in *BackendSecurityPolicyStatus) DeepCopy() *BackendSecurityPolicyStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BedrockGuardrailProvider) DeepCopyInto(out *BedrockGuardrailProvider) { + *out = *in + if in.CredentialsSecretRef != nil { + in, out := &in.CredentialsSecretRef, &out.CredentialsSecretRef + *out = new(v1.SecretObjectReference) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BedrockGuardrailProvider. +func (in *BedrockGuardrailProvider) DeepCopy() *BedrockGuardrailProvider { + if in == nil { + return nil + } + out := new(BedrockGuardrailProvider) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CredentialOverrideFromDynamicMetadata) DeepCopyInto(out *CredentialOverrideFromDynamicMetadata) { *out = *in @@ -1094,7 +1139,9 @@ func (in *GuardrailPolicySpec) DeepCopyInto(out *GuardrailPolicySpec) { if in.Rules != nil { in, out := &in.Rules, &out.Rules *out = make([]GuardrailRule, len(*in)) - copy(*out, *in) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } } } @@ -1133,6 +1180,26 @@ func (in *GuardrailPolicyStatus) DeepCopy() *GuardrailPolicyStatus { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GuardrailProvider) DeepCopyInto(out *GuardrailProvider) { *out = *in + if in.Presidio != nil { + in, out := &in.Presidio, &out.Presidio + *out = new(PresidioGuardrailProvider) + (*in).DeepCopyInto(*out) + } + if in.Bedrock != nil { + in, out := &in.Bedrock, &out.Bedrock + *out = new(BedrockGuardrailProvider) + (*in).DeepCopyInto(*out) + } + if in.AzureContentSafety != nil { + in, out := &in.AzureContentSafety, &out.AzureContentSafety + *out = new(AzureContentSafetyGuardrailProvider) + (*in).DeepCopyInto(*out) + } + if in.TimeoutSeconds != nil { + in, out := &in.TimeoutSeconds, &out.TimeoutSeconds + *out = new(int32) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GuardrailProvider. @@ -1148,7 +1215,7 @@ func (in *GuardrailProvider) DeepCopy() *GuardrailProvider { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GuardrailRule) DeepCopyInto(out *GuardrailRule) { *out = *in - out.Provider = in.Provider + in.Provider.DeepCopyInto(&out.Provider) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GuardrailRule. @@ -1904,6 +1971,31 @@ func (in *MCPToolFilter) DeepCopy() *MCPToolFilter { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PresidioGuardrailProvider) DeepCopyInto(out *PresidioGuardrailProvider) { + *out = *in + if in.ScoreThresholdPercent != nil { + in, out := &in.ScoreThresholdPercent, &out.ScoreThresholdPercent + *out = new(int32) + **out = **in + } + if in.APIKeySecretRef != nil { + in, out := &in.APIKeySecretRef, &out.APIKeySecretRef + *out = new(v1.SecretObjectReference) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PresidioGuardrailProvider. +func (in *PresidioGuardrailProvider) DeepCopy() *PresidioGuardrailProvider { + if in == nil { + return nil + } + out := new(PresidioGuardrailProvider) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ProtectedResourceMetadata) DeepCopyInto(out *ProtectedResourceMetadata) { *out = *in diff --git a/examples/README.md b/examples/README.md index 5aa788de8f..b4299977ed 100644 --- a/examples/README.md +++ b/examples/README.md @@ -40,3 +40,7 @@ Shows how to use prompt caching with Claude models to reduce costs and improve p ### [Monitoring](./monitoring/) Example setup for comprehensive monitoring and observability with Prometheus and Grafana. + +### [Guardrails](./guardrails/) + +Shows how to enforce request and response content policies with regex, Presidio, AWS Bedrock Guardrails, and Azure AI Content Safety. diff --git a/examples/guardrails/README.md b/examples/guardrails/README.md new file mode 100644 index 0000000000..aed1c41ba1 --- /dev/null +++ b/examples/guardrails/README.md @@ -0,0 +1,24 @@ +# Guardrails + +These examples attach content-safety rules to an existing `AIServiceBackend` named `openai`. +Update `spec.targetRefs[].name` when your backend has a different name. + +Apply the deterministic regex example: + +```shell +kubectl apply -f regex.yaml +``` + +External providers require their service endpoint and, where applicable, a Kubernetes Secret: + +```shell +kubectl apply -f providers.yaml +``` + +The expected Secret keys are: + +- Presidio: optional `apiKey` +- Azure Content Safety: required `apiKey` +- AWS Bedrock Guardrails: optional `credentials` containing an AWS shared credentials file; when omitted, the ext-proc uses the standard AWS credential chain. + +Provider failures block traffic by default. Set `failureMode: FailOpen` on a rule to continue traffic when the external provider is unavailable. Response guardrails force buffered response processing so blocked content is not partially delivered. diff --git a/examples/guardrails/providers.yaml b/examples/guardrails/providers.yaml new file mode 100644 index 0000000000..fba8478bc8 --- /dev/null +++ b/examples/guardrails/providers.yaml @@ -0,0 +1,58 @@ +# Copyright Envoy AI Gateway Authors +# SPDX-License-Identifier: Apache-2.0 +# The full text of the Apache license is available in the LICENSE file at +# the root of the repo. + +apiVersion: v1 +kind: Secret +metadata: + name: guardrail-provider-keys +type: Opaque +stringData: + apiKey: replace-me +--- +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: GuardrailPolicy +metadata: + name: external-content-guardrails +spec: + targetRefs: + - group: aigateway.envoyproxy.io + kind: AIServiceBackend + name: openai + rules: + - name: detect-pii-with-presidio + phase: Request + provider: + type: Presidio + timeoutSeconds: 5 + failureMode: FailClosed + message: Request contains personally identifiable information + presidio: + endpoint: http://presidio-analyzer.presidio.svc.cluster.local:3000 + language: en + scoreThresholdPercent: 70 + - name: apply-bedrock-guardrail + phase: Response + provider: + type: Bedrock + timeoutSeconds: 10 + failureMode: FailClosed + message: Response was rejected by the configured Bedrock guardrail + bedrock: + region: us-east-1 + guardrailIdentifier: replace-me + guardrailVersion: "1" + - name: analyze-with-azure-content-safety + phase: Response + provider: + type: AzureContentSafety + timeoutSeconds: 10 + failureMode: FailOpen + message: Response violates the configured content-safety threshold + azureContentSafety: + endpoint: https://replace-me.cognitiveservices.azure.com + apiVersion: "2024-09-01" + severityThreshold: 4 + apiKeySecretRef: + name: guardrail-provider-keys diff --git a/examples/guardrails/regex.yaml b/examples/guardrails/regex.yaml new file mode 100644 index 0000000000..23df71f228 --- /dev/null +++ b/examples/guardrails/regex.yaml @@ -0,0 +1,29 @@ +# Copyright Envoy AI Gateway Authors +# SPDX-License-Identifier: Apache-2.0 +# The full text of the Apache license is available in the LICENSE file at +# the root of the repo. + +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: GuardrailPolicy +metadata: + name: local-content-guardrails +spec: + targetRefs: + - group: aigateway.envoyproxy.io + kind: AIServiceBackend + name: openai + rules: + - name: block-sensitive-request + phase: Request + provider: + type: Regex + pattern: "(?i)password|social security number" + action: Block + message: Request contains sensitive content + - name: block-sensitive-response + phase: Response + provider: + type: Regex + pattern: "(?i)internal[- ]only" + action: Block + message: Response contains restricted content diff --git a/internal/controller/controller.go b/internal/controller/controller.go index 07eb6c7121..c730d53635 100644 --- a/internal/controller/controller.go +++ b/internal/controller/controller.go @@ -261,6 +261,7 @@ func StartControllers(ctx context.Context, mgr manager.Manager, config *rest.Con guardrailPolicyC := NewGuardrailPolicyController(c, kube, logger.WithName("guardrail-policy"), aiGatewayRouteEventChan) if err = TypedControllerBuilderForCRD(mgr, &aigv1b1.GuardrailPolicy{}). Watches(&aigv1b1.AIServiceBackend{}, handler.EnqueueRequestsFromMapFunc(guardrailPolicyC.BackendToGuardrailPolicy)). + Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(guardrailPolicyC.SecretToGuardrailPolicy)). Complete(guardrailPolicyC); err != nil { return fmt.Errorf("failed to create controller for GuardrailPolicy: %w", err) } @@ -322,6 +323,8 @@ const ( // k8sClientIndexAIServiceBackendToTargetingGuardrailPolicy is the index name that maps from an AIServiceBackend // to the GuardrailPolicy whose targetRefs contains the AIServiceBackend. k8sClientIndexAIServiceBackendToTargetingGuardrailPolicy = "AIServiceBackendToTargetingGuardrailPolicy" + // k8sClientIndexSecretToReferencingGuardrailPolicy maps a Secret to GuardrailPolicies using it. + k8sClientIndexSecretToReferencingGuardrailPolicy = "SecretToReferencingGuardrailPolicy" // k8sClientIndexGatewayToGatewayConfig maps from a GatewayConfig name to Gateways referencing it. k8sClientIndexGatewayToGatewayConfig = "GatewayToGatewayConfig" @@ -376,6 +379,11 @@ func ApplyIndexing(ctx context.Context, indexer func(ctx context.Context, obj cl if err != nil { return fmt.Errorf("failed to index field for GuardrailPolicy targetRefs: %w", err) } + err = indexer(ctx, &aigv1b1.GuardrailPolicy{}, + k8sClientIndexSecretToReferencingGuardrailPolicy, guardrailPolicySecretRefsIndexFunc) + if err != nil { + return fmt.Errorf("failed to index field for GuardrailPolicy secretRefs: %w", err) + } err = indexer(ctx, &gwapiv1.Gateway{}, k8sClientIndexGatewayToGatewayConfig, gatewayToGatewayConfigIndexFunc) @@ -548,6 +556,37 @@ func guardrailPolicyTargetRefsIndexFunc(o client.Object) []string { return ret } +func guardrailPolicySecretRefsIndexFunc(o client.Object) []string { + policy := o.(*aigv1b1.GuardrailPolicy) + var keys []string + for i := range policy.Spec.Rules { + provider := &policy.Spec.Rules[i].Provider + var ref *gwapiv1.SecretObjectReference + switch provider.Type { + case aigv1b1.GuardrailProviderTypePresidio: + if provider.Presidio != nil { + ref = provider.Presidio.APIKeySecretRef + } + case aigv1b1.GuardrailProviderTypeBedrockGuardrails: + if provider.Bedrock != nil { + ref = provider.Bedrock.CredentialsSecretRef + } + case aigv1b1.GuardrailProviderTypeAzureContentSafety: + if provider.AzureContentSafety != nil { + ref = provider.AzureContentSafety.APIKeySecretRef + } + } + if ref != nil { + namespace := policy.Namespace + if ref.Namespace != nil { + namespace = string(*ref.Namespace) + } + keys = append(keys, fmt.Sprintf("%s.%s", ref.Name, namespace)) + } + } + return keys +} + func getSecretNameAndNamespace(secretRef *gwapiv1.SecretObjectReference, namespace string) string { if secretRef.Namespace != nil { return fmt.Sprintf("%s.%s", secretRef.Name, *secretRef.Namespace) diff --git a/internal/controller/gateway.go b/internal/controller/gateway.go index e3d904da9f..cca850fc69 100644 --- a/internal/controller/gateway.go +++ b/internal/controller/gateway.go @@ -431,6 +431,7 @@ func (c *GatewayController) reconcileFilterConfigSecret( routeBackendNamesSet := map[string]struct{}{} routeBackendNames := []string{} injectedQuotaCosts := make(map[string]struct{}) + injectedGuardrails := make(map[string]struct{}) for ruleIndex := range spec.Rules { rule := &spec.Rules[ruleIndex] for _, m := range rule.Matches { @@ -558,6 +559,9 @@ func (c *GatewayController) reconcileFilterConfigSecret( // Inject QuotaPolicy cost expressions as LLMRequestCost entries so ext_proc // computes and stores them in metadata for the HitsAddend to read. c.injectQuotaPolicyCostExpressions(ctx, aiGatewayRoute, ec, injectedQuotaCosts, routeName) + if guardrailErr := c.injectGuardrails(ctx, aiGatewayRoute, ec, injectedGuardrails); guardrailErr != nil { + return false, fmt.Errorf("failed to inject guardrails for route %s: %w", aiGatewayRoute.Name, guardrailErr) + } for _, fc := range dedup { ec.LLMRequestCosts = append(ec.LLMRequestCosts, fc) @@ -594,6 +598,175 @@ func (c *GatewayController) reconcileFilterConfigSecret( return hasEffectiveRoute, nil } +func (c *GatewayController) injectGuardrails( + ctx context.Context, + route *aigv1b1.AIGatewayRoute, + ec *filterapi.Config, + injected map[string]struct{}, +) error { + var policies aigv1b1.GuardrailPolicyList + if err := c.client.List(ctx, &policies, client.InNamespace(route.Namespace)); err != nil { + return fmt.Errorf("failed to list GuardrailPolicies: %w", err) + } + + routeBackends := make(map[string]struct{}) + for _, routeRule := range route.Spec.Rules { + for _, backendRef := range routeRule.BackendRefs { + backendNamespace := backendRef.GetNamespace(route.Namespace) + routeBackends[backendNamespace+"/"+backendRef.Name] = struct{}{} + } + } + + for i := range policies.Items { + policy := &policies.Items[i] + targetsRoute := false + for _, targetRef := range policy.Spec.TargetRefs { + if _, ok := routeBackends[policy.Namespace+"/"+string(targetRef.Name)]; ok { + targetsRoute = true + break + } + } + if !targetsRoute { + continue + } + + for ruleIndex := range policy.Spec.Rules { + rule := &policy.Spec.Rules[ruleIndex] + key := policy.Namespace + "/" + policy.Name + "/" + rule.Name + if _, ok := injected[key]; ok { + continue + } + backends := guardrailBackendNames(route, policy) + provider, err := c.guardrailProviderToFilterAPI(ctx, policy.Namespace, &rule.Provider) + if err != nil { + if rule.Provider.FailureMode == aigv1b1.GuardrailFailureModeFailOpen { + c.logger.Error(err, "guardrail configuration failed open", "policy", policy.Name, "rule", rule.Name) + injected[key] = struct{}{} + continue + } + c.logger.Error(err, "guardrail configuration failed closed", "policy", policy.Name, "rule", rule.Name) + provider = filterapi.GuardrailProvider{ + Type: filterapi.GuardrailProviderTypeRegex, + Pattern: `(?s).*`, + Action: filterapi.GuardrailActionBlock, + Message: "request blocked because guardrail configuration is unavailable", + } + } + ec.Guardrails = append(ec.Guardrails, filterapi.Guardrail{ + Name: key, + Phase: filterapi.GuardrailPhase(rule.Phase), + Provider: provider, + Backends: backends, + }) + injected[key] = struct{}{} + } + } + return nil +} + +func guardrailBackendNames(route *aigv1b1.AIGatewayRoute, policy *aigv1b1.GuardrailPolicy) []string { + targets := make(map[string]struct{}, len(policy.Spec.TargetRefs)) + for _, targetRef := range policy.Spec.TargetRefs { + targets[string(targetRef.Name)] = struct{}{} + } + var names []string + for ruleIndex := range route.Spec.Rules { + for backendRefIndex := range route.Spec.Rules[ruleIndex].BackendRefs { + backendRef := &route.Spec.Rules[ruleIndex].BackendRefs[backendRefIndex] + if backendRef.GetNamespace(route.Namespace) != policy.Namespace { + continue + } + if _, ok := targets[backendRef.Name]; !ok { + continue + } + names = append(names, internalapi.PerRouteRuleRefBackendName( + route.Namespace, backendRef.Name, route.Name, ruleIndex, backendRefIndex)) + } + } + return names +} + +func (c *GatewayController) guardrailProviderToFilterAPI(ctx context.Context, namespace string, provider *aigv1b1.GuardrailProvider) (filterapi.GuardrailProvider, error) { + converted := filterapi.GuardrailProvider{ + Type: filterapi.GuardrailProviderType(provider.Type), + Pattern: provider.Pattern, + Action: filterapi.GuardrailAction(provider.Action), + Message: provider.Message, + FailureMode: filterapi.GuardrailFailureMode(provider.FailureMode), + } + if provider.TimeoutSeconds != nil { + converted.TimeoutSeconds = *provider.TimeoutSeconds + } + switch provider.Type { + case aigv1b1.GuardrailProviderTypeRegex: + case aigv1b1.GuardrailProviderTypePresidio: + if provider.Presidio == nil { + return converted, fmt.Errorf("presidio configuration is required") + } + config := provider.Presidio + converted.Presidio = &filterapi.PresidioGuardrailProvider{ + Endpoint: config.Endpoint, + Language: config.Language, + } + if config.ScoreThresholdPercent != nil { + converted.Presidio.ScoreThresholdPercent = *config.ScoreThresholdPercent + } + if config.APIKeySecretRef != nil { + apiKey, err := c.getGuardrailSecretData(ctx, namespace, config.APIKeySecretRef, "apiKey") + if err != nil { + return converted, err + } + converted.Presidio.APIKey = apiKey + } + case aigv1b1.GuardrailProviderTypeBedrockGuardrails: + if provider.Bedrock == nil { + return converted, fmt.Errorf("bedrock configuration is required") + } + config := provider.Bedrock + converted.Bedrock = &filterapi.BedrockGuardrailProvider{ + Endpoint: config.Endpoint, + Region: config.Region, + GuardrailIdentifier: config.GuardrailIdentifier, + GuardrailVersion: config.GuardrailVersion, + } + if config.CredentialsSecretRef != nil { + credentials, err := c.getGuardrailSecretData(ctx, namespace, config.CredentialsSecretRef, "credentials") + if err != nil { + return converted, err + } + converted.Bedrock.CredentialFileLiteral = credentials + } + case aigv1b1.GuardrailProviderTypeAzureContentSafety: + if provider.AzureContentSafety == nil { + return converted, fmt.Errorf("azure Content Safety configuration is required") + } + config := provider.AzureContentSafety + apiKey, err := c.getGuardrailSecretData(ctx, namespace, config.APIKeySecretRef, "apiKey") + if err != nil { + return converted, err + } + converted.AzureContentSafety = &filterapi.AzureContentSafetyGuardrailProvider{ + Endpoint: config.Endpoint, + APIVersion: config.APIVersion, + SeverityThreshold: config.SeverityThreshold, + APIKey: apiKey, + } + default: + return converted, fmt.Errorf("unsupported provider type %q", provider.Type) + } + return converted, nil +} + +func (c *GatewayController) getGuardrailSecretData(ctx context.Context, namespace string, ref *gwapiv1.SecretObjectReference, key string) (string, error) { + if ref == nil { + return "", fmt.Errorf("secret reference is required") + } + if ref.Namespace != nil && string(*ref.Namespace) != namespace { + return "", fmt.Errorf("cross-namespace guardrail secret references are not supported") + } + return c.getSecretData(ctx, namespace, string(ref.Name), key) +} + // reconcileFilterConfigSecretForMCPGateway updates the filter config secret for the external processor. func mcpConfig(mcpRoutes []aigv1b1.MCPRoute) (_ *filterapi.MCPConfig, hasEffectiveRoute bool) { if len(mcpRoutes) == 0 { diff --git a/internal/controller/guardrail_policy.go b/internal/controller/guardrail_policy.go index 08ef5f3fed..b636080087 100644 --- a/internal/controller/guardrail_policy.go +++ b/internal/controller/guardrail_policy.go @@ -8,15 +8,19 @@ package controller import ( "context" "fmt" + "net/url" + "regexp" "github.com/go-logr/logr" apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/util/retry" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/reconcile" + gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" aigv1b1 "github.com/envoyproxy/ai-gateway/api/v1beta1" ) @@ -52,16 +56,16 @@ func (c *GuardrailPolicyController) Reconcile(ctx context.Context, req reconcile } c.logger.Info("Reconciling GuardrailPolicy", "namespace", req.Namespace, "name", req.Name) - if handleFinalizer(ctx, c.client, c.logger, &policy, nil) { - return ctrl.Result{}, nil - } - if len(policy.Spec.TargetRefs) == 0 { - c.updateGuardrailPolicyStatus(ctx, &policy, aigv1b1.ConditionTypeAccepted, "GuardrailPolicy reconciled successfully") + if handleFinalizer(ctx, c.client, c.logger, &policy, func(ctx context.Context, policy *aigv1b1.GuardrailPolicy) error { + c.notifyAIGatewayRoutesForGuardrailPolicy(ctx, policy) + return nil + }) { return ctrl.Result{}, nil } if err := c.syncGuardrailPolicy(ctx, &policy); err != nil { c.logger.Error(err, "failed to sync GuardrailPolicy") c.updateGuardrailPolicyStatus(ctx, &policy, aigv1b1.ConditionTypeNotAccepted, err.Error()) + c.notifyAIGatewayRoutesForGuardrailPolicy(ctx, &policy) return ctrl.Result{}, err } @@ -70,20 +74,35 @@ func (c *GuardrailPolicyController) Reconcile(ctx context.Context, req reconcile return ctrl.Result{}, nil } -func (c *GuardrailPolicyController) syncGuardrailPolicy(ctx context.Context, policy *aigv1b1.GuardrailPolicy) error { - if len(policy.Spec.TargetRefs) == 0 { +// SecretToGuardrailPolicy maps Secret changes to GuardrailPolicy reconcile requests. +func (c *GuardrailPolicyController) SecretToGuardrailPolicy(ctx context.Context, obj client.Object) []reconcile.Request { + var policies aigv1b1.GuardrailPolicyList + key := fmt.Sprintf("%s.%s", obj.GetName(), obj.GetNamespace()) + if err := c.client.List(ctx, &policies, + client.MatchingFields{k8sClientIndexSecretToReferencingGuardrailPolicy: key}); err != nil { + c.logger.Error(err, "failed to list GuardrailPolicies for secret", "secret", key) return nil } + requests := make([]reconcile.Request, 0, len(policies.Items)) + for i := range policies.Items { + requests = append(requests, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&policies.Items[i])}) + } + return requests +} + +func (c *GuardrailPolicyController) syncGuardrailPolicy(ctx context.Context, policy *aigv1b1.GuardrailPolicy) error { + for i := range policy.Spec.Rules { + if err := c.validateGuardrailProvider(ctx, policy.Namespace, &policy.Spec.Rules[i].Provider); err != nil { + return fmt.Errorf("rule %q: %w", policy.Spec.Rules[i].Name, err) + } + } for _, ref := range policy.Spec.TargetRefs { var backend aigv1b1.AIServiceBackend key := client.ObjectKey{Namespace: policy.Namespace, Name: string(ref.Name)} if err := c.client.Get(ctx, key, &backend); err != nil { if apierrors.IsNotFound(err) { - c.logger.Info("AIServiceBackend not found, skipping guardrail policy target", - "namespace", key.Namespace, "name", key.Name, - "guardrailPolicy", policy.Name) - continue + return fmt.Errorf("target AIServiceBackend %s not found", key) } return fmt.Errorf("failed to get AIServiceBackend %s: %w", key, err) } @@ -92,6 +111,84 @@ func (c *GuardrailPolicyController) syncGuardrailPolicy(ctx context.Context, pol return nil } +func (c *GuardrailPolicyController) validateGuardrailProvider(ctx context.Context, namespace string, provider *aigv1b1.GuardrailProvider) error { + if provider.Action != "" && provider.Action != aigv1b1.GuardrailActionBlock { + return fmt.Errorf("unsupported action %q", provider.Action) + } + if provider.FailureMode != "" && provider.FailureMode != aigv1b1.GuardrailFailureModeFailClosed && provider.FailureMode != aigv1b1.GuardrailFailureModeFailOpen { + return fmt.Errorf("unsupported failureMode %q", provider.FailureMode) + } + switch provider.Type { + case aigv1b1.GuardrailProviderTypeRegex: + if provider.Pattern == "" { + return fmt.Errorf("regex pattern is required") + } + if _, err := regexp.Compile(provider.Pattern); err != nil { + return fmt.Errorf("invalid regex pattern: %w", err) + } + case aigv1b1.GuardrailProviderTypePresidio: + if provider.Presidio == nil { + return fmt.Errorf("presidio configuration is required") + } + if err := validateGuardrailEndpoint(provider.Presidio.Endpoint); err != nil { + return err + } + if provider.Presidio.APIKeySecretRef != nil { + return c.validateGuardrailSecret(ctx, namespace, provider.Presidio.APIKeySecretRef, "apiKey") + } + case aigv1b1.GuardrailProviderTypeBedrockGuardrails: + if provider.Bedrock == nil || provider.Bedrock.Region == "" || provider.Bedrock.GuardrailIdentifier == "" || provider.Bedrock.GuardrailVersion == "" { + return fmt.Errorf("bedrock region, guardrailIdentifier, and guardrailVersion are required") + } + if provider.Bedrock.Endpoint != "" { + if err := validateGuardrailEndpoint(provider.Bedrock.Endpoint); err != nil { + return err + } + } + if provider.Bedrock.CredentialsSecretRef != nil { + return c.validateGuardrailSecret(ctx, namespace, provider.Bedrock.CredentialsSecretRef, "credentials") + } + case aigv1b1.GuardrailProviderTypeAzureContentSafety: + if provider.AzureContentSafety == nil { + return fmt.Errorf("azure Content Safety configuration is required") + } + if err := validateGuardrailEndpoint(provider.AzureContentSafety.Endpoint); err != nil { + return err + } + return c.validateGuardrailSecret(ctx, namespace, provider.AzureContentSafety.APIKeySecretRef, "apiKey") + default: + return fmt.Errorf("unsupported provider type %q", provider.Type) + } + return nil +} + +func validateGuardrailEndpoint(endpoint string) error { + parsed, err := url.ParseRequestURI(endpoint) + if err != nil || parsed.Scheme == "" || parsed.Host == "" { + return fmt.Errorf("valid provider endpoint is required") + } + return nil +} + +func (c *GuardrailPolicyController) validateGuardrailSecret(ctx context.Context, namespace string, ref *gwapiv1.SecretObjectReference, key string) error { + if ref == nil { + return fmt.Errorf("secret reference is required") + } + if ref.Namespace != nil && string(*ref.Namespace) != namespace { + return fmt.Errorf("cross-namespace guardrail secret references are not supported") + } + secret, err := c.kube.CoreV1().Secrets(namespace).Get(ctx, string(ref.Name), metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get secret %s: %w", ref.Name, err) + } + if _, ok := secret.Data[key]; !ok { + if _, ok = secret.StringData[key]; !ok { + return fmt.Errorf("secret %s does not contain key %s", ref.Name, key) + } + } + return nil +} + // BackendToGuardrailPolicy maps AIServiceBackend changes to GuardrailPolicy reconcile requests. func (c *GuardrailPolicyController) BackendToGuardrailPolicy(ctx context.Context, obj client.Object) []reconcile.Request { var policies aigv1b1.GuardrailPolicyList diff --git a/internal/controller/guardrail_policy_test.go b/internal/controller/guardrail_policy_test.go new file mode 100644 index 0000000000..e1c30db58b --- /dev/null +++ b/internal/controller/guardrail_policy_test.go @@ -0,0 +1,325 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package controller + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + fakekube "k8s.io/client-go/kubernetes/fake" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" + gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + + aigv1b1 "github.com/envoyproxy/ai-gateway/api/v1beta1" + "github.com/envoyproxy/ai-gateway/internal/filterapi" + "github.com/envoyproxy/ai-gateway/internal/guardrails" +) + +func newGuardrailPolicyTestClient(t *testing.T) client.Client { + t.Helper() + builder := fake.NewClientBuilder().WithScheme(Scheme). + WithStatusSubresource(&aigv1b1.GuardrailPolicy{}) + require.NoError(t, ApplyIndexing(t.Context(), func(_ context.Context, obj client.Object, field string, extractValue client.IndexerFunc) error { + builder = builder.WithIndex(obj, field, extractValue) + return nil + })) + return builder.Build() +} + +func TestGuardrailPolicyControllerReconcileProviderReadiness(t *testing.T) { + const namespace = "default" + tests := []struct { + name string + withSecret bool + wantCondition string + wantError bool + }{ + {name: "accepted with provider secret", withSecret: true, wantCondition: aigv1b1.ConditionTypeAccepted}, + {name: "not accepted without provider secret", wantCondition: aigv1b1.ConditionTypeNotAccepted, wantError: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + kube := fakekube.NewClientset() + if test.withSecret { + _, err := kube.CoreV1().Secrets(namespace).Create(t.Context(), &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "azure-key", Namespace: namespace}, + Data: map[string][]byte{"apiKey": []byte("secret")}, + }, metav1.CreateOptions{}) + require.NoError(t, err) + } + controllerClient := newGuardrailPolicyTestClient(t) + require.NoError(t, controllerClient.Create(t.Context(), &aigv1b1.AIServiceBackend{ + ObjectMeta: metav1.ObjectMeta{Name: "backend", Namespace: namespace}, + })) + policy := &aigv1b1.GuardrailPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: "policy", Namespace: namespace}, + Spec: aigv1b1.GuardrailPolicySpec{ + TargetRefs: []gwapiv1a2.LocalPolicyTargetReference{{ + Group: "aigateway.envoyproxy.io", Kind: "AIServiceBackend", Name: "backend", + }}, + Rules: []aigv1b1.GuardrailRule{{ + Name: "azure", Phase: aigv1b1.GuardrailPhaseRequest, + Provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeAzureContentSafety, + AzureContentSafety: &aigv1b1.AzureContentSafetyGuardrailProvider{ + Endpoint: "https://content-safety.example.com", + APIKeySecretRef: &gwapiv1.SecretObjectReference{Name: "azure-key"}, + }, + }, + }}, + }, + } + require.NoError(t, controllerClient.Create(t.Context(), policy)) + controller := NewGuardrailPolicyController(controllerClient, kube, ctrl.Log, make(chan event.GenericEvent, 1)) + _, err := controller.Reconcile(t.Context(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: namespace, Name: policy.Name}}) + if test.wantError { + require.Error(t, err) + } else { + require.NoError(t, err) + } + + var updated aigv1b1.GuardrailPolicy + require.NoError(t, controllerClient.Get(t.Context(), client.ObjectKeyFromObject(policy), &updated)) + require.Len(t, updated.Status.Conditions, 1) + require.Equal(t, test.wantCondition, updated.Status.Conditions[0].Type) + }) + } +} + +func TestGuardrailPolicyControllerDeletionNotifiesRoutes(t *testing.T) { + const namespace = "default" + controllerClient := newGuardrailPolicyTestClient(t) + route := &aigv1b1.AIGatewayRoute{ + ObjectMeta: metav1.ObjectMeta{Name: "route", Namespace: namespace}, + Spec: aigv1b1.AIGatewayRouteSpec{Rules: []aigv1b1.AIGatewayRouteRule{{ + BackendRefs: []aigv1b1.AIGatewayRouteRuleBackendRef{{Name: "backend"}}, + }}}, + } + require.NoError(t, controllerClient.Create(t.Context(), route)) + require.NoError(t, controllerClient.Create(t.Context(), &aigv1b1.AIServiceBackend{ + ObjectMeta: metav1.ObjectMeta{Name: "backend", Namespace: namespace}, + })) + policy := &aigv1b1.GuardrailPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "policy", + Namespace: namespace, + }, + Spec: aigv1b1.GuardrailPolicySpec{ + TargetRefs: []gwapiv1a2.LocalPolicyTargetReference{{Name: "backend"}}, + Rules: []aigv1b1.GuardrailRule{{ + Name: "deny", Phase: aigv1b1.GuardrailPhaseRequest, + Provider: aigv1b1.GuardrailProvider{Type: aigv1b1.GuardrailProviderTypeRegex, Pattern: "secret"}, + }}, + }, + } + require.NoError(t, controllerClient.Create(t.Context(), policy)) + routeEvents := make(chan event.GenericEvent, 1) + controller := NewGuardrailPolicyController(controllerClient, fakekube.NewClientset(), ctrl.Log, routeEvents) + request := reconcile.Request{NamespacedName: client.ObjectKeyFromObject(policy)} + _, err := controller.Reconcile(t.Context(), request) + require.NoError(t, err) + <-routeEvents + require.NoError(t, controllerClient.Delete(t.Context(), policy)) + + _, err = controller.Reconcile(t.Context(), request) + require.NoError(t, err) + select { + case got := <-routeEvents: + require.Equal(t, route.Name, got.Object.GetName()) + require.Equal(t, route.Namespace, got.Object.GetNamespace()) + default: + t.Fatal("expected route notification when GuardrailPolicy is deleted") + } + + var updated aigv1b1.GuardrailPolicy + err = controllerClient.Get(t.Context(), client.ObjectKeyFromObject(policy), &updated) + if err == nil { + require.NotContains(t, updated.Finalizers, aiGatewayControllerFinalizer) + } else { + require.True(t, apierrors.IsNotFound(err), "unexpected get error: %v", err) + } +} + +func TestGuardrailProviderToFilterAPIResolvesSecret(t *testing.T) { + const namespace = "default" + kube := fakekube.NewClientset(&corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "presidio-key", Namespace: namespace}, + Data: map[string][]byte{"apiKey": []byte("secret-value")}, + }) + controller := &GatewayController{kube: kube} + provider, err := controller.guardrailProviderToFilterAPI(t.Context(), namespace, &aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypePresidio, + Presidio: &aigv1b1.PresidioGuardrailProvider{ + Endpoint: "https://presidio.example.com", + APIKeySecretRef: &gwapiv1.SecretObjectReference{Name: "presidio-key"}, + }, + }) + require.NoError(t, err) + require.Equal(t, "secret-value", provider.Presidio.APIKey) +} + +func TestInjectGuardrailsTargetsRouteBackends(t *testing.T) { + const namespace = "default" + controllerClient := newGuardrailPolicyTestClient(t) + for _, policy := range []*aigv1b1.GuardrailPolicy{ + { + ObjectMeta: metav1.ObjectMeta{Name: "selected", Namespace: namespace}, + Spec: aigv1b1.GuardrailPolicySpec{ + TargetRefs: []gwapiv1a2.LocalPolicyTargetReference{{Name: "backend"}}, + Rules: []aigv1b1.GuardrailRule{{ + Name: "deny", Phase: aigv1b1.GuardrailPhaseRequest, + Provider: aigv1b1.GuardrailProvider{Type: aigv1b1.GuardrailProviderTypeRegex, Pattern: "secret"}, + }}, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "ignored", Namespace: namespace}, + Spec: aigv1b1.GuardrailPolicySpec{ + TargetRefs: []gwapiv1a2.LocalPolicyTargetReference{{Name: "other-backend"}}, + Rules: []aigv1b1.GuardrailRule{{ + Name: "ignored", Phase: aigv1b1.GuardrailPhaseRequest, + Provider: aigv1b1.GuardrailProvider{Type: aigv1b1.GuardrailProviderTypeRegex, Pattern: "ignored"}, + }}, + }, + }, + } { + require.NoError(t, controllerClient.Create(t.Context(), policy)) + } + + controller := &GatewayController{client: controllerClient, kube: fakekube.NewClientset(), logger: ctrl.Log} + config := &filterapi.Config{} + route := &aigv1b1.AIGatewayRoute{ + ObjectMeta: metav1.ObjectMeta{Name: "route", Namespace: namespace}, + Spec: aigv1b1.AIGatewayRouteSpec{Rules: []aigv1b1.AIGatewayRouteRule{{ + BackendRefs: []aigv1b1.AIGatewayRouteRuleBackendRef{{Name: "backend"}}, + }}}, + } + require.NoError(t, controller.injectGuardrails(t.Context(), route, config, map[string]struct{}{})) + require.Len(t, config.Guardrails, 1) + require.Equal(t, "default/selected/deny", config.Guardrails[0].Name) + require.Equal(t, "secret", config.Guardrails[0].Provider.Pattern) + require.Equal(t, []string{"default/backend/route/route/rule/0/ref/0"}, config.Guardrails[0].Backends) +} + +func TestSecretToGuardrailPolicy(t *testing.T) { + const namespace = "default" + controllerClient := newGuardrailPolicyTestClient(t) + policy := &aigv1b1.GuardrailPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: "policy", Namespace: namespace}, + Spec: aigv1b1.GuardrailPolicySpec{Rules: []aigv1b1.GuardrailRule{{ + Name: "presidio", Phase: aigv1b1.GuardrailPhaseRequest, + Provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypePresidio, + Presidio: &aigv1b1.PresidioGuardrailProvider{ + Endpoint: "https://presidio.example.com", + APIKeySecretRef: &gwapiv1.SecretObjectReference{Name: "provider-key"}, + }, + }, + }}}, + } + require.NoError(t, controllerClient.Create(t.Context(), policy)) + controller := NewGuardrailPolicyController(controllerClient, fakekube.NewClientset(), ctrl.Log, make(chan event.GenericEvent, 1)) + + requests := controller.SecretToGuardrailPolicy(t.Context(), &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "provider-key", Namespace: namespace}, + }) + require.Equal(t, []reconcile.Request{{NamespacedName: types.NamespacedName{ + Namespace: namespace, Name: policy.Name, + }}}, requests) +} + +func TestInjectGuardrailsConfigurationFailureModes(t *testing.T) { + const namespace = "default" + for _, test := range []struct { + name string + failureMode aigv1b1.GuardrailFailureMode + wantGuardrail bool + }{ + {name: "fail closed publishes blocking fallback", wantGuardrail: true}, + {name: "fail open omits unavailable rule", failureMode: aigv1b1.GuardrailFailureModeFailOpen}, + } { + t.Run(test.name, func(t *testing.T) { + controllerClient := newGuardrailPolicyTestClient(t) + policy := &aigv1b1.GuardrailPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: "policy", Namespace: namespace}, + Spec: aigv1b1.GuardrailPolicySpec{ + TargetRefs: []gwapiv1a2.LocalPolicyTargetReference{{Name: "backend"}}, + Rules: []aigv1b1.GuardrailRule{{ + Name: "azure", Phase: aigv1b1.GuardrailPhaseRequest, + Provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeAzureContentSafety, + FailureMode: test.failureMode, + AzureContentSafety: &aigv1b1.AzureContentSafetyGuardrailProvider{ + Endpoint: "https://content-safety.example.com", + APIKeySecretRef: &gwapiv1.SecretObjectReference{Name: "missing"}, + }, + }, + }}, + }, + } + require.NoError(t, controllerClient.Create(t.Context(), policy)) + controller := &GatewayController{client: controllerClient, kube: fakekube.NewClientset(), logger: ctrl.Log} + config := &filterapi.Config{} + route := &aigv1b1.AIGatewayRoute{ + ObjectMeta: metav1.ObjectMeta{Name: "route", Namespace: namespace}, + Spec: aigv1b1.AIGatewayRouteSpec{Rules: []aigv1b1.AIGatewayRouteRule{{ + BackendRefs: []aigv1b1.AIGatewayRouteRuleBackendRef{{Name: "backend"}}, + }}}, + } + require.NoError(t, controller.injectGuardrails(t.Context(), route, config, map[string]struct{}{})) + if !test.wantGuardrail { + require.Empty(t, config.Guardrails) + return + } + require.Len(t, config.Guardrails, 1) + require.Equal(t, filterapi.GuardrailProviderTypeRegex, config.Guardrails[0].Provider.Type) + require.Equal(t, `(?s).*`, config.Guardrails[0].Provider.Pattern) + }) + } +} + +func TestGuardrailPolicyToRuntimeIntegration(t *testing.T) { + providerServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`[{"entity_type":"EMAIL_ADDRESS","score":0.99}]`)) + })) + t.Cleanup(providerServer.Close) + + kube := fakekube.NewClientset(&corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "presidio-key", Namespace: "default"}, + Data: map[string][]byte{"apiKey": []byte("secret")}, + }) + controller := &GatewayController{kube: kube} + converted, err := controller.guardrailProviderToFilterAPI(t.Context(), "default", &aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypePresidio, + Presidio: &aigv1b1.PresidioGuardrailProvider{ + Endpoint: providerServer.URL, + APIKeySecretRef: &gwapiv1.SecretObjectReference{Name: "presidio-key"}, + }, + }) + require.NoError(t, err) + runtimeConfig, err := filterapi.NewRuntimeConfig(t.Context(), &filterapi.Config{ + Guardrails: []filterapi.Guardrail{{Name: "pii", Phase: filterapi.GuardrailPhaseRequest, Provider: converted}}, + }, func(context.Context, *filterapi.BackendAuth) (filterapi.BackendAuthHandler, error) { + return nil, nil + }, guardrails.NewEvaluator) + require.NoError(t, err) + require.Len(t, runtimeConfig.Guardrails, 1) + blocked, err := runtimeConfig.Guardrails[0].Evaluator.Evaluate(t.Context(), []byte("user@example.com"), filterapi.GuardrailPhaseRequest) + require.NoError(t, err) + require.True(t, blocked) +} diff --git a/internal/extproc/guardrails.go b/internal/extproc/guardrails.go index 0f0e1f625b..3e8ef0abed 100644 --- a/internal/extproc/guardrails.go +++ b/internal/extproc/guardrails.go @@ -6,7 +6,10 @@ package extproc import ( + "context" + "errors" "fmt" + "slices" "github.com/envoyproxy/ai-gateway/internal/filterapi" ) @@ -17,41 +20,84 @@ type guardrailViolation struct { Message string } -func guardrailsConfiguredForPhase(guardrails []filterapi.RuntimeGuardrail, phase filterapi.GuardrailPhase) bool { +type guardrailFailOpenError struct { + errors []error +} + +func (e *guardrailFailOpenError) Error() string { + return fmt.Sprintf("%d guardrail provider evaluation(s) failed open: %v", len(e.errors), errors.Join(e.errors...)) +} + +func isGuardrailFailOpenError(err error) bool { + var failOpenError *guardrailFailOpenError + return errors.As(err, &failOpenError) +} + +func guardrailsConfiguredForPhase(guardrails []filterapi.RuntimeGuardrail, phase filterapi.GuardrailPhase, backendName string, includeGlobal bool) bool { for i := range guardrails { - if guardrails[i].Phase == phase { + if guardrails[i].Phase == phase && guardrailAppliesToBackend(&guardrails[i], backendName, includeGlobal) { return true } } return false } -func evaluateGuardrailsForPhase(guardrails []filterapi.RuntimeGuardrail, phase filterapi.GuardrailPhase, body []byte) (*guardrailViolation, error) { +func evaluateGuardrailsForPhase(ctx context.Context, guardrails []filterapi.RuntimeGuardrail, phase filterapi.GuardrailPhase, body []byte, backendName string, includeGlobal bool) (*guardrailViolation, error) { + var failOpenErrors []error for i := range guardrails { g := &guardrails[i] - if g.Phase != phase { + if g.Phase != phase || !guardrailAppliesToBackend(g, backendName, includeGlobal) { continue } + var blocked bool if g.Provider.Type == filterapi.GuardrailProviderTypeRegex { if g.Matcher == nil { return nil, fmt.Errorf("guardrail %q uses regex provider without a compiled matcher", g.Name) } - if g.Matcher.Match(body) { - msg := g.Provider.Message - if msg == "" { - msg = fmt.Sprintf("request blocked by guardrail %q", g.Name) + blocked = g.Matcher.Match(body) + } else { + if g.Evaluator == nil { + return nil, fmt.Errorf("guardrail %q uses provider %q without an evaluator", g.Name, g.Provider.Type) + } + var err error + blocked, err = g.Evaluator.Evaluate(ctx, body, phase) + if err != nil { + if g.Provider.FailureMode == filterapi.GuardrailFailureModeFailOpen { + failOpenErrors = append(failOpenErrors, fmt.Errorf("guardrail %q evaluation failed: %w", g.Name, err)) + continue } - return &guardrailViolation{Name: g.Name, Message: msg}, nil + return nil, fmt.Errorf("guardrail %q evaluation failed: %w", g.Name, err) } } + if blocked { + msg := g.Provider.Message + if msg == "" { + msg = fmt.Sprintf("request blocked by guardrail %q", g.Name) + } + return &guardrailViolation{Name: g.Name, Message: msg}, nil + } + } + if len(failOpenErrors) > 0 { + return nil, &guardrailFailOpenError{errors: failOpenErrors} } return nil, nil } -func evaluateRequestGuardrails(guardrails []filterapi.RuntimeGuardrail, body []byte) (*guardrailViolation, error) { - return evaluateGuardrailsForPhase(guardrails, filterapi.GuardrailPhaseRequest, body) +func guardrailAppliesToBackend(guardrail *filterapi.RuntimeGuardrail, backendName string, includeGlobal bool) bool { + if len(guardrail.Backends) == 0 { + return includeGlobal + } + return backendName != "" && slices.Contains(guardrail.Backends, backendName) +} + +func evaluateRequestGuardrails(ctx context.Context, guardrails []filterapi.RuntimeGuardrail, body []byte) (*guardrailViolation, error) { + return evaluateGuardrailsForPhase(ctx, guardrails, filterapi.GuardrailPhaseRequest, body, "", true) +} + +func evaluateBackendRequestGuardrails(ctx context.Context, guardrails []filterapi.RuntimeGuardrail, body []byte, backendName string) (*guardrailViolation, error) { + return evaluateGuardrailsForPhase(ctx, guardrails, filterapi.GuardrailPhaseRequest, body, backendName, false) } -func evaluateResponseGuardrails(guardrails []filterapi.RuntimeGuardrail, body []byte) (*guardrailViolation, error) { - return evaluateGuardrailsForPhase(guardrails, filterapi.GuardrailPhaseResponse, body) +func evaluateResponseGuardrails(ctx context.Context, guardrails []filterapi.RuntimeGuardrail, body []byte, backendName string) (*guardrailViolation, error) { + return evaluateGuardrailsForPhase(ctx, guardrails, filterapi.GuardrailPhaseResponse, body, backendName, true) } diff --git a/internal/extproc/guardrails_test.go b/internal/extproc/guardrails_test.go index 7c28b7542f..87cf6fea4e 100644 --- a/internal/extproc/guardrails_test.go +++ b/internal/extproc/guardrails_test.go @@ -7,6 +7,7 @@ package extproc import ( "context" + "errors" "log/slog" "regexp" "testing" @@ -26,6 +27,12 @@ type recordingGuardrailMetrics struct { count int } +type failingGuardrailEvaluator struct{} + +func (*failingGuardrailEvaluator) Evaluate(context.Context, []byte, filterapi.GuardrailPhase) (bool, error) { + return false, errors.New("provider unavailable") +} + func (m *recordingGuardrailMetrics) RecordEvaluation(_ context.Context, phase string, result metrics.GuardrailResult) { m.phase = phase m.result = result @@ -44,7 +51,7 @@ func TestEvaluateGuardrailsForPhase(t *testing.T) { Matcher: regexp.MustCompile(`\bSSN\b`), }} - violation, err := evaluateRequestGuardrails(guardrails, []byte("customer SSN is present")) + violation, err := evaluateRequestGuardrails(t.Context(), guardrails, []byte("customer SSN is present")) require.NoError(t, err) require.NotNil(t, violation) require.Equal(t, "deny-pii", violation.Name) @@ -61,7 +68,7 @@ func TestEvaluateGuardrailsForPhase(t *testing.T) { Matcher: regexp.MustCompile(`forbidden`), }} - violation, err := evaluateRequestGuardrails(guardrails, []byte("forbidden")) + violation, err := evaluateRequestGuardrails(t.Context(), guardrails, []byte("forbidden")) require.NoError(t, err) require.Nil(t, violation) }) @@ -75,7 +82,7 @@ func TestEvaluateGuardrailsForPhase(t *testing.T) { }, }} - violation, err := evaluateRequestGuardrails(guardrails, []byte("forbidden")) + violation, err := evaluateRequestGuardrails(t.Context(), guardrails, []byte("forbidden")) require.Error(t, err) require.Nil(t, violation) require.Contains(t, err.Error(), "uses regex provider without a compiled matcher") @@ -120,6 +127,41 @@ func TestRecordGuardrailEvaluationIgnoresUnconfiguredPhase(t *testing.T) { guardrailMetrics: recorder, } - processor.recordGuardrailEvaluation(t.Context(), filterapi.GuardrailPhaseRequest, metrics.GuardrailResultAllowed) + processor.recordGuardrailEvaluation(t.Context(), filterapi.GuardrailPhaseRequest, metrics.GuardrailResultAllowed, "", true) require.Zero(t, recorder.count) } + +func TestBackendScopedGuardrail(t *testing.T) { + guardrails := []filterapi.RuntimeGuardrail{{ + Name: "backend-only", Phase: filterapi.GuardrailPhaseRequest, + Backends: []string{"selected-backend"}, + Provider: filterapi.GuardrailProvider{Type: filterapi.GuardrailProviderTypeRegex}, + Matcher: regexp.MustCompile("blocked"), + }} + + violation, err := evaluateBackendRequestGuardrails(t.Context(), guardrails, []byte("blocked"), "other-backend") + require.NoError(t, err) + require.Nil(t, violation) + + violation, err = evaluateBackendRequestGuardrails(t.Context(), guardrails, []byte("blocked"), "selected-backend") + require.NoError(t, err) + require.NotNil(t, violation) +} + +func TestGuardrailFailureModes(t *testing.T) { + guardrail := filterapi.RuntimeGuardrail{ + Name: "external", Phase: filterapi.GuardrailPhaseRequest, + Provider: filterapi.GuardrailProvider{Type: filterapi.GuardrailProviderTypePresidio}, + Evaluator: &failingGuardrailEvaluator{}, + } + + _, err := evaluateRequestGuardrails(t.Context(), []filterapi.RuntimeGuardrail{guardrail}, []byte("payload")) + require.Error(t, err) + require.False(t, isGuardrailFailOpenError(err)) + + guardrail.Provider.FailureMode = filterapi.GuardrailFailureModeFailOpen + violation, err := evaluateRequestGuardrails(t.Context(), []filterapi.RuntimeGuardrail{guardrail}, []byte("payload")) + require.Nil(t, violation) + require.Error(t, err) + require.True(t, isGuardrailFailOpenError(err)) +} diff --git a/internal/extproc/processor_impl.go b/internal/extproc/processor_impl.go index 720c3a9a60..be6f00f7c1 100644 --- a/internal/extproc/processor_impl.go +++ b/internal/extproc/processor_impl.go @@ -13,6 +13,7 @@ import ( "fmt" "io" "log/slog" + "net/http" "strconv" "strings" @@ -163,13 +164,21 @@ func (r *routerProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) recordGuardrai ctx context.Context, phase filterapi.GuardrailPhase, result metrics.GuardrailResult, + backendName string, + includeGlobal bool, ) { - if r.guardrailMetrics == nil || !guardrailsConfiguredForPhase(r.config.Guardrails, phase) { + if r.guardrailMetrics == nil || r.config == nil || !guardrailsConfiguredForPhase(r.config.Guardrails, phase, backendName, includeGlobal) { return } r.guardrailMetrics.RecordEvaluation(ctx, string(phase), result) } +func (r *routerProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) recordGuardrailTrace(name string, phase filterapi.GuardrailPhase, result metrics.GuardrailResult) { + if span, ok := r.span.(tracingapi.GuardrailSpan); ok { + span.RecordGuardrail(name, string(phase), string(result)) + } +} + func newUpstreamProcessor[ReqT, RespT, RespChunkT any, EndpointSpecT endpointspec.Spec[ReqT, RespT, RespChunkT]]( reqHeader map[string]string, metrics metrics.Metrics, logger *slog.Logger, @@ -228,6 +237,10 @@ func (u *upstreamProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) respondLocal // createUserFacingErrorResponse creates an ImmediateResponse for user-facing errors with JSON body. func createUserFacingErrorResponse(statusCode int, errorType string, message string) *extprocv3.ProcessingResponse { body := formatUserFacingErrorJSON(errorType, statusCode, message) + grpcStatus := codes.InvalidArgument + if statusCode == http.StatusForbidden { + grpcStatus = codes.PermissionDenied + } headerMutation := &extprocv3.HeaderMutation{} setHeader(headerMutation, "content-type", "application/json") setHeader(headerMutation, "content-length", strconv.Itoa(len(body))) @@ -238,7 +251,7 @@ func createUserFacingErrorResponse(statusCode int, errorType string, message str Status: &typev3.HttpStatus{Code: typev3.StatusCode(statusCode)}, // #nosec G115 - HTTP status codes are always in valid int32 range Headers: headerMutation, Body: body, - GrpcStatus: &extprocv3.GrpcStatus{Status: uint32(codes.InvalidArgument)}, + GrpcStatus: &extprocv3.GrpcStatus{Status: uint32(grpcStatus)}, }, }, } @@ -268,18 +281,6 @@ func (r *routerProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) ProcessRequest } return nil, fmt.Errorf("failed to parse request body: %w", err) } - if violation, err := evaluateRequestGuardrails(r.config.Guardrails, rawBody.Body); err != nil { - r.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultError) - return nil, fmt.Errorf("failed to evaluate request guardrails: %w", err) - } else if violation != nil { - r.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultBlocked) - r.logger.Warn("request blocked by guardrail", - slog.String("guardrail.name", violation.Name), - slog.String("guardrail.phase", string(filterapi.GuardrailPhaseRequest))) - return createUserFacingErrorResponse(400, "BadRequest", violation.Message), nil - } - r.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultAllowed) - // Use the request-scoped logger from context if available, otherwise fall back to processor logger logger := loggerFromContext(ctx) if logger == nil { @@ -345,6 +346,30 @@ func (r *routerProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) ProcessRequest body, rawBody.Body, ) + violation, guardrailErr := evaluateRequestGuardrails(ctx, r.config.Guardrails, rawBody.Body) + if guardrailErr != nil { + r.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultError, "", true) + r.recordGuardrailTrace("", filterapi.GuardrailPhaseRequest, metrics.GuardrailResultError) + if !isGuardrailFailOpenError(guardrailErr) { + return nil, fmt.Errorf("failed to evaluate request guardrails: %w", guardrailErr) + } + r.logger.Warn("request guardrail provider failed open", slog.String("error", guardrailErr.Error())) + } else if violation != nil { + r.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultBlocked, "", true) + r.recordGuardrailTrace(violation.Name, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultBlocked) + r.logger.Warn("request blocked by guardrail", + slog.String("guardrail.name", violation.Name), + slog.String("guardrail.phase", string(filterapi.GuardrailPhaseRequest))) + response := createUserFacingErrorResponse(http.StatusForbidden, "GuardrailViolation", violation.Message) + if r.span != nil { + r.span.EndSpanOnError(http.StatusForbidden, response.GetImmediateResponse().GetBody()) + } + return response, nil + } + if guardrailErr == nil { + r.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultAllowed, "", true) + r.recordGuardrailTrace("", filterapi.GuardrailPhaseRequest, metrics.GuardrailResultAllowed) + } return &extprocv3.ProcessingResponse{ Response: &extprocv3.ProcessingResponse_RequestBody{ @@ -382,6 +407,28 @@ func (u *upstreamProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) ProcessReque // Set the request model for metrics from the original model or override if applied. reqModel := cmp.Or(u.requestHeaders[internalapi.ModelNameHeaderKeyDefault], u.parent.originalModel) u.metrics.SetRequestModel(reqModel) + var configuredGuardrails []filterapi.RuntimeGuardrail + if u.parent.config != nil { + configuredGuardrails = u.parent.config.Guardrails + } + violation, guardrailErr := evaluateBackendRequestGuardrails(ctx, configuredGuardrails, u.parent.originalRequestBodyRaw, u.backendName) + if guardrailErr != nil { + u.parent.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultError, u.backendName, false) + u.parent.recordGuardrailTrace("", filterapi.GuardrailPhaseRequest, metrics.GuardrailResultError) + if !isGuardrailFailOpenError(guardrailErr) { + return nil, fmt.Errorf("failed to evaluate backend request guardrails: %w", guardrailErr) + } + u.logger.Warn("request guardrail provider failed open", slog.String("error", guardrailErr.Error())) + } else if violation != nil { + u.parent.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultBlocked, u.backendName, false) + u.parent.recordGuardrailTrace(violation.Name, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultBlocked) + u.logger.Warn("request blocked by guardrail", slog.String("guardrail.name", violation.Name), slog.String("guardrail.phase", string(filterapi.GuardrailPhaseRequest))) + return u.respondLocally(ctx, http.StatusForbidden, "GuardrailViolation", violation.Message), nil + } + if guardrailErr == nil { + u.parent.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultAllowed, u.backendName, false) + u.parent.recordGuardrailTrace("", filterapi.GuardrailPhaseRequest, metrics.GuardrailResultAllowed) + } // We force the body mutation in the following cases: // * The request is a retry request because the body mutation might have happened the previous iteration. @@ -533,7 +580,9 @@ func (u *upstreamProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) ProcessRespo return nil, fmt.Errorf("failed to transform response headers: %w", err) } var mode *extprocv3http.ProcessingMode - if u.parent.stream && u.responseHeaders[":status"] == "200" { + hasResponseGuardrails := u.parent.config != nil && guardrailsConfiguredForPhase( + u.parent.config.Guardrails, filterapi.GuardrailPhaseResponse, u.backendName, true) + if u.parent.stream && u.responseHeaders[":status"] == "200" && !hasResponseGuardrails { // We only stream the response if the status code is 200 and the response is a stream. mode = &extprocv3http.ProcessingMode{ResponseBodyMode: extprocv3http.ProcessingMode_STREAMED} } @@ -645,17 +694,26 @@ func (u *upstreamProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) ProcessRespo // Translator reports the latest cumulative token usage which we use to override existing costs. u.costs.Override(tokenUsage) - if violation, err := evaluateResponseGuardrails(u.parent.config.Guardrails, body.Body); err != nil { - u.parent.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseResponse, metrics.GuardrailResultError) - return nil, fmt.Errorf("failed to evaluate response guardrails: %w", err) + violation, guardrailErr := evaluateResponseGuardrails(ctx, u.parent.config.Guardrails, body.Body, u.backendName) + if guardrailErr != nil { + u.parent.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseResponse, metrics.GuardrailResultError, u.backendName, true) + u.parent.recordGuardrailTrace("", filterapi.GuardrailPhaseResponse, metrics.GuardrailResultError) + if !isGuardrailFailOpenError(guardrailErr) { + return nil, fmt.Errorf("failed to evaluate response guardrails: %w", guardrailErr) + } + u.logger.Warn("response guardrail provider failed open", slog.String("error", guardrailErr.Error())) } else if violation != nil { - u.parent.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseResponse, metrics.GuardrailResultBlocked) + u.parent.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseResponse, metrics.GuardrailResultBlocked, u.backendName, true) + u.parent.recordGuardrailTrace(violation.Name, filterapi.GuardrailPhaseResponse, metrics.GuardrailResultBlocked) u.logger.Warn("response blocked by guardrail", slog.String("guardrail.name", violation.Name), slog.String("guardrail.phase", string(filterapi.GuardrailPhaseResponse))) - return u.respondLocally(ctx, 400, "BadRequest", violation.Message), nil + return u.respondLocally(ctx, http.StatusForbidden, "GuardrailViolation", violation.Message), nil + } + if guardrailErr == nil { + u.parent.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseResponse, metrics.GuardrailResultAllowed, u.backendName, true) + u.parent.recordGuardrailTrace("", filterapi.GuardrailPhaseResponse, metrics.GuardrailResultAllowed) } - u.parent.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseResponse, metrics.GuardrailResultAllowed) // Set the response model for metrics u.metrics.SetResponseModel(responseModel) diff --git a/internal/extproc/processor_impl_test.go b/internal/extproc/processor_impl_test.go index faa3e3cd93..048f09ef6b 100644 --- a/internal/extproc/processor_impl_test.go +++ b/internal/extproc/processor_impl_test.go @@ -325,6 +325,23 @@ func Test_chatCompletionProcessorUpstreamFilter_ProcessResponseHeaders(t *testin require.Equal(t, []string{"content-length"}, commonRes.HeaderMutation.RemoveHeaders) require.Equal(t, &extprocv3http.ProcessingMode{ResponseBodyMode: extprocv3http.ProcessingMode_STREAMED}, res.ModeOverride) }) + t.Run("streaming with response guardrail remains buffered", func(t *testing.T) { + inHeaders := &corev3.HeaderMap{Headers: []*corev3.HeaderValue{{Key: ":status", Value: "200"}}} + mt := &mockTranslator{t: t, expHeaders: map[string]string{":status": "200"}} + p := &chatCompletionProcessorUpstreamFilter{ + translator: mt, + metrics: &mockMetrics{}, + parent: &chatCompletionProcessorRouterFilter{ + stream: true, + config: &filterapi.RuntimeConfig{Guardrails: []filterapi.RuntimeGuardrail{{ + Phase: filterapi.GuardrailPhaseResponse, + }}}, + }, + } + res, err := p.ProcessResponseHeaders(t.Context(), inHeaders) + require.NoError(t, err) + require.Nil(t, res.ModeOverride) + }) t.Run("error/streaming", func(t *testing.T) { inHeaders := &corev3.HeaderMap{ Headers: []*corev3.HeaderValue{{Key: ":status", Value: "500"}, {Key: "dog", RawValue: []byte("cat")}}, diff --git a/internal/extproc/server.go b/internal/extproc/server.go index 1ef537cf2b..a70ec7e66a 100644 --- a/internal/extproc/server.go +++ b/internal/extproc/server.go @@ -27,6 +27,7 @@ import ( "github.com/envoyproxy/ai-gateway/internal/backendauth" "github.com/envoyproxy/ai-gateway/internal/filterapi" + "github.com/envoyproxy/ai-gateway/internal/guardrails" "github.com/envoyproxy/ai-gateway/internal/internalapi" "github.com/envoyproxy/ai-gateway/internal/redaction" ) @@ -79,7 +80,7 @@ func NewServer(logger *slog.Logger, enableRedaction bool) (*Server, error) { // LoadConfig updates the configuration of the external processor. func (s *Server) LoadConfig(ctx context.Context, config *filterapi.Config) error { - newConfig, err := filterapi.NewRuntimeConfig(ctx, config, backendauth.NewHandler) + newConfig, err := filterapi.NewRuntimeConfig(ctx, config, backendauth.NewHandler, guardrails.NewEvaluator) if err != nil { return fmt.Errorf("cannot create runtime filter config: %w", err) } diff --git a/internal/filterapi/filterconfig.go b/internal/filterapi/filterconfig.go index 698ff87588..94acd7b2a0 100644 --- a/internal/filterapi/filterconfig.go +++ b/internal/filterapi/filterconfig.go @@ -61,6 +61,8 @@ type Guardrail struct { Name string `json:"name"` Phase GuardrailPhase `json:"phase"` Provider GuardrailProvider `json:"provider"` + // Backends scopes this guardrail to generated backend names. Empty means global. + Backends []string `json:"backends,omitempty"` } // GuardrailPhase determines when the rule is evaluated. @@ -73,10 +75,48 @@ const ( // GuardrailProvider describes the implementation used to evaluate a guardrail. type GuardrailProvider struct { - Type GuardrailProviderType `json:"type"` - Pattern string `json:"pattern,omitempty"` - Action GuardrailAction `json:"action,omitempty"` - Message string `json:"message,omitempty"` + Type GuardrailProviderType `json:"type"` + Pattern string `json:"pattern,omitempty"` + Action GuardrailAction `json:"action,omitempty"` + Message string `json:"message,omitempty"` + Presidio *PresidioGuardrailProvider `json:"presidio,omitempty"` + Bedrock *BedrockGuardrailProvider `json:"bedrock,omitempty"` + AzureContentSafety *AzureContentSafetyGuardrailProvider `json:"azureContentSafety,omitempty"` + TimeoutSeconds int32 `json:"timeoutSeconds,omitempty"` + FailureMode GuardrailFailureMode `json:"failureMode,omitempty"` +} + +// GuardrailFailureMode determines how external provider failures are handled. +type GuardrailFailureMode string + +const ( + GuardrailFailureModeFailClosed GuardrailFailureMode = "FailClosed" + GuardrailFailureModeFailOpen GuardrailFailureMode = "FailOpen" +) + +// PresidioGuardrailProvider configures a Presidio analyzer request. +type PresidioGuardrailProvider struct { + Endpoint string `json:"endpoint"` + Language string `json:"language,omitempty"` + ScoreThresholdPercent int32 `json:"scoreThresholdPercent,omitempty"` + APIKey string `json:"apiKey,omitempty"` +} + +// BedrockGuardrailProvider configures an AWS Bedrock ApplyGuardrail request. +type BedrockGuardrailProvider struct { + Endpoint string `json:"endpoint,omitempty"` + Region string `json:"region"` + GuardrailIdentifier string `json:"guardrailIdentifier"` + GuardrailVersion string `json:"guardrailVersion"` + CredentialFileLiteral string `json:"credentialFileLiteral,omitempty"` +} + +// AzureContentSafetyGuardrailProvider configures an Azure AI Content Safety request. +type AzureContentSafetyGuardrailProvider struct { + Endpoint string `json:"endpoint"` + APIVersion string `json:"apiVersion,omitempty"` + SeverityThreshold *int32 `json:"severityThreshold,omitempty"` + APIKey string `json:"apiKey"` } // GuardrailProviderType identifies a guardrail implementation. diff --git a/internal/filterapi/runtime.go b/internal/filterapi/runtime.go index c08b5ac12d..c0c0b118c1 100644 --- a/internal/filterapi/runtime.go +++ b/internal/filterapi/runtime.go @@ -26,6 +26,14 @@ type BackendAuthHandler interface { // NewBackendAuthHandlerFunc is a function type that creates a new BackendAuthHandler for a given BackendAuth configuration. type NewBackendAuthHandlerFunc func(ctx context.Context, auth *BackendAuth) (BackendAuthHandler, error) +// GuardrailEvaluator evaluates a request or response payload using a configured provider. +type GuardrailEvaluator interface { + Evaluate(ctx context.Context, body []byte, phase GuardrailPhase) (blocked bool, err error) +} + +// NewGuardrailEvaluatorFunc creates an evaluator for an external guardrail provider. +type NewGuardrailEvaluatorFunc func(ctx context.Context, provider *GuardrailProvider) (GuardrailEvaluator, error) + // RuntimeConfig is the runtime filter configuration that is derived from the filterapi.Config. type RuntimeConfig struct { // UUID is the unique identifier of the filter configuration, inherited from filterapi.Config. @@ -74,14 +82,20 @@ type RuntimeRequestCost struct { // RuntimeGuardrail is a compiled guardrail rule for runtime evaluation. type RuntimeGuardrail struct { - Name string - Phase GuardrailPhase - Provider GuardrailProvider - Matcher *regexp.Regexp + Name string + Phase GuardrailPhase + Provider GuardrailProvider + Backends []string + Matcher *regexp.Regexp + Evaluator GuardrailEvaluator } // NewRuntimeConfig creates a new runtime filter configuration from the given filterapi.Config and a function to create backend auth handlers. -func NewRuntimeConfig(ctx context.Context, config *Config, fn NewBackendAuthHandlerFunc) (*RuntimeConfig, error) { +func NewRuntimeConfig(ctx context.Context, config *Config, fn NewBackendAuthHandlerFunc, guardrailEvaluatorFactories ...NewGuardrailEvaluatorFunc) (*RuntimeConfig, error) { + var newGuardrailEvaluator NewGuardrailEvaluatorFunc + if len(guardrailEvaluatorFactories) > 0 { + newGuardrailEvaluator = guardrailEvaluatorFactories[0] + } backends := make(map[string]*RuntimeBackend, len(config.Backends)) for i := range config.Backends { b := &config.Backends[i] @@ -149,14 +163,24 @@ func NewRuntimeConfig(ctx context.Context, config *Config, fn NewBackendAuthHand Name: g.Name, Phase: g.Phase, Provider: g.Provider, + Backends: g.Backends, Matcher: re, }) continue } + if newGuardrailEvaluator == nil { + return nil, fmt.Errorf("guardrail %q uses provider %q but no evaluator factory is configured", g.Name, g.Provider.Type) + } + evaluator, err := newGuardrailEvaluator(ctx, &g.Provider) + if err != nil { + return nil, fmt.Errorf("cannot create evaluator for guardrail %q: %w", g.Name, err) + } guardrails = append(guardrails, RuntimeGuardrail{ - Name: g.Name, - Phase: g.Phase, - Provider: g.Provider, + Name: g.Name, + Phase: g.Phase, + Provider: g.Provider, + Backends: g.Backends, + Evaluator: evaluator, }) } diff --git a/internal/filterapi/runtime_test.go b/internal/filterapi/runtime_test.go index 9e1758c2d6..0d562a1e1e 100644 --- a/internal/filterapi/runtime_test.go +++ b/internal/filterapi/runtime_test.go @@ -138,6 +138,19 @@ func TestServer_LoadConfig(t *testing.T) { require.True(t, rc.Guardrails[0].Matcher.MatchString("SSN")) }) + t.Run("external guardrail creates evaluator", func(t *testing.T) { + config := &Config{Guardrails: []Guardrail{{ + Name: "presidio", Phase: GuardrailPhaseRequest, + Provider: GuardrailProvider{Type: GuardrailProviderTypePresidio}, + }}} + evaluator := &testGuardrailEvaluator{} + rc, err := NewRuntimeConfig(t.Context(), config, + func(_ context.Context, _ *BackendAuth) (BackendAuthHandler, error) { return nil, nil }, + func(_ context.Context, _ *GuardrailProvider) (GuardrailEvaluator, error) { return evaluator, nil }) + require.NoError(t, err) + require.Same(t, evaluator, rc.Guardrails[0].Evaluator) + }) + t.Run("error - route cost with empty RouteName", func(t *testing.T) { config := &Config{ LLMRequestCosts: []LLMRequestCost{ @@ -152,3 +165,9 @@ func TestServer_LoadConfig(t *testing.T) { require.Contains(t, err.Error(), "missing_route") }) } + +type testGuardrailEvaluator struct{} + +func (*testGuardrailEvaluator) Evaluate(context.Context, []byte, GuardrailPhase) (bool, error) { + return false, nil +} diff --git a/internal/guardrails/adapters.go b/internal/guardrails/adapters.go new file mode 100644 index 0000000000..1982b50526 --- /dev/null +++ b/internal/guardrails/adapters.go @@ -0,0 +1,76 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +// Package guardrails implements external content-safety provider adapters. +package guardrails + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/envoyproxy/ai-gateway/internal/filterapi" + "github.com/envoyproxy/ai-gateway/internal/json" +) + +const defaultTimeoutSeconds int32 = 10 + +// NewEvaluator creates an evaluator for the configured external provider. +func NewEvaluator(ctx context.Context, provider *filterapi.GuardrailProvider) (filterapi.GuardrailEvaluator, error) { + client := &http.Client{Timeout: time.Duration(timeoutSeconds(provider.TimeoutSeconds)) * time.Second} + switch provider.Type { + case filterapi.GuardrailProviderTypePresidio: + return newPresidioEvaluator(provider.Presidio, client) + case filterapi.GuardrailProviderTypeBedrockGuardrails: + return newBedrockEvaluator(ctx, provider.Bedrock, client) + case filterapi.GuardrailProviderTypeAzureContentSafety: + return newAzureContentSafetyEvaluator(provider.AzureContentSafety, client) + default: + return nil, fmt.Errorf("unsupported external guardrail provider %q", provider.Type) + } +} + +func timeoutSeconds(configured int32) int32 { + if configured <= 0 { + return defaultTimeoutSeconds + } + return configured +} + +func doJSON(ctx context.Context, client *http.Client, method, endpoint string, body any, mutate func(*http.Request), response any) error { + payload, err := json.Marshal(body) + if err != nil { + return err + } + req, err := http.NewRequestWithContext(ctx, method, endpoint, bytes.NewReader(payload)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + if mutate != nil { + mutate(req) + } + return sendJSON(req, client, response) +} + +func sendJSON(req *http.Request, client *http.Client, response any) error { + resp, err := client.Do(req) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + return fmt.Errorf("provider returned HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(body))) + } + if err = json.NewDecoder(resp.Body).Decode(response); err != nil { + return fmt.Errorf("cannot decode provider response: %w", err) + } + return nil +} diff --git a/internal/guardrails/adapters_live_test.go b/internal/guardrails/adapters_live_test.go new file mode 100644 index 0000000000..ce27cd6b22 --- /dev/null +++ b/internal/guardrails/adapters_live_test.go @@ -0,0 +1,76 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package guardrails + +import ( + "os" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/envoyproxy/ai-gateway/internal/filterapi" +) + +func TestLivePresidio(t *testing.T) { + endpoint := os.Getenv("TEST_PRESIDIO_ENDPOINT") + blockedText := os.Getenv("TEST_PRESIDIO_BLOCKED_TEXT") + if endpoint == "" || blockedText == "" { + t.Skip("TEST_PRESIDIO_ENDPOINT and TEST_PRESIDIO_BLOCKED_TEXT are not set") + } + provider := &filterapi.GuardrailProvider{ + Type: filterapi.GuardrailProviderTypePresidio, + Presidio: &filterapi.PresidioGuardrailProvider{ + Endpoint: endpoint, + APIKey: os.Getenv("TEST_PRESIDIO_API_KEY"), + }, + } + evaluator, err := NewEvaluator(t.Context(), provider) + require.NoError(t, err) + blocked, err := evaluator.Evaluate(t.Context(), []byte(blockedText), filterapi.GuardrailPhaseRequest) + require.NoError(t, err) + require.True(t, blocked) +} + +func TestLiveAzureContentSafety(t *testing.T) { + endpoint, apiKey := os.Getenv("TEST_AZURE_CONTENT_SAFETY_ENDPOINT"), os.Getenv("TEST_AZURE_CONTENT_SAFETY_API_KEY") + blockedText := os.Getenv("TEST_AZURE_CONTENT_SAFETY_BLOCKED_TEXT") + if endpoint == "" || apiKey == "" || blockedText == "" { + t.Skip("TEST_AZURE_CONTENT_SAFETY_ENDPOINT, TEST_AZURE_CONTENT_SAFETY_API_KEY, and TEST_AZURE_CONTENT_SAFETY_BLOCKED_TEXT are not set") + } + threshold := int32(4) + provider := &filterapi.GuardrailProvider{ + Type: filterapi.GuardrailProviderTypeAzureContentSafety, + AzureContentSafety: &filterapi.AzureContentSafetyGuardrailProvider{ + Endpoint: endpoint, APIKey: apiKey, SeverityThreshold: &threshold, + }, + } + evaluator, err := NewEvaluator(t.Context(), provider) + require.NoError(t, err) + blocked, err := evaluator.Evaluate(t.Context(), []byte(blockedText), filterapi.GuardrailPhaseRequest) + require.NoError(t, err) + require.True(t, blocked) +} + +func TestLiveBedrockGuardrail(t *testing.T) { + region := os.Getenv("TEST_AWS_BEDROCK_GUARDRAIL_REGION") + identifier := os.Getenv("TEST_AWS_BEDROCK_GUARDRAIL_ID") + version := os.Getenv("TEST_AWS_BEDROCK_GUARDRAIL_VERSION") + blockedText := os.Getenv("TEST_AWS_BEDROCK_GUARDRAIL_BLOCKED_TEXT") + if region == "" || identifier == "" || version == "" || blockedText == "" { + t.Skip("Bedrock guardrail configuration and TEST_AWS_BEDROCK_GUARDRAIL_BLOCKED_TEXT are not set") + } + provider := &filterapi.GuardrailProvider{ + Type: filterapi.GuardrailProviderTypeBedrockGuardrails, + Bedrock: &filterapi.BedrockGuardrailProvider{ + Region: region, GuardrailIdentifier: identifier, GuardrailVersion: version, + }, + } + evaluator, err := NewEvaluator(t.Context(), provider) + require.NoError(t, err) + blocked, err := evaluator.Evaluate(t.Context(), []byte(blockedText), filterapi.GuardrailPhaseRequest) + require.NoError(t, err) + require.True(t, blocked) +} diff --git a/internal/guardrails/adapters_test.go b/internal/guardrails/adapters_test.go new file mode 100644 index 0000000000..2b7209f748 --- /dev/null +++ b/internal/guardrails/adapters_test.go @@ -0,0 +1,122 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package guardrails + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/envoyproxy/ai-gateway/internal/filterapi" + "github.com/envoyproxy/ai-gateway/internal/json" +) + +func TestPresidioEvaluatorHTTP(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + require.Equal(t, "/analyze", req.URL.Path) + require.Equal(t, "Bearer secret", req.Header.Get("Authorization")) + var body struct { + Text string `json:"text"` + Language string `json:"language"` + ScoreThreshold float64 `json:"score_threshold"` + } + require.NoError(t, json.NewDecoder(req.Body).Decode(&body)) + require.Equal(t, "customer SSN", body.Text) + require.Equal(t, "es", body.Language) + require.Equal(t, 0.75, body.ScoreThreshold) + _, _ = w.Write([]byte(`[{"entity_type":"US_SSN","score":0.98}]`)) + })) + t.Cleanup(server.Close) + + evaluator, err := newPresidioEvaluator(&filterapi.PresidioGuardrailProvider{ + Endpoint: server.URL, Language: "es", ScoreThresholdPercent: 75, APIKey: "secret", + }, server.Client()) + require.NoError(t, err) + blocked, err := evaluator.Evaluate(t.Context(), []byte("customer SSN"), filterapi.GuardrailPhaseRequest) + require.NoError(t, err) + require.True(t, blocked) +} + +func TestAzureContentSafetyEvaluatorHTTP(t *testing.T) { + severityThreshold := int32(4) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + require.Equal(t, "/contentsafety/text:analyze", req.URL.Path) + require.Equal(t, "2024-09-01", req.URL.Query().Get("api-version")) + require.Equal(t, "azure-secret", req.Header.Get("Ocp-Apim-Subscription-Key")) + _, _ = w.Write([]byte(`{"categoriesAnalysis":[{"category":"Violence","severity":6}]}`)) + })) + t.Cleanup(server.Close) + + evaluator, err := newAzureContentSafetyEvaluator(&filterapi.AzureContentSafetyGuardrailProvider{ + Endpoint: server.URL, APIKey: "azure-secret", SeverityThreshold: &severityThreshold, + }, server.Client()) + require.NoError(t, err) + blocked, err := evaluator.Evaluate(t.Context(), []byte("unsafe response"), filterapi.GuardrailPhaseResponse) + require.NoError(t, err) + require.True(t, blocked) +} + +func TestBedrockEvaluatorHTTP(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + require.Equal(t, "/guardrail/guardrail-id/version/1/apply", req.URL.Path) + require.Contains(t, req.Header.Get("Authorization"), "Credential=AKIDEXAMPLE/") + require.NotEmpty(t, req.Header.Get("X-Amz-Date")) + var body struct { + Source string `json:"source"` + Content []struct { + Text struct { + Text string `json:"text"` + } `json:"text"` + } `json:"content"` + } + require.NoError(t, json.NewDecoder(req.Body).Decode(&body)) + require.Equal(t, "OUTPUT", body.Source) + require.Equal(t, "unsafe response", body.Content[0].Text.Text) + _, _ = w.Write([]byte(`{"action":"GUARDRAIL_INTERVENED"}`)) + })) + t.Cleanup(server.Close) + + evaluator, err := newBedrockEvaluator(t.Context(), &filterapi.BedrockGuardrailProvider{ + Endpoint: server.URL, Region: "us-east-1", GuardrailIdentifier: "guardrail-id", GuardrailVersion: "1", + CredentialFileLiteral: strings.TrimSpace(` +[default] +aws_access_key_id = AKIDEXAMPLE +aws_secret_access_key = secret +`), + }, server.Client()) + require.NoError(t, err) + blocked, err := evaluator.Evaluate(t.Context(), []byte("unsafe response"), filterapi.GuardrailPhaseResponse) + require.NoError(t, err) + require.True(t, blocked) +} + +func TestEvaluatorProviderError(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "provider unavailable", http.StatusServiceUnavailable) + })) + t.Cleanup(server.Close) + + evaluator, err := newPresidioEvaluator(&filterapi.PresidioGuardrailProvider{Endpoint: server.URL}, server.Client()) + require.NoError(t, err) + blocked, err := evaluator.Evaluate(t.Context(), []byte("payload"), filterapi.GuardrailPhaseRequest) + require.False(t, blocked) + require.ErrorContains(t, err, "HTTP 503") + require.ErrorContains(t, err, "provider unavailable") +} + +func TestNewEvaluatorConfiguresTimeout(t *testing.T) { + evaluator, err := NewEvaluator(t.Context(), &filterapi.GuardrailProvider{ + Type: filterapi.GuardrailProviderTypePresidio, + TimeoutSeconds: 3, + Presidio: &filterapi.PresidioGuardrailProvider{Endpoint: "https://presidio.example.com"}, + }) + require.NoError(t, err) + require.Equal(t, 3*time.Second, evaluator.(*presidioEvaluator).client.Timeout) +} diff --git a/internal/guardrails/azure.go b/internal/guardrails/azure.go new file mode 100644 index 0000000000..68a62af374 --- /dev/null +++ b/internal/guardrails/azure.go @@ -0,0 +1,62 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package guardrails + +import ( + "context" + "fmt" + "net/http" + "net/url" + "strings" + + "github.com/envoyproxy/ai-gateway/internal/filterapi" +) + +type azureContentSafetyEvaluator struct { + config *filterapi.AzureContentSafetyGuardrailProvider + client *http.Client +} + +func newAzureContentSafetyEvaluator(config *filterapi.AzureContentSafetyGuardrailProvider, client *http.Client) (filterapi.GuardrailEvaluator, error) { + if config == nil || config.Endpoint == "" { + return nil, fmt.Errorf("azure Content Safety endpoint is required") + } + if config.APIKey == "" { + return nil, fmt.Errorf("azure Content Safety API key is required") + } + configCopy := *config + if configCopy.APIVersion == "" { + configCopy.APIVersion = "2024-09-01" + } + if configCopy.SeverityThreshold == nil { + threshold := int32(4) + configCopy.SeverityThreshold = &threshold + } + return &azureContentSafetyEvaluator{config: &configCopy, client: client}, nil +} + +func (e *azureContentSafetyEvaluator) Evaluate(ctx context.Context, body []byte, _ filterapi.GuardrailPhase) (bool, error) { + payload := struct { + Text string `json:"text"` + }{Text: string(body)} + result := struct { + CategoriesAnalysis []struct { + Severity int32 `json:"severity"` + } `json:"categoriesAnalysis"` + }{} + endpoint := strings.TrimRight(e.config.Endpoint, "/") + "/contentsafety/text:analyze?api-version=" + url.QueryEscape(e.config.APIVersion) + if err := doJSON(ctx, e.client, http.MethodPost, endpoint, payload, func(req *http.Request) { + req.Header.Set("Ocp-Apim-Subscription-Key", e.config.APIKey) + }, &result); err != nil { + return false, fmt.Errorf("azure Content Safety analyze request failed: %w", err) + } + for _, category := range result.CategoriesAnalysis { + if category.Severity >= *e.config.SeverityThreshold { + return true, nil + } + } + return false, nil +} diff --git a/internal/guardrails/bedrock.go b/internal/guardrails/bedrock.go new file mode 100644 index 0000000000..7186092982 --- /dev/null +++ b/internal/guardrails/bedrock.go @@ -0,0 +1,127 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package guardrails + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "net/http" + "net/url" + "os" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + awsv4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + + "github.com/envoyproxy/ai-gateway/internal/filterapi" + "github.com/envoyproxy/ai-gateway/internal/json" +) + +type bedrockEvaluator struct { + config *filterapi.BedrockGuardrailProvider + client *http.Client + credentials aws.CredentialsProvider + signer *awsv4.Signer +} + +func newBedrockEvaluator(ctx context.Context, config *filterapi.BedrockGuardrailProvider, client *http.Client) (filterapi.GuardrailEvaluator, error) { + if config == nil || config.Region == "" || config.GuardrailIdentifier == "" || config.GuardrailVersion == "" { + return nil, fmt.Errorf("bedrock region, guardrailIdentifier, and guardrailVersion are required") + } + credentials, err := loadAWSCredentials(ctx, config) + if err != nil { + return nil, err + } + configCopy := *config + if configCopy.Endpoint == "" { + configCopy.Endpoint = fmt.Sprintf("https://bedrock-runtime.%s.amazonaws.com", configCopy.Region) + } + return &bedrockEvaluator{config: &configCopy, client: client, credentials: credentials, signer: awsv4.NewSigner()}, nil +} + +func loadAWSCredentials(ctx context.Context, guardrailConfig *filterapi.BedrockGuardrailProvider) (aws.CredentialsProvider, error) { + options := []func(*awsconfig.LoadOptions) error{awsconfig.WithRegion(guardrailConfig.Region)} + var credentialsFile *os.File + if guardrailConfig.CredentialFileLiteral != "" { + var err error + credentialsFile, err = os.CreateTemp("", "guardrail-aws-credentials") + if err != nil { + return nil, fmt.Errorf("cannot create temporary AWS credentials file: %w", err) + } + defer func() { + _ = credentialsFile.Close() + _ = os.Remove(credentialsFile.Name()) + }() + if _, err = credentialsFile.WriteString(guardrailConfig.CredentialFileLiteral); err != nil { + return nil, fmt.Errorf("cannot write AWS credentials file: %w", err) + } + options = append(options, awsconfig.WithSharedCredentialsFiles([]string{credentialsFile.Name()})) + } + cfg, err := awsconfig.LoadDefaultConfig(ctx, options...) + if err != nil { + return nil, fmt.Errorf("cannot load AWS config: %w", err) + } + if credentialsFile == nil { + return cfg.Credentials, nil + } + credentials, err := cfg.Credentials.Retrieve(ctx) + if err != nil { + return nil, fmt.Errorf("cannot load AWS credentials: %w", err) + } + return aws.CredentialsProviderFunc(func(context.Context) (aws.Credentials, error) { + return credentials, nil + }), nil +} + +func (e *bedrockEvaluator) Evaluate(ctx context.Context, body []byte, phase filterapi.GuardrailPhase) (bool, error) { + source := "INPUT" + if phase == filterapi.GuardrailPhaseResponse { + source = "OUTPUT" + } + payload, err := json.Marshal(struct { + Source string `json:"source"` + Content []struct { + Text struct { + Text string `json:"text"` + } `json:"text"` + } `json:"content"` + }{Source: source, Content: []struct { + Text struct { + Text string `json:"text"` + } `json:"text"` + }{{Text: struct { + Text string `json:"text"` + }{Text: string(body)}}}}) + if err != nil { + return false, err + } + endpoint := strings.TrimRight(e.config.Endpoint, "/") + "/guardrail/" + url.PathEscape(e.config.GuardrailIdentifier) + "/version/" + url.PathEscape(e.config.GuardrailVersion) + "/apply" + req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(payload)) + if err != nil { + return false, err + } + req.Header.Set("Content-Type", "application/json") + credentials, err := e.credentials.Retrieve(ctx) + if err != nil { + return false, fmt.Errorf("cannot retrieve AWS credentials: %w", err) + } + payloadHash := sha256.Sum256(payload) + if err = e.signer.SignHTTP(ctx, credentials, req, hex.EncodeToString(payloadHash[:]), "bedrock", e.config.Region, time.Now()); err != nil { + return false, fmt.Errorf("cannot sign Bedrock request: %w", err) + } + result := struct { + Action string `json:"action"` + }{} + if err = sendJSON(req, e.client, &result); err != nil { + return false, fmt.Errorf("bedrock ApplyGuardrail request failed: %w", err) + } + return result.Action == "GUARDRAIL_INTERVENED", nil +} diff --git a/internal/guardrails/presidio.go b/internal/guardrails/presidio.go new file mode 100644 index 0000000000..fb0a69e5a4 --- /dev/null +++ b/internal/guardrails/presidio.go @@ -0,0 +1,55 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package guardrails + +import ( + "context" + "fmt" + "net/http" + "strings" + + "github.com/envoyproxy/ai-gateway/internal/filterapi" +) + +type presidioEvaluator struct { + config *filterapi.PresidioGuardrailProvider + client *http.Client +} + +func newPresidioEvaluator(config *filterapi.PresidioGuardrailProvider, client *http.Client) (filterapi.GuardrailEvaluator, error) { + if config == nil || config.Endpoint == "" { + return nil, fmt.Errorf("presidio endpoint is required") + } + configCopy := *config + if configCopy.Language == "" { + configCopy.Language = "en" + } + return &presidioEvaluator{config: &configCopy, client: client}, nil +} + +func (e *presidioEvaluator) Evaluate(ctx context.Context, body []byte, _ filterapi.GuardrailPhase) (bool, error) { + payload := struct { + Text string `json:"text"` + Language string `json:"language"` + ScoreThreshold *float64 `json:"score_threshold,omitempty"` + }{Text: string(body), Language: e.config.Language} + if e.config.ScoreThresholdPercent > 0 { + threshold := float64(e.config.ScoreThresholdPercent) / 100 + payload.ScoreThreshold = &threshold + } + + var result []struct { + Score float64 `json:"score"` + } + if err := doJSON(ctx, e.client, http.MethodPost, strings.TrimRight(e.config.Endpoint, "/")+"/analyze", payload, func(req *http.Request) { + if e.config.APIKey != "" { + req.Header.Set("Authorization", "Bearer "+e.config.APIKey) + } + }, &result); err != nil { + return false, fmt.Errorf("presidio analyze request failed: %w", err) + } + return len(result) > 0, nil +} diff --git a/internal/guardrails/presidio_container_test.go b/internal/guardrails/presidio_container_test.go new file mode 100644 index 0000000000..e63df792ae --- /dev/null +++ b/internal/guardrails/presidio_container_test.go @@ -0,0 +1,59 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package guardrails + +import ( + "context" + "fmt" + "net" + "net/http" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" + + "github.com/envoyproxy/ai-gateway/internal/filterapi" +) + +const presidioAnalyzerImage = "ghcr.io/data-privacy-stack/presidio-analyzer:2.2.364" + +func TestPresidioEvaluatorContainer(t *testing.T) { + testcontainers.SkipIfProviderIsNotHealthy(t) + + ctx := t.Context() + container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: testcontainers.ContainerRequest{ + Image: presidioAnalyzerImage, + ExposedPorts: []string{"3000/tcp"}, + WaitingFor: wait.ForHTTP("/health"). + WithPort("3000/tcp"). + WithStartupTimeout(2 * time.Minute), + }, + Started: true, + }) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, container.Terminate(context.WithoutCancel(ctx))) + }) + + host, err := container.Host(ctx) + require.NoError(t, err) + port, err := container.MappedPort(ctx, "3000/tcp") + require.NoError(t, err) + + evaluator, err := newPresidioEvaluator(&filterapi.PresidioGuardrailProvider{ + Endpoint: fmt.Sprintf("http://%s", net.JoinHostPort(host, port.Port())), + Language: "en", + ScoreThresholdPercent: 80, + }, &http.Client{Timeout: 10 * time.Second}) + require.NoError(t, err) + + blocked, err := evaluator.Evaluate(ctx, []byte("Contact me at alice@example.com"), filterapi.GuardrailPhaseRequest) + require.NoError(t, err) + require.True(t, blocked) +} diff --git a/internal/tracing/span.go b/internal/tracing/span.go index ef4935c65d..e08bd5fddb 100644 --- a/internal/tracing/span.go +++ b/internal/tracing/span.go @@ -6,6 +6,7 @@ package tracing import ( + "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" anthropicschema "github.com/envoyproxy/ai-gateway/internal/apischema/anthropic" @@ -21,6 +22,15 @@ type span[RespT, ChunkT any] struct { chunks []*ChunkT } +// RecordGuardrail records a guardrail evaluation event without payload content. +func (s *span[RespT, ChunkT]) RecordGuardrail(name, phase, result string) { + s.span.AddEvent("guardrail.evaluation", trace.WithAttributes( + attribute.String("aigateway.guardrail.name", name), + attribute.String("aigateway.guardrail.phase", phase), + attribute.String("aigateway.guardrail.result", result), + )) +} + // RecordResponseChunk implements [tracingapi.Span.RecordResponseChunk] func (s *span[RespT, ChunkT]) RecordResponseChunk(resp *ChunkT) { s.chunks = append(s.chunks, resp) diff --git a/internal/tracing/span_test.go b/internal/tracing/span_test.go index 9a19435712..517fae4584 100644 --- a/internal/tracing/span_test.go +++ b/internal/tracing/span_test.go @@ -48,6 +48,22 @@ func TestChatCompletionSpan_RecordResponse(t *testing.T) { }, actualSpan.Attributes) } +func TestChatCompletionSpan_RecordGuardrail(t *testing.T) { + s := &chatCompletionSpan{} + actualSpan := testotel.RecordWithSpan(t, func(span oteltrace.Span) bool { + s.span = span + s.RecordGuardrail("deny-pii", "Request", "blocked") + return false + }) + require.Len(t, actualSpan.Events, 1) + require.Equal(t, "guardrail.evaluation", actualSpan.Events[0].Name) + require.ElementsMatch(t, []attribute.KeyValue{ + attribute.String("aigateway.guardrail.name", "deny-pii"), + attribute.String("aigateway.guardrail.phase", "Request"), + attribute.String("aigateway.guardrail.result", "blocked"), + }, actualSpan.Events[0].Attributes) +} + // TestChatCompletionSpan_RecordBackend pins that the backend only reaches the // span when the convention's recorder asks for it. The backend is resolved for // every request, so a recorder whose convention defines no backend attributes diff --git a/internal/tracing/tracingapi/api.go b/internal/tracing/tracingapi/api.go index ad16153244..b28ee64178 100644 --- a/internal/tracing/tracingapi/api.go +++ b/internal/tracing/tracingapi/api.go @@ -102,6 +102,10 @@ type ( ) type ( + // GuardrailSpan optionally records guardrail evaluations on a request span. + GuardrailSpan interface { + RecordGuardrail(name, phase, result string) + } // Span standardizes span interfaces, supporting both streaming and non-streaming endpoints. Span[RespT any, RespChunkT any] interface { // RecordResponseChunk records streaming response chunks. Implementations that do not support streaming should provide a no-op implementation. diff --git a/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_guardrailpolicies.yaml b/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_guardrailpolicies.yaml index 261f2cea08..02c20af2b7 100644 --- a/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_guardrailpolicies.yaml +++ b/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_guardrailpolicies.yaml @@ -64,6 +64,7 @@ spec: properties: name: description: Name is a stable identifier for the rule. + maxLength: 63 minLength: 1 type: string phase: @@ -80,6 +81,144 @@ spec: default: Block description: Action is the action taken when the rule is matched. + enum: + - Block + type: string + azureContentSafety: + description: AzureContentSafety configures Azure AI Content + Safety. + properties: + apiKeySecretRef: + description: APIKeySecretRef references a Secret whose + apiKey entry is sent to Azure. + properties: + group: + default: "" + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Secret + description: Kind is kind of the referent. For example + "Secret". + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the referenced object. When unspecified, the local + namespace is inferred. + + Note that when a namespace different than the local namespace is specified, + a ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - name + type: object + apiVersion: + default: "2024-09-01" + type: string + endpoint: + format: uri + type: string + severityThreshold: + format: int32 + maximum: 7 + minimum: 0 + type: integer + required: + - apiKeySecretRef + - endpoint + type: object + bedrock: + description: Bedrock configures AWS Bedrock Guardrails. + properties: + credentialsSecretRef: + description: CredentialsSecretRef optionally references + a Secret whose credentials entry contains an AWS shared + credentials file. + properties: + group: + default: "" + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Secret + description: Kind is kind of the referent. For example + "Secret". + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the referenced object. When unspecified, the local + namespace is inferred. + + Note that when a namespace different than the local namespace is specified, + a ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - name + type: object + endpoint: + description: Endpoint overrides the Bedrock runtime + endpoint, primarily for private endpoints and testing. + format: uri + type: string + guardrailIdentifier: + minLength: 1 + type: string + guardrailVersion: + minLength: 1 + type: string + region: + minLength: 1 + type: string + required: + - guardrailIdentifier + - guardrailVersion + - region + type: object + failureMode: + default: FailClosed + description: FailureMode determines whether provider errors + block or allow the request. + enum: + - FailClosed + - FailOpen type: string message: description: Message is returned to the caller when the @@ -89,6 +228,74 @@ spec: description: Pattern is used for deterministic regex-based evaluations. type: string + presidio: + description: Presidio configures the Presidio analyzer provider. + properties: + apiKeySecretRef: + description: APIKeySecretRef optionally references a + Secret whose apiKey entry is sent as a Bearer token. + properties: + group: + default: "" + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Secret + description: Kind is kind of the referent. For example + "Secret". + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the referenced object. When unspecified, the local + namespace is inferred. + + Note that when a namespace different than the local namespace is specified, + a ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - name + type: object + endpoint: + format: uri + type: string + language: + default: en + type: string + scoreThresholdPercent: + format: int32 + maximum: 100 + minimum: 0 + type: integer + required: + - endpoint + type: object + timeoutSeconds: + default: 10 + description: TimeoutSeconds limits each external provider + evaluation. + format: int32 + maximum: 60 + minimum: 1 + type: integer type: description: Type identifies the guardrail implementation. enum: @@ -100,13 +307,32 @@ spec: required: - type type: object + x-kubernetes-validations: + - message: Regex requires pattern and no external provider configuration + rule: self.type != 'Regex' || (has(self.pattern) && self.pattern + != ” && !has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety)) + - message: Presidio requires only presidio provider configuration + rule: self.type != 'Presidio' || (!has(self.pattern) && has(self.presidio) + && !has(self.bedrock) && !has(self.azureContentSafety)) + - message: Bedrock requires only bedrock provider configuration + rule: self.type != 'Bedrock' || (!has(self.pattern) && has(self.bedrock) + && !has(self.presidio) && !has(self.azureContentSafety)) + - message: AzureContentSafety requires only azureContentSafety + provider configuration + rule: self.type != 'AzureContentSafety' || (!has(self.pattern) + && has(self.azureContentSafety) && !has(self.presidio) && + !has(self.bedrock)) required: - name - phase - provider type: object maxItems: 32 + minItems: 1 type: array + x-kubernetes-validations: + - message: rule name must be unique within the policy + rule: self.all(rule, self.exists_one(other, other.name == rule.name)) targetRefs: description: TargetRefs are the names of the AIServiceBackend resources this GuardrailPolicy is attached to. @@ -140,6 +366,7 @@ spec: - name type: object maxItems: 16 + minItems: 1 type: array x-kubernetes-validations: - message: targetRefs must reference AIServiceBackend resources @@ -256,6 +483,7 @@ spec: properties: name: description: Name is a stable identifier for the rule. + maxLength: 63 minLength: 1 type: string phase: @@ -272,6 +500,144 @@ spec: default: Block description: Action is the action taken when the rule is matched. + enum: + - Block + type: string + azureContentSafety: + description: AzureContentSafety configures Azure AI Content + Safety. + properties: + apiKeySecretRef: + description: APIKeySecretRef references a Secret whose + apiKey entry is sent to Azure. + properties: + group: + default: "" + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Secret + description: Kind is kind of the referent. For example + "Secret". + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the referenced object. When unspecified, the local + namespace is inferred. + + Note that when a namespace different than the local namespace is specified, + a ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - name + type: object + apiVersion: + default: "2024-09-01" + type: string + endpoint: + format: uri + type: string + severityThreshold: + format: int32 + maximum: 7 + minimum: 0 + type: integer + required: + - apiKeySecretRef + - endpoint + type: object + bedrock: + description: Bedrock configures AWS Bedrock Guardrails. + properties: + credentialsSecretRef: + description: CredentialsSecretRef optionally references + a Secret whose credentials entry contains an AWS shared + credentials file. + properties: + group: + default: "" + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Secret + description: Kind is kind of the referent. For example + "Secret". + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the referenced object. When unspecified, the local + namespace is inferred. + + Note that when a namespace different than the local namespace is specified, + a ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - name + type: object + endpoint: + description: Endpoint overrides the Bedrock runtime + endpoint, primarily for private endpoints and testing. + format: uri + type: string + guardrailIdentifier: + minLength: 1 + type: string + guardrailVersion: + minLength: 1 + type: string + region: + minLength: 1 + type: string + required: + - guardrailIdentifier + - guardrailVersion + - region + type: object + failureMode: + default: FailClosed + description: FailureMode determines whether provider errors + block or allow the request. + enum: + - FailClosed + - FailOpen type: string message: description: Message is returned to the caller when the @@ -281,6 +647,74 @@ spec: description: Pattern is used for deterministic regex-based evaluations. type: string + presidio: + description: Presidio configures the Presidio analyzer provider. + properties: + apiKeySecretRef: + description: APIKeySecretRef optionally references a + Secret whose apiKey entry is sent as a Bearer token. + properties: + group: + default: "" + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Secret + description: Kind is kind of the referent. For example + "Secret". + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the referenced object. When unspecified, the local + namespace is inferred. + + Note that when a namespace different than the local namespace is specified, + a ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - name + type: object + endpoint: + format: uri + type: string + language: + default: en + type: string + scoreThresholdPercent: + format: int32 + maximum: 100 + minimum: 0 + type: integer + required: + - endpoint + type: object + timeoutSeconds: + default: 10 + description: TimeoutSeconds limits each external provider + evaluation. + format: int32 + maximum: 60 + minimum: 1 + type: integer type: description: Type identifies the guardrail implementation. enum: @@ -292,13 +726,32 @@ spec: required: - type type: object + x-kubernetes-validations: + - message: Regex requires pattern and no external provider configuration + rule: self.type != 'Regex' || (has(self.pattern) && self.pattern + != ” && !has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety)) + - message: Presidio requires only presidio provider configuration + rule: self.type != 'Presidio' || (!has(self.pattern) && has(self.presidio) + && !has(self.bedrock) && !has(self.azureContentSafety)) + - message: Bedrock requires only bedrock provider configuration + rule: self.type != 'Bedrock' || (!has(self.pattern) && has(self.bedrock) + && !has(self.presidio) && !has(self.azureContentSafety)) + - message: AzureContentSafety requires only azureContentSafety + provider configuration + rule: self.type != 'AzureContentSafety' || (!has(self.pattern) + && has(self.azureContentSafety) && !has(self.presidio) && + !has(self.bedrock)) required: - name - phase - provider type: object maxItems: 32 + minItems: 1 type: array + x-kubernetes-validations: + - message: rule name must be unique within the policy + rule: self.all(rule, self.exists_one(other, other.name == rule.name)) targetRefs: description: TargetRefs are the names of the AIServiceBackend resources this GuardrailPolicy is attached to. @@ -332,6 +785,7 @@ spec: - name type: object maxItems: 16 + minItems: 1 type: array x-kubernetes-validations: - message: targetRefs must reference AIServiceBackend resources diff --git a/site/docs/api/api.mdx b/site/docs/api/api.mdx index 172061f0ab..4b903d5156 100644 --- a/site/docs/api/api.mdx +++ b/site/docs/api/api.mdx @@ -649,6 +649,7 @@ QuotaPolicyList contains a list of QuotaPolicy - [APISchema](#github-com-envoyproxy-ai-gateway-api-v1alpha1-apischema) - [AWSCredentialsFile](#github-com-envoyproxy-ai-gateway-api-v1alpha1-awscredentialsfile) - [AWSOIDCExchangeToken](#github-com-envoyproxy-ai-gateway-api-v1alpha1-awsoidcexchangetoken) +- [AzureContentSafetyGuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1alpha1-azurecontentsafetyguardrailprovider) - [AzureOIDCExchangeToken](#github-com-envoyproxy-ai-gateway-api-v1alpha1-azureoidcexchangetoken) - [BackendSecurityPolicyAPIKey](#github-com-envoyproxy-ai-gateway-api-v1alpha1-backendsecuritypolicyapikey) - [BackendSecurityPolicyAWSCredentials](#github-com-envoyproxy-ai-gateway-api-v1alpha1-backendsecuritypolicyawscredentials) @@ -660,6 +661,7 @@ QuotaPolicyList contains a list of QuotaPolicy - [BackendSecurityPolicySpec](#github-com-envoyproxy-ai-gateway-api-v1alpha1-backendsecuritypolicyspec) - [BackendSecurityPolicyStatus](#github-com-envoyproxy-ai-gateway-api-v1alpha1-backendsecuritypolicystatus) - [BackendSecurityPolicyType](#github-com-envoyproxy-ai-gateway-api-v1alpha1-backendsecuritypolicytype) +- [BedrockGuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1alpha1-bedrockguardrailprovider) - [GCPCredentialsFile](#github-com-envoyproxy-ai-gateway-api-v1alpha1-gcpcredentialsfile) - [GCPOIDCExchangeToken](#github-com-envoyproxy-ai-gateway-api-v1alpha1-gcpoidcexchangetoken) - [GCPServiceAccountImpersonationConfig](#github-com-envoyproxy-ai-gateway-api-v1alpha1-gcpserviceaccountimpersonationconfig) @@ -670,6 +672,7 @@ QuotaPolicyList contains a list of QuotaPolicy - [GatewayConfigSpec](#github-com-envoyproxy-ai-gateway-api-v1alpha1-gatewayconfigspec) - [GatewayConfigStatus](#github-com-envoyproxy-ai-gateway-api-v1alpha1-gatewayconfigstatus) - [GuardrailAction](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailaction) +- [GuardrailFailureMode](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailfailuremode) - [GuardrailPhase](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailphase) - [GuardrailPolicySpec](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailpolicyspec) - [GuardrailPolicyStatus](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailpolicystatus) @@ -700,6 +703,7 @@ QuotaPolicyList contains a list of QuotaPolicy - [MCPRouteStatus](#github-com-envoyproxy-ai-gateway-api-v1alpha1-mcproutestatus) - [MCPToolFilter](#github-com-envoyproxy-ai-gateway-api-v1alpha1-mcptoolfilter) - [PerModelQuota](#github-com-envoyproxy-ai-gateway-api-v1alpha1-permodelquota) +- [PresidioGuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1alpha1-presidioguardrailprovider) - [ProtectedResourceMetadata](#github-com-envoyproxy-ai-gateway-api-v1alpha1-protectedresourcemetadata) - [QuotaBucketMode](#github-com-envoyproxy-ai-gateway-api-v1alpha1-quotabucketmode) - [QuotaDefinition](#github-com-envoyproxy-ai-gateway-api-v1alpha1-quotadefinition) @@ -1088,6 +1092,43 @@ and store them in a temporary credentials file. /> +#### AzureContentSafetyGuardrailProvider + + + +**Appears in:** +- [GuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailprovider) + +AzureContentSafetyGuardrailProvider configures calls to Azure AI Content Safety. + +##### Fields + + + + + + #### AzureOIDCExchangeToken @@ -1459,6 +1500,47 @@ BackendSecurityPolicyType specifies the type of auth mechanism used to access a required="false" description="" /> +#### BedrockGuardrailProvider + + + +**Appears in:** +- [GuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailprovider) + +BedrockGuardrailProvider configures calls to the AWS Bedrock ApplyGuardrail API. + +##### Fields + + + + + + #### GCPCredentialsFile @@ -1687,6 +1769,30 @@ GuardrailAction defines the safeguard action. required="false" description="" /> +#### GuardrailFailureMode + +**Underlying type:** string + +**Appears in:** +- [GuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailprovider) + +GuardrailFailureMode defines behavior when an external provider cannot evaluate content. + + + +##### Possible Values + + #### GuardrailPhase **Underlying type:** string @@ -1727,12 +1833,12 @@ GuardrailPolicySpec contains the configured checks attached to an AIServiceBacke @@ -1792,6 +1898,33 @@ GuardrailProvider describes the implementation used to evaluate a rule. type="string" required="false" description="Message is returned to the caller when the rule blocks a request or response." +/> @@ -2631,6 +2764,43 @@ a tool must match an include rule AND not match any exclude rule to be allowed. /> +#### PresidioGuardrailProvider + + + +**Appears in:** +- [GuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailprovider) + +PresidioGuardrailProvider configures calls to a Presidio analyzer service. + +##### Fields + + + + + + #### ProtectedResourceMetadata @@ -3494,6 +3664,7 @@ MCPRouteList contains a list of MCPRoute. - [APISchema](#github-com-envoyproxy-ai-gateway-api-v1beta1-apischema) - [AWSCredentialsFile](#github-com-envoyproxy-ai-gateway-api-v1beta1-awscredentialsfile) - [AWSOIDCExchangeToken](#github-com-envoyproxy-ai-gateway-api-v1beta1-awsoidcexchangetoken) +- [AzureContentSafetyGuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1beta1-azurecontentsafetyguardrailprovider) - [AzureOIDCExchangeToken](#github-com-envoyproxy-ai-gateway-api-v1beta1-azureoidcexchangetoken) - [BackendSecurityPolicyAPIKey](#github-com-envoyproxy-ai-gateway-api-v1beta1-backendsecuritypolicyapikey) - [BackendSecurityPolicyAWSCredentials](#github-com-envoyproxy-ai-gateway-api-v1beta1-backendsecuritypolicyawscredentials) @@ -3506,6 +3677,7 @@ MCPRouteList contains a list of MCPRoute. - [BackendSecurityPolicySpec](#github-com-envoyproxy-ai-gateway-api-v1beta1-backendsecuritypolicyspec) - [BackendSecurityPolicyStatus](#github-com-envoyproxy-ai-gateway-api-v1beta1-backendsecuritypolicystatus) - [BackendSecurityPolicyType](#github-com-envoyproxy-ai-gateway-api-v1beta1-backendsecuritypolicytype) +- [BedrockGuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1beta1-bedrockguardrailprovider) - [CredentialOverrideFromDynamicMetadata](#github-com-envoyproxy-ai-gateway-api-v1beta1-credentialoverridefromdynamicmetadata) - [CredentialOverrideFromRequestHeaders](#github-com-envoyproxy-ai-gateway-api-v1beta1-credentialoverridefromrequestheaders) - [GCPCredentialsFile](#github-com-envoyproxy-ai-gateway-api-v1beta1-gcpcredentialsfile) @@ -3518,6 +3690,7 @@ MCPRouteList contains a list of MCPRoute. - [GatewayConfigSpec](#github-com-envoyproxy-ai-gateway-api-v1beta1-gatewayconfigspec) - [GatewayConfigStatus](#github-com-envoyproxy-ai-gateway-api-v1beta1-gatewayconfigstatus) - [GuardrailAction](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailaction) +- [GuardrailFailureMode](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailfailuremode) - [GuardrailPhase](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailphase) - [GuardrailPolicySpec](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailpolicyspec) - [GuardrailPolicyStatus](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailpolicystatus) @@ -3552,6 +3725,7 @@ MCPRouteList contains a list of MCPRoute. - [MCPRouteSpec](#github-com-envoyproxy-ai-gateway-api-v1beta1-mcproutespec) - [MCPRouteStatus](#github-com-envoyproxy-ai-gateway-api-v1beta1-mcproutestatus) - [MCPToolFilter](#github-com-envoyproxy-ai-gateway-api-v1beta1-mcptoolfilter) +- [PresidioGuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1beta1-presidioguardrailprovider) - [ProtectedResourceMetadata](#github-com-envoyproxy-ai-gateway-api-v1beta1-protectedresourcemetadata) - [ToolCall](#github-com-envoyproxy-ai-gateway-api-v1beta1-toolcall) - [VersionedAPISchema](#github-com-envoyproxy-ai-gateway-api-v1beta1-versionedapischema) @@ -3938,6 +4112,43 @@ and store them in a temporary credentials file. /> +#### AzureContentSafetyGuardrailProvider + + + +**Appears in:** +- [GuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailprovider) + +AzureContentSafetyGuardrailProvider configures calls to Azure AI Content Safety. + +##### Fields + + + + + + #### AzureOIDCExchangeToken @@ -4347,6 +4558,47 @@ BackendSecurityPolicyType specifies the type of auth mechanism used to access a required="false" description="" /> +#### BedrockGuardrailProvider + + + +**Appears in:** +- [GuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailprovider) + +BedrockGuardrailProvider configures calls to the AWS Bedrock ApplyGuardrail API. + +##### Fields + + + + + + #### CredentialOverrideFromDynamicMetadata @@ -4640,6 +4892,30 @@ GuardrailAction defines the safeguard action. required="false" description="" /> +#### GuardrailFailureMode + +**Underlying type:** string + +**Appears in:** +- [GuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailprovider) + +GuardrailFailureMode defines behavior when an external provider cannot evaluate content. + + + +##### Possible Values + + #### GuardrailPhase **Underlying type:** string @@ -4680,12 +4956,12 @@ GuardrailPolicySpec contains the configured checks attached to an AIServiceBacke @@ -4745,6 +5021,33 @@ GuardrailProvider describes the implementation used to evaluate a rule. type="string" required="false" description="Message is returned to the caller when the rule blocks a request or response." +/> @@ -5738,6 +6041,43 @@ a tool must match an include rule AND not match any exclude rule to be allowed. /> +#### PresidioGuardrailProvider + + + +**Appears in:** +- [GuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailprovider) + +PresidioGuardrailProvider configures calls to a Presidio analyzer service. + +##### Fields + + + + + + #### ProtectedResourceMetadata diff --git a/site/docs/capabilities/security/guardrails.md b/site/docs/capabilities/security/guardrails.md new file mode 100644 index 0000000000..fd43482e3e --- /dev/null +++ b/site/docs/capabilities/security/guardrails.md @@ -0,0 +1,137 @@ +--- +id: guardrails +title: Content Guardrails +sidebar_position: 9 +--- + +# Content Guardrails + +`GuardrailPolicy` evaluates request or response payloads for an `AIServiceBackend`. Rules can use local regular expressions or external content-safety providers. + +## Apply a regex guardrail + +```yaml +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: GuardrailPolicy +metadata: + name: pii-guardrails +spec: + targetRefs: + - group: aigateway.envoyproxy.io + kind: AIServiceBackend + name: openai + rules: + - name: block-sensitive-input + phase: Request + provider: + type: Regex + pattern: "(?i)password|social security number" + action: Block + message: Request contains sensitive content +``` + +A matched rule returns HTTP `403` with error type `GuardrailViolation`. Rules are scoped to the generated backends that reference the targeted `AIServiceBackend`. + +## External providers + +### Presidio + +Presidio calls the analyzer `POST /analyze` endpoint. `scoreThresholdPercent` accepts values from 0 through 100. Authentication is optional. + +```yaml +provider: + type: Presidio + timeoutSeconds: 5 + failureMode: FailClosed + presidio: + endpoint: http://presidio-analyzer.presidio.svc.cluster.local:3000 + language: en + scoreThresholdPercent: 70 + apiKeySecretRef: + name: presidio-key +``` + +When configured, the Secret must contain an `apiKey` entry, sent as a bearer token. + +### AWS Bedrock Guardrails + +Bedrock uses the ApplyGuardrail API and SigV4 signing. By default, the ext-proc uses the standard AWS credential chain, including IRSA and EKS Pod Identity. + +```yaml +provider: + type: Bedrock + bedrock: + region: us-east-1 + guardrailIdentifier: my-guardrail + guardrailVersion: "1" +``` + +For static credentials, set `credentialsSecretRef` to a Secret whose `credentials` entry contains an AWS shared credentials file. `endpoint` can override the public Bedrock runtime endpoint for a private endpoint. + +### Azure AI Content Safety + +```yaml +provider: + type: AzureContentSafety + azureContentSafety: + endpoint: https://my-resource.cognitiveservices.azure.com + apiVersion: "2024-09-01" + severityThreshold: 4 + apiKeySecretRef: + name: azure-content-safety-key +``` + +The referenced Secret must contain an `apiKey` entry. + +## Failure behavior + +External providers default to `failureMode: FailClosed`. Provider errors, missing Secrets, and revoked credentials prevent unchecked traffic. Use `FailOpen` only when availability is more important than enforcement: + +```yaml +provider: + type: Presidio + failureMode: FailOpen + timeoutSeconds: 3 + presidio: + endpoint: http://presidio-analyzer.presidio.svc.cluster.local:3000 +``` + +Credential Secret changes automatically requeue the policy and regenerate affected gateway configuration. + +## Streaming responses + +When an applicable response guardrail exists, the gateway keeps the upstream response buffered until evaluation completes. This prevents unsafe content from being partially delivered before a blocking decision. Routes without response guardrails retain normal streaming behavior. + +## Observability + +The ext-proc emits structured block and provider-failure logs without payload content. It also records `guardrail.evaluation` span events and the counter: + +```text +aigateway.guardrail.evaluation.count +``` + +The counter attributes are `aigateway.guardrail.phase` (`Request` or `Response`) and `aigateway.guardrail.result` (`allowed`, `blocked`, or `error`). With the Prometheus exporter, dots are converted to underscores. + +## More examples + +See [`examples/guardrails`](https://github.com/envoyproxy/ai-gateway/tree/main/examples/guardrails) for local and external-provider manifests. + +## Optional live-provider tests + +Presidio is tested against its official analyzer image with Testcontainers. The test runs automatically when Docker is available and skips otherwise: + +```shell +go test ./internal/guardrails -run '^TestPresidioEvaluatorContainer$' -v +``` + +The Azure and Bedrock live tests are disabled unless all required environment variables for a provider are set: + +- Presidio managed/external deployment: `TEST_PRESIDIO_ENDPOINT`, `TEST_PRESIDIO_BLOCKED_TEXT`, and optionally `TEST_PRESIDIO_API_KEY`. +- Azure: `TEST_AZURE_CONTENT_SAFETY_ENDPOINT`, `TEST_AZURE_CONTENT_SAFETY_API_KEY`, and `TEST_AZURE_CONTENT_SAFETY_BLOCKED_TEXT`. +- Bedrock: `TEST_AWS_BEDROCK_GUARDRAIL_REGION`, `TEST_AWS_BEDROCK_GUARDRAIL_ID`, `TEST_AWS_BEDROCK_GUARDRAIL_VERSION`, and `TEST_AWS_BEDROCK_GUARDRAIL_BLOCKED_TEXT`. AWS credentials use the standard credential chain. + +Run them with: + +```shell +go test ./internal/guardrails -run '^TestLive' -v +``` diff --git a/tests/crdcel/main_test.go b/tests/crdcel/main_test.go index e238e04080..6f0de2de71 100644 --- a/tests/crdcel/main_test.go +++ b/tests/crdcel/main_test.go @@ -160,6 +160,37 @@ func TestGatewayConfigs(t *testing.T) { } } +func TestGuardrailPolicies(t *testing.T) { + c, _, _ := testsinternal.NewEnvTest(t) + ctx := t.Context() + + for _, tc := range []struct { + name string + expErr string + }{ + {name: "basic.yaml"}, + {name: "regex_missing_pattern.yaml", expErr: "Regex requires pattern and no external provider configuration"}, + {name: "provider_mismatch.yaml", expErr: "Presidio requires only presidio provider configuration"}, + {name: "invalid_action.yaml", expErr: "Unsupported value"}, + {name: "invalid_endpoint.yaml", expErr: "Invalid value"}, + {name: "duplicate_rule_names.yaml", expErr: "rule name must be unique within the policy"}, + } { + t.Run(tc.name, func(t *testing.T) { + data, err := testdata.ReadFile(path.Join("testdata/guardrailpolicies", tc.name)) + require.NoError(t, err) + + policy := &aigv1b1.GuardrailPolicy{} + require.NoError(t, yaml.UnmarshalStrict(data, policy)) + if tc.expErr != "" { + require.ErrorContains(t, c.Create(ctx, policy), tc.expErr) + } else { + require.NoError(t, c.Create(ctx, policy)) + require.NoError(t, c.Delete(ctx, policy)) + } + }) + } +} + func TestBackendSecurityPolicies(t *testing.T) { c, _, _ := testsinternal.NewEnvTest(t) ctx := t.Context() diff --git a/tests/crdcel/testdata/guardrailpolicies/basic.yaml b/tests/crdcel/testdata/guardrailpolicies/basic.yaml new file mode 100644 index 0000000000..34b6658e65 --- /dev/null +++ b/tests/crdcel/testdata/guardrailpolicies/basic.yaml @@ -0,0 +1,47 @@ +# Copyright Envoy AI Gateway Authors +# SPDX-License-Identifier: Apache-2.0 +# The full text of the Apache license is available in the LICENSE file at +# the root of the repo. + +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: GuardrailPolicy +metadata: + name: valid-guardrails + namespace: default +spec: + targetRefs: + - group: aigateway.envoyproxy.io + kind: AIServiceBackend + name: backend + rules: + - name: regex + phase: Request + provider: + type: Regex + pattern: secret + action: Block + - name: presidio + phase: Request + provider: + type: Presidio + timeoutSeconds: 5 + failureMode: FailOpen + presidio: + endpoint: https://presidio.example.com + scoreThresholdPercent: 70 + - name: bedrock + phase: Response + provider: + type: Bedrock + bedrock: + region: us-east-1 + guardrailIdentifier: guardrail-id + guardrailVersion: "1" + - name: azure + phase: Response + provider: + type: AzureContentSafety + azureContentSafety: + endpoint: https://content-safety.example.com + apiKeySecretRef: + name: azure-key diff --git a/tests/crdcel/testdata/guardrailpolicies/duplicate_rule_names.yaml b/tests/crdcel/testdata/guardrailpolicies/duplicate_rule_names.yaml new file mode 100644 index 0000000000..ff32953938 --- /dev/null +++ b/tests/crdcel/testdata/guardrailpolicies/duplicate_rule_names.yaml @@ -0,0 +1,26 @@ +# Copyright Envoy AI Gateway Authors +# SPDX-License-Identifier: Apache-2.0 +# The full text of the Apache license is available in the LICENSE file at +# the root of the repo. + +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: GuardrailPolicy +metadata: + name: duplicate-rule-names + namespace: default +spec: + targetRefs: + - group: aigateway.envoyproxy.io + kind: AIServiceBackend + name: backend + rules: + - name: duplicate + phase: Request + provider: + type: Regex + pattern: request + - name: duplicate + phase: Response + provider: + type: Regex + pattern: response diff --git a/tests/crdcel/testdata/guardrailpolicies/invalid_action.yaml b/tests/crdcel/testdata/guardrailpolicies/invalid_action.yaml new file mode 100644 index 0000000000..b201b64007 --- /dev/null +++ b/tests/crdcel/testdata/guardrailpolicies/invalid_action.yaml @@ -0,0 +1,22 @@ +# Copyright Envoy AI Gateway Authors +# SPDX-License-Identifier: Apache-2.0 +# The full text of the Apache license is available in the LICENSE file at +# the root of the repo. + +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: GuardrailPolicy +metadata: + name: invalid-action + namespace: default +spec: + targetRefs: + - group: aigateway.envoyproxy.io + kind: AIServiceBackend + name: backend + rules: + - name: invalid + phase: Request + provider: + type: Regex + pattern: secret + action: Log diff --git a/tests/crdcel/testdata/guardrailpolicies/invalid_endpoint.yaml b/tests/crdcel/testdata/guardrailpolicies/invalid_endpoint.yaml new file mode 100644 index 0000000000..197909a4b2 --- /dev/null +++ b/tests/crdcel/testdata/guardrailpolicies/invalid_endpoint.yaml @@ -0,0 +1,22 @@ +# Copyright Envoy AI Gateway Authors +# SPDX-License-Identifier: Apache-2.0 +# The full text of the Apache license is available in the LICENSE file at +# the root of the repo. + +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: GuardrailPolicy +metadata: + name: invalid-endpoint + namespace: default +spec: + targetRefs: + - group: aigateway.envoyproxy.io + kind: AIServiceBackend + name: backend + rules: + - name: invalid + phase: Request + provider: + type: Presidio + presidio: + endpoint: not-a-uri diff --git a/tests/crdcel/testdata/guardrailpolicies/provider_mismatch.yaml b/tests/crdcel/testdata/guardrailpolicies/provider_mismatch.yaml new file mode 100644 index 0000000000..ef323fc426 --- /dev/null +++ b/tests/crdcel/testdata/guardrailpolicies/provider_mismatch.yaml @@ -0,0 +1,24 @@ +# Copyright Envoy AI Gateway Authors +# SPDX-License-Identifier: Apache-2.0 +# The full text of the Apache license is available in the LICENSE file at +# the root of the repo. + +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: GuardrailPolicy +metadata: + name: provider-mismatch + namespace: default +spec: + targetRefs: + - group: aigateway.envoyproxy.io + kind: AIServiceBackend + name: backend + rules: + - name: invalid + phase: Request + provider: + type: Presidio + bedrock: + region: us-east-1 + guardrailIdentifier: guardrail-id + guardrailVersion: "1" diff --git a/tests/crdcel/testdata/guardrailpolicies/regex_missing_pattern.yaml b/tests/crdcel/testdata/guardrailpolicies/regex_missing_pattern.yaml new file mode 100644 index 0000000000..9ec3045cfb --- /dev/null +++ b/tests/crdcel/testdata/guardrailpolicies/regex_missing_pattern.yaml @@ -0,0 +1,20 @@ +# Copyright Envoy AI Gateway Authors +# SPDX-License-Identifier: Apache-2.0 +# The full text of the Apache license is available in the LICENSE file at +# the root of the repo. + +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: GuardrailPolicy +metadata: + name: regex-missing-pattern + namespace: default +spec: + targetRefs: + - group: aigateway.envoyproxy.io + kind: AIServiceBackend + name: backend + rules: + - name: invalid + phase: Request + provider: + type: Regex diff --git a/tests/data-plane/testupstream_test.go b/tests/data-plane/testupstream_test.go index f46da64ee0..197ceb2682 100644 --- a/tests/data-plane/testupstream_test.go +++ b/tests/data-plane/testupstream_test.go @@ -40,6 +40,60 @@ func failIf5xx(t *testing.T, resp *http.Response, was5xx *bool) { } } +func TestGuardrailsWithTestUpstream(t *testing.T) { + configBytes, err := yaml.Marshal(&filterapi.Config{ + Version: version.Parse(), + Backends: []filterapi.Backend{testUpstreamOpenAIBackend}, + Guardrails: []filterapi.Guardrail{ + { + Name: "e2e-request-guardrail", Phase: filterapi.GuardrailPhaseRequest, + Provider: filterapi.GuardrailProvider{ + Type: filterapi.GuardrailProviderTypeRegex, Pattern: "E2E_REQUEST_GUARDRAIL", Message: "request blocked by e2e guardrail", + }, + }, + { + Name: "e2e-response-guardrail", Phase: filterapi.GuardrailPhaseResponse, + Provider: filterapi.GuardrailProvider{ + Type: filterapi.GuardrailProviderTypeRegex, Pattern: "E2E_RESPONSE_GUARDRAIL", Message: "response blocked by e2e guardrail", + }, + }, + }, + }) + require.NoError(t, err) + env := startTestEnvironment(t, string(configBytes), true, false) + endpoint := fmt.Sprintf("http://localhost:%d/v1/chat/completions", env.EnvoyListenerPort()) + + for _, test := range []struct { + name, requestBody, responseBody, wantBody string + }{ + { + name: "request", requestBody: `{"model":"something","messages":[{"role":"user","content":"E2E_REQUEST_GUARDRAIL"}]}`, + wantBody: `{"type":"error","error":{"type":"GuardrailViolation","code":"403","message":"request blocked by e2e guardrail"}}`, + }, + { + name: "response", requestBody: `{"model":"something","messages":[{"role":"user","content":"hello"}]}`, + responseBody: `{"id":"guardrail-test","object":"chat.completion","created":123,"model":"something","choices":[{"index":0,"message":{"role":"assistant","content":"E2E_RESPONSE_GUARDRAIL"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}`, + wantBody: `{"type":"error","error":{"type":"GuardrailViolation","code":"403","message":"response blocked by e2e guardrail"}}`, + }, + } { + t.Run(test.name, func(t *testing.T) { + req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, endpoint, strings.NewReader(test.requestBody)) + require.NoError(t, err) + req.Header.Set("x-test-backend", "openai") + if test.responseBody != "" { + req.Header.Set(testupstreamlib.ResponseBodyHeaderKey, base64.StdEncoding.EncodeToString([]byte(test.responseBody))) + } + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + defer func() { _ = resp.Body.Close() }() + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.Equal(t, http.StatusForbidden, resp.StatusCode) + require.JSONEq(t, test.wantBody, string(body)) + }) + } +} + // TestWithTestUpstream tests the end-to-end flow of the external processor with Envoy and the test upstream. // // This does not require any environment variables to be set as it relies on the test upstream. From 4341f5c47f69fafae4627f2876f94b9e408932bb Mon Sep 17 00:00:00 2001 From: Fernando Escolar Date: Wed, 16 Sep 2026 19:14:33 +0200 Subject: [PATCH 05/18] refactor: clarify TargetRefs description in GuardrailPolicy and related documentation Signed-off-by: Fernando Escolar --- api/v1alpha1/guardrail_policy.go | 2 +- api/v1beta1/guardrail_policy.go | 2 +- internal/controller/gateway.go | 4 ++-- .../aigateway.envoyproxy.io_guardrailpolicies.yaml | 8 ++++---- site/docs/api/api.mdx | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/api/v1alpha1/guardrail_policy.go b/api/v1alpha1/guardrail_policy.go index a5f86ca7c2..40556e4607 100644 --- a/api/v1alpha1/guardrail_policy.go +++ b/api/v1alpha1/guardrail_policy.go @@ -30,7 +30,7 @@ type GuardrailPolicy struct { // GuardrailPolicySpec contains the configured checks attached to an AIServiceBackend. type GuardrailPolicySpec struct { - // TargetRefs are the names of the AIServiceBackend resources this GuardrailPolicy is attached to. + // TargetRefs identify the AIServiceBackend resources this GuardrailPolicy is attached to. // // +kubebuilder:validation:MinItems=1 // +kubebuilder:validation:MaxItems=16 diff --git a/api/v1beta1/guardrail_policy.go b/api/v1beta1/guardrail_policy.go index 33f4af0cae..4ecdf6a7ef 100644 --- a/api/v1beta1/guardrail_policy.go +++ b/api/v1beta1/guardrail_policy.go @@ -30,7 +30,7 @@ type GuardrailPolicy struct { // GuardrailPolicySpec contains the configured checks attached to an AIServiceBackend. type GuardrailPolicySpec struct { - // TargetRefs are the names of the AIServiceBackend resources this GuardrailPolicy is attached to. + // TargetRefs identify the AIServiceBackend resources this GuardrailPolicy is attached to. // // +kubebuilder:validation:MinItems=1 // +kubebuilder:validation:MaxItems=16 diff --git a/internal/controller/gateway.go b/internal/controller/gateway.go index cca850fc69..4609319bf7 100644 --- a/internal/controller/gateway.go +++ b/internal/controller/gateway.go @@ -431,7 +431,7 @@ func (c *GatewayController) reconcileFilterConfigSecret( routeBackendNamesSet := map[string]struct{}{} routeBackendNames := []string{} injectedQuotaCosts := make(map[string]struct{}) - injectedGuardrails := make(map[string]struct{}) + injectedGuardrailKeys := make(map[string]struct{}) for ruleIndex := range spec.Rules { rule := &spec.Rules[ruleIndex] for _, m := range rule.Matches { @@ -559,7 +559,7 @@ func (c *GatewayController) reconcileFilterConfigSecret( // Inject QuotaPolicy cost expressions as LLMRequestCost entries so ext_proc // computes and stores them in metadata for the HitsAddend to read. c.injectQuotaPolicyCostExpressions(ctx, aiGatewayRoute, ec, injectedQuotaCosts, routeName) - if guardrailErr := c.injectGuardrails(ctx, aiGatewayRoute, ec, injectedGuardrails); guardrailErr != nil { + if guardrailErr := c.injectGuardrails(ctx, aiGatewayRoute, ec, injectedGuardrailKeys); guardrailErr != nil { return false, fmt.Errorf("failed to inject guardrails for route %s: %w", aiGatewayRoute.Name, guardrailErr) } diff --git a/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_guardrailpolicies.yaml b/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_guardrailpolicies.yaml index 02c20af2b7..b2ce12ddd6 100644 --- a/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_guardrailpolicies.yaml +++ b/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_guardrailpolicies.yaml @@ -334,8 +334,8 @@ spec: - message: rule name must be unique within the policy rule: self.all(rule, self.exists_one(other, other.name == rule.name)) targetRefs: - description: TargetRefs are the names of the AIServiceBackend resources - this GuardrailPolicy is attached to. + description: TargetRefs identify the AIServiceBackend resources this + GuardrailPolicy is attached to. items: description: |- LocalPolicyTargetReference identifies an API object to apply a direct or @@ -753,8 +753,8 @@ spec: - message: rule name must be unique within the policy rule: self.all(rule, self.exists_one(other, other.name == rule.name)) targetRefs: - description: TargetRefs are the names of the AIServiceBackend resources - this GuardrailPolicy is attached to. + description: TargetRefs identify the AIServiceBackend resources this + GuardrailPolicy is attached to. items: description: |- LocalPolicyTargetReference identifies an API object to apply a direct or diff --git a/site/docs/api/api.mdx b/site/docs/api/api.mdx index 4b903d5156..5dc13d6b82 100644 --- a/site/docs/api/api.mdx +++ b/site/docs/api/api.mdx @@ -1834,7 +1834,7 @@ GuardrailPolicySpec contains the configured checks attached to an AIServiceBacke name="targetRefs" type="[LocalPolicyTargetReference](https://gateway-api.sigs.k8s.io/reference/spec/#gateway.networking.k8s.io/v1alpha2.LocalPolicyTargetReference) array" required="true" - description="TargetRefs are the names of the AIServiceBackend resources this GuardrailPolicy is attached to." + description="TargetRefs identify the AIServiceBackend resources this GuardrailPolicy is attached to." /> Date: Wed, 16 Sep 2026 21:08:34 +0200 Subject: [PATCH 06/18] - Added support for Monitor and Mask actions in guardrail policies. - Introduced maxRequestBodyBytes and maxResponseBodyBytes to limit payload sizes. - Updated GuardrailEvaluator interface to return GuardrailEvaluationResult. - Enhanced Presidio and Bedrock evaluators to support masking functionality. Signed-off-by: Fernando Escolar --- api/v1alpha1/guardrail_policy.go | 23 ++- api/v1alpha1/zz_generated.deepcopy.go | 10 ++ api/v1beta1/guardrail_policy.go | 23 ++- api/v1beta1/zz_generated.deepcopy.go | 10 ++ examples/guardrails/regex.yaml | 9 ++ internal/controller/gateway.go | 41 +++-- internal/controller/guardrail_policy.go | 5 +- internal/controller/guardrail_policy_test.go | 37 ++++- internal/extproc/guardrail_content.go | 90 +++++++++++ internal/extproc/guardrail_content_test.go | 27 ++++ internal/extproc/guardrails.go | 147 ++++++++++++++---- internal/extproc/guardrails_test.go | 109 +++++++++++-- internal/extproc/processor_impl.go | 121 ++++++++++---- internal/extproc/processor_impl_test.go | 18 +++ internal/filterapi/filterconfig.go | 7 +- internal/filterapi/runtime.go | 52 +++++-- internal/filterapi/runtime_test.go | 4 +- internal/guardrails/adapters.go | 2 +- internal/guardrails/adapters_live_test.go | 12 +- internal/guardrails/adapters_test.go | 26 ++-- internal/guardrails/azure.go | 8 +- internal/guardrails/bedrock.go | 23 ++- internal/guardrails/presidio.go | 45 +++++- .../guardrails/presidio_container_test.go | 6 +- internal/metrics/guardrail_metrics.go | 8 +- ...teway.envoyproxy.io_guardrailpolicies.yaml | 50 ++++++ site/docs/api/api.mdx | 56 +++++++ site/docs/capabilities/security/guardrails.md | 14 +- tests/crdcel/main_test.go | 1 + .../guardrailpolicies/azure_mask.yaml | 25 +++ .../testdata/guardrailpolicies/basic.yaml | 9 +- tests/data-plane/testupstream_test.go | 49 +++++- 32 files changed, 909 insertions(+), 158 deletions(-) create mode 100644 internal/extproc/guardrail_content.go create mode 100644 internal/extproc/guardrail_content_test.go create mode 100644 tests/crdcel/testdata/guardrailpolicies/azure_mask.yaml diff --git a/api/v1alpha1/guardrail_policy.go b/api/v1alpha1/guardrail_policy.go index 40556e4607..016aca2f39 100644 --- a/api/v1alpha1/guardrail_policy.go +++ b/api/v1alpha1/guardrail_policy.go @@ -42,6 +42,18 @@ type GuardrailPolicySpec struct { // +kubebuilder:validation:MaxItems=32 // +kubebuilder:validation:XValidation:rule="self.all(rule, self.exists_one(other, other.name == rule.name))",message="rule name must be unique within the policy" Rules []GuardrailRule `json:"rules,omitempty"` + // MaxRequestBodyBytes is the largest request body evaluated by this policy. + // +optional + // +kubebuilder:default=10485760 + // +kubebuilder:validation:Minimum=1024 + // +kubebuilder:validation:Maximum=52428800 + MaxRequestBodyBytes *int64 `json:"maxRequestBodyBytes,omitempty"` + // MaxResponseBodyBytes is the largest response body buffered and evaluated by this policy. + // +optional + // +kubebuilder:default=10485760 + // +kubebuilder:validation:Minimum=1024 + // +kubebuilder:validation:Maximum=52428800 + MaxResponseBodyBytes *int64 `json:"maxResponseBodyBytes,omitempty"` } // GuardrailRule defines one content-safety check to apply to a request or response. @@ -73,6 +85,7 @@ const ( // +kubebuilder:validation:XValidation:rule="self.type != 'Presidio' || (!has(self.pattern) && has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety))",message="Presidio requires only presidio provider configuration" // +kubebuilder:validation:XValidation:rule="self.type != 'Bedrock' || (!has(self.pattern) && has(self.bedrock) && !has(self.presidio) && !has(self.azureContentSafety))",message="Bedrock requires only bedrock provider configuration" // +kubebuilder:validation:XValidation:rule="self.type != 'AzureContentSafety' || (!has(self.pattern) && has(self.azureContentSafety) && !has(self.presidio) && !has(self.bedrock))",message="AzureContentSafety requires only azureContentSafety provider configuration" +// +kubebuilder:validation:XValidation:rule="self.action != 'Mask' || self.type != 'AzureContentSafety'",message="AzureContentSafety does not support Mask" type GuardrailProvider struct { // Type identifies the guardrail implementation. // @@ -86,8 +99,12 @@ type GuardrailProvider struct { // // +optional // +kubebuilder:default=Block - // +kubebuilder:validation:Enum=Block + // +kubebuilder:validation:Enum=Block;Monitor;Mask Action GuardrailAction `json:"action,omitempty"` + // MaskReplacement is used by Regex and Presidio Mask actions. + // +optional + // +kubebuilder:default="[REDACTED]" + MaskReplacement string `json:"maskReplacement,omitempty"` // Message is returned to the caller when the rule blocks a request or response. // // +optional @@ -179,7 +196,9 @@ const ( type GuardrailAction string const ( - GuardrailActionBlock GuardrailAction = "Block" + GuardrailActionBlock GuardrailAction = "Block" + GuardrailActionMonitor GuardrailAction = "Monitor" + GuardrailActionMask GuardrailAction = "Mask" ) // GuardrailFailureMode defines behavior when an external provider cannot evaluate content. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 8375e211a0..83a25a0017 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -1061,6 +1061,16 @@ func (in *GuardrailPolicySpec) DeepCopyInto(out *GuardrailPolicySpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.MaxRequestBodyBytes != nil { + in, out := &in.MaxRequestBodyBytes, &out.MaxRequestBodyBytes + *out = new(int64) + **out = **in + } + if in.MaxResponseBodyBytes != nil { + in, out := &in.MaxResponseBodyBytes, &out.MaxResponseBodyBytes + *out = new(int64) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GuardrailPolicySpec. diff --git a/api/v1beta1/guardrail_policy.go b/api/v1beta1/guardrail_policy.go index 4ecdf6a7ef..2a1c9c6274 100644 --- a/api/v1beta1/guardrail_policy.go +++ b/api/v1beta1/guardrail_policy.go @@ -42,6 +42,18 @@ type GuardrailPolicySpec struct { // +kubebuilder:validation:MaxItems=32 // +kubebuilder:validation:XValidation:rule="self.all(rule, self.exists_one(other, other.name == rule.name))",message="rule name must be unique within the policy" Rules []GuardrailRule `json:"rules,omitempty"` + // MaxRequestBodyBytes is the largest request body evaluated by this policy. + // +optional + // +kubebuilder:default=10485760 + // +kubebuilder:validation:Minimum=1024 + // +kubebuilder:validation:Maximum=52428800 + MaxRequestBodyBytes *int64 `json:"maxRequestBodyBytes,omitempty"` + // MaxResponseBodyBytes is the largest response body buffered and evaluated by this policy. + // +optional + // +kubebuilder:default=10485760 + // +kubebuilder:validation:Minimum=1024 + // +kubebuilder:validation:Maximum=52428800 + MaxResponseBodyBytes *int64 `json:"maxResponseBodyBytes,omitempty"` } // GuardrailRule defines one content-safety check to apply to a request or response. @@ -73,6 +85,7 @@ const ( // +kubebuilder:validation:XValidation:rule="self.type != 'Presidio' || (!has(self.pattern) && has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety))",message="Presidio requires only presidio provider configuration" // +kubebuilder:validation:XValidation:rule="self.type != 'Bedrock' || (!has(self.pattern) && has(self.bedrock) && !has(self.presidio) && !has(self.azureContentSafety))",message="Bedrock requires only bedrock provider configuration" // +kubebuilder:validation:XValidation:rule="self.type != 'AzureContentSafety' || (!has(self.pattern) && has(self.azureContentSafety) && !has(self.presidio) && !has(self.bedrock))",message="AzureContentSafety requires only azureContentSafety provider configuration" +// +kubebuilder:validation:XValidation:rule="self.action != 'Mask' || self.type != 'AzureContentSafety'",message="AzureContentSafety does not support Mask" type GuardrailProvider struct { // Type identifies the guardrail implementation. // @@ -86,8 +99,12 @@ type GuardrailProvider struct { // // +optional // +kubebuilder:default=Block - // +kubebuilder:validation:Enum=Block + // +kubebuilder:validation:Enum=Block;Monitor;Mask Action GuardrailAction `json:"action,omitempty"` + // MaskReplacement is used by Regex and Presidio Mask actions. + // +optional + // +kubebuilder:default="[REDACTED]" + MaskReplacement string `json:"maskReplacement,omitempty"` // Message is returned to the caller when the rule blocks a request or response. // // +optional @@ -179,7 +196,9 @@ const ( type GuardrailAction string const ( - GuardrailActionBlock GuardrailAction = "Block" + GuardrailActionBlock GuardrailAction = "Block" + GuardrailActionMonitor GuardrailAction = "Monitor" + GuardrailActionMask GuardrailAction = "Mask" ) // GuardrailFailureMode defines behavior when an external provider cannot evaluate content. diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 0732566768..be41209e14 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -1143,6 +1143,16 @@ func (in *GuardrailPolicySpec) DeepCopyInto(out *GuardrailPolicySpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.MaxRequestBodyBytes != nil { + in, out := &in.MaxRequestBodyBytes, &out.MaxRequestBodyBytes + *out = new(int64) + **out = **in + } + if in.MaxResponseBodyBytes != nil { + in, out := &in.MaxResponseBodyBytes, &out.MaxResponseBodyBytes + *out = new(int64) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GuardrailPolicySpec. diff --git a/examples/guardrails/regex.yaml b/examples/guardrails/regex.yaml index 23df71f228..99877acc23 100644 --- a/examples/guardrails/regex.yaml +++ b/examples/guardrails/regex.yaml @@ -8,6 +8,8 @@ kind: GuardrailPolicy metadata: name: local-content-guardrails spec: + maxRequestBodyBytes: 10485760 + maxResponseBodyBytes: 10485760 targetRefs: - group: aigateway.envoyproxy.io kind: AIServiceBackend @@ -27,3 +29,10 @@ spec: pattern: "(?i)internal[- ]only" action: Block message: Response contains restricted content + - name: mask-email-addresses + phase: Response + provider: + type: Regex + pattern: '[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}' + action: Mask + maskReplacement: "[EMAIL REDACTED]" diff --git a/internal/controller/gateway.go b/internal/controller/gateway.go index 4609319bf7..36620bb8a2 100644 --- a/internal/controller/gateway.go +++ b/internal/controller/gateway.go @@ -608,6 +608,12 @@ func (c *GatewayController) injectGuardrails( if err := c.client.List(ctx, &policies, client.InNamespace(route.Namespace)); err != nil { return fmt.Errorf("failed to list GuardrailPolicies: %w", err) } + sort.Slice(policies.Items, func(i, j int) bool { + if policies.Items[i].Namespace != policies.Items[j].Namespace { + return policies.Items[i].Namespace < policies.Items[j].Namespace + } + return policies.Items[i].Name < policies.Items[j].Name + }) routeBackends := make(map[string]struct{}) for _, routeRule := range route.Spec.Rules { @@ -639,7 +645,7 @@ func (c *GatewayController) injectGuardrails( backends := guardrailBackendNames(route, policy) provider, err := c.guardrailProviderToFilterAPI(ctx, policy.Namespace, &rule.Provider) if err != nil { - if rule.Provider.FailureMode == aigv1b1.GuardrailFailureModeFailOpen { + if rule.Provider.FailureMode == aigv1b1.GuardrailFailureModeFailOpen || rule.Provider.Action == aigv1b1.GuardrailActionMonitor { c.logger.Error(err, "guardrail configuration failed open", "policy", policy.Name, "rule", rule.Name) injected[key] = struct{}{} continue @@ -653,10 +659,11 @@ func (c *GatewayController) injectGuardrails( } } ec.Guardrails = append(ec.Guardrails, filterapi.Guardrail{ - Name: key, - Phase: filterapi.GuardrailPhase(rule.Phase), - Provider: provider, - Backends: backends, + Name: key, + Phase: filterapi.GuardrailPhase(rule.Phase), + Provider: provider, + Backends: backends, + MaxPayloadBytes: guardrailMaxPayloadBytes(policy, rule.Phase), }) injected[key] = struct{}{} } @@ -664,6 +671,19 @@ func (c *GatewayController) injectGuardrails( return nil } +const defaultGuardrailMaxPayloadBytes int64 = 10 * 1024 * 1024 + +func guardrailMaxPayloadBytes(policy *aigv1b1.GuardrailPolicy, phase aigv1b1.GuardrailPhase) int64 { + configured := policy.Spec.MaxRequestBodyBytes + if phase == aigv1b1.GuardrailPhaseResponse { + configured = policy.Spec.MaxResponseBodyBytes + } + if configured == nil { + return defaultGuardrailMaxPayloadBytes + } + return *configured +} + func guardrailBackendNames(route *aigv1b1.AIGatewayRoute, policy *aigv1b1.GuardrailPolicy) []string { targets := make(map[string]struct{}, len(policy.Spec.TargetRefs)) for _, targetRef := range policy.Spec.TargetRefs { @@ -688,11 +708,12 @@ func guardrailBackendNames(route *aigv1b1.AIGatewayRoute, policy *aigv1b1.Guardr func (c *GatewayController) guardrailProviderToFilterAPI(ctx context.Context, namespace string, provider *aigv1b1.GuardrailProvider) (filterapi.GuardrailProvider, error) { converted := filterapi.GuardrailProvider{ - Type: filterapi.GuardrailProviderType(provider.Type), - Pattern: provider.Pattern, - Action: filterapi.GuardrailAction(provider.Action), - Message: provider.Message, - FailureMode: filterapi.GuardrailFailureMode(provider.FailureMode), + Type: filterapi.GuardrailProviderType(provider.Type), + Pattern: provider.Pattern, + Action: filterapi.GuardrailAction(provider.Action), + MaskReplacement: provider.MaskReplacement, + Message: provider.Message, + FailureMode: filterapi.GuardrailFailureMode(provider.FailureMode), } if provider.TimeoutSeconds != nil { converted.TimeoutSeconds = *provider.TimeoutSeconds diff --git a/internal/controller/guardrail_policy.go b/internal/controller/guardrail_policy.go index b636080087..84517e7c37 100644 --- a/internal/controller/guardrail_policy.go +++ b/internal/controller/guardrail_policy.go @@ -112,9 +112,12 @@ func (c *GuardrailPolicyController) syncGuardrailPolicy(ctx context.Context, pol } func (c *GuardrailPolicyController) validateGuardrailProvider(ctx context.Context, namespace string, provider *aigv1b1.GuardrailProvider) error { - if provider.Action != "" && provider.Action != aigv1b1.GuardrailActionBlock { + if provider.Action != "" && provider.Action != aigv1b1.GuardrailActionBlock && provider.Action != aigv1b1.GuardrailActionMonitor && provider.Action != aigv1b1.GuardrailActionMask { return fmt.Errorf("unsupported action %q", provider.Action) } + if provider.Action == aigv1b1.GuardrailActionMask && provider.Type == aigv1b1.GuardrailProviderTypeAzureContentSafety { + return fmt.Errorf("azure Content Safety does not support mask") + } if provider.FailureMode != "" && provider.FailureMode != aigv1b1.GuardrailFailureModeFailClosed && provider.FailureMode != aigv1b1.GuardrailFailureModeFailOpen { return fmt.Errorf("unsupported failureMode %q", provider.FailureMode) } diff --git a/internal/controller/guardrail_policy_test.go b/internal/controller/guardrail_policy_test.go index e1c30db58b..c4d2e10ed3 100644 --- a/internal/controller/guardrail_policy_test.go +++ b/internal/controller/guardrail_policy_test.go @@ -216,6 +216,36 @@ func TestInjectGuardrailsTargetsRouteBackends(t *testing.T) { require.Equal(t, []string{"default/backend/route/route/rule/0/ref/0"}, config.Guardrails[0].Backends) } +func TestInjectGuardrailsUsesDeterministicPolicyOrder(t *testing.T) { + const namespace = "default" + controllerClient := newGuardrailPolicyTestClient(t) + for _, name := range []string{"zeta", "alpha"} { + require.NoError(t, controllerClient.Create(t.Context(), &aigv1b1.GuardrailPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Spec: aigv1b1.GuardrailPolicySpec{ + TargetRefs: []gwapiv1a2.LocalPolicyTargetReference{{Name: "backend"}}, + Rules: []aigv1b1.GuardrailRule{{ + Name: "rule", Phase: aigv1b1.GuardrailPhaseRequest, + Provider: aigv1b1.GuardrailProvider{Type: aigv1b1.GuardrailProviderTypeRegex, Pattern: name}, + }}, + }, + })) + } + controller := &GatewayController{client: controllerClient, kube: fakekube.NewClientset(), logger: ctrl.Log} + config := &filterapi.Config{} + route := &aigv1b1.AIGatewayRoute{ + ObjectMeta: metav1.ObjectMeta{Name: "route", Namespace: namespace}, + Spec: aigv1b1.AIGatewayRouteSpec{Rules: []aigv1b1.AIGatewayRouteRule{{ + BackendRefs: []aigv1b1.AIGatewayRouteRuleBackendRef{{Name: "backend"}}, + }}}, + } + require.NoError(t, controller.injectGuardrails(t.Context(), route, config, map[string]struct{}{})) + require.Len(t, config.Guardrails, 2) + require.Equal(t, "default/alpha/rule", config.Guardrails[0].Name) + require.Equal(t, "default/zeta/rule", config.Guardrails[1].Name) + require.Equal(t, defaultGuardrailMaxPayloadBytes, config.Guardrails[0].MaxPayloadBytes) +} + func TestSecretToGuardrailPolicy(t *testing.T) { const namespace = "default" controllerClient := newGuardrailPolicyTestClient(t) @@ -248,10 +278,12 @@ func TestInjectGuardrailsConfigurationFailureModes(t *testing.T) { for _, test := range []struct { name string failureMode aigv1b1.GuardrailFailureMode + action aigv1b1.GuardrailAction wantGuardrail bool }{ {name: "fail closed publishes blocking fallback", wantGuardrail: true}, {name: "fail open omits unavailable rule", failureMode: aigv1b1.GuardrailFailureModeFailOpen}, + {name: "monitor omits unavailable rule", action: aigv1b1.GuardrailActionMonitor}, } { t.Run(test.name, func(t *testing.T) { controllerClient := newGuardrailPolicyTestClient(t) @@ -263,6 +295,7 @@ func TestInjectGuardrailsConfigurationFailureModes(t *testing.T) { Name: "azure", Phase: aigv1b1.GuardrailPhaseRequest, Provider: aigv1b1.GuardrailProvider{ Type: aigv1b1.GuardrailProviderTypeAzureContentSafety, + Action: test.action, FailureMode: test.failureMode, AzureContentSafety: &aigv1b1.AzureContentSafetyGuardrailProvider{ Endpoint: "https://content-safety.example.com", @@ -319,7 +352,7 @@ func TestGuardrailPolicyToRuntimeIntegration(t *testing.T) { }, guardrails.NewEvaluator) require.NoError(t, err) require.Len(t, runtimeConfig.Guardrails, 1) - blocked, err := runtimeConfig.Guardrails[0].Evaluator.Evaluate(t.Context(), []byte("user@example.com"), filterapi.GuardrailPhaseRequest) + evaluation, err := runtimeConfig.Guardrails[0].Evaluator.Evaluate(t.Context(), []byte("user@example.com"), filterapi.GuardrailPhaseRequest) require.NoError(t, err) - require.True(t, blocked) + require.True(t, evaluation.Matched) } diff --git a/internal/extproc/guardrail_content.go b/internal/extproc/guardrail_content.go new file mode 100644 index 0000000000..1e5d21067a --- /dev/null +++ b/internal/extproc/guardrail_content.go @@ -0,0 +1,90 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package extproc + +import ( + "fmt" + "sort" + "strconv" + "strings" + + "github.com/tidwall/sjson" + + "github.com/envoyproxy/ai-gateway/internal/json" +) + +type guardrailContent struct { + text []byte + path string +} + +var guardrailTextFields = map[string]struct{}{ + "content": {}, "input": {}, "instructions": {}, "output_text": {}, + "prompt": {}, "refusal": {}, "system": {}, "text": {}, +} + +var guardrailContainerFields = map[string]struct{}{ + "choices": {}, "contents": {}, "delta": {}, "message": {}, "messages": {}, + "output": {}, "parts": {}, +} + +func extractGuardrailContents(body []byte) ([]guardrailContent, error) { + var value any + if err := json.Unmarshal(body, &value); err != nil { + return nil, fmt.Errorf("cannot extract guardrail text from JSON body: %w", err) + } + var contents []guardrailContent + walkGuardrailContent(value, "", false, &contents) + return contents, nil +} + +func walkGuardrailContent(value any, path string, capture bool, contents *[]guardrailContent) { + switch typed := value.(type) { + case string: + if capture && typed != "" { + *contents = append(*contents, guardrailContent{text: []byte(typed), path: path}) + } + case []any: + for i := range typed { + walkGuardrailContent(typed[i], appendGuardrailPath(path, strconv.Itoa(i)), capture, contents) + } + case map[string]any: + keys := make([]string, 0, len(typed)) + for key := range typed { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + childPath := appendGuardrailPath(path, escapeGuardrailPathKey(key)) + if _, ok := guardrailTextFields[key]; ok { + walkGuardrailContent(typed[key], childPath, true, contents) + continue + } + if _, ok := guardrailContainerFields[key]; ok { + walkGuardrailContent(typed[key], childPath, false, contents) + } + } + } +} + +func appendGuardrailPath(path, component string) string { + if path == "" { + return component + } + return path + "." + component +} + +func escapeGuardrailPathKey(key string) string { + replacer := strings.NewReplacer("\\", "\\\\", ".", "\\.", "*", "\\*", "?", "\\?", "#", "\\#") + return replacer.Replace(key) +} + +func replaceGuardrailContent(body []byte, content guardrailContent, replacement []byte) ([]byte, error) { + if content.path == "" { + return replacement, nil + } + return sjson.SetBytesOptions(body, content.path, string(replacement), &sjson.Options{ReplaceInPlace: true}) +} diff --git a/internal/extproc/guardrail_content_test.go b/internal/extproc/guardrail_content_test.go new file mode 100644 index 0000000000..e18428793d --- /dev/null +++ b/internal/extproc/guardrail_content_test.go @@ -0,0 +1,27 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package extproc + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestExtractGuardrailContents(t *testing.T) { + contents, err := extractGuardrailContents([]byte(`{ + "model":"must-not-be-scanned", + "messages":[ + {"role":"system","content":"system text"}, + {"role":"user","content":[{"type":"text","text":"user text"},{"type":"image_url","image_url":{"url":"https://example.com"}}]} + ] + }`)) + require.NoError(t, err) + require.Equal(t, []guardrailContent{ + {text: []byte("system text"), path: "messages.0.content"}, + {text: []byte("user text"), path: "messages.1.content.0.text"}, + }, contents) +} diff --git a/internal/extproc/guardrails.go b/internal/extproc/guardrails.go index 3e8ef0abed..eefd797e51 100644 --- a/internal/extproc/guardrails.go +++ b/internal/extproc/guardrails.go @@ -20,6 +20,14 @@ type guardrailViolation struct { Message string } +type guardrailOutcome struct { + Violation *guardrailViolation + Body []byte + Masked bool + Monitored bool + RuleName string +} + type guardrailFailOpenError struct { errors []error } @@ -42,45 +50,132 @@ func guardrailsConfiguredForPhase(guardrails []filterapi.RuntimeGuardrail, phase return false } -func evaluateGuardrailsForPhase(ctx context.Context, guardrails []filterapi.RuntimeGuardrail, phase filterapi.GuardrailPhase, body []byte, backendName string, includeGlobal bool) (*guardrailViolation, error) { +func guardrailsRequireBufferedResponse(guardrails []filterapi.RuntimeGuardrail, backendName string) bool { + for i := range guardrails { + guardrail := &guardrails[i] + if guardrail.Phase == filterapi.GuardrailPhaseResponse && + guardrailAppliesToBackend(guardrail, backendName, true) { + return true + } + } + return false +} + +func evaluateGuardrailsForPhase(ctx context.Context, guardrails []filterapi.RuntimeGuardrail, phase filterapi.GuardrailPhase, body []byte, backendName string, includeGlobal bool) (guardrailOutcome, error) { + outcome := guardrailOutcome{Body: body} var failOpenErrors []error for i := range guardrails { g := &guardrails[i] if g.Phase != phase || !guardrailAppliesToBackend(g, backendName, includeGlobal) { continue } - var blocked bool - if g.Provider.Type == filterapi.GuardrailProviderTypeRegex { + maxPayloadBytes := g.MaxPayloadBytes + if maxPayloadBytes <= 0 { + maxPayloadBytes = 10 * 1024 * 1024 + } + if int64(len(outcome.Body)) > maxPayloadBytes { + err := fmt.Errorf("guardrail %q payload is %d bytes, exceeding the %d-byte limit", g.Name, len(outcome.Body), maxPayloadBytes) + if g.Provider.FailureMode == filterapi.GuardrailFailureModeFailOpen || guardrailAction(g.Provider.Action) == filterapi.GuardrailActionMonitor { + failOpenErrors = append(failOpenErrors, err) + continue + } + return outcome, err + } + + action := guardrailAction(g.Provider.Action) + if g.Provider.Type == filterapi.GuardrailProviderTypeRegex && action != filterapi.GuardrailActionMask { if g.Matcher == nil { - return nil, fmt.Errorf("guardrail %q uses regex provider without a compiled matcher", g.Name) + return outcome, fmt.Errorf("guardrail %q uses regex provider without a compiled matcher", g.Name) } - blocked = g.Matcher.Match(body) - } else { - if g.Evaluator == nil { - return nil, fmt.Errorf("guardrail %q uses provider %q without an evaluator", g.Name, g.Provider.Type) + if !g.Matcher.Match(outcome.Body) { + continue } - var err error - blocked, err = g.Evaluator.Evaluate(ctx, body, phase) - if err != nil { - if g.Provider.FailureMode == filterapi.GuardrailFailureModeFailOpen { - failOpenErrors = append(failOpenErrors, fmt.Errorf("guardrail %q evaluation failed: %w", g.Name, err)) - continue - } - return nil, fmt.Errorf("guardrail %q evaluation failed: %w", g.Name, err) + if action == filterapi.GuardrailActionMonitor { + outcome.Monitored = true + outcome.RuleName = g.Name + continue } + outcome.Violation = newGuardrailViolation(g) + return outcome, nil } - if blocked { - msg := g.Provider.Message - if msg == "" { - msg = fmt.Sprintf("request blocked by guardrail %q", g.Name) + + contents, err := extractGuardrailContents(outcome.Body) + if err != nil { + if g.Provider.FailureMode == filterapi.GuardrailFailureModeFailOpen || action == filterapi.GuardrailActionMonitor { + failOpenErrors = append(failOpenErrors, fmt.Errorf("guardrail %q text extraction failed: %w", g.Name, err)) + continue + } + return outcome, fmt.Errorf("guardrail %q text extraction failed: %w", g.Name, err) + } + for _, content := range contents { + var evaluation filterapi.GuardrailEvaluationResult + if g.Provider.Type == filterapi.GuardrailProviderTypeRegex { + if g.Matcher == nil { + return outcome, fmt.Errorf("guardrail %q uses regex provider without a compiled matcher", g.Name) + } + evaluation.Matched = g.Matcher.Match(content.text) + if evaluation.Matched { + replacement := g.Provider.MaskReplacement + if replacement == "" { + replacement = "[REDACTED]" + } + evaluation.Replacement = g.Matcher.ReplaceAll(content.text, []byte(replacement)) + } + } else { + if g.Evaluator == nil { + return outcome, fmt.Errorf("guardrail %q uses provider %q without an evaluator", g.Name, g.Provider.Type) + } + evaluation, err = g.Evaluator.Evaluate(ctx, content.text, phase) + if err != nil { + if g.Provider.FailureMode == filterapi.GuardrailFailureModeFailOpen || action == filterapi.GuardrailActionMonitor { + failOpenErrors = append(failOpenErrors, fmt.Errorf("guardrail %q evaluation failed: %w", g.Name, err)) + continue + } + return outcome, fmt.Errorf("guardrail %q evaluation failed: %w", g.Name, err) + } + } + if !evaluation.Matched { + continue + } + switch action { + case filterapi.GuardrailActionMonitor: + outcome.Monitored = true + outcome.RuleName = g.Name + case filterapi.GuardrailActionMask: + if len(evaluation.Replacement) == 0 { + return outcome, fmt.Errorf("guardrail %q matched but provider %q returned no masked content", g.Name, g.Provider.Type) + } + outcome.Body, err = replaceGuardrailContent(outcome.Body, content, evaluation.Replacement) + if err != nil { + return outcome, fmt.Errorf("guardrail %q failed to mask content: %w", g.Name, err) + } + outcome.Masked = true + outcome.RuleName = g.Name + default: + outcome.Violation = newGuardrailViolation(g) + return outcome, nil } - return &guardrailViolation{Name: g.Name, Message: msg}, nil } } if len(failOpenErrors) > 0 { - return nil, &guardrailFailOpenError{errors: failOpenErrors} + return outcome, &guardrailFailOpenError{errors: failOpenErrors} + } + return outcome, nil +} + +func guardrailAction(action filterapi.GuardrailAction) filterapi.GuardrailAction { + if action == "" { + return filterapi.GuardrailActionBlock + } + return action +} + +func newGuardrailViolation(guardrail *filterapi.RuntimeGuardrail) *guardrailViolation { + message := guardrail.Provider.Message + if message == "" { + message = fmt.Sprintf("request blocked by guardrail %q", guardrail.Name) } - return nil, nil + return &guardrailViolation{Name: guardrail.Name, Message: message} } func guardrailAppliesToBackend(guardrail *filterapi.RuntimeGuardrail, backendName string, includeGlobal bool) bool { @@ -90,14 +185,14 @@ func guardrailAppliesToBackend(guardrail *filterapi.RuntimeGuardrail, backendNam return backendName != "" && slices.Contains(guardrail.Backends, backendName) } -func evaluateRequestGuardrails(ctx context.Context, guardrails []filterapi.RuntimeGuardrail, body []byte) (*guardrailViolation, error) { +func evaluateRequestGuardrails(ctx context.Context, guardrails []filterapi.RuntimeGuardrail, body []byte) (guardrailOutcome, error) { return evaluateGuardrailsForPhase(ctx, guardrails, filterapi.GuardrailPhaseRequest, body, "", true) } -func evaluateBackendRequestGuardrails(ctx context.Context, guardrails []filterapi.RuntimeGuardrail, body []byte, backendName string) (*guardrailViolation, error) { +func evaluateBackendRequestGuardrails(ctx context.Context, guardrails []filterapi.RuntimeGuardrail, body []byte, backendName string) (guardrailOutcome, error) { return evaluateGuardrailsForPhase(ctx, guardrails, filterapi.GuardrailPhaseRequest, body, backendName, false) } -func evaluateResponseGuardrails(ctx context.Context, guardrails []filterapi.RuntimeGuardrail, body []byte, backendName string) (*guardrailViolation, error) { +func evaluateResponseGuardrails(ctx context.Context, guardrails []filterapi.RuntimeGuardrail, body []byte, backendName string) (guardrailOutcome, error) { return evaluateGuardrailsForPhase(ctx, guardrails, filterapi.GuardrailPhaseResponse, body, backendName, true) } diff --git a/internal/extproc/guardrails_test.go b/internal/extproc/guardrails_test.go index 87cf6fea4e..8b5a8498b1 100644 --- a/internal/extproc/guardrails_test.go +++ b/internal/extproc/guardrails_test.go @@ -29,8 +29,14 @@ type recordingGuardrailMetrics struct { type failingGuardrailEvaluator struct{} -func (*failingGuardrailEvaluator) Evaluate(context.Context, []byte, filterapi.GuardrailPhase) (bool, error) { - return false, errors.New("provider unavailable") +func (*failingGuardrailEvaluator) Evaluate(context.Context, []byte, filterapi.GuardrailPhase) (filterapi.GuardrailEvaluationResult, error) { + return filterapi.GuardrailEvaluationResult{}, errors.New("provider unavailable") +} + +type maskingGuardrailEvaluator struct{} + +func (*maskingGuardrailEvaluator) Evaluate(_ context.Context, body []byte, _ filterapi.GuardrailPhase) (filterapi.GuardrailEvaluationResult, error) { + return filterapi.GuardrailEvaluationResult{Matched: true, Replacement: []byte("masked:" + string(body))}, nil } func (m *recordingGuardrailMetrics) RecordEvaluation(_ context.Context, phase string, result metrics.GuardrailResult) { @@ -51,11 +57,11 @@ func TestEvaluateGuardrailsForPhase(t *testing.T) { Matcher: regexp.MustCompile(`\bSSN\b`), }} - violation, err := evaluateRequestGuardrails(t.Context(), guardrails, []byte("customer SSN is present")) + outcome, err := evaluateRequestGuardrails(t.Context(), guardrails, []byte("customer SSN is present")) require.NoError(t, err) - require.NotNil(t, violation) - require.Equal(t, "deny-pii", violation.Name) - require.Equal(t, "PII detected in request", violation.Message) + require.NotNil(t, outcome.Violation) + require.Equal(t, "deny-pii", outcome.Violation.Name) + require.Equal(t, "PII detected in request", outcome.Violation.Message) }) t.Run("response guardrail ignores different phase", func(t *testing.T) { @@ -68,9 +74,9 @@ func TestEvaluateGuardrailsForPhase(t *testing.T) { Matcher: regexp.MustCompile(`forbidden`), }} - violation, err := evaluateRequestGuardrails(t.Context(), guardrails, []byte("forbidden")) + outcome, err := evaluateRequestGuardrails(t.Context(), guardrails, []byte("forbidden")) require.NoError(t, err) - require.Nil(t, violation) + require.Nil(t, outcome.Violation) }) t.Run("regex guardrail without compiled matcher returns error on matching phase", func(t *testing.T) { @@ -82,9 +88,9 @@ func TestEvaluateGuardrailsForPhase(t *testing.T) { }, }} - violation, err := evaluateRequestGuardrails(t.Context(), guardrails, []byte("forbidden")) + outcome, err := evaluateRequestGuardrails(t.Context(), guardrails, []byte("forbidden")) require.Error(t, err) - require.Nil(t, violation) + require.Nil(t, outcome.Violation) require.Contains(t, err.Error(), "uses regex provider without a compiled matcher") }) } @@ -118,6 +124,34 @@ func TestRequestGuardrailBlockRecordsMetric(t *testing.T) { require.Equal(t, metrics.GuardrailResultBlocked, recorder.result) } +func TestRequestGuardrailMaskMutatesBody(t *testing.T) { + recorder := &recordingGuardrailMetrics{} + config := &filterapi.RuntimeConfig{Guardrails: []filterapi.RuntimeGuardrail{{ + Name: "mask-email", Phase: filterapi.GuardrailPhaseRequest, + Provider: filterapi.GuardrailProvider{ + Type: filterapi.GuardrailProviderTypeRegex, Action: filterapi.GuardrailActionMask, + MaskReplacement: "[EMAIL]", + }, + Matcher: regexp.MustCompile(`alice@example\.com`), + }}} + factory := NewFactory(nil, recorder, tracingapi.NoopChatCompletionTracer{}, endpointspec.ChatCompletionsEndpointSpec{}) + processor, err := factory(config, map[string]string{ + "content-type": "application/json", + ":path": "/v1/chat/completions", + }, slog.Default(), false, false) + require.NoError(t, err) + + response, err := processor.ProcessRequestBody(t.Context(), &extprocv3.HttpBody{ + Body: []byte(`{"model":"test","messages":[{"role":"user","content":"email alice@example.com"}]}`), + }) + require.NoError(t, err) + require.Nil(t, response.GetImmediateResponse()) + require.Nil(t, response.GetRequestBody().Response.BodyMutation) + processorImpl := processor.(*chatCompletionProcessorRouterFilter) + require.JSONEq(t, `{"model":"test","messages":[{"role":"user","content":"email [EMAIL]"}]}`, string(processorImpl.originalRequestBodyRaw)) + require.Equal(t, metrics.GuardrailResultMasked, recorder.result) +} + func TestRecordGuardrailEvaluationIgnoresUnconfiguredPhase(t *testing.T) { recorder := &recordingGuardrailMetrics{} processor := &chatCompletionProcessorRouterFilter{ @@ -139,13 +173,13 @@ func TestBackendScopedGuardrail(t *testing.T) { Matcher: regexp.MustCompile("blocked"), }} - violation, err := evaluateBackendRequestGuardrails(t.Context(), guardrails, []byte("blocked"), "other-backend") + outcome, err := evaluateBackendRequestGuardrails(t.Context(), guardrails, []byte("blocked"), "other-backend") require.NoError(t, err) - require.Nil(t, violation) + require.Nil(t, outcome.Violation) - violation, err = evaluateBackendRequestGuardrails(t.Context(), guardrails, []byte("blocked"), "selected-backend") + outcome, err = evaluateBackendRequestGuardrails(t.Context(), guardrails, []byte("blocked"), "selected-backend") require.NoError(t, err) - require.NotNil(t, violation) + require.NotNil(t, outcome.Violation) } func TestGuardrailFailureModes(t *testing.T) { @@ -160,8 +194,51 @@ func TestGuardrailFailureModes(t *testing.T) { require.False(t, isGuardrailFailOpenError(err)) guardrail.Provider.FailureMode = filterapi.GuardrailFailureModeFailOpen - violation, err := evaluateRequestGuardrails(t.Context(), []filterapi.RuntimeGuardrail{guardrail}, []byte("payload")) - require.Nil(t, violation) + outcome, err := evaluateRequestGuardrails(t.Context(), []filterapi.RuntimeGuardrail{guardrail}, []byte("payload")) + require.Nil(t, outcome.Violation) require.Error(t, err) require.True(t, isGuardrailFailOpenError(err)) } + +func TestGuardrailMonitorAndMask(t *testing.T) { + body := []byte(`{"model":"safe-model","messages":[{"role":"user","content":"secret"}]}`) + + t.Run("monitor detects without changing body", func(t *testing.T) { + outcome, err := evaluateRequestGuardrails(t.Context(), []filterapi.RuntimeGuardrail{{ + Name: "monitor", Phase: filterapi.GuardrailPhaseRequest, + Provider: filterapi.GuardrailProvider{Type: filterapi.GuardrailProviderTypeRegex, Action: filterapi.GuardrailActionMonitor}, + Matcher: regexp.MustCompile("secret"), + }}, body) + require.NoError(t, err) + require.True(t, outcome.Monitored) + require.False(t, outcome.Masked) + require.Nil(t, outcome.Violation) + require.Equal(t, body, outcome.Body) + }) + + t.Run("mask replaces extracted content only", func(t *testing.T) { + outcome, err := evaluateRequestGuardrails(t.Context(), []filterapi.RuntimeGuardrail{{ + Name: "mask", Phase: filterapi.GuardrailPhaseRequest, + Provider: filterapi.GuardrailProvider{Type: filterapi.GuardrailProviderTypePresidio, Action: filterapi.GuardrailActionMask}, + Evaluator: &maskingGuardrailEvaluator{}, + }}, body) + require.NoError(t, err) + require.True(t, outcome.Masked) + require.JSONEq(t, `{"model":"safe-model","messages":[{"role":"user","content":"masked:secret"}]}`, string(outcome.Body)) + require.NotContains(t, string(outcome.Body), "masked:safe-model") + }) +} + +func TestGuardrailPayloadLimit(t *testing.T) { + guardrail := filterapi.RuntimeGuardrail{ + Name: "small", Phase: filterapi.GuardrailPhaseRequest, MaxPayloadBytes: 8, + Provider: filterapi.GuardrailProvider{Type: filterapi.GuardrailProviderTypeRegex}, + Matcher: regexp.MustCompile("secret"), + } + _, err := evaluateRequestGuardrails(t.Context(), []filterapi.RuntimeGuardrail{guardrail}, []byte(`{"content":"secret"}`)) + require.ErrorContains(t, err, "exceeding the 8-byte limit") + + guardrail.Provider.FailureMode = filterapi.GuardrailFailureModeFailOpen + _, err = evaluateRequestGuardrails(t.Context(), []filterapi.RuntimeGuardrail{guardrail}, []byte(`{"content":"secret"}`)) + require.True(t, isGuardrailFailOpenError(err)) +} diff --git a/internal/extproc/processor_impl.go b/internal/extproc/processor_impl.go index be6f00f7c1..1a9a316142 100644 --- a/internal/extproc/processor_impl.go +++ b/internal/extproc/processor_impl.go @@ -289,11 +289,11 @@ func (r *routerProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) ProcessRequest // Only log parsed request body when redaction is enabled if r.debugLogEnabled && r.enableRedaction { - if redactedBody, err := r.eh.RedactSensitiveInfoFromRequest(body); err != nil { - logger.Warn("failed to redact sensitive info from request, ignoring and continuing", slog.Any("error", err)) + if redactedBody, redactionErr := r.eh.RedactSensitiveInfoFromRequest(body); redactionErr != nil { + logger.Warn("failed to redact sensitive info from request, ignoring and continuing", slog.Any("error", redactionErr)) } else { - if jsonBody, err := json.Marshal(redactedBody); err != nil { - logger.Error("failed to marshal redacted request for logging, ignoring and continuing", slog.Any("error", err)) + if jsonBody, marshalErr := json.Marshal(redactedBody); marshalErr != nil { + logger.Error("failed to marshal redacted request for logging, ignoring and continuing", slog.Any("error", marshalErr)) } else { logger.Debug("request body processing", slog.Any("request", string(jsonBody))) } @@ -346,7 +346,7 @@ func (r *routerProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) ProcessRequest body, rawBody.Body, ) - violation, guardrailErr := evaluateRequestGuardrails(ctx, r.config.Guardrails, rawBody.Body) + outcome, guardrailErr := evaluateRequestGuardrails(ctx, r.config.Guardrails, rawBody.Body) if guardrailErr != nil { r.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultError, "", true) r.recordGuardrailTrace("", filterapi.GuardrailPhaseRequest, metrics.GuardrailResultError) @@ -354,19 +354,31 @@ func (r *routerProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) ProcessRequest return nil, fmt.Errorf("failed to evaluate request guardrails: %w", guardrailErr) } r.logger.Warn("request guardrail provider failed open", slog.String("error", guardrailErr.Error())) - } else if violation != nil { + } else if outcome.Violation != nil { r.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultBlocked, "", true) - r.recordGuardrailTrace(violation.Name, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultBlocked) + r.recordGuardrailTrace(outcome.Violation.Name, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultBlocked) r.logger.Warn("request blocked by guardrail", - slog.String("guardrail.name", violation.Name), + slog.String("guardrail.name", outcome.Violation.Name), slog.String("guardrail.phase", string(filterapi.GuardrailPhaseRequest))) - response := createUserFacingErrorResponse(http.StatusForbidden, "GuardrailViolation", violation.Message) + response := createUserFacingErrorResponse(http.StatusForbidden, "GuardrailViolation", outcome.Violation.Message) if r.span != nil { r.span.EndSpanOnError(http.StatusForbidden, response.GetImmediateResponse().GetBody()) } return response, nil } - if guardrailErr == nil { + switch { + case outcome.Masked: + if err = r.updateParsedRequestAfterGuardrailMask(outcome.Body, costConfigured); err != nil { + return nil, err + } + r.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultMasked, "", true) + r.recordGuardrailTrace(outcome.RuleName, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultMasked) + r.logger.Info("request masked by guardrail", slog.String("guardrail.name", outcome.RuleName)) + case outcome.Monitored: + r.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultMonitored, "", true) + r.recordGuardrailTrace(outcome.RuleName, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultMonitored) + r.logger.Info("request detected by monitor guardrail", slog.String("guardrail.name", outcome.RuleName)) + case guardrailErr == nil: r.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultAllowed, "", true) r.recordGuardrailTrace("", filterapi.GuardrailPhaseRequest, metrics.GuardrailResultAllowed) } @@ -383,6 +395,22 @@ func (r *routerProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) ProcessRequest }, nil } +func (r *routerProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) updateParsedRequestAfterGuardrailMask(maskedBody []byte, costConfigured bool) error { + originalModel, parsedBody, stream, mutatedBody, err := r.eh.ParseBody(maskedBody, costConfigured) + if err != nil { + return fmt.Errorf("failed to parse guardrail-masked request body: %w", err) + } + r.originalModel = originalModel + r.originalRequestBody = parsedBody + r.stream = stream + r.originalRequestBodyRaw = maskedBody + if mutatedBody != nil { + r.originalRequestBodyRaw = mutatedBody + } + r.forceBodyMutation = true + return nil +} + func (u *upstreamProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) onRetry() bool { return u.parent.upstreamFilterCount > 1 } @@ -411,7 +439,7 @@ func (u *upstreamProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) ProcessReque if u.parent.config != nil { configuredGuardrails = u.parent.config.Guardrails } - violation, guardrailErr := evaluateBackendRequestGuardrails(ctx, configuredGuardrails, u.parent.originalRequestBodyRaw, u.backendName) + outcome, guardrailErr := evaluateBackendRequestGuardrails(ctx, configuredGuardrails, u.parent.originalRequestBodyRaw, u.backendName) if guardrailErr != nil { u.parent.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultError, u.backendName, false) u.parent.recordGuardrailTrace("", filterapi.GuardrailPhaseRequest, metrics.GuardrailResultError) @@ -419,13 +447,26 @@ func (u *upstreamProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) ProcessReque return nil, fmt.Errorf("failed to evaluate backend request guardrails: %w", guardrailErr) } u.logger.Warn("request guardrail provider failed open", slog.String("error", guardrailErr.Error())) - } else if violation != nil { + } else if outcome.Violation != nil { u.parent.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultBlocked, u.backendName, false) - u.parent.recordGuardrailTrace(violation.Name, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultBlocked) - u.logger.Warn("request blocked by guardrail", slog.String("guardrail.name", violation.Name), slog.String("guardrail.phase", string(filterapi.GuardrailPhaseRequest))) - return u.respondLocally(ctx, http.StatusForbidden, "GuardrailViolation", violation.Message), nil - } - if guardrailErr == nil { + u.parent.recordGuardrailTrace(outcome.Violation.Name, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultBlocked) + u.logger.Warn("request blocked by guardrail", slog.String("guardrail.name", outcome.Violation.Name), slog.String("guardrail.phase", string(filterapi.GuardrailPhaseRequest))) + return u.respondLocally(ctx, http.StatusForbidden, "GuardrailViolation", outcome.Violation.Message), nil + } + switch { + case outcome.Masked: + costConfigured := len(u.parent.config.RequestCosts) > 0 || len(u.parent.config.GlobalRequestCosts) > 0 + if err = u.parent.updateParsedRequestAfterGuardrailMask(outcome.Body, costConfigured); err != nil { + return nil, err + } + u.parent.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultMasked, u.backendName, false) + u.parent.recordGuardrailTrace(outcome.RuleName, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultMasked) + u.logger.Info("request masked by guardrail", slog.String("guardrail.name", outcome.RuleName)) + case outcome.Monitored: + u.parent.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultMonitored, u.backendName, false) + u.parent.recordGuardrailTrace(outcome.RuleName, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultMonitored) + u.logger.Info("request detected by monitor guardrail", slog.String("guardrail.name", outcome.RuleName)) + case guardrailErr == nil: u.parent.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultAllowed, u.backendName, false) u.parent.recordGuardrailTrace("", filterapi.GuardrailPhaseRequest, metrics.GuardrailResultAllowed) } @@ -580,8 +621,7 @@ func (u *upstreamProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) ProcessRespo return nil, fmt.Errorf("failed to transform response headers: %w", err) } var mode *extprocv3http.ProcessingMode - hasResponseGuardrails := u.parent.config != nil && guardrailsConfiguredForPhase( - u.parent.config.Guardrails, filterapi.GuardrailPhaseResponse, u.backendName, true) + hasResponseGuardrails := u.parent.config != nil && guardrailsRequireBufferedResponse(u.parent.config.Guardrails, u.backendName) if u.parent.stream && u.responseHeaders[":status"] == "200" && !hasResponseGuardrails { // We only stream the response if the status code is 200 and the response is a stream. mode = &extprocv3http.ProcessingMode{ResponseBodyMode: extprocv3http.ProcessingMode_STREAMED} @@ -671,15 +711,16 @@ func (u *upstreamProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) ProcessRespo }, nil } - newHeaders, newBody, tokenUsage, responseModel, err := u.translator.ResponseBody(u.responseHeaders, decodingResult.reader, body.EndOfStream, u.parent.span) + decodedBody, err := io.ReadAll(decodingResult.reader) + if err != nil { + return nil, fmt.Errorf("failed to read decoded response body: %w", err) + } + newHeaders, newBody, tokenUsage, responseModel, err := u.translator.ResponseBody(u.responseHeaders, bytes.NewReader(decodedBody), body.EndOfStream, u.parent.span) if err != nil { return nil, fmt.Errorf("failed to transform response: %w", err) } headerMutation, bodyMutation := mutationsFromTranslationResult(newHeaders, newBody) - // Remove content-encoding header if original body encoded but was mutated in the processor. - headerMutation = removeContentEncodingIfNeeded(headerMutation, bodyMutation, decodingResult.isEncoded) - resp := &extprocv3.ProcessingResponse{ Response: &extprocv3.ProcessingResponse_ResponseBody{ ResponseBody: &extprocv3.BodyResponse{ @@ -694,7 +735,11 @@ func (u *upstreamProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) ProcessRespo // Translator reports the latest cumulative token usage which we use to override existing costs. u.costs.Override(tokenUsage) - violation, guardrailErr := evaluateResponseGuardrails(ctx, u.parent.config.Guardrails, body.Body, u.backendName) + guardrailBody := decodedBody + if len(newBody) > 0 { + guardrailBody = newBody + } + outcome, guardrailErr := evaluateResponseGuardrails(ctx, u.parent.config.Guardrails, guardrailBody, u.backendName) if guardrailErr != nil { u.parent.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseResponse, metrics.GuardrailResultError, u.backendName, true) u.parent.recordGuardrailTrace("", filterapi.GuardrailPhaseResponse, metrics.GuardrailResultError) @@ -702,19 +747,35 @@ func (u *upstreamProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) ProcessRespo return nil, fmt.Errorf("failed to evaluate response guardrails: %w", guardrailErr) } u.logger.Warn("response guardrail provider failed open", slog.String("error", guardrailErr.Error())) - } else if violation != nil { + } else if outcome.Violation != nil { u.parent.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseResponse, metrics.GuardrailResultBlocked, u.backendName, true) - u.parent.recordGuardrailTrace(violation.Name, filterapi.GuardrailPhaseResponse, metrics.GuardrailResultBlocked) + u.parent.recordGuardrailTrace(outcome.Violation.Name, filterapi.GuardrailPhaseResponse, metrics.GuardrailResultBlocked) u.logger.Warn("response blocked by guardrail", - slog.String("guardrail.name", violation.Name), + slog.String("guardrail.name", outcome.Violation.Name), slog.String("guardrail.phase", string(filterapi.GuardrailPhaseResponse))) - return u.respondLocally(ctx, http.StatusForbidden, "GuardrailViolation", violation.Message), nil - } - if guardrailErr == nil { + return u.respondLocally(ctx, http.StatusForbidden, "GuardrailViolation", outcome.Violation.Message), nil + } + switch { + case outcome.Masked: + bodyMutation = &extprocv3.BodyMutation{Mutation: &extprocv3.BodyMutation_Body{Body: outcome.Body}} + resp.GetResponseBody().Response.BodyMutation = bodyMutation + setHeader(headerMutation, "content-length", strconv.Itoa(len(outcome.Body))) + u.parent.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseResponse, metrics.GuardrailResultMasked, u.backendName, true) + u.parent.recordGuardrailTrace(outcome.RuleName, filterapi.GuardrailPhaseResponse, metrics.GuardrailResultMasked) + u.logger.Info("response masked by guardrail", slog.String("guardrail.name", outcome.RuleName)) + case outcome.Monitored: + u.parent.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseResponse, metrics.GuardrailResultMonitored, u.backendName, true) + u.parent.recordGuardrailTrace(outcome.RuleName, filterapi.GuardrailPhaseResponse, metrics.GuardrailResultMonitored) + u.logger.Info("response detected by monitor guardrail", slog.String("guardrail.name", outcome.RuleName)) + case guardrailErr == nil: u.parent.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseResponse, metrics.GuardrailResultAllowed, u.backendName, true) u.parent.recordGuardrailTrace("", filterapi.GuardrailPhaseResponse, metrics.GuardrailResultAllowed) } + // Remove content-encoding when translation or guardrail masking produces an uncompressed body. + headerMutation = removeContentEncodingIfNeeded(headerMutation, bodyMutation, decodingResult.isEncoded) + resp.GetResponseBody().Response.HeaderMutation = headerMutation + // Set the response model for metrics u.metrics.SetResponseModel(responseModel) diff --git a/internal/extproc/processor_impl_test.go b/internal/extproc/processor_impl_test.go index 048f09ef6b..3f574474ec 100644 --- a/internal/extproc/processor_impl_test.go +++ b/internal/extproc/processor_impl_test.go @@ -342,6 +342,24 @@ func Test_chatCompletionProcessorUpstreamFilter_ProcessResponseHeaders(t *testin require.NoError(t, err) require.Nil(t, res.ModeOverride) }) + t.Run("streaming with monitor-only response guardrail remains buffered", func(t *testing.T) { + inHeaders := &corev3.HeaderMap{Headers: []*corev3.HeaderValue{{Key: ":status", Value: "200"}}} + mt := &mockTranslator{t: t, expHeaders: map[string]string{":status": "200"}} + p := &chatCompletionProcessorUpstreamFilter{ + translator: mt, + metrics: &mockMetrics{}, + parent: &chatCompletionProcessorRouterFilter{ + stream: true, + config: &filterapi.RuntimeConfig{Guardrails: []filterapi.RuntimeGuardrail{{ + Phase: filterapi.GuardrailPhaseResponse, + Provider: filterapi.GuardrailProvider{Action: filterapi.GuardrailActionMonitor}, + }}}, + }, + } + res, err := p.ProcessResponseHeaders(t.Context(), inHeaders) + require.NoError(t, err) + require.Nil(t, res.ModeOverride) + }) t.Run("error/streaming", func(t *testing.T) { inHeaders := &corev3.HeaderMap{ Headers: []*corev3.HeaderValue{{Key: ":status", Value: "500"}, {Key: "dog", RawValue: []byte("cat")}}, diff --git a/internal/filterapi/filterconfig.go b/internal/filterapi/filterconfig.go index 94acd7b2a0..b08217d39e 100644 --- a/internal/filterapi/filterconfig.go +++ b/internal/filterapi/filterconfig.go @@ -63,6 +63,8 @@ type Guardrail struct { Provider GuardrailProvider `json:"provider"` // Backends scopes this guardrail to generated backend names. Empty means global. Backends []string `json:"backends,omitempty"` + // MaxPayloadBytes bounds evaluation and response buffering for this rule. + MaxPayloadBytes int64 `json:"maxPayloadBytes,omitempty"` } // GuardrailPhase determines when the rule is evaluated. @@ -78,6 +80,7 @@ type GuardrailProvider struct { Type GuardrailProviderType `json:"type"` Pattern string `json:"pattern,omitempty"` Action GuardrailAction `json:"action,omitempty"` + MaskReplacement string `json:"maskReplacement,omitempty"` Message string `json:"message,omitempty"` Presidio *PresidioGuardrailProvider `json:"presidio,omitempty"` Bedrock *BedrockGuardrailProvider `json:"bedrock,omitempty"` @@ -133,7 +136,9 @@ const ( type GuardrailAction string const ( - GuardrailActionBlock GuardrailAction = "Block" + GuardrailActionBlock GuardrailAction = "Block" + GuardrailActionMonitor GuardrailAction = "Monitor" + GuardrailActionMask GuardrailAction = "Mask" ) // Model corresponds to the OpenAI model object in the OpenAI-compatible APIs diff --git a/internal/filterapi/runtime.go b/internal/filterapi/runtime.go index c0c0b118c1..9f60e208db 100644 --- a/internal/filterapi/runtime.go +++ b/internal/filterapi/runtime.go @@ -28,7 +28,13 @@ type NewBackendAuthHandlerFunc func(ctx context.Context, auth *BackendAuth) (Bac // GuardrailEvaluator evaluates a request or response payload using a configured provider. type GuardrailEvaluator interface { - Evaluate(ctx context.Context, body []byte, phase GuardrailPhase) (blocked bool, err error) + Evaluate(ctx context.Context, body []byte, phase GuardrailPhase) (GuardrailEvaluationResult, error) +} + +// GuardrailEvaluationResult is the provider-neutral outcome for one text fragment. +type GuardrailEvaluationResult struct { + Matched bool + Replacement []byte } // NewGuardrailEvaluatorFunc creates an evaluator for an external guardrail provider. @@ -82,12 +88,13 @@ type RuntimeRequestCost struct { // RuntimeGuardrail is a compiled guardrail rule for runtime evaluation. type RuntimeGuardrail struct { - Name string - Phase GuardrailPhase - Provider GuardrailProvider - Backends []string - Matcher *regexp.Regexp - Evaluator GuardrailEvaluator + Name string + Phase GuardrailPhase + Provider GuardrailProvider + Backends []string + MaxPayloadBytes int64 + Matcher *regexp.Regexp + Evaluator GuardrailEvaluator } // NewRuntimeConfig creates a new runtime filter configuration from the given filterapi.Config and a function to create backend auth handlers. @@ -160,11 +167,12 @@ func NewRuntimeConfig(ctx context.Context, config *Config, fn NewBackendAuthHand return nil, fmt.Errorf("guardrail %q has an invalid regex pattern: %w", g.Name, err) } guardrails = append(guardrails, RuntimeGuardrail{ - Name: g.Name, - Phase: g.Phase, - Provider: g.Provider, - Backends: g.Backends, - Matcher: re, + Name: g.Name, + Phase: g.Phase, + Provider: g.Provider, + Backends: g.Backends, + MaxPayloadBytes: guardrailMaxPayloadBytes(g.MaxPayloadBytes), + Matcher: re, }) continue } @@ -176,11 +184,12 @@ func NewRuntimeConfig(ctx context.Context, config *Config, fn NewBackendAuthHand return nil, fmt.Errorf("cannot create evaluator for guardrail %q: %w", g.Name, err) } guardrails = append(guardrails, RuntimeGuardrail{ - Name: g.Name, - Phase: g.Phase, - Provider: g.Provider, - Backends: g.Backends, - Evaluator: evaluator, + Name: g.Name, + Phase: g.Phase, + Provider: g.Provider, + Backends: g.Backends, + MaxPayloadBytes: guardrailMaxPayloadBytes(g.MaxPayloadBytes), + Evaluator: evaluator, }) } @@ -195,3 +204,12 @@ func NewRuntimeConfig(ctx context.Context, config *Config, fn NewBackendAuthHand Guardrails: guardrails, }, nil } + +const defaultGuardrailMaxPayloadBytes int64 = 10 * 1024 * 1024 + +func guardrailMaxPayloadBytes(configured int64) int64 { + if configured <= 0 { + return defaultGuardrailMaxPayloadBytes + } + return configured +} diff --git a/internal/filterapi/runtime_test.go b/internal/filterapi/runtime_test.go index 0d562a1e1e..8bb15d63af 100644 --- a/internal/filterapi/runtime_test.go +++ b/internal/filterapi/runtime_test.go @@ -168,6 +168,6 @@ func TestServer_LoadConfig(t *testing.T) { type testGuardrailEvaluator struct{} -func (*testGuardrailEvaluator) Evaluate(context.Context, []byte, GuardrailPhase) (bool, error) { - return false, nil +func (*testGuardrailEvaluator) Evaluate(context.Context, []byte, GuardrailPhase) (GuardrailEvaluationResult, error) { + return GuardrailEvaluationResult{}, nil } diff --git a/internal/guardrails/adapters.go b/internal/guardrails/adapters.go index 1982b50526..017ebf38f6 100644 --- a/internal/guardrails/adapters.go +++ b/internal/guardrails/adapters.go @@ -26,7 +26,7 @@ func NewEvaluator(ctx context.Context, provider *filterapi.GuardrailProvider) (f client := &http.Client{Timeout: time.Duration(timeoutSeconds(provider.TimeoutSeconds)) * time.Second} switch provider.Type { case filterapi.GuardrailProviderTypePresidio: - return newPresidioEvaluator(provider.Presidio, client) + return newPresidioEvaluator(provider.Presidio, provider.MaskReplacement, client) case filterapi.GuardrailProviderTypeBedrockGuardrails: return newBedrockEvaluator(ctx, provider.Bedrock, client) case filterapi.GuardrailProviderTypeAzureContentSafety: diff --git a/internal/guardrails/adapters_live_test.go b/internal/guardrails/adapters_live_test.go index ce27cd6b22..79a4d07ea9 100644 --- a/internal/guardrails/adapters_live_test.go +++ b/internal/guardrails/adapters_live_test.go @@ -29,9 +29,9 @@ func TestLivePresidio(t *testing.T) { } evaluator, err := NewEvaluator(t.Context(), provider) require.NoError(t, err) - blocked, err := evaluator.Evaluate(t.Context(), []byte(blockedText), filterapi.GuardrailPhaseRequest) + evaluation, err := evaluator.Evaluate(t.Context(), []byte(blockedText), filterapi.GuardrailPhaseRequest) require.NoError(t, err) - require.True(t, blocked) + require.True(t, evaluation.Matched) } func TestLiveAzureContentSafety(t *testing.T) { @@ -49,9 +49,9 @@ func TestLiveAzureContentSafety(t *testing.T) { } evaluator, err := NewEvaluator(t.Context(), provider) require.NoError(t, err) - blocked, err := evaluator.Evaluate(t.Context(), []byte(blockedText), filterapi.GuardrailPhaseRequest) + evaluation, err := evaluator.Evaluate(t.Context(), []byte(blockedText), filterapi.GuardrailPhaseRequest) require.NoError(t, err) - require.True(t, blocked) + require.True(t, evaluation.Matched) } func TestLiveBedrockGuardrail(t *testing.T) { @@ -70,7 +70,7 @@ func TestLiveBedrockGuardrail(t *testing.T) { } evaluator, err := NewEvaluator(t.Context(), provider) require.NoError(t, err) - blocked, err := evaluator.Evaluate(t.Context(), []byte(blockedText), filterapi.GuardrailPhaseRequest) + evaluation, err := evaluator.Evaluate(t.Context(), []byte(blockedText), filterapi.GuardrailPhaseRequest) require.NoError(t, err) - require.True(t, blocked) + require.True(t, evaluation.Matched) } diff --git a/internal/guardrails/adapters_test.go b/internal/guardrails/adapters_test.go index 2b7209f748..75341c5a27 100644 --- a/internal/guardrails/adapters_test.go +++ b/internal/guardrails/adapters_test.go @@ -31,17 +31,18 @@ func TestPresidioEvaluatorHTTP(t *testing.T) { require.Equal(t, "customer SSN", body.Text) require.Equal(t, "es", body.Language) require.Equal(t, 0.75, body.ScoreThreshold) - _, _ = w.Write([]byte(`[{"entity_type":"US_SSN","score":0.98}]`)) + _, _ = w.Write([]byte(`[{"entity_type":"US_SSN","start":9,"end":12,"score":0.98}]`)) })) t.Cleanup(server.Close) evaluator, err := newPresidioEvaluator(&filterapi.PresidioGuardrailProvider{ Endpoint: server.URL, Language: "es", ScoreThresholdPercent: 75, APIKey: "secret", - }, server.Client()) + }, "[REDACTED]", server.Client()) require.NoError(t, err) - blocked, err := evaluator.Evaluate(t.Context(), []byte("customer SSN"), filterapi.GuardrailPhaseRequest) + evaluation, err := evaluator.Evaluate(t.Context(), []byte("customer SSN"), filterapi.GuardrailPhaseRequest) require.NoError(t, err) - require.True(t, blocked) + require.True(t, evaluation.Matched) + require.Equal(t, "customer [REDACTED]", string(evaluation.Replacement)) } func TestAzureContentSafetyEvaluatorHTTP(t *testing.T) { @@ -58,9 +59,9 @@ func TestAzureContentSafetyEvaluatorHTTP(t *testing.T) { Endpoint: server.URL, APIKey: "azure-secret", SeverityThreshold: &severityThreshold, }, server.Client()) require.NoError(t, err) - blocked, err := evaluator.Evaluate(t.Context(), []byte("unsafe response"), filterapi.GuardrailPhaseResponse) + evaluation, err := evaluator.Evaluate(t.Context(), []byte("unsafe response"), filterapi.GuardrailPhaseResponse) require.NoError(t, err) - require.True(t, blocked) + require.True(t, evaluation.Matched) } func TestBedrockEvaluatorHTTP(t *testing.T) { @@ -79,7 +80,7 @@ func TestBedrockEvaluatorHTTP(t *testing.T) { require.NoError(t, json.NewDecoder(req.Body).Decode(&body)) require.Equal(t, "OUTPUT", body.Source) require.Equal(t, "unsafe response", body.Content[0].Text.Text) - _, _ = w.Write([]byte(`{"action":"GUARDRAIL_INTERVENED"}`)) + _, _ = w.Write([]byte(`{"action":"GUARDRAIL_INTERVENED","outputs":[{"text":"safe response"}]}`)) })) t.Cleanup(server.Close) @@ -92,9 +93,10 @@ aws_secret_access_key = secret `), }, server.Client()) require.NoError(t, err) - blocked, err := evaluator.Evaluate(t.Context(), []byte("unsafe response"), filterapi.GuardrailPhaseResponse) + evaluation, err := evaluator.Evaluate(t.Context(), []byte("unsafe response"), filterapi.GuardrailPhaseResponse) require.NoError(t, err) - require.True(t, blocked) + require.True(t, evaluation.Matched) + require.Equal(t, "safe response", string(evaluation.Replacement)) } func TestEvaluatorProviderError(t *testing.T) { @@ -103,10 +105,10 @@ func TestEvaluatorProviderError(t *testing.T) { })) t.Cleanup(server.Close) - evaluator, err := newPresidioEvaluator(&filterapi.PresidioGuardrailProvider{Endpoint: server.URL}, server.Client()) + evaluator, err := newPresidioEvaluator(&filterapi.PresidioGuardrailProvider{Endpoint: server.URL}, "", server.Client()) require.NoError(t, err) - blocked, err := evaluator.Evaluate(t.Context(), []byte("payload"), filterapi.GuardrailPhaseRequest) - require.False(t, blocked) + evaluation, err := evaluator.Evaluate(t.Context(), []byte("payload"), filterapi.GuardrailPhaseRequest) + require.False(t, evaluation.Matched) require.ErrorContains(t, err, "HTTP 503") require.ErrorContains(t, err, "provider unavailable") } diff --git a/internal/guardrails/azure.go b/internal/guardrails/azure.go index 68a62af374..8c8b03d1e9 100644 --- a/internal/guardrails/azure.go +++ b/internal/guardrails/azure.go @@ -38,7 +38,7 @@ func newAzureContentSafetyEvaluator(config *filterapi.AzureContentSafetyGuardrai return &azureContentSafetyEvaluator{config: &configCopy, client: client}, nil } -func (e *azureContentSafetyEvaluator) Evaluate(ctx context.Context, body []byte, _ filterapi.GuardrailPhase) (bool, error) { +func (e *azureContentSafetyEvaluator) Evaluate(ctx context.Context, body []byte, _ filterapi.GuardrailPhase) (filterapi.GuardrailEvaluationResult, error) { payload := struct { Text string `json:"text"` }{Text: string(body)} @@ -51,12 +51,12 @@ func (e *azureContentSafetyEvaluator) Evaluate(ctx context.Context, body []byte, if err := doJSON(ctx, e.client, http.MethodPost, endpoint, payload, func(req *http.Request) { req.Header.Set("Ocp-Apim-Subscription-Key", e.config.APIKey) }, &result); err != nil { - return false, fmt.Errorf("azure Content Safety analyze request failed: %w", err) + return filterapi.GuardrailEvaluationResult{}, fmt.Errorf("azure Content Safety analyze request failed: %w", err) } for _, category := range result.CategoriesAnalysis { if category.Severity >= *e.config.SeverityThreshold { - return true, nil + return filterapi.GuardrailEvaluationResult{Matched: true}, nil } } - return false, nil + return filterapi.GuardrailEvaluationResult{}, nil } diff --git a/internal/guardrails/bedrock.go b/internal/guardrails/bedrock.go index 7186092982..af43810bec 100644 --- a/internal/guardrails/bedrock.go +++ b/internal/guardrails/bedrock.go @@ -81,7 +81,7 @@ func loadAWSCredentials(ctx context.Context, guardrailConfig *filterapi.BedrockG }), nil } -func (e *bedrockEvaluator) Evaluate(ctx context.Context, body []byte, phase filterapi.GuardrailPhase) (bool, error) { +func (e *bedrockEvaluator) Evaluate(ctx context.Context, body []byte, phase filterapi.GuardrailPhase) (filterapi.GuardrailEvaluationResult, error) { source := "INPUT" if phase == filterapi.GuardrailPhaseResponse { source = "OUTPUT" @@ -101,27 +101,34 @@ func (e *bedrockEvaluator) Evaluate(ctx context.Context, body []byte, phase filt Text string `json:"text"` }{Text: string(body)}}}}) if err != nil { - return false, err + return filterapi.GuardrailEvaluationResult{}, err } endpoint := strings.TrimRight(e.config.Endpoint, "/") + "/guardrail/" + url.PathEscape(e.config.GuardrailIdentifier) + "/version/" + url.PathEscape(e.config.GuardrailVersion) + "/apply" req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(payload)) if err != nil { - return false, err + return filterapi.GuardrailEvaluationResult{}, err } req.Header.Set("Content-Type", "application/json") credentials, err := e.credentials.Retrieve(ctx) if err != nil { - return false, fmt.Errorf("cannot retrieve AWS credentials: %w", err) + return filterapi.GuardrailEvaluationResult{}, fmt.Errorf("cannot retrieve AWS credentials: %w", err) } payloadHash := sha256.Sum256(payload) if err = e.signer.SignHTTP(ctx, credentials, req, hex.EncodeToString(payloadHash[:]), "bedrock", e.config.Region, time.Now()); err != nil { - return false, fmt.Errorf("cannot sign Bedrock request: %w", err) + return filterapi.GuardrailEvaluationResult{}, fmt.Errorf("cannot sign Bedrock request: %w", err) } result := struct { - Action string `json:"action"` + Action string `json:"action"` + Outputs []struct { + Text string `json:"text"` + } `json:"outputs"` }{} if err = sendJSON(req, e.client, &result); err != nil { - return false, fmt.Errorf("bedrock ApplyGuardrail request failed: %w", err) + return filterapi.GuardrailEvaluationResult{}, fmt.Errorf("bedrock ApplyGuardrail request failed: %w", err) + } + evaluation := filterapi.GuardrailEvaluationResult{Matched: result.Action == "GUARDRAIL_INTERVENED"} + if len(result.Outputs) > 0 { + evaluation.Replacement = []byte(result.Outputs[0].Text) } - return result.Action == "GUARDRAIL_INTERVENED", nil + return evaluation, nil } diff --git a/internal/guardrails/presidio.go b/internal/guardrails/presidio.go index fb0a69e5a4..2b1253ab2a 100644 --- a/internal/guardrails/presidio.go +++ b/internal/guardrails/presidio.go @@ -9,17 +9,19 @@ import ( "context" "fmt" "net/http" + "sort" "strings" "github.com/envoyproxy/ai-gateway/internal/filterapi" ) type presidioEvaluator struct { - config *filterapi.PresidioGuardrailProvider - client *http.Client + config *filterapi.PresidioGuardrailProvider + maskReplacement string + client *http.Client } -func newPresidioEvaluator(config *filterapi.PresidioGuardrailProvider, client *http.Client) (filterapi.GuardrailEvaluator, error) { +func newPresidioEvaluator(config *filterapi.PresidioGuardrailProvider, maskReplacement string, client *http.Client) (filterapi.GuardrailEvaluator, error) { if config == nil || config.Endpoint == "" { return nil, fmt.Errorf("presidio endpoint is required") } @@ -27,10 +29,13 @@ func newPresidioEvaluator(config *filterapi.PresidioGuardrailProvider, client *h if configCopy.Language == "" { configCopy.Language = "en" } - return &presidioEvaluator{config: &configCopy, client: client}, nil + if maskReplacement == "" { + maskReplacement = "[REDACTED]" + } + return &presidioEvaluator{config: &configCopy, maskReplacement: maskReplacement, client: client}, nil } -func (e *presidioEvaluator) Evaluate(ctx context.Context, body []byte, _ filterapi.GuardrailPhase) (bool, error) { +func (e *presidioEvaluator) Evaluate(ctx context.Context, body []byte, _ filterapi.GuardrailPhase) (filterapi.GuardrailEvaluationResult, error) { payload := struct { Text string `json:"text"` Language string `json:"language"` @@ -42,6 +47,8 @@ func (e *presidioEvaluator) Evaluate(ctx context.Context, body []byte, _ filtera } var result []struct { + Start int `json:"start"` + End int `json:"end"` Score float64 `json:"score"` } if err := doJSON(ctx, e.client, http.MethodPost, strings.TrimRight(e.config.Endpoint, "/")+"/analyze", payload, func(req *http.Request) { @@ -49,7 +56,31 @@ func (e *presidioEvaluator) Evaluate(ctx context.Context, body []byte, _ filtera req.Header.Set("Authorization", "Bearer "+e.config.APIKey) } }, &result); err != nil { - return false, fmt.Errorf("presidio analyze request failed: %w", err) + return filterapi.GuardrailEvaluationResult{}, fmt.Errorf("presidio analyze request failed: %w", err) + } + if len(result) == 0 { + return filterapi.GuardrailEvaluationResult{}, nil + } + return filterapi.GuardrailEvaluationResult{ + Matched: true, + Replacement: maskPresidioMatches(body, result, e.maskReplacement), + }, nil +} + +func maskPresidioMatches(body []byte, matches []struct { + Start int `json:"start"` + End int `json:"end"` + Score float64 `json:"score"` +}, replacement string, +) []byte { + sort.Slice(matches, func(i, j int) bool { return matches[i].Start < matches[j].Start }) + runes := []rune(string(body)) + for i := len(matches) - 1; i >= 0; i-- { + match := matches[i] + if match.Start < 0 || match.End > len(runes) || match.Start >= match.End { + continue + } + runes = append(runes[:match.Start], append([]rune(replacement), runes[match.End:]...)...) } - return len(result) > 0, nil + return []byte(string(runes)) } diff --git a/internal/guardrails/presidio_container_test.go b/internal/guardrails/presidio_container_test.go index e63df792ae..a898349732 100644 --- a/internal/guardrails/presidio_container_test.go +++ b/internal/guardrails/presidio_container_test.go @@ -50,10 +50,10 @@ func TestPresidioEvaluatorContainer(t *testing.T) { Endpoint: fmt.Sprintf("http://%s", net.JoinHostPort(host, port.Port())), Language: "en", ScoreThresholdPercent: 80, - }, &http.Client{Timeout: 10 * time.Second}) + }, "[REDACTED]", &http.Client{Timeout: 10 * time.Second}) require.NoError(t, err) - blocked, err := evaluator.Evaluate(ctx, []byte("Contact me at alice@example.com"), filterapi.GuardrailPhaseRequest) + evaluation, err := evaluator.Evaluate(ctx, []byte("Contact me at alice@example.com"), filterapi.GuardrailPhaseRequest) require.NoError(t, err) - require.True(t, blocked) + require.True(t, evaluation.Matched) } diff --git a/internal/metrics/guardrail_metrics.go b/internal/metrics/guardrail_metrics.go index 95bfe6170d..b285f63994 100644 --- a/internal/metrics/guardrail_metrics.go +++ b/internal/metrics/guardrail_metrics.go @@ -22,9 +22,11 @@ const ( type GuardrailResult string const ( - GuardrailResultAllowed GuardrailResult = "allowed" - GuardrailResultBlocked GuardrailResult = "blocked" - GuardrailResultError GuardrailResult = "error" + GuardrailResultAllowed GuardrailResult = "allowed" + GuardrailResultBlocked GuardrailResult = "blocked" + GuardrailResultError GuardrailResult = "error" + GuardrailResultMasked GuardrailResult = "masked" + GuardrailResultMonitored GuardrailResult = "monitored" ) // GuardrailMetrics records guardrail evaluation outcomes. diff --git a/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_guardrailpolicies.yaml b/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_guardrailpolicies.yaml index b2ce12ddd6..328af8ae34 100644 --- a/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_guardrailpolicies.yaml +++ b/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_guardrailpolicies.yaml @@ -55,6 +55,22 @@ spec: description: GuardrailPolicySpec contains the configured checks attached to an AIServiceBackend. properties: + maxRequestBodyBytes: + default: 10485760 + description: MaxRequestBodyBytes is the largest request body evaluated + by this policy. + format: int64 + maximum: 52428800 + minimum: 1024 + type: integer + maxResponseBodyBytes: + default: 10485760 + description: MaxResponseBodyBytes is the largest response body buffered + and evaluated by this policy. + format: int64 + maximum: 52428800 + minimum: 1024 + type: integer rules: description: Rules are executed in order and can evaluate request or response payloads. @@ -83,6 +99,8 @@ spec: matched. enum: - Block + - Monitor + - Mask type: string azureContentSafety: description: AzureContentSafety configures Azure AI Content @@ -220,6 +238,11 @@ spec: - FailClosed - FailOpen type: string + maskReplacement: + default: '[REDACTED]' + description: MaskReplacement is used by Regex and Presidio + Mask actions. + type: string message: description: Message is returned to the caller when the rule blocks a request or response. @@ -322,6 +345,8 @@ spec: rule: self.type != 'AzureContentSafety' || (!has(self.pattern) && has(self.azureContentSafety) && !has(self.presidio) && !has(self.bedrock)) + - message: AzureContentSafety does not support Mask + rule: self.action != 'Mask' || self.type != 'AzureContentSafety' required: - name - phase @@ -474,6 +499,22 @@ spec: description: GuardrailPolicySpec contains the configured checks attached to an AIServiceBackend. properties: + maxRequestBodyBytes: + default: 10485760 + description: MaxRequestBodyBytes is the largest request body evaluated + by this policy. + format: int64 + maximum: 52428800 + minimum: 1024 + type: integer + maxResponseBodyBytes: + default: 10485760 + description: MaxResponseBodyBytes is the largest response body buffered + and evaluated by this policy. + format: int64 + maximum: 52428800 + minimum: 1024 + type: integer rules: description: Rules are executed in order and can evaluate request or response payloads. @@ -502,6 +543,8 @@ spec: matched. enum: - Block + - Monitor + - Mask type: string azureContentSafety: description: AzureContentSafety configures Azure AI Content @@ -639,6 +682,11 @@ spec: - FailClosed - FailOpen type: string + maskReplacement: + default: '[REDACTED]' + description: MaskReplacement is used by Regex and Presidio + Mask actions. + type: string message: description: Message is returned to the caller when the rule blocks a request or response. @@ -741,6 +789,8 @@ spec: rule: self.type != 'AzureContentSafety' || (!has(self.pattern) && has(self.azureContentSafety) && !has(self.presidio) && !has(self.bedrock)) + - message: AzureContentSafety does not support Mask + rule: self.action != 'Mask' || self.type != 'AzureContentSafety' required: - name - phase diff --git a/site/docs/api/api.mdx b/site/docs/api/api.mdx index 5dc13d6b82..df3c07d9da 100644 --- a/site/docs/api/api.mdx +++ b/site/docs/api/api.mdx @@ -1768,6 +1768,16 @@ GuardrailAction defines the safeguard action. type="enum" required="false" description="" +/> #### GuardrailFailureMode @@ -1840,6 +1850,18 @@ GuardrailPolicySpec contains the configured checks attached to an AIServiceBacke type="[GuardrailRule](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailrule) array" required="true" description="Rules are executed in order and can evaluate request or response payloads." +/> @@ -1893,6 +1915,12 @@ GuardrailProvider describes the implementation used to evaluate a rule. required="false" defaultValue="Block" description="Action is the action taken when the rule is matched." +/> #### GuardrailFailureMode @@ -4963,6 +5001,18 @@ GuardrailPolicySpec contains the configured checks attached to an AIServiceBacke type="[GuardrailRule](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailrule) array" required="true" description="Rules are executed in order and can evaluate request or response payloads." +/> @@ -5016,6 +5066,12 @@ GuardrailProvider describes the implementation used to evaluate a rule. required="false" defaultValue="Block" description="Action is the action taken when the rule is matched." +/> Date: Wed, 16 Sep 2026 23:15:40 +0200 Subject: [PATCH 07/18] add Guardrails Proposal Signed-off-by: Fernando Escolar --- docs/proposals/013-guardrails/proposal.md | 469 ++++++++++++++++++++++ 1 file changed, 469 insertions(+) create mode 100644 docs/proposals/013-guardrails/proposal.md diff --git a/docs/proposals/013-guardrails/proposal.md b/docs/proposals/013-guardrails/proposal.md new file mode 100644 index 0000000000..fce776b294 --- /dev/null +++ b/docs/proposals/013-guardrails/proposal.md @@ -0,0 +1,469 @@ +# Guardrails Proposal + +## Table of Contents + +- [Guardrails Proposal](#guardrails-proposal) + - [Table of Contents](#table-of-contents) + - [Summary](#summary) + - [Motivation](#motivation) + - [Goals](#goals) + - [Non-Goals](#non-goals) + - [Prior Art](#prior-art) + - [Built on Envoy Bedrock Guardrails](#built-on-envoy-bedrock-guardrails) + - [Built on Envoy Azure Content Safety](#built-on-envoy-azure-content-safety) + - [Lessons Applied to Agent Router](#lessons-applied-to-agent-router) + - [Proposed Architecture](#proposed-architecture) + - [Why ExtProc](#why-extproc) + - [Policy Attachment and Backend Scoping](#policy-attachment-and-backend-scoping) + - [Evaluation Lifecycle](#evaluation-lifecycle) + - [Streaming Responses](#streaming-responses) + - [API Design](#api-design) + - [GuardrailPolicy](#guardrailpolicy) + - [Provider Configuration](#provider-configuration) + - [Actions](#actions) + - [Failure Modes and Timeouts](#failure-modes-and-timeouts) + - [Provider Behavior](#provider-behavior) + - [Regex](#regex) + - [Presidio](#presidio) + - [AWS Bedrock Guardrails](#aws-bedrock-guardrails) + - [Azure AI Content Safety](#azure-ai-content-safety) + - [Payload Extraction](#payload-extraction) + - [Credentials and Security](#credentials-and-security) + - [Status and Reconciliation](#status-and-reconciliation) + - [Observability](#observability) + - [Performance and Reliability](#performance-and-reliability) + - [Alternatives Considered](#alternatives-considered) + - [Use Built on Envoy Dynamic Modules Directly](#use-built-on-envoy-dynamic-modules-directly) + - [Configure Guardrails Directly on AIGatewayRoute](#configure-guardrails-directly-on-aigatewayroute) + - [Use BackendSecurityPolicy](#use-backendsecuritypolicy) + - [Use Only Provider-Native Model Guardrails](#use-only-provider-native-model-guardrails) + - [Implementation Plan](#implementation-plan) + - [Testing Strategy](#testing-strategy) + - [Current Implementation and Gaps](#current-implementation-and-gaps) + - [References](#references) + +## Summary + +This proposal introduces `GuardrailPolicy`, a backend-attached policy for evaluating LLM request and response content before it is sent to an AI provider or returned to a client. The policy provides a common API for local regular-expression checks and external safety providers, initially Presidio, AWS Bedrock Guardrails, and Azure AI Content Safety. + +The proposed implementation uses the existing Agent Router external processor. The controller resolves policies and credentials into the filter configuration, while ext-proc buffers the relevant body, extracts content, invokes the configured evaluator, and applies the selected action. + +This design is informed by the Built on Envoy Bedrock Guardrails and Azure Content Safety dynamic-module extensions. It adopts their strongest behavioral patterns while retaining Agent Router's Kubernetes policy model, backend scoping, provider-neutral configuration, existing ext-proc deployment, and observability conventions. + +## Motivation + +AI Gateway users need a consistent way to enforce content-safety requirements across different models and providers. Today, users must deploy an additional filter, application middleware, or provider-specific integration. Those approaches make policy attachment, credentials, observability, and failure behavior inconsistent across backends. + +A native policy should allow platform administrators to: + +- attach safety checks to one or more `AIServiceBackend` resources; +- inspect prompts before upstream routing and completions before downstream delivery; +- use deterministic local checks or managed safety services; +- choose blocking or observational behavior independently from provider failures; +- rotate credentials without restarting gateways; and +- observe allowed, blocked, and failed evaluations without recording sensitive payloads. + +## Goals + +- Define a Kubernetes-native `GuardrailPolicy` consistent with the repository's direct-policy conventions. +- Support both `v1alpha1` and `v1beta1`, with `v1beta1` as the storage version. +- Scope policies to `AIServiceBackend` resources and preserve route/backend isolation. +- Support request and response evaluation. +- Provide a provider-neutral runtime evaluator interface. +- Initially support Regex, Presidio, AWS Bedrock Guardrails, and Azure AI Content Safety. +- Support explicit timeout and fail-open/fail-closed behavior. +- Prevent partial delivery of blocked streaming responses. +- Integrate with existing logs, metrics, traces, config distribution, Secret watches, and status conditions. +- Keep provider credentials out of the CRD and user-visible status. + +## Non-Goals + +- Replacing provider-native model safety configuration. +- Defining a general-purpose Web Application Firewall API. +- Supporting arbitrary executable plugins supplied by users. +- Performing image, audio, or video moderation in the initial API. +- Guaranteeing identical safety classifications across providers. +- Retrying external safety calls in the initial implementation. +- Exposing provider response details or sensitive matched content to downstream clients. + +## Prior Art + +### Built on Envoy Bedrock Guardrails + +The [Built on Envoy Bedrock Guardrails] extension is an Envoy dynamic-module HTTP filter. It recognizes OpenAI Chat Completions requests, extracts user prompts, invokes each configured Bedrock guardrail, and can apply block, mask, or no-op outcomes. + +Notable design choices are: + +- semantic extraction of user messages instead of sending the serialized request envelope; +- multiple Bedrock guardrails evaluated for one request; +- configurable request timeout; +- response outputs used to replace masked prompt content; and +- an Envoy cluster used for provider callouts. + +The extension authenticates with a Bedrock API key. Agent Router already supports AWS workload credentials and SigV4, so this proposal uses the standard AWS credential chain or a referenced Kubernetes Secret instead. + +### Built on Envoy Azure Content Safety + +The [Built on Envoy Azure Content Safety] extension is also an Envoy dynamic-module HTTP filter. It auto-detects OpenAI Chat Completions, OpenAI Responses, and Anthropic Messages payloads and uses phase-specific Azure APIs: + +- Prompt Shield for request prompt-injection detection; +- Task Adherence for request tool-use alignment; +- Text Analysis for harmful response content; and +- Protected Material Detection for response content. + +It supports block and monitor modes, fail-open behavior, selected categories, and independent severity thresholds for hate, self-harm, sexual, and violent content. + +The extension demonstrates that vendor integration is more useful when it understands the LLM API schema and chooses a provider operation based on the intended safety check, rather than treating every JSON body as undifferentiated text. + +### Lessons Applied to Agent Router + +The following ideas should be adopted: + +- buffer content before a blocking decision; +- extract semantically relevant text from known LLM schemas; +- distinguish enforcement mode from provider failure behavior; +- support provider-specific checks without leaking them into unrelated providers; +- allow per-category thresholds where the provider exposes them; +- short-circuit after a blocking result; and +- make provider latency and errors observable. + +At the time of this proposal, both extension pages identify the implementation as version `0.12.0-dev` with an Envoy compatibility range of 1.38 through 1.39. The following choices remain specific to Agent Router: + +- policy attachment through Kubernetes `targetRefs`; +- backend-specific scoping after route selection; +- credentials resolved through Kubernetes Secrets or cloud workload identity; +- configuration distributed through the existing ext-proc bundle; +- a provider-neutral evaluator interface; and +- no additional Envoy binary or dynamic-module deployment requirement. + +## Proposed Architecture + +```text +GuardrailPolicy + Secret + | + v +Gateway controller + - validates targets and provider configuration + - resolves credential references + - scopes rules to generated backend identities + | + v +Filter configuration bundle + | + v +Envoy ext_proc + - parses the known LLM request/response schema + - selects rules for phase and chosen backend + - invokes local or external evaluators + - records logs, metrics, and span events + | + +--> allow / monitor + | + +--> block with HTTP 403 +``` + +### Why ExtProc + +Agent Router already uses ext-proc for request parsing, provider translation, credential handling, token accounting, and response processing. Guardrails require the same parsed content and backend identity. + +Using the existing ext-proc has these advantages: + +- no second extension packaging or compatibility lifecycle; +- no dependency on a narrow Envoy dynamic-module version range; +- direct reuse of endpoint schemas and translators; +- one configuration and observability path; and +- consistent behavior in controller-managed and standalone deployments. + +A dynamic module may reduce cross-process overhead and can use Envoy clusters for callouts. However, adopting the [Built on Envoy repository] extensions directly would require reconciling their configuration, credentials, deployment, API parsing, and release compatibility with Agent Router. This proposal therefore uses them as prior art rather than as runtime dependencies. + +### Policy Attachment and Backend Scoping + +`GuardrailPolicy.spec.targetRefs` references one or more `AIServiceBackend` resources in the policy namespace. During gateway reconciliation, the controller maps each target to generated per-route backend identities. Runtime rules are evaluated only when the matching backend is selected. + +Request rules that do not depend on backend selection may run at router level. Backend-attached rules run after selection and before upstream forwarding. Response rules run against the selected backend's response. + +Multiple policies targeting the same backend compose additively. The controller sorts policies lexicographically by namespace and name, then preserves declaration order within each policy. Evaluation stops at the first blocking rule. Monitor and Mask rules continue to later rules after recording or applying their result. + +### Evaluation Lifecycle + +Rules execute in order for their phase and backend: + +1. Select rules matching the request or response phase. +2. Filter rules by the selected generated backend identity. +3. Extract provider-relevant content from the parsed LLM payload. +4. Invoke the evaluator with the request context and configured timeout. +5. Record the result. +6. Stop at the first blocking result. +7. Continue after an allowed result or a fail-open provider error. + +A provider error under `FailClosed` prevents unchecked traffic. A provider error under `FailOpen` records an error and allows evaluation to continue. Provider failures are not counted as allowed evaluations. + +### Streaming Responses + +A response cannot be safely blocked after bytes have already reached the client. When any applicable response rule can block, ext-proc must keep response-body processing buffered. Only routes without applicable blocking response rules may switch to streamed response processing. + +This increases latency and memory use for guarded streaming responses. All response rules, including Monitor, use buffering so providers receive a complete semantic payload rather than partial SSE events. Each policy defaults to a 10 MiB response evaluation limit and may configure up to 50 MiB, matching the Envoy per-connection buffer ceiling. The limit is checked before provider evaluation; exceeding it follows the rule's failure mode. + +## API Design + +### GuardrailPolicy + +```yaml +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: GuardrailPolicy +metadata: + name: content-safety + namespace: ai-gateway +spec: + maxRequestBodyBytes: 10485760 + maxResponseBodyBytes: 10485760 + targetRefs: + - group: aigateway.envoyproxy.io + kind: AIServiceBackend + name: openai + rules: + - name: block-pii + phase: Request + provider: + type: Presidio + action: Block + failureMode: FailClosed + timeoutSeconds: 5 + message: Request contains sensitive information + presidio: + endpoint: http://presidio-analyzer.presidio.svc.cluster.local:3000 + language: en + scoreThresholdPercent: 70 +``` + +Rules have stable unique names and execute in order. The initial action is `Block`. The API reserves room for additional actions without forcing all providers to support identical capabilities. + +### Provider Configuration + +The provider is a tagged union. Admission validation requires exactly the configuration matching `provider.type`. + +- `Regex` requires `pattern`. +- `Presidio` requires `presidio`. +- `Bedrock` requires `bedrock`. +- `AzureContentSafety` requires `azureContentSafety`. + +Provider-specific configuration remains nested so new provider capabilities can be added without adding unrelated fields to every rule. + +### Actions + +The portable actions are: + +- `Block`: return HTTP 403 with error type `GuardrailViolation`. +- `Monitor`: record a detection and continue without modifying traffic. +- `Mask`: replace detected text and continue evaluation. + +Mask behavior depends on provider capability: + +- Regex replaces matching substrings with `maskReplacement`, defaulting to `[REDACTED]`. +- Presidio replaces detected spans with `maskReplacement`. +- Bedrock uses transformed text returned by ApplyGuardrail. +- Azure Text Analysis does not return transformed content, so admission rejects `Mask` for Azure rules. + +Mask is applied only to extracted text fields and never replaces unrelated model or configuration fields. Masked streaming responses remain buffered until the complete body has been evaluated and rewritten. + +### Failure Modes and Timeouts + +`failureMode` controls provider errors, not policy matches: + +- `FailClosed` is the default and prevents unchecked traffic. +- `FailOpen` records the provider error and continues processing. + +`timeoutSeconds` applies to each external evaluation. The initial implementation does not retry. This avoids multiplying tail latency and prevents ambiguous duplicate calls. If retries are added, they should be limited to clearly retryable transport and 5xx failures and remain inside the rule's total timeout budget. + +## Provider Behavior + +### Regex + +Regex rules compile when the runtime configuration is loaded. Invalid patterns reject the configuration. Evaluation is local and deterministic. + +Regex is useful for tests, organization-specific markers, and simple deny patterns. It is not a replacement for semantic safety classification. + +### Presidio + +[Presidio] calls the Analyzer `/analyze` endpoint with extracted text, language, and an optional score threshold. An optional Secret-provided API key can be sent as a bearer token for deployments that place authentication in front of Presidio. + +The official Presidio analyzer image is suitable for deterministic integration tests through Testcontainers. + +Future configuration may add entity allow/deny lists. Those should use Presidio entity names and avoid exposing matched text in status, logs, or metrics. + +### AWS Bedrock Guardrails + +Bedrock uses the [AWS Bedrock ApplyGuardrail] API. The request source is `INPUT` for requests and `OUTPUT` for responses. Authentication uses SigV4 with either: + +- the standard AWS credential chain, including workload identity; or +- an AWS shared credentials file read from a Kubernetes Secret. + +The Built on Envoy extension additionally supports Bedrock API-key authentication and mask results. API-key support can be added as another credential source if users require it. Mask support requires a separate action and schema-aware mutation contract. + +Each rule invokes one Bedrock guardrail identifier/version. Multiple rules may target the same backend, but each external call adds latency and cost. Administrators should prefer consolidating related checks into one managed Bedrock guardrail where possible. + +### Azure AI Content Safety + +The initial [Azure AI Content Safety] provider configuration includes endpoint, API version, API key Secret, and severity threshold. + +The Built on Envoy implementation highlights that Azure has multiple distinct safety operations. The API should eventually model the check explicitly instead of inferring it only from request/response phase: + +- request Prompt Shield; +- request Task Adherence; +- request or response Text Analysis; +- response Protected Material Detection. + +A future shape could add `azureContentSafety.check` and check-specific configuration. Per-category thresholds should also replace a single threshold when Text Analysis is selected. Until then, the implementation should be documented as Text Analysis over the configured evaluation input, not as complete Azure Content Safety feature parity. + +## Payload Extraction + +Provider callouts receive relevant user or assistant text, not an arbitrary serialized JSON envelope. Schema-aware extraction reduces false matches, removes model/configuration metadata, lowers provider cost, and avoids provider input-size limits. + +The implementation walks known LLM text and container fields in deterministic path order and records each JSON path so Mask can write transformed text back into the same field. It recognizes fields such as `messages`, `content`, `text`, `input`, `prompt`, `instructions`, `system`, `choices`, and `output` while ignoring unrelated values such as model names and image URLs. + +A future richer provider-neutral input may add structured fields such as: + +```go +type EvaluationInput struct { + Phase GuardrailPhase + Text []string + Documents []string + Tools []Tool + Messages []Message +} +``` + +External providers and Mask rules consume extracted fragments. Regex Block and Monitor preserve raw-body matching for backward compatibility. Unknown or malformed structured formats produce an evaluation error governed by `failureMode`; they are not silently interpreted as a known chat schema. + +## Credentials and Security + +Provider credentials are referenced from Kubernetes Secrets and resolved into the ext-proc configuration using the repository's existing bundle mechanism. The controller watches referenced Secrets so rotation or deletion triggers policy and route reconciliation. + +Security requirements are: + +- never include credentials in `GuardrailPolicy` status, logs, metrics, traces, examples, or error responses; +- never include evaluated payloads or matched text in default logs, metrics, or traces; +- restrict cross-namespace Secret references unless `ReferenceGrant` support is explicitly designed; +- preserve Secret file permissions in local tests; +- use workload identity instead of static AWS credentials where possible; and +- bound provider error bodies before logging or returning errors. + +A fail-closed policy whose credentials become unavailable must not leave a stale accepted configuration active. A fail-open policy may omit the unavailable evaluator while recording the configuration failure. + +## Status and Reconciliation + +The controller sets `Accepted` only when: + +- all target backends exist; +- all provider configuration is valid; +- regular expressions compile; and +- all required Secrets and keys are present. + +Backend changes, Secret changes, policy updates, and policy deletion trigger reconciliation. Deletion uses the finalizer to notify affected routes before the policy disappears, ensuring stale rules are removed from generated configuration. + +Status does not perform a live provider health check. Provider availability is a runtime concern represented by metrics, traces, logs, and failure mode. + +## Observability + +The runtime emits: + +- structured logs for blocks and provider failures without payload content; +- `aigateway.guardrail.evaluation.count` with phase and result attributes; and +- `guardrail.evaluation` span events with rule name, phase, and result. + +Metric attributes intentionally exclude rule names and error text to avoid unbounded cardinality. Result values are `allowed`, `blocked`, and `error`. Provider type may be added because it is bounded, subject to consistency with the project's metric conventions. + +The design should also expose provider-call latency in a future histogram so operators can measure the cost of each external integration. + +## Performance and Reliability + +External rules add network latency and may add provider cost. Rules currently execute sequentially and short-circuit on block. Sequential execution preserves deterministic ordering but makes worst-case latency the sum of rule timeouts. + +The policy defines separate `maxRequestBodyBytes` and `maxResponseBodyBytes` values. Both default to 10 MiB and are capped at 50 MiB. The cap aligns with the configured Envoy per-connection buffer limit. These are evaluation limits rather than streaming truncation: fail-closed rules reject oversized payloads, while fail-open and Monitor rules record the error and continue. + +Before increasing rule limits or adding many external checks, the implementation should further define: + +- a total guardrail evaluation budget per phase; +- whether independent monitor-only rules can run concurrently; +- provider rate-limit handling. + +No automatic retries are proposed initially. Connection reuse is delegated to `http.Client`. Provider contexts are canceled with the ext-proc request. + +## Alternatives Considered + +### Use Built on Envoy Dynamic Modules Directly + +This provides mature provider-specific behavior close to Envoy and avoids ext-proc RPC overhead. It also provides Azure semantic parsing and Bedrock mask behavior today. + +It is not selected for the initial implementation because it would introduce: + +- a second extension deployment and release lifecycle; +- Envoy dynamic-module ABI/version compatibility constraints; +- separate configuration and credential models; +- duplicated endpoint parsing and observability; and +- additional integration work for controller-managed backend scoping. + +The extensions remain valuable prior art, and sharing provider-neutral parsing or client libraries upstream may be preferable to independently maintaining equivalent logic. + +### Configure Guardrails Directly on AIGatewayRoute + +This makes route-level differences easy but duplicates policy across routes and couples safety configuration to routing. Backend attachment better represents a requirement that should follow a provider endpoint wherever it is referenced. + +### Use BackendSecurityPolicy + +`BackendSecurityPolicy` controls upstream authentication. Mixing content evaluation into it would combine unrelated ownership and status semantics. + +### Use Only Provider-Native Model Guardrails + +Provider-native model configuration does not cover cross-provider policy, request checks independent of inference, local Presidio deployments, or consistent gateway observability. + +## Implementation Plan + +1. Introduce the dual-version `GuardrailPolicy`, generated clients, CRD, and status. +2. Add target and Secret indexes, reconciliation, deletion propagation, and filter-config translation. +3. Add runtime compilation and backend-scoped request/response evaluation. +4. Add Regex, Presidio, Bedrock, and Azure Text Analysis evaluators. +5. Add failure modes, timeouts, logs, metrics, traces, and guarded-response buffering. +6. Add CRD admission, controller, HTTP-stub, Testcontainers, live-provider, and dataplane tests. +7. Add semantic payload extraction and schema-aware Mask mutation. +8. Add Monitor mode and deterministic multi-policy composition. +9. Add explicit request and response evaluation limits. +10. Add provider-specific Azure check selection. +11. Evaluate Bedrock API-key authentication. + +Steps 1 through 9 describe the current implementation. Steps 10 and 11 are proposed follow-up work informed by the Built on Envoy extensions. + +## Testing Strategy + +- CRD admission tests for valid and invalid provider unions, endpoints, actions, limits, and unique rule names. +- Controller tests for target validation, Secret resolution and rotation, status, backend scoping, failure modes, and deletion. +- Runtime tests for regex compilation and evaluator construction. +- HTTP-stub tests for provider paths, payloads, headers, authentication, responses, malformed responses, and errors. +- Testcontainers integration with the pinned official Presidio analyzer image. +- Credential-gated live tests for Azure AI Content Safety and AWS Bedrock Guardrails. +- Envoy dataplane tests for request blocking, response blocking, allowed traffic, and streaming mode behavior. +- Staging tests for policy creation, update, Secret rotation, deletion, and multi-policy attachment. +- Load tests for provider latency, concurrency, and buffered response limits. + +## Current Implementation and Gaps + +The current implementation includes the policy resource, backend scoping, lifecycle reconciliation, external adapters, blocking, failure modes, timeout, response buffering, observability, and layered tests. + +The following gaps remain intentionally visible for design review: + +- Azure currently uses Text Analysis and does not yet expose Prompt Shield, Task Adherence, Protected Material Detection, categories, or per-category thresholds; +- schema extraction is field-oriented and does not yet expose structured tools/documents to providers; +- Mask is unsupported for Azure Text Analysis and for non-JSON payloads; +- there is no total per-phase timeout budget or provider latency histogram; and +- load testing is still needed to validate the selected 10 MiB default under production concurrency. + +These gaps do not require changing the core policy-to-runtime architecture, but some require API additions before the feature is declared complete. + +## References + +[Built on Envoy Bedrock Guardrails]: https://builtonenvoy.io/extensions/bedrock-guardrails/ +[Built on Envoy Azure Content Safety]: https://builtonenvoy.io/extensions/azure-content-safety/ +[Built on Envoy repository]: https://github.com/tetratelabs/built-on-envoy +[AWS Bedrock ApplyGuardrail]: https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ApplyGuardrail.html +[Azure AI Content Safety]: https://learn.microsoft.com/azure/ai-services/content-safety/ +[Presidio]: https://presidio.dataprivacystack.org/ From 837affe88b6f4b4594257f9a54344a150e57b4bf Mon Sep 17 00:00:00 2001 From: Fernando Escolar Date: Wed, 16 Sep 2026 23:23:35 +0200 Subject: [PATCH 08/18] clarify response handling and default actions in guardrail policies Signed-off-by: Fernando Escolar --- docs/proposals/013-guardrails/proposal.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/proposals/013-guardrails/proposal.md b/docs/proposals/013-guardrails/proposal.md index fce776b294..2eef5f3d40 100644 --- a/docs/proposals/013-guardrails/proposal.md +++ b/docs/proposals/013-guardrails/proposal.md @@ -200,7 +200,7 @@ A provider error under `FailClosed` prevents unchecked traffic. A provider error ### Streaming Responses -A response cannot be safely blocked after bytes have already reached the client. When any applicable response rule can block, ext-proc must keep response-body processing buffered. Only routes without applicable blocking response rules may switch to streamed response processing. +A response cannot be safely blocked or rewritten after bytes have already reached the client. When any applicable response rule is configured, ext-proc must keep response-body processing buffered. Only routes without applicable response guardrails may switch to streamed response processing. This increases latency and memory use for guarded streaming responses. All response rules, including Monitor, use buffering so providers receive a complete semantic payload rather than partial SSE events. Each policy defaults to a 10 MiB response evaluation limit and may configure up to 50 MiB, matching the Envoy per-connection buffer ceiling. The limit is checked before provider evaluation; exceeding it follows the rule's failure mode. @@ -236,7 +236,7 @@ spec: scoreThresholdPercent: 70 ``` -Rules have stable unique names and execute in order. The initial action is `Block`. The API reserves room for additional actions without forcing all providers to support identical capabilities. +Rules have stable unique names and execute in order. The default action is `Block`; users may explicitly select `Monitor` or `Mask` when the provider supports it. The API reserves room for additional actions without forcing all providers to support identical capabilities. ### Provider Configuration @@ -371,7 +371,7 @@ The runtime emits: - `aigateway.guardrail.evaluation.count` with phase and result attributes; and - `guardrail.evaluation` span events with rule name, phase, and result. -Metric attributes intentionally exclude rule names and error text to avoid unbounded cardinality. Result values are `allowed`, `blocked`, and `error`. Provider type may be added because it is bounded, subject to consistency with the project's metric conventions. +Metric attributes intentionally exclude rule names and error text to avoid unbounded cardinality. Result values are `allowed`, `blocked`, `monitored`, `masked`, and `error`. Provider type may be added because it is bounded, subject to consistency with the project's metric conventions. The design should also expose provider-call latency in a future histogram so operators can measure the cost of each external integration. From 9dde236a5b60734cf2bd9b7033615b7c627fdee6 Mon Sep 17 00:00:00 2001 From: Fernando Escolar Date: Mon, 28 Sep 2026 12:49:25 +0200 Subject: [PATCH 09/18] add custom HTTP guardrail provider for custom content safety services Signed-off-by: Fernando Escolar --- api/v1alpha1/guardrail_policy.go | 39 +++- api/v1alpha1/zz_generated.deepcopy.go | 25 +++ api/v1beta1/guardrail_policy.go | 39 +++- api/v1beta1/zz_generated.deepcopy.go | 25 +++ docs/proposals/013-guardrails/proposal.md | 83 ++++++- examples/README.md | 2 +- examples/guardrails/README.md | 3 + examples/guardrails/providers.yaml | 21 ++ internal/controller/controller.go | 4 + internal/controller/gateway.go | 16 ++ internal/controller/guardrail_policy.go | 14 ++ internal/controller/guardrail_policy_test.go | 81 +++++++ internal/extproc/guardrails_test.go | 63 ++++++ internal/filterapi/filterconfig.go | 9 + internal/guardrails/adapters.go | 42 ++++ internal/guardrails/http.go | 128 +++++++++++ internal/guardrails/http_test.go | 202 ++++++++++++++++++ internal/guardrails/presidio.go | 27 +-- ...teway.envoyproxy.io_guardrailpolicies.yaml | 164 ++++++++++++-- site/docs/api/api.mdx | 102 ++++++++- site/docs/capabilities/security/guardrails.md | 102 ++++++++- tests/crdcel/main_test.go | 2 + .../testdata/guardrailpolicies/basic.yaml | 10 + .../guardrailpolicies/http_invalid_path.yaml | 22 ++ .../guardrailpolicies/http_mismatch.yaml | 21 ++ 25 files changed, 1186 insertions(+), 60 deletions(-) create mode 100644 internal/guardrails/http.go create mode 100644 internal/guardrails/http_test.go create mode 100644 tests/crdcel/testdata/guardrailpolicies/http_invalid_path.yaml create mode 100644 tests/crdcel/testdata/guardrailpolicies/http_mismatch.yaml diff --git a/api/v1alpha1/guardrail_policy.go b/api/v1alpha1/guardrail_policy.go index 016aca2f39..9594231897 100644 --- a/api/v1alpha1/guardrail_policy.go +++ b/api/v1alpha1/guardrail_policy.go @@ -81,15 +81,16 @@ const ( ) // GuardrailProvider describes the implementation used to evaluate a rule. -// +kubebuilder:validation:XValidation:rule="self.type != 'Regex' || (has(self.pattern) && self.pattern != ” && !has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety))",message="Regex requires pattern and no external provider configuration" -// +kubebuilder:validation:XValidation:rule="self.type != 'Presidio' || (!has(self.pattern) && has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety))",message="Presidio requires only presidio provider configuration" -// +kubebuilder:validation:XValidation:rule="self.type != 'Bedrock' || (!has(self.pattern) && has(self.bedrock) && !has(self.presidio) && !has(self.azureContentSafety))",message="Bedrock requires only bedrock provider configuration" -// +kubebuilder:validation:XValidation:rule="self.type != 'AzureContentSafety' || (!has(self.pattern) && has(self.azureContentSafety) && !has(self.presidio) && !has(self.bedrock))",message="AzureContentSafety requires only azureContentSafety provider configuration" +// +kubebuilder:validation:XValidation:rule="self.type != 'Regex' || (has(self.pattern) && size(self.pattern) > 0 && !has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety) && !has(self.http))",message="Regex requires pattern and no external provider configuration" +// +kubebuilder:validation:XValidation:rule="self.type != 'Presidio' || (!has(self.pattern) && has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety) && !has(self.http))",message="Presidio requires only presidio provider configuration" +// +kubebuilder:validation:XValidation:rule="self.type != 'Bedrock' || (!has(self.pattern) && has(self.bedrock) && !has(self.presidio) && !has(self.azureContentSafety) && !has(self.http))",message="Bedrock requires only bedrock provider configuration" +// +kubebuilder:validation:XValidation:rule="self.type != 'AzureContentSafety' || (!has(self.pattern) && has(self.azureContentSafety) && !has(self.presidio) && !has(self.bedrock) && !has(self.http))",message="AzureContentSafety requires only azureContentSafety provider configuration" +// +kubebuilder:validation:XValidation:rule="self.type != 'HTTP' || (!has(self.pattern) && has(self.http) && !has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety))",message="HTTP requires only http provider configuration" // +kubebuilder:validation:XValidation:rule="self.action != 'Mask' || self.type != 'AzureContentSafety'",message="AzureContentSafety does not support Mask" type GuardrailProvider struct { // Type identifies the guardrail implementation. // - // +kubebuilder:validation:Enum=Regex;Presidio;Bedrock;AzureContentSafety + // +kubebuilder:validation:Enum=Regex;Presidio;Bedrock;AzureContentSafety;HTTP Type GuardrailProviderType `json:"type"` // Pattern is used for deterministic regex-based evaluations. // @@ -101,7 +102,7 @@ type GuardrailProvider struct { // +kubebuilder:default=Block // +kubebuilder:validation:Enum=Block;Monitor;Mask Action GuardrailAction `json:"action,omitempty"` - // MaskReplacement is used by Regex and Presidio Mask actions. + // MaskReplacement is used by Regex, Presidio, and HTTP Mask actions. // +optional // +kubebuilder:default="[REDACTED]" MaskReplacement string `json:"maskReplacement,omitempty"` @@ -121,6 +122,10 @@ type GuardrailProvider struct { // // +optional AzureContentSafety *AzureContentSafetyGuardrailProvider `json:"azureContentSafety,omitempty"` + // HTTP configures a custom guardrail service that implements the generic guardrail HTTP contract. + // + // +optional + HTTP *HTTPGuardrailProvider `json:"http,omitempty"` // TimeoutSeconds limits each external provider evaluation. // +optional // +kubebuilder:default=10 @@ -182,6 +187,27 @@ type AzureContentSafetyGuardrailProvider struct { APIKeySecretRef *gwapiv1.SecretObjectReference `json:"apiKeySecretRef"` } +// HTTPGuardrailProvider configures calls to a custom guardrail service. +// +// The gateway sends a JSON request of the form +// {"text": "...", "context": {"stage": "input"}} to endpoint+path and expects a +// normalized response of the form +// {"action": "allow|block|modify", "findings": [{"type": "PII", "start": 0, "end": 4, "score": 0.9}], "replacement": "..."}. +// Finding offsets are Unicode code point positions in text; end is exclusive. +type HTTPGuardrailProvider struct { + // Endpoint is the base URL of the guardrail service. + // +kubebuilder:validation:Format=uri + Endpoint string `json:"endpoint"` + // Path is appended to the endpoint for each evaluation. + // +optional + // +kubebuilder:default="/analyze" + // +kubebuilder:validation:Pattern=`^/.*$` + Path string `json:"path,omitempty"` + // APIKeySecretRef optionally references a Secret whose apiKey entry is sent as a Bearer token. + // +optional + APIKeySecretRef *gwapiv1.SecretObjectReference `json:"apiKeySecretRef,omitempty"` +} + // GuardrailProviderType is the guardrail implementation. type GuardrailProviderType string @@ -190,6 +216,7 @@ const ( GuardrailProviderTypePresidio GuardrailProviderType = "Presidio" GuardrailProviderTypeBedrockGuardrails GuardrailProviderType = "Bedrock" GuardrailProviderTypeAzureContentSafety GuardrailProviderType = "AzureContentSafety" + GuardrailProviderTypeHTTP GuardrailProviderType = "HTTP" ) // GuardrailAction defines the safeguard action. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index f0e7aea7f5..1b29aa1747 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -1123,6 +1123,11 @@ func (in *GuardrailProvider) DeepCopyInto(out *GuardrailProvider) { *out = new(AzureContentSafetyGuardrailProvider) (*in).DeepCopyInto(*out) } + if in.HTTP != nil { + in, out := &in.HTTP, &out.HTTP + *out = new(HTTPGuardrailProvider) + (*in).DeepCopyInto(*out) + } if in.TimeoutSeconds != nil { in, out := &in.TimeoutSeconds, &out.TimeoutSeconds *out = new(int32) @@ -1196,6 +1201,26 @@ func (in *HTTPBodyMutation) DeepCopy() *HTTPBodyMutation { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HTTPGuardrailProvider) DeepCopyInto(out *HTTPGuardrailProvider) { + *out = *in + if in.APIKeySecretRef != nil { + in, out := &in.APIKeySecretRef, &out.APIKeySecretRef + *out = new(v1.SecretObjectReference) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HTTPGuardrailProvider. +func (in *HTTPGuardrailProvider) DeepCopy() *HTTPGuardrailProvider { + if in == nil { + return nil + } + out := new(HTTPGuardrailProvider) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HTTPHeaderMutation) DeepCopyInto(out *HTTPHeaderMutation) { *out = *in diff --git a/api/v1beta1/guardrail_policy.go b/api/v1beta1/guardrail_policy.go index 2a1c9c6274..4ab37569de 100644 --- a/api/v1beta1/guardrail_policy.go +++ b/api/v1beta1/guardrail_policy.go @@ -81,15 +81,16 @@ const ( ) // GuardrailProvider describes the implementation used to evaluate a rule. -// +kubebuilder:validation:XValidation:rule="self.type != 'Regex' || (has(self.pattern) && self.pattern != ” && !has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety))",message="Regex requires pattern and no external provider configuration" -// +kubebuilder:validation:XValidation:rule="self.type != 'Presidio' || (!has(self.pattern) && has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety))",message="Presidio requires only presidio provider configuration" -// +kubebuilder:validation:XValidation:rule="self.type != 'Bedrock' || (!has(self.pattern) && has(self.bedrock) && !has(self.presidio) && !has(self.azureContentSafety))",message="Bedrock requires only bedrock provider configuration" -// +kubebuilder:validation:XValidation:rule="self.type != 'AzureContentSafety' || (!has(self.pattern) && has(self.azureContentSafety) && !has(self.presidio) && !has(self.bedrock))",message="AzureContentSafety requires only azureContentSafety provider configuration" +// +kubebuilder:validation:XValidation:rule="self.type != 'Regex' || (has(self.pattern) && size(self.pattern) > 0 && !has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety) && !has(self.http))",message="Regex requires pattern and no external provider configuration" +// +kubebuilder:validation:XValidation:rule="self.type != 'Presidio' || (!has(self.pattern) && has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety) && !has(self.http))",message="Presidio requires only presidio provider configuration" +// +kubebuilder:validation:XValidation:rule="self.type != 'Bedrock' || (!has(self.pattern) && has(self.bedrock) && !has(self.presidio) && !has(self.azureContentSafety) && !has(self.http))",message="Bedrock requires only bedrock provider configuration" +// +kubebuilder:validation:XValidation:rule="self.type != 'AzureContentSafety' || (!has(self.pattern) && has(self.azureContentSafety) && !has(self.presidio) && !has(self.bedrock) && !has(self.http))",message="AzureContentSafety requires only azureContentSafety provider configuration" +// +kubebuilder:validation:XValidation:rule="self.type != 'HTTP' || (!has(self.pattern) && has(self.http) && !has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety))",message="HTTP requires only http provider configuration" // +kubebuilder:validation:XValidation:rule="self.action != 'Mask' || self.type != 'AzureContentSafety'",message="AzureContentSafety does not support Mask" type GuardrailProvider struct { // Type identifies the guardrail implementation. // - // +kubebuilder:validation:Enum=Regex;Presidio;Bedrock;AzureContentSafety + // +kubebuilder:validation:Enum=Regex;Presidio;Bedrock;AzureContentSafety;HTTP Type GuardrailProviderType `json:"type"` // Pattern is used for deterministic regex-based evaluations. // @@ -101,7 +102,7 @@ type GuardrailProvider struct { // +kubebuilder:default=Block // +kubebuilder:validation:Enum=Block;Monitor;Mask Action GuardrailAction `json:"action,omitempty"` - // MaskReplacement is used by Regex and Presidio Mask actions. + // MaskReplacement is used by Regex, Presidio, and HTTP Mask actions. // +optional // +kubebuilder:default="[REDACTED]" MaskReplacement string `json:"maskReplacement,omitempty"` @@ -121,6 +122,10 @@ type GuardrailProvider struct { // // +optional AzureContentSafety *AzureContentSafetyGuardrailProvider `json:"azureContentSafety,omitempty"` + // HTTP configures a custom guardrail service that implements the generic guardrail HTTP contract. + // + // +optional + HTTP *HTTPGuardrailProvider `json:"http,omitempty"` // TimeoutSeconds limits each external provider evaluation. // +optional // +kubebuilder:default=10 @@ -182,6 +187,27 @@ type AzureContentSafetyGuardrailProvider struct { APIKeySecretRef *gwapiv1.SecretObjectReference `json:"apiKeySecretRef"` } +// HTTPGuardrailProvider configures calls to a custom guardrail service. +// +// The gateway sends a JSON request of the form +// {"text": "...", "context": {"stage": "input"}} to endpoint+path and expects a +// normalized response of the form +// {"action": "allow|block|modify", "findings": [{"type": "PII", "start": 0, "end": 4, "score": 0.9}], "replacement": "..."}. +// Finding offsets are Unicode code point positions in text; end is exclusive. +type HTTPGuardrailProvider struct { + // Endpoint is the base URL of the guardrail service. + // +kubebuilder:validation:Format=uri + Endpoint string `json:"endpoint"` + // Path is appended to the endpoint for each evaluation. + // +optional + // +kubebuilder:default="/analyze" + // +kubebuilder:validation:Pattern=`^/.*$` + Path string `json:"path,omitempty"` + // APIKeySecretRef optionally references a Secret whose apiKey entry is sent as a Bearer token. + // +optional + APIKeySecretRef *gwapiv1.SecretObjectReference `json:"apiKeySecretRef,omitempty"` +} + // GuardrailProviderType is the guardrail implementation. type GuardrailProviderType string @@ -190,6 +216,7 @@ const ( GuardrailProviderTypePresidio GuardrailProviderType = "Presidio" GuardrailProviderTypeBedrockGuardrails GuardrailProviderType = "Bedrock" GuardrailProviderTypeAzureContentSafety GuardrailProviderType = "AzureContentSafety" + GuardrailProviderTypeHTTP GuardrailProviderType = "HTTP" ) // GuardrailAction defines the safeguard action. diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index d6f0dd646f..a3929fa067 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -1205,6 +1205,11 @@ func (in *GuardrailProvider) DeepCopyInto(out *GuardrailProvider) { *out = new(AzureContentSafetyGuardrailProvider) (*in).DeepCopyInto(*out) } + if in.HTTP != nil { + in, out := &in.HTTP, &out.HTTP + *out = new(HTTPGuardrailProvider) + (*in).DeepCopyInto(*out) + } if in.TimeoutSeconds != nil { in, out := &in.TimeoutSeconds, &out.TimeoutSeconds *out = new(int32) @@ -1278,6 +1283,26 @@ func (in *HTTPBodyMutation) DeepCopy() *HTTPBodyMutation { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HTTPGuardrailProvider) DeepCopyInto(out *HTTPGuardrailProvider) { + *out = *in + if in.APIKeySecretRef != nil { + in, out := &in.APIKeySecretRef, &out.APIKeySecretRef + *out = new(v1.SecretObjectReference) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HTTPGuardrailProvider. +func (in *HTTPGuardrailProvider) DeepCopy() *HTTPGuardrailProvider { + if in == nil { + return nil + } + out := new(HTTPGuardrailProvider) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HTTPHeaderMutation) DeepCopyInto(out *HTTPHeaderMutation) { *out = *in diff --git a/docs/proposals/013-guardrails/proposal.md b/docs/proposals/013-guardrails/proposal.md index 2eef5f3d40..30924cd7ba 100644 --- a/docs/proposals/013-guardrails/proposal.md +++ b/docs/proposals/013-guardrails/proposal.md @@ -27,6 +27,7 @@ - [Presidio](#presidio) - [AWS Bedrock Guardrails](#aws-bedrock-guardrails) - [Azure AI Content Safety](#azure-ai-content-safety) + - [Custom HTTP Guardrails](#custom-http-guardrails) - [Payload Extraction](#payload-extraction) - [Credentials and Security](#credentials-and-security) - [Status and Reconciliation](#status-and-reconciliation) @@ -37,6 +38,7 @@ - [Configure Guardrails Directly on AIGatewayRoute](#configure-guardrails-directly-on-aigatewayroute) - [Use BackendSecurityPolicy](#use-backendsecuritypolicy) - [Use Only Provider-Native Model Guardrails](#use-only-provider-native-model-guardrails) + - [Invoke Local Executables for Custom Guardrails](#invoke-local-executables-for-custom-guardrails) - [Implementation Plan](#implementation-plan) - [Testing Strategy](#testing-strategy) - [Current Implementation and Gaps](#current-implementation-and-gaps) @@ -44,7 +46,7 @@ ## Summary -This proposal introduces `GuardrailPolicy`, a backend-attached policy for evaluating LLM request and response content before it is sent to an AI provider or returned to a client. The policy provides a common API for local regular-expression checks and external safety providers, initially Presidio, AWS Bedrock Guardrails, and Azure AI Content Safety. +This proposal introduces `GuardrailPolicy`, a backend-attached policy for evaluating LLM request and response content before it is sent to an AI provider or returned to a client. The policy provides a common API for local regular-expression checks and external safety providers, initially Presidio, AWS Bedrock Guardrails, and Azure AI Content Safety. A generic HTTP provider defines a small, normalized contract that any custom guardrail service can implement, so providers without a native integration can be used without code changes to the gateway. The proposed implementation uses the existing Agent Router external processor. The controller resolves policies and credentials into the filter configuration, while ext-proc buffers the relevant body, extracts content, invokes the configured evaluator, and applies the selected action. @@ -71,6 +73,7 @@ A native policy should allow platform administrators to: - Support request and response evaluation. - Provide a provider-neutral runtime evaluator interface. - Initially support Regex, Presidio, AWS Bedrock Guardrails, and Azure AI Content Safety. +- Define a portable HTTP contract for custom guardrail services that are not natively supported. - Support explicit timeout and fail-open/fail-closed behavior. - Prevent partial delivery of blocked streaming responses. - Integrate with existing logs, metrics, traces, config distribution, Secret watches, and status conditions. @@ -246,6 +249,7 @@ The provider is a tagged union. Admission validation requires exactly the config - `Presidio` requires `presidio`. - `Bedrock` requires `bedrock`. - `AzureContentSafety` requires `azureContentSafety`. +- `HTTP` requires `http`. Provider-specific configuration remains nested so new provider capabilities can be added without adding unrelated fields to every rule. @@ -263,6 +267,7 @@ Mask behavior depends on provider capability: - Presidio replaces detected spans with `maskReplacement`. - Bedrock uses transformed text returned by ApplyGuardrail. - Azure Text Analysis does not return transformed content, so admission rejects `Mask` for Azure rules. +- HTTP uses the `replacement` text returned by the service, or replaces returned finding spans with `maskReplacement`. Mask is applied only to extracted text fields and never replaces unrelated model or configuration fields. Masked streaming responses remain buffered until the complete body has been evaluated and rewritten. @@ -315,6 +320,70 @@ The Built on Envoy implementation highlights that Azure has multiple distinct sa A future shape could add `azureContentSafety.check` and check-specific configuration. Per-category thresholds should also replace a single threshold when Text Analysis is selected. Until then, the implementation should be documented as Text Analysis over the configured evaluation input, not as complete Azure Content Safety feature parity. +### Custom HTTP Guardrails + +Native integrations cover common providers, but organizations frequently run in-house classifiers or services that the gateway does not support yet. The `HTTP` provider defines a small, provider-neutral interface, similar in spirit to the Presidio Analyzer API, that such services implement. It lets new guardrails be adopted without gateway changes and without the security and portability concerns of invoking arbitrary local executables. + +```yaml +provider: + type: HTTP + action: Mask + timeoutSeconds: 5 + failureMode: FailClosed + http: + endpoint: http://custom-guardrail.guardrails.svc.cluster.local:8080 + path: /analyze # default + apiKeySecretRef: # optional; sent as a bearer token + name: custom-guardrail-key +``` + +For each extracted text fragment, the gateway sends `POST {endpoint}{path}` with `Content-Type: application/json`: + +```json +{ + "text": "some user input", + "context": { + "stage": "input" + } +} +``` + +`context.stage` is `input` for `Request` rules and `output` for `Response` rules. The `context` object is extensible; services must ignore unknown fields so the gateway can add information such as the endpoint schema or message role later. + +The service returns HTTP 2xx with a normalized response: + +```json +{ + "action": "allow", + "findings": [ + { + "type": "PII", + "start": 10, + "end": 20, + "score": 0.92 + } + ] +} +``` + +| Field | Required | Description | +| ------------- | -------- | --------------------------------------------------------------------------------------------------------------------------------------- | +| `action` | yes | `allow`, `block`, or `modify` (case-insensitive). | +| `findings` | no | Detected spans. `start` and `end` are Unicode code point offsets into `text`; `end` is exclusive. `type` and `score` are informational. | +| `replacement` | no | Full replacement for `text`, used with `modify`. | + +The response is mapped to the provider-neutral evaluation result: + +- `allow`: the rule does not match. Findings are ignored. +- `block`: the rule matches. Findings, if present, are masked with `maskReplacement` so the result can also drive a `Mask` rule. +- `modify`: the rule matches. `replacement` is used when present; otherwise findings are masked with `maskReplacement`. A `modify` response with neither is an error. + +The service reports whether content violates its policy; the rule's `action` remains authoritative for enforcement. A `block` or `modify` result is therefore blocked by `Block`, recorded by `Monitor`, and rewritten by `Mask`. A `Mask` rule returns an error when the service matches content without providing a replacement or valid findings, so unmasked content is never forwarded. Invalid spans are ignored and overlapping spans are merged before masking. + +Non-2xx responses, malformed JSON, missing or unknown actions, and timeouts are provider errors governed by `failureMode`. Provider error bodies are truncated before they are logged. Findings, scores, and replacement text are never returned to downstream clients or recorded in metrics. + +Native integrations remain preferable when the provider exposes richer semantics, such as Bedrock transformed output or Azure check selection. Services can also implement this contract as a thin adapter in front of an existing provider. + ## Payload Extraction Provider callouts receive relevant user or assistant text, not an arbitrary serialized JSON envelope. Schema-aware extraction reduces false matches, removes model/configuration metadata, lowers provider cost, and avoids provider input-size limits. @@ -417,6 +486,10 @@ This makes route-level differences easy but duplicates policy across routes and Provider-native model configuration does not cover cross-provider policy, request checks independent of inference, local Presidio deployments, or consistent gateway observability. +### Invoke Local Executables for Custom Guardrails + +Running a user-supplied binary or script inside the ext-proc for each evaluation would allow arbitrary custom logic. It is rejected because it expands the ext-proc attack surface, requires distributing binaries into gateway pods, couples guardrail code to the gateway image and platform, and makes resource isolation, timeouts, and upgrades harder to control. The HTTP contract provides the same extensibility while keeping custom logic in an independently deployed and secured service. + ## Implementation Plan 1. Introduce the dual-version `GuardrailPolicy`, generated clients, CRD, and status. @@ -428,10 +501,11 @@ Provider-native model configuration does not cover cross-provider policy, reques 7. Add semantic payload extraction and schema-aware Mask mutation. 8. Add Monitor mode and deterministic multi-policy composition. 9. Add explicit request and response evaluation limits. -10. Add provider-specific Azure check selection. -11. Evaluate Bedrock API-key authentication. +10. Add the generic HTTP provider and its normalized request/response contract. +11. Add provider-specific Azure check selection. +12. Evaluate Bedrock API-key authentication. -Steps 1 through 9 describe the current implementation. Steps 10 and 11 are proposed follow-up work informed by the Built on Envoy extensions. +Steps 1 through 10 describe the current implementation. Steps 11 and 12 are proposed follow-up work informed by the Built on Envoy extensions. ## Testing Strategy @@ -439,6 +513,7 @@ Steps 1 through 9 describe the current implementation. Steps 10 and 11 are propo - Controller tests for target validation, Secret resolution and rotation, status, backend scoping, failure modes, and deletion. - Runtime tests for regex compilation and evaluator construction. - HTTP-stub tests for provider paths, payloads, headers, authentication, responses, malformed responses, and errors. +- HTTP-stub tests for the custom HTTP contract: stages, paths, bearer authentication, each action, replacement and finding masking, and invalid responses. - Testcontainers integration with the pinned official Presidio analyzer image. - Credential-gated live tests for Azure AI Content Safety and AWS Bedrock Guardrails. - Envoy dataplane tests for request blocking, response blocking, allowed traffic, and streaming mode behavior. diff --git a/examples/README.md b/examples/README.md index b4299977ed..5e5637d6a7 100644 --- a/examples/README.md +++ b/examples/README.md @@ -43,4 +43,4 @@ Example setup for comprehensive monitoring and observability with Prometheus and ### [Guardrails](./guardrails/) -Shows how to enforce request and response content policies with regex, Presidio, AWS Bedrock Guardrails, and Azure AI Content Safety. +Shows how to enforce request and response content policies with regex, Presidio, AWS Bedrock Guardrails, Azure AI Content Safety, and custom HTTP guardrail services. diff --git a/examples/guardrails/README.md b/examples/guardrails/README.md index aed1c41ba1..27a9d88e1c 100644 --- a/examples/guardrails/README.md +++ b/examples/guardrails/README.md @@ -20,5 +20,8 @@ The expected Secret keys are: - Presidio: optional `apiKey` - Azure Content Safety: required `apiKey` - AWS Bedrock Guardrails: optional `credentials` containing an AWS shared credentials file; when omitted, the ext-proc uses the standard AWS credential chain. +- Custom HTTP guardrail: optional `apiKey`, sent as a bearer token. + +The `HTTP` provider calls any service that implements the generic guardrail contract: the gateway sends `{"text": "...", "context": {"stage": "input"}}` to `POST /analyze` and expects `{"action": "allow" | "block" | "modify", "findings": [...], "replacement": "..."}`. See the [guardrails documentation](../../site/docs/capabilities/security/guardrails.md#custom-http-guardrails) for the full contract. Provider failures block traffic by default. Set `failureMode: FailOpen` on a rule to continue traffic when the external provider is unavailable. Response guardrails force buffered response processing so blocked content is not partially delivered. diff --git a/examples/guardrails/providers.yaml b/examples/guardrails/providers.yaml index fba8478bc8..55ddeaa098 100644 --- a/examples/guardrails/providers.yaml +++ b/examples/guardrails/providers.yaml @@ -11,6 +11,14 @@ type: Opaque stringData: apiKey: replace-me --- +apiVersion: v1 +kind: Secret +metadata: + name: custom-guardrail-key +type: Opaque +stringData: + apiKey: replace-me +--- apiVersion: aigateway.envoyproxy.io/v1beta1 kind: GuardrailPolicy metadata: @@ -56,3 +64,16 @@ spec: severityThreshold: 4 apiKeySecretRef: name: guardrail-provider-keys + - name: mask-with-custom-http-guardrail + phase: Request + provider: + type: HTTP + action: Mask + maskReplacement: "[REDACTED]" + timeoutSeconds: 5 + failureMode: FailClosed + http: + endpoint: http://custom-guardrail.guardrails.svc.cluster.local:8080 + path: /analyze + apiKeySecretRef: + name: custom-guardrail-key diff --git a/internal/controller/controller.go b/internal/controller/controller.go index c730d53635..463e7e7b27 100644 --- a/internal/controller/controller.go +++ b/internal/controller/controller.go @@ -575,6 +575,10 @@ func guardrailPolicySecretRefsIndexFunc(o client.Object) []string { if provider.AzureContentSafety != nil { ref = provider.AzureContentSafety.APIKeySecretRef } + case aigv1b1.GuardrailProviderTypeHTTP: + if provider.HTTP != nil { + ref = provider.HTTP.APIKeySecretRef + } } if ref != nil { namespace := policy.Namespace diff --git a/internal/controller/gateway.go b/internal/controller/gateway.go index 5d35c8bb87..d5fea63798 100644 --- a/internal/controller/gateway.go +++ b/internal/controller/gateway.go @@ -783,6 +783,22 @@ func (c *GatewayController) guardrailProviderToFilterAPI(ctx context.Context, na SeverityThreshold: config.SeverityThreshold, APIKey: apiKey, } + case aigv1b1.GuardrailProviderTypeHTTP: + if provider.HTTP == nil { + return converted, fmt.Errorf("http guardrail configuration is required") + } + config := provider.HTTP + converted.HTTP = &filterapi.HTTPGuardrailProvider{ + Endpoint: config.Endpoint, + Path: config.Path, + } + if config.APIKeySecretRef != nil { + apiKey, err := c.getGuardrailSecretData(ctx, namespace, config.APIKeySecretRef, "apiKey") + if err != nil { + return converted, err + } + converted.HTTP.APIKey = apiKey + } default: return converted, fmt.Errorf("unsupported provider type %q", provider.Type) } diff --git a/internal/controller/guardrail_policy.go b/internal/controller/guardrail_policy.go index 84517e7c37..1ce52ea904 100644 --- a/internal/controller/guardrail_policy.go +++ b/internal/controller/guardrail_policy.go @@ -10,6 +10,7 @@ import ( "fmt" "net/url" "regexp" + "strings" "github.com/go-logr/logr" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -159,6 +160,19 @@ func (c *GuardrailPolicyController) validateGuardrailProvider(ctx context.Contex return err } return c.validateGuardrailSecret(ctx, namespace, provider.AzureContentSafety.APIKeySecretRef, "apiKey") + case aigv1b1.GuardrailProviderTypeHTTP: + if provider.HTTP == nil { + return fmt.Errorf("http guardrail configuration is required") + } + if err := validateGuardrailEndpoint(provider.HTTP.Endpoint); err != nil { + return err + } + if provider.HTTP.Path != "" && !strings.HasPrefix(provider.HTTP.Path, "/") { + return fmt.Errorf("http guardrail path must start with /") + } + if provider.HTTP.APIKeySecretRef != nil { + return c.validateGuardrailSecret(ctx, namespace, provider.HTTP.APIKeySecretRef, "apiKey") + } default: return fmt.Errorf("unsupported provider type %q", provider.Type) } diff --git a/internal/controller/guardrail_policy_test.go b/internal/controller/guardrail_policy_test.go index c4d2e10ed3..84812a8ac0 100644 --- a/internal/controller/guardrail_policy_test.go +++ b/internal/controller/guardrail_policy_test.go @@ -356,3 +356,84 @@ func TestGuardrailPolicyToRuntimeIntegration(t *testing.T) { require.NoError(t, err) require.True(t, evaluation.Matched) } + +func TestGuardrailHTTPProviderToRuntimeEvaluator(t *testing.T) { + providerServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + require.Equal(t, "/v1/check", req.URL.Path) + require.Equal(t, "Bearer custom-secret", req.Header.Get("Authorization")) + _, _ = w.Write([]byte(`{"action":"block","findings":[{"type":"PII","start":0,"end":16,"score":0.92}]}`)) + })) + t.Cleanup(providerServer.Close) + + kube := fakekube.NewClientset(&corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "custom-guardrail-key", Namespace: "default"}, + Data: map[string][]byte{"apiKey": []byte("custom-secret")}, + }) + controller := &GatewayController{kube: kube} + converted, err := controller.guardrailProviderToFilterAPI(t.Context(), "default", &aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeHTTP, + HTTP: &aigv1b1.HTTPGuardrailProvider{ + Endpoint: providerServer.URL, + Path: "/v1/check", + APIKeySecretRef: &gwapiv1.SecretObjectReference{Name: "custom-guardrail-key"}, + }, + }) + require.NoError(t, err) + require.Equal(t, "custom-secret", converted.HTTP.APIKey) + runtimeConfig, err := filterapi.NewRuntimeConfig(t.Context(), &filterapi.Config{ + Guardrails: []filterapi.Guardrail{{Name: "custom", Phase: filterapi.GuardrailPhaseRequest, Provider: converted}}, + }, func(context.Context, *filterapi.BackendAuth) (filterapi.BackendAuthHandler, error) { + return nil, nil + }, guardrails.NewEvaluator) + require.NoError(t, err) + require.Len(t, runtimeConfig.Guardrails, 1) + evaluation, err := runtimeConfig.Guardrails[0].Evaluator.Evaluate(t.Context(), []byte("user@example.com"), filterapi.GuardrailPhaseRequest) + require.NoError(t, err) + require.True(t, evaluation.Matched) + require.Equal(t, "[REDACTED]", string(evaluation.Replacement)) +} + +func TestGuardrailHTTPProviderValidation(t *testing.T) { + kube := fakekube.NewClientset() + controller := &GuardrailPolicyController{kube: kube} + require.ErrorContains(t, controller.validateGuardrailProvider(t.Context(), "default", &aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeHTTP, + }), "http guardrail configuration is required") + require.ErrorContains(t, controller.validateGuardrailProvider(t.Context(), "default", &aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeHTTP, + HTTP: &aigv1b1.HTTPGuardrailProvider{Endpoint: "not-a-url"}, + }), "valid provider endpoint is required") + require.ErrorContains(t, controller.validateGuardrailProvider(t.Context(), "default", &aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeHTTP, + HTTP: &aigv1b1.HTTPGuardrailProvider{Endpoint: "https://guardrail.example.com", Path: "analyze"}, + }), "path must start with /") + require.ErrorContains(t, controller.validateGuardrailProvider(t.Context(), "default", &aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeHTTP, + HTTP: &aigv1b1.HTTPGuardrailProvider{ + Endpoint: "https://guardrail.example.com", + APIKeySecretRef: &gwapiv1.SecretObjectReference{Name: "missing"}, + }, + }), "failed to get secret missing") + require.NoError(t, controller.validateGuardrailProvider(t.Context(), "default", &aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeHTTP, + Action: aigv1b1.GuardrailActionMask, + HTTP: &aigv1b1.HTTPGuardrailProvider{Endpoint: "https://guardrail.example.com"}, + })) +} + +func TestGuardrailPolicySecretRefsIndexIncludesHTTPProvider(t *testing.T) { + policy := &aigv1b1.GuardrailPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: "custom", Namespace: "default"}, + Spec: aigv1b1.GuardrailPolicySpec{Rules: []aigv1b1.GuardrailRule{{ + Name: "custom", Phase: aigv1b1.GuardrailPhaseRequest, + Provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeHTTP, + HTTP: &aigv1b1.HTTPGuardrailProvider{ + Endpoint: "https://guardrail.example.com", + APIKeySecretRef: &gwapiv1.SecretObjectReference{Name: "custom-guardrail-key"}, + }, + }, + }}}, + } + require.Equal(t, []string{"custom-guardrail-key.default"}, guardrailPolicySecretRefsIndexFunc(policy)) +} diff --git a/internal/extproc/guardrails_test.go b/internal/extproc/guardrails_test.go index 8b5a8498b1..d6883f86f8 100644 --- a/internal/extproc/guardrails_test.go +++ b/internal/extproc/guardrails_test.go @@ -9,6 +9,8 @@ import ( "context" "errors" "log/slog" + "net/http" + "net/http/httptest" "regexp" "testing" @@ -17,6 +19,7 @@ import ( "github.com/envoyproxy/ai-gateway/internal/endpointspec" "github.com/envoyproxy/ai-gateway/internal/filterapi" + "github.com/envoyproxy/ai-gateway/internal/guardrails" "github.com/envoyproxy/ai-gateway/internal/metrics" "github.com/envoyproxy/ai-gateway/internal/tracing/tracingapi" ) @@ -242,3 +245,63 @@ func TestGuardrailPayloadLimit(t *testing.T) { _, err = evaluateRequestGuardrails(t.Context(), []filterapi.RuntimeGuardrail{guardrail}, []byte(`{"content":"secret"}`)) require.True(t, isGuardrailFailOpenError(err)) } + +func TestGuardrailHTTPProviderActions(t *testing.T) { + body := []byte(`{"model":"safe-model","messages":[{"role":"user","content":"call me at 555-0100"}]}`) + for _, tc := range []struct { + name string + response string + action filterapi.GuardrailAction + expViolation bool + expMonitored bool + expMasked bool + expBody string + expErr string + }{ + {name: "allow passes through", response: `{"action":"allow"}`, expBody: string(body)}, + {name: "block blocks", response: `{"action":"block"}`, expViolation: true, expBody: string(body)}, + {name: "block with monitor records", response: `{"action":"block"}`, action: filterapi.GuardrailActionMonitor, expMonitored: true, expBody: string(body)}, + { + name: "findings masked", + response: `{"action":"modify","findings":[{"type":"PHONE","start":11,"end":19,"score":0.9}]}`, + action: filterapi.GuardrailActionMask, + expMasked: true, + expBody: `{"model":"safe-model","messages":[{"role":"user","content":"call me at [REDACTED]"}]}`, + }, + { + name: "replacement applied", + response: `{"action":"modify","replacement":"call me later"}`, + action: filterapi.GuardrailActionMask, + expMasked: true, + expBody: `{"model":"safe-model","messages":[{"role":"user","content":"call me later"}]}`, + }, + {name: "block without findings cannot mask", response: `{"action":"block"}`, action: filterapi.GuardrailActionMask, expErr: "returned no masked content"}, + } { + t.Run(tc.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(tc.response)) + })) + t.Cleanup(server.Close) + provider := filterapi.GuardrailProvider{ + Type: filterapi.GuardrailProviderTypeHTTP, + Action: tc.action, + HTTP: &filterapi.HTTPGuardrailProvider{Endpoint: server.URL}, + } + evaluator, err := guardrails.NewEvaluator(t.Context(), &provider) + require.NoError(t, err) + + outcome, err := evaluateRequestGuardrails(t.Context(), []filterapi.RuntimeGuardrail{{ + Name: "custom", Phase: filterapi.GuardrailPhaseRequest, Provider: provider, Evaluator: evaluator, + }}, body) + if tc.expErr != "" { + require.ErrorContains(t, err, tc.expErr) + return + } + require.NoError(t, err) + require.Equal(t, tc.expViolation, outcome.Violation != nil) + require.Equal(t, tc.expMonitored, outcome.Monitored) + require.Equal(t, tc.expMasked, outcome.Masked) + require.JSONEq(t, tc.expBody, string(outcome.Body)) + }) + } +} diff --git a/internal/filterapi/filterconfig.go b/internal/filterapi/filterconfig.go index b30961cfbb..c160f0f7d4 100644 --- a/internal/filterapi/filterconfig.go +++ b/internal/filterapi/filterconfig.go @@ -85,6 +85,7 @@ type GuardrailProvider struct { Presidio *PresidioGuardrailProvider `json:"presidio,omitempty"` Bedrock *BedrockGuardrailProvider `json:"bedrock,omitempty"` AzureContentSafety *AzureContentSafetyGuardrailProvider `json:"azureContentSafety,omitempty"` + HTTP *HTTPGuardrailProvider `json:"http,omitempty"` TimeoutSeconds int32 `json:"timeoutSeconds,omitempty"` FailureMode GuardrailFailureMode `json:"failureMode,omitempty"` } @@ -122,6 +123,13 @@ type AzureContentSafetyGuardrailProvider struct { APIKey string `json:"apiKey"` } +// HTTPGuardrailProvider configures a request to a custom guardrail service implementing the generic HTTP contract. +type HTTPGuardrailProvider struct { + Endpoint string `json:"endpoint"` + Path string `json:"path,omitempty"` + APIKey string `json:"apiKey,omitempty"` +} + // GuardrailProviderType identifies a guardrail implementation. type GuardrailProviderType string @@ -130,6 +138,7 @@ const ( GuardrailProviderTypePresidio GuardrailProviderType = "Presidio" GuardrailProviderTypeBedrockGuardrails GuardrailProviderType = "Bedrock" GuardrailProviderTypeAzureContentSafety GuardrailProviderType = "AzureContentSafety" + GuardrailProviderTypeHTTP GuardrailProviderType = "HTTP" ) // GuardrailAction is the action taken when the rule matches. diff --git a/internal/guardrails/adapters.go b/internal/guardrails/adapters.go index 017ebf38f6..3040049d77 100644 --- a/internal/guardrails/adapters.go +++ b/internal/guardrails/adapters.go @@ -12,6 +12,7 @@ import ( "fmt" "io" "net/http" + "sort" "strings" "time" @@ -31,6 +32,8 @@ func NewEvaluator(ctx context.Context, provider *filterapi.GuardrailProvider) (f return newBedrockEvaluator(ctx, provider.Bedrock, client) case filterapi.GuardrailProviderTypeAzureContentSafety: return newAzureContentSafetyEvaluator(provider.AzureContentSafety, client) + case filterapi.GuardrailProviderTypeHTTP: + return newHTTPEvaluator(provider.HTTP, provider.MaskReplacement, client) default: return nil, fmt.Errorf("unsupported external guardrail provider %q", provider.Type) } @@ -74,3 +77,42 @@ func sendJSON(req *http.Request, client *http.Client, response any) error { } return nil } + +// textSpan is a detected region of evaluated text. Offsets are Unicode code point +// positions and End is exclusive. +type textSpan struct { + Start int `json:"start"` + End int `json:"end"` + Score float64 `json:"score"` +} + +// maskSpans replaces each valid span in body with replacement. Invalid spans are +// ignored and overlapping or adjacent spans are merged into a single replacement. +// It returns nil when no span is valid so Mask rules never forward unmasked content. +func maskSpans(body []byte, spans []textSpan, replacement string) []byte { + runes := []rune(string(body)) + valid := make([]textSpan, 0, len(spans)) + for _, span := range spans { + if span.Start < 0 || span.End > len(runes) || span.Start >= span.End { + continue + } + valid = append(valid, span) + } + if len(valid) == 0 { + return nil + } + sort.Slice(valid, func(i, j int) bool { return valid[i].Start < valid[j].Start }) + merged := valid[:0] + for _, span := range valid { + if n := len(merged); n > 0 && span.Start <= merged[n-1].End { + merged[n-1].End = max(merged[n-1].End, span.End) + continue + } + merged = append(merged, span) + } + for i := len(merged) - 1; i >= 0; i-- { + span := merged[i] + runes = append(runes[:span.Start], append([]rune(replacement), runes[span.End:]...)...) + } + return []byte(string(runes)) +} diff --git a/internal/guardrails/http.go b/internal/guardrails/http.go new file mode 100644 index 0000000000..2d8b8753be --- /dev/null +++ b/internal/guardrails/http.go @@ -0,0 +1,128 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package guardrails + +import ( + "context" + "fmt" + "net/http" + "strings" + + "github.com/envoyproxy/ai-gateway/internal/filterapi" +) + +const defaultHTTPGuardrailPath = "/analyze" + +// HTTP guardrail contract stages sent in the request context. +const ( + httpGuardrailStageInput = "input" + httpGuardrailStageOutput = "output" +) + +// HTTP guardrail contract actions returned by the custom service. +const ( + httpGuardrailActionAllow = "allow" + httpGuardrailActionBlock = "block" + httpGuardrailActionModify = "modify" +) + +// httpGuardrailRequest is the payload sent to a custom guardrail service. +type httpGuardrailRequest struct { + Text string `json:"text"` + Context httpGuardrailRequestContext `json:"context"` +} + +type httpGuardrailRequestContext struct { + Stage string `json:"stage"` +} + +// httpGuardrailResponse is the normalized response returned by a custom guardrail service. +type httpGuardrailResponse struct { + Action string `json:"action"` + Findings []httpGuardrailFinding `json:"findings,omitempty"` + Replacement *string `json:"replacement,omitempty"` +} + +type httpGuardrailFinding struct { + Type string `json:"type,omitempty"` + textSpan +} + +type httpEvaluator struct { + url string + apiKey string + maskReplacement string + client *http.Client +} + +func newHTTPEvaluator(config *filterapi.HTTPGuardrailProvider, maskReplacement string, client *http.Client) (filterapi.GuardrailEvaluator, error) { + if config == nil || config.Endpoint == "" { + return nil, fmt.Errorf("http guardrail endpoint is required") + } + path := config.Path + if path == "" { + path = defaultHTTPGuardrailPath + } + if !strings.HasPrefix(path, "/") { + return nil, fmt.Errorf("http guardrail path must start with /") + } + if maskReplacement == "" { + maskReplacement = "[REDACTED]" + } + return &httpEvaluator{ + url: strings.TrimRight(config.Endpoint, "/") + path, + apiKey: config.APIKey, + maskReplacement: maskReplacement, + client: client, + }, nil +} + +func (e *httpEvaluator) Evaluate(ctx context.Context, body []byte, phase filterapi.GuardrailPhase) (filterapi.GuardrailEvaluationResult, error) { + stage := httpGuardrailStageInput + if phase == filterapi.GuardrailPhaseResponse { + stage = httpGuardrailStageOutput + } + payload := httpGuardrailRequest{Text: string(body), Context: httpGuardrailRequestContext{Stage: stage}} + + var result httpGuardrailResponse + if err := doJSON(ctx, e.client, http.MethodPost, e.url, payload, func(req *http.Request) { + if e.apiKey != "" { + req.Header.Set("Authorization", "Bearer "+e.apiKey) + } + }, &result); err != nil { + return filterapi.GuardrailEvaluationResult{}, fmt.Errorf("http guardrail analyze request failed: %w", err) + } + + switch strings.ToLower(result.Action) { + case httpGuardrailActionAllow: + return filterapi.GuardrailEvaluationResult{}, nil + case httpGuardrailActionBlock: + return filterapi.GuardrailEvaluationResult{Matched: true, Replacement: e.maskFindings(body, result.Findings)}, nil + case httpGuardrailActionModify: + if result.Replacement != nil { + return filterapi.GuardrailEvaluationResult{Matched: true, Replacement: []byte(*result.Replacement)}, nil + } + if len(result.Findings) == 0 { + return filterapi.GuardrailEvaluationResult{}, fmt.Errorf("http guardrail returned modify without replacement or findings") + } + return filterapi.GuardrailEvaluationResult{Matched: true, Replacement: e.maskFindings(body, result.Findings)}, nil + default: + return filterapi.GuardrailEvaluationResult{}, fmt.Errorf("http guardrail returned unsupported action %q", result.Action) + } +} + +// maskFindings masks the finding spans in body. It returns nil when there are no findings, +// so Mask rules fail instead of forwarding unchanged content. +func (e *httpEvaluator) maskFindings(body []byte, findings []httpGuardrailFinding) []byte { + if len(findings) == 0 { + return nil + } + spans := make([]textSpan, len(findings)) + for i := range findings { + spans[i] = findings[i].textSpan + } + return maskSpans(body, spans, e.maskReplacement) +} diff --git a/internal/guardrails/http_test.go b/internal/guardrails/http_test.go new file mode 100644 index 0000000000..cdb8685a23 --- /dev/null +++ b/internal/guardrails/http_test.go @@ -0,0 +1,202 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package guardrails + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/envoyproxy/ai-gateway/internal/filterapi" + "github.com/envoyproxy/ai-gateway/internal/json" +) + +func newHTTPGuardrailTestServer(t *testing.T, response string, check func(*http.Request, httpGuardrailRequest)) *httptest.Server { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + require.Equal(t, http.MethodPost, req.Method) + require.Equal(t, "application/json", req.Header.Get("Content-Type")) + var body httpGuardrailRequest + require.NoError(t, json.NewDecoder(req.Body).Decode(&body)) + if check != nil { + check(req, body) + } + _, _ = w.Write([]byte(response)) + })) + t.Cleanup(server.Close) + return server +} + +func TestHTTPEvaluatorRequestContract(t *testing.T) { + for _, tc := range []struct { + name string + path string + apiKey string + phase filterapi.GuardrailPhase + expPath string + expStage string + expHeader string + }{ + {name: "default path request", phase: filterapi.GuardrailPhaseRequest, expPath: "/analyze", expStage: "input"}, + {name: "custom path response with api key", path: "/v1/check", apiKey: "secret", phase: filterapi.GuardrailPhaseResponse, expPath: "/v1/check", expStage: "output", expHeader: "Bearer secret"}, + } { + t.Run(tc.name, func(t *testing.T) { + server := newHTTPGuardrailTestServer(t, `{"action":"allow"}`, func(req *http.Request, body httpGuardrailRequest) { + require.Equal(t, tc.expPath, req.URL.Path) + require.Equal(t, tc.expHeader, req.Header.Get("Authorization")) + require.Equal(t, "some user input", body.Text) + require.Equal(t, tc.expStage, body.Context.Stage) + }) + evaluator, err := newHTTPEvaluator(&filterapi.HTTPGuardrailProvider{ + Endpoint: server.URL + "/", Path: tc.path, APIKey: tc.apiKey, + }, "", server.Client()) + require.NoError(t, err) + evaluation, err := evaluator.Evaluate(t.Context(), []byte("some user input"), tc.phase) + require.NoError(t, err) + require.False(t, evaluation.Matched) + require.Nil(t, evaluation.Replacement) + }) + } +} + +func TestHTTPEvaluatorResponseContract(t *testing.T) { + const text = "my email is john@example.com ok" + for _, tc := range []struct { + name string + response string + expMatched bool + expReplacement string + expErr string + }{ + { + name: "allow ignores findings", + response: `{"action":"allow","findings":[{"type":"PII","start":12,"end":28,"score":0.92}]}`, + }, + { + name: "block without findings", + response: `{"action":"block"}`, + expMatched: true, + }, + { + name: "block with findings provides mask", + response: `{"action":"block","findings":[{"type":"EMAIL","start":12,"end":28,"score":0.92}]}`, + expMatched: true, + expReplacement: "my email is [MASKED] ok", + }, + { + name: "modify with replacement", + response: `{"action":"modify","replacement":"my email is ok"}`, + expMatched: true, + expReplacement: "my email is ok", + }, + { + name: "modify with findings only", + response: `{"action":"modify","findings":[{"type":"EMAIL","start":12,"end":28}]}`, + expMatched: true, + expReplacement: "my email is [MASKED] ok", + }, + { + name: "action is case insensitive", + response: `{"action":"MODIFY","findings":[{"start":0,"end":2}]}`, + expMatched: true, + expReplacement: "[MASKED] email is john@example.com ok", + }, + { + name: "modify without replacement or findings", + response: `{"action":"modify"}`, + expErr: "modify without replacement or findings", + }, + { + name: "unknown action", + response: `{"action":"quarantine"}`, + expErr: `unsupported action "quarantine"`, + }, + { + name: "missing action", + response: `{"findings":[]}`, + expErr: `unsupported action ""`, + }, + { + name: "malformed response", + response: `[1,2,3]`, + expErr: "cannot decode provider response", + }, + } { + t.Run(tc.name, func(t *testing.T) { + server := newHTTPGuardrailTestServer(t, tc.response, nil) + evaluator, err := newHTTPEvaluator(&filterapi.HTTPGuardrailProvider{Endpoint: server.URL}, "[MASKED]", server.Client()) + require.NoError(t, err) + evaluation, err := evaluator.Evaluate(t.Context(), []byte(text), filterapi.GuardrailPhaseRequest) + if tc.expErr != "" { + require.ErrorContains(t, err, tc.expErr) + require.False(t, evaluation.Matched) + return + } + require.NoError(t, err) + require.Equal(t, tc.expMatched, evaluation.Matched) + require.Equal(t, tc.expReplacement, string(evaluation.Replacement)) + }) + } +} + +func TestHTTPEvaluatorProviderError(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "guardrail unavailable", http.StatusBadGateway) + })) + t.Cleanup(server.Close) + + evaluator, err := newHTTPEvaluator(&filterapi.HTTPGuardrailProvider{Endpoint: server.URL}, "", server.Client()) + require.NoError(t, err) + evaluation, err := evaluator.Evaluate(t.Context(), []byte("payload"), filterapi.GuardrailPhaseRequest) + require.False(t, evaluation.Matched) + require.ErrorContains(t, err, "HTTP 502") + require.ErrorContains(t, err, "guardrail unavailable") +} + +func TestNewHTTPEvaluatorValidation(t *testing.T) { + _, err := newHTTPEvaluator(nil, "", http.DefaultClient) + require.ErrorContains(t, err, "endpoint is required") + _, err = newHTTPEvaluator(&filterapi.HTTPGuardrailProvider{Endpoint: "https://guardrail.example.com", Path: "analyze"}, "", http.DefaultClient) + require.ErrorContains(t, err, "path must start with /") +} + +func TestNewEvaluatorHTTPProvider(t *testing.T) { + evaluator, err := NewEvaluator(t.Context(), &filterapi.GuardrailProvider{ + Type: filterapi.GuardrailProviderTypeHTTP, + TimeoutSeconds: 2, + HTTP: &filterapi.HTTPGuardrailProvider{Endpoint: "https://guardrail.example.com/", Path: "/v1/analyze"}, + }) + require.NoError(t, err) + httpEval := evaluator.(*httpEvaluator) + require.Equal(t, "https://guardrail.example.com/v1/analyze", httpEval.url) + require.Equal(t, "[REDACTED]", httpEval.maskReplacement) + require.Equal(t, 2*time.Second, httpEval.client.Timeout) +} + +func TestMaskSpans(t *testing.T) { + for _, tc := range []struct { + name string + body string + spans []textSpan + exp string + }{ + {name: "single", body: "abcdef", spans: []textSpan{{Start: 1, End: 3}}, exp: "a#def"}, + {name: "unsorted", body: "abcdef", spans: []textSpan{{Start: 4, End: 5}, {Start: 0, End: 1}}, exp: "#bcd#f"}, + {name: "overlapping merged", body: "abcdef", spans: []textSpan{{Start: 1, End: 4}, {Start: 2, End: 5}}, exp: "a#f"}, + {name: "adjacent merged", body: "abcdef", spans: []textSpan{{Start: 1, End: 2}, {Start: 2, End: 3}}, exp: "a#def"}, + {name: "unicode offsets", body: "héllo wörld", spans: []textSpan{{Start: 6, End: 11}}, exp: "héllo #"}, + {name: "invalid spans ignored", body: "abcdef", spans: []textSpan{{Start: -1, End: 2}, {Start: 2, End: 99}, {Start: 3, End: 3}, {Start: 0, End: 1}}, exp: "#bcdef"}, + } { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.exp, string(maskSpans([]byte(tc.body), tc.spans, "#"))) + }) + } + require.Nil(t, maskSpans([]byte("abc"), []textSpan{{Start: 5, End: 6}}, "#")) + require.Nil(t, maskSpans([]byte("abc"), nil, "#")) +} diff --git a/internal/guardrails/presidio.go b/internal/guardrails/presidio.go index 2b1253ab2a..0830d800c3 100644 --- a/internal/guardrails/presidio.go +++ b/internal/guardrails/presidio.go @@ -9,7 +9,6 @@ import ( "context" "fmt" "net/http" - "sort" "strings" "github.com/envoyproxy/ai-gateway/internal/filterapi" @@ -46,11 +45,7 @@ func (e *presidioEvaluator) Evaluate(ctx context.Context, body []byte, _ filtera payload.ScoreThreshold = &threshold } - var result []struct { - Start int `json:"start"` - End int `json:"end"` - Score float64 `json:"score"` - } + var result []textSpan if err := doJSON(ctx, e.client, http.MethodPost, strings.TrimRight(e.config.Endpoint, "/")+"/analyze", payload, func(req *http.Request) { if e.config.APIKey != "" { req.Header.Set("Authorization", "Bearer "+e.config.APIKey) @@ -63,24 +58,6 @@ func (e *presidioEvaluator) Evaluate(ctx context.Context, body []byte, _ filtera } return filterapi.GuardrailEvaluationResult{ Matched: true, - Replacement: maskPresidioMatches(body, result, e.maskReplacement), + Replacement: maskSpans(body, result, e.maskReplacement), }, nil } - -func maskPresidioMatches(body []byte, matches []struct { - Start int `json:"start"` - End int `json:"end"` - Score float64 `json:"score"` -}, replacement string, -) []byte { - sort.Slice(matches, func(i, j int) bool { return matches[i].Start < matches[j].Start }) - runes := []rune(string(body)) - for i := len(matches) - 1; i >= 0; i-- { - match := matches[i] - if match.Start < 0 || match.End > len(runes) || match.Start >= match.End { - continue - } - runes = append(runes[:match.Start], append([]rune(replacement), runes[match.End:]...)...) - } - return []byte(string(runes)) -} diff --git a/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_guardrailpolicies.yaml b/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_guardrailpolicies.yaml index 328af8ae34..c5c7ddfbf7 100644 --- a/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_guardrailpolicies.yaml +++ b/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_guardrailpolicies.yaml @@ -238,10 +238,71 @@ spec: - FailClosed - FailOpen type: string + http: + description: HTTP configures a custom guardrail service + that implements the generic guardrail HTTP contract. + properties: + apiKeySecretRef: + description: APIKeySecretRef optionally references a + Secret whose apiKey entry is sent as a Bearer token. + properties: + group: + default: "" + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Secret + description: Kind is kind of the referent. For example + "Secret". + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the referenced object. When unspecified, the local + namespace is inferred. + + Note that when a namespace different than the local namespace is specified, + a ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - name + type: object + endpoint: + description: Endpoint is the base URL of the guardrail + service. + format: uri + type: string + path: + default: /analyze + description: Path is appended to the endpoint for each + evaluation. + pattern: ^/.*$ + type: string + required: + - endpoint + type: object maskReplacement: default: '[REDACTED]' - description: MaskReplacement is used by Regex and Presidio - Mask actions. + description: MaskReplacement is used by Regex, Presidio, + and HTTP Mask actions. type: string message: description: Message is returned to the caller when the @@ -326,25 +387,32 @@ spec: - Presidio - Bedrock - AzureContentSafety + - HTTP type: string required: - type type: object x-kubernetes-validations: - message: Regex requires pattern and no external provider configuration - rule: self.type != 'Regex' || (has(self.pattern) && self.pattern - != ” && !has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety)) + rule: self.type != 'Regex' || (has(self.pattern) && size(self.pattern) + > 0 && !has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety) + && !has(self.http)) - message: Presidio requires only presidio provider configuration rule: self.type != 'Presidio' || (!has(self.pattern) && has(self.presidio) - && !has(self.bedrock) && !has(self.azureContentSafety)) + && !has(self.bedrock) && !has(self.azureContentSafety) && + !has(self.http)) - message: Bedrock requires only bedrock provider configuration rule: self.type != 'Bedrock' || (!has(self.pattern) && has(self.bedrock) - && !has(self.presidio) && !has(self.azureContentSafety)) + && !has(self.presidio) && !has(self.azureContentSafety) + && !has(self.http)) - message: AzureContentSafety requires only azureContentSafety provider configuration rule: self.type != 'AzureContentSafety' || (!has(self.pattern) && has(self.azureContentSafety) && !has(self.presidio) && - !has(self.bedrock)) + !has(self.bedrock) && !has(self.http)) + - message: HTTP requires only http provider configuration + rule: self.type != 'HTTP' || (!has(self.pattern) && has(self.http) + && !has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety)) - message: AzureContentSafety does not support Mask rule: self.action != 'Mask' || self.type != 'AzureContentSafety' required: @@ -682,10 +750,71 @@ spec: - FailClosed - FailOpen type: string + http: + description: HTTP configures a custom guardrail service + that implements the generic guardrail HTTP contract. + properties: + apiKeySecretRef: + description: APIKeySecretRef optionally references a + Secret whose apiKey entry is sent as a Bearer token. + properties: + group: + default: "" + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Secret + description: Kind is kind of the referent. For example + "Secret". + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the referenced object. When unspecified, the local + namespace is inferred. + + Note that when a namespace different than the local namespace is specified, + a ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - name + type: object + endpoint: + description: Endpoint is the base URL of the guardrail + service. + format: uri + type: string + path: + default: /analyze + description: Path is appended to the endpoint for each + evaluation. + pattern: ^/.*$ + type: string + required: + - endpoint + type: object maskReplacement: default: '[REDACTED]' - description: MaskReplacement is used by Regex and Presidio - Mask actions. + description: MaskReplacement is used by Regex, Presidio, + and HTTP Mask actions. type: string message: description: Message is returned to the caller when the @@ -770,25 +899,32 @@ spec: - Presidio - Bedrock - AzureContentSafety + - HTTP type: string required: - type type: object x-kubernetes-validations: - message: Regex requires pattern and no external provider configuration - rule: self.type != 'Regex' || (has(self.pattern) && self.pattern - != ” && !has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety)) + rule: self.type != 'Regex' || (has(self.pattern) && size(self.pattern) + > 0 && !has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety) + && !has(self.http)) - message: Presidio requires only presidio provider configuration rule: self.type != 'Presidio' || (!has(self.pattern) && has(self.presidio) - && !has(self.bedrock) && !has(self.azureContentSafety)) + && !has(self.bedrock) && !has(self.azureContentSafety) && + !has(self.http)) - message: Bedrock requires only bedrock provider configuration rule: self.type != 'Bedrock' || (!has(self.pattern) && has(self.bedrock) - && !has(self.presidio) && !has(self.azureContentSafety)) + && !has(self.presidio) && !has(self.azureContentSafety) + && !has(self.http)) - message: AzureContentSafety requires only azureContentSafety provider configuration rule: self.type != 'AzureContentSafety' || (!has(self.pattern) && has(self.azureContentSafety) && !has(self.presidio) && - !has(self.bedrock)) + !has(self.bedrock) && !has(self.http)) + - message: HTTP requires only http provider configuration + rule: self.type != 'HTTP' || (!has(self.pattern) && has(self.http) + && !has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety)) - message: AzureContentSafety does not support Mask rule: self.action != 'Mask' || self.type != 'AzureContentSafety' required: diff --git a/site/docs/api/api.mdx b/site/docs/api/api.mdx index 55cf21aaea..94c086cd4f 100644 --- a/site/docs/api/api.mdx +++ b/site/docs/api/api.mdx @@ -681,6 +681,7 @@ QuotaPolicyList contains a list of QuotaPolicy - [GuardrailRule](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailrule) - [HTTPBodyField](#github-com-envoyproxy-ai-gateway-api-v1alpha1-httpbodyfield) - [HTTPBodyMutation](#github-com-envoyproxy-ai-gateway-api-v1alpha1-httpbodymutation) +- [HTTPGuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1alpha1-httpguardrailprovider) - [HTTPHeaderMutation](#github-com-envoyproxy-ai-gateway-api-v1alpha1-httpheadermutation) - [JWKS](#github-com-envoyproxy-ai-gateway-api-v1alpha1-jwks) - [JWTSource](#github-com-envoyproxy-ai-gateway-api-v1alpha1-jwtsource) @@ -1930,7 +1931,7 @@ GuardrailProvider describes the implementation used to evaluate a rule. type="string" required="false" defaultValue="[REDACTED]" - description="MaskReplacement is used by Regex and Presidio Mask actions." + description="MaskReplacement is used by Regex, Presidio, and HTTP Mask actions." /> #### GuardrailRule @@ -2084,6 +2095,44 @@ HTTPBodyMutation defines the mutation of HTTP request body JSON fields that will /> +#### HTTPGuardrailProvider + + + +**Appears in:** +- [GuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailprovider) + +HTTPGuardrailProvider configures calls to a custom guardrail service. + +The gateway sends a JSON request of the form +{"text": "...", "context": {"stage": "input"}} to endpoint+path and expects a +normalized response of the form +{"action": "allow|block|modify", "findings": [{"type": "PII", "start": 0, "end": 4, "score": 0.9}], "replacement": "..."}. +Finding offsets are Unicode code point positions in text; end is exclusive. + +##### Fields + + + + + + #### HTTPHeaderMutation @@ -3742,6 +3791,7 @@ MCPRouteList contains a list of MCPRoute. - [GuardrailRule](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailrule) - [HTTPBodyField](#github-com-envoyproxy-ai-gateway-api-v1beta1-httpbodyfield) - [HTTPBodyMutation](#github-com-envoyproxy-ai-gateway-api-v1beta1-httpbodymutation) +- [HTTPGuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1beta1-httpguardrailprovider) - [HTTPHeaderMutation](#github-com-envoyproxy-ai-gateway-api-v1beta1-httpheadermutation) - [HTTPHeaderValueFilter](#github-com-envoyproxy-ai-gateway-api-v1beta1-httpheadervaluefilter) - [HTTPHeaderValueFilterMode](#github-com-envoyproxy-ai-gateway-api-v1beta1-httpheadervaluefiltermode) @@ -5096,7 +5146,7 @@ GuardrailProvider describes the implementation used to evaluate a rule. type="string" required="false" defaultValue="[REDACTED]" - description="MaskReplacement is used by Regex and Presidio Mask actions." + description="MaskReplacement is used by Regex, Presidio, and HTTP Mask actions." /> #### GuardrailRule @@ -5250,6 +5310,44 @@ HTTPBodyMutation defines the mutation of HTTP request body JSON fields that will /> +#### HTTPGuardrailProvider + + + +**Appears in:** +- [GuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailprovider) + +HTTPGuardrailProvider configures calls to a custom guardrail service. + +The gateway sends a JSON request of the form +{"text": "...", "context": {"stage": "input"}} to endpoint+path and expects a +normalized response of the form +{"action": "allow|block|modify", "findings": [{"type": "PII", "start": 0, "end": 4, "score": 0.9}], "replacement": "..."}. +Finding offsets are Unicode code point positions in text; end is exclusive. + +##### Fields + + + + + + #### HTTPHeaderMutation diff --git a/site/docs/capabilities/security/guardrails.md b/site/docs/capabilities/security/guardrails.md index 64902f04b2..9c7fea8b03 100644 --- a/site/docs/capabilities/security/guardrails.md +++ b/site/docs/capabilities/security/guardrails.md @@ -6,7 +6,7 @@ sidebar_position: 9 # Content Guardrails -`GuardrailPolicy` evaluates request or response payloads for an `AIServiceBackend`. Rules can use local regular expressions or external content-safety providers. +`GuardrailPolicy` evaluates request or response payloads for an `AIServiceBackend`. Rules can use local regular expressions, native integrations with external content-safety providers, or any custom guardrail service that implements a small HTTP contract. ## Apply a regex guardrail @@ -38,7 +38,7 @@ Rules support three actions: - `Block` rejects matching traffic. - `Monitor` records matching traffic without blocking or changing it. -- `Mask` replaces detected text. Regex and Presidio use `maskReplacement`; Bedrock uses transformed output returned by the provider. Azure Text Analysis does not support Mask. +- `Mask` replaces detected text. Regex and Presidio use `maskReplacement`; Bedrock uses transformed output returned by the provider; HTTP uses the returned `replacement` or masks returned findings with `maskReplacement`. Azure Text Analysis does not support Mask. When multiple policies target one backend, policies are evaluated in namespace/name order and rules retain declaration order. The first Block result stops evaluation. @@ -93,6 +93,104 @@ provider: The referenced Secret must contain an `apiKey` entry. +### Custom HTTP guardrails + +Use the `HTTP` provider to integrate a guardrail service that has no native integration, such as an in-house classifier. The gateway calls the service over HTTP instead of running local executables. + +```yaml +provider: + type: HTTP + action: Mask + timeoutSeconds: 5 + failureMode: FailClosed + http: + endpoint: http://custom-guardrail.guardrails.svc.cluster.local:8080 + path: /analyze + apiKeySecretRef: + name: custom-guardrail-key +``` + +`path` defaults to `/analyze`. When `apiKeySecretRef` is set, the Secret must contain an `apiKey` entry, sent as a bearer token. + +#### Request + +For each extracted text fragment, the gateway sends `POST {endpoint}{path}`: + +```json +{ + "text": "some user input", + "context": { + "stage": "input" + } +} +``` + +`context.stage` is `input` for `Request` rules and `output` for `Response` rules. Services should ignore unknown fields, because more context may be added later. + +#### Response + +The service must return HTTP 2xx with: + +```json +{ + "action": "allow", + "findings": [ + { + "type": "PII", + "start": 10, + "end": 20, + "score": 0.92 + } + ] +} +``` + +| Field | Required | Description | +| ------------- | -------- | ----------------------------------------------------------------------------------------------------------------------------------------- | +| `action` | yes | `allow`, `block`, or `modify`. | +| `findings` | no | Detected spans. `start` and `end` are Unicode code point offsets into `text`, with `end` exclusive. `type` and `score` are informational. | +| `replacement` | no | Replacement text for the whole fragment, used with `modify`. | + +The gateway interprets the response as follows: + +| Service `action` | Result | +| ---------------- | ----------------------------------------------------------------------------------------------------------- | +| `allow` | The rule does not match. | +| `block` | The rule matches. Findings, if present, are masked with `maskReplacement` for `Mask` rules. | +| `modify` | The rule matches. `replacement` is used when present; otherwise findings are masked with `maskReplacement`. | + +The rule's `action` decides what happens on a match: `Block` rejects the request or response, `Monitor` records it, and `Mask` rewrites the fragment. A `Mask` rule fails when the service reports a match without a replacement or valid findings, so unmasked content is never forwarded. + +Non-2xx responses, invalid JSON, and missing or unknown actions are provider errors handled by `failureMode`. + +A minimal service that blocks prompts containing a keyword could look like this: + +```python +from fastapi import FastAPI +from pydantic import BaseModel + +app = FastAPI() + +class Context(BaseModel): + stage: str + +class Request(BaseModel): + text: str + context: Context + +@app.post("/analyze") +def analyze(req: Request): + start = req.text.lower().find("confidential") + if start < 0: + return {"action": "allow"} + return { + "action": "block", + "findings": [{"type": "KEYWORD", "start": start, "end": start + len("confidential"), "score": 1.0}], + } +``` + +Python string indexes are Unicode code points, which matches the offsets expected by the gateway. + ## Failure behavior External providers default to `failureMode: FailClosed`. Provider errors, missing Secrets, and revoked credentials prevent unchecked traffic. Use `FailOpen` only when availability is more important than enforcement: diff --git a/tests/crdcel/main_test.go b/tests/crdcel/main_test.go index 45f21e60b5..2a79bd8df6 100644 --- a/tests/crdcel/main_test.go +++ b/tests/crdcel/main_test.go @@ -175,6 +175,8 @@ func TestGuardrailPolicies(t *testing.T) { {name: "invalid_endpoint.yaml", expErr: "Invalid value"}, {name: "duplicate_rule_names.yaml", expErr: "rule name must be unique within the policy"}, {name: "azure_mask.yaml", expErr: "AzureContentSafety does not support Mask"}, + {name: "http_mismatch.yaml", expErr: "HTTP requires only http provider configuration"}, + {name: "http_invalid_path.yaml", expErr: "should match '^/.*$'"}, } { t.Run(tc.name, func(t *testing.T) { data, err := testdata.ReadFile(path.Join("testdata/guardrailpolicies", tc.name)) diff --git a/tests/crdcel/testdata/guardrailpolicies/basic.yaml b/tests/crdcel/testdata/guardrailpolicies/basic.yaml index dd2f427aba..fb589a3db8 100644 --- a/tests/crdcel/testdata/guardrailpolicies/basic.yaml +++ b/tests/crdcel/testdata/guardrailpolicies/basic.yaml @@ -52,3 +52,13 @@ spec: endpoint: https://content-safety.example.com apiKeySecretRef: name: azure-key + - name: custom-http + phase: Request + provider: + type: HTTP + action: Mask + http: + endpoint: https://guardrail.example.com + path: /v1/analyze + apiKeySecretRef: + name: custom-guardrail-key diff --git a/tests/crdcel/testdata/guardrailpolicies/http_invalid_path.yaml b/tests/crdcel/testdata/guardrailpolicies/http_invalid_path.yaml new file mode 100644 index 0000000000..2d85b17ed9 --- /dev/null +++ b/tests/crdcel/testdata/guardrailpolicies/http_invalid_path.yaml @@ -0,0 +1,22 @@ +# Copyright Envoy AI Gateway Authors +# SPDX-License-Identifier: Apache-2.0 +# The full text of the Apache license is available in the LICENSE file at +# the root of the repo. +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: GuardrailPolicy +metadata: + name: http-invalid-path + namespace: default +spec: + targetRefs: + - group: aigateway.envoyproxy.io + kind: AIServiceBackend + name: backend + rules: + - name: invalid + phase: Request + provider: + type: HTTP + http: + endpoint: https://guardrail.example.com + path: analyze diff --git a/tests/crdcel/testdata/guardrailpolicies/http_mismatch.yaml b/tests/crdcel/testdata/guardrailpolicies/http_mismatch.yaml new file mode 100644 index 0000000000..2bb14f08d3 --- /dev/null +++ b/tests/crdcel/testdata/guardrailpolicies/http_mismatch.yaml @@ -0,0 +1,21 @@ +# Copyright Envoy AI Gateway Authors +# SPDX-License-Identifier: Apache-2.0 +# The full text of the Apache license is available in the LICENSE file at +# the root of the repo. +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: GuardrailPolicy +metadata: + name: http-mismatch + namespace: default +spec: + targetRefs: + - group: aigateway.envoyproxy.io + kind: AIServiceBackend + name: backend + rules: + - name: invalid + phase: Request + provider: + type: HTTP + presidio: + endpoint: https://presidio.example.com From f8c9d92da361fe8a3362046c0e266c314c9322e3 Mon Sep 17 00:00:00 2001 From: Fernando Escolar Date: Mon, 28 Sep 2026 13:13:55 +0200 Subject: [PATCH 10/18] update HTTPGuardrailProvider documentation without code examples Signed-off-by: Fernando Escolar --- api/v1alpha1/guardrail_policy.go | 8 ++++---- api/v1beta1/guardrail_policy.go | 8 ++++---- site/docs/api/api.mdx | 16 ++++++++-------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/api/v1alpha1/guardrail_policy.go b/api/v1alpha1/guardrail_policy.go index 9594231897..5433392670 100644 --- a/api/v1alpha1/guardrail_policy.go +++ b/api/v1alpha1/guardrail_policy.go @@ -189,10 +189,10 @@ type AzureContentSafetyGuardrailProvider struct { // HTTPGuardrailProvider configures calls to a custom guardrail service. // -// The gateway sends a JSON request of the form -// {"text": "...", "context": {"stage": "input"}} to endpoint+path and expects a -// normalized response of the form -// {"action": "allow|block|modify", "findings": [{"type": "PII", "start": 0, "end": 4, "score": 0.9}], "replacement": "..."}. +// For each extracted text fragment, the gateway sends a JSON POST request with the +// text and an evaluation context (stage "input" or "output") to endpoint+path. The +// service returns a normalized response with an action ("allow", "block", or "modify"), +// optional findings (type, start, end, score), and an optional replacement text. // Finding offsets are Unicode code point positions in text; end is exclusive. type HTTPGuardrailProvider struct { // Endpoint is the base URL of the guardrail service. diff --git a/api/v1beta1/guardrail_policy.go b/api/v1beta1/guardrail_policy.go index 4ab37569de..697c11f15a 100644 --- a/api/v1beta1/guardrail_policy.go +++ b/api/v1beta1/guardrail_policy.go @@ -189,10 +189,10 @@ type AzureContentSafetyGuardrailProvider struct { // HTTPGuardrailProvider configures calls to a custom guardrail service. // -// The gateway sends a JSON request of the form -// {"text": "...", "context": {"stage": "input"}} to endpoint+path and expects a -// normalized response of the form -// {"action": "allow|block|modify", "findings": [{"type": "PII", "start": 0, "end": 4, "score": 0.9}], "replacement": "..."}. +// For each extracted text fragment, the gateway sends a JSON POST request with the +// text and an evaluation context (stage "input" or "output") to endpoint+path. The +// service returns a normalized response with an action ("allow", "block", or "modify"), +// optional findings (type, start, end, score), and an optional replacement text. // Finding offsets are Unicode code point positions in text; end is exclusive. type HTTPGuardrailProvider struct { // Endpoint is the base URL of the guardrail service. diff --git a/site/docs/api/api.mdx b/site/docs/api/api.mdx index 94c086cd4f..633620f9f8 100644 --- a/site/docs/api/api.mdx +++ b/site/docs/api/api.mdx @@ -2104,10 +2104,10 @@ HTTPBodyMutation defines the mutation of HTTP request body JSON fields that will HTTPGuardrailProvider configures calls to a custom guardrail service. -The gateway sends a JSON request of the form -{"text": "...", "context": {"stage": "input"}} to endpoint+path and expects a -normalized response of the form -{"action": "allow|block|modify", "findings": [{"type": "PII", "start": 0, "end": 4, "score": 0.9}], "replacement": "..."}. +For each extracted text fragment, the gateway sends a JSON POST request with the +text and an evaluation context (stage "input" or "output") to endpoint+path. The +service returns a normalized response with an action ("allow", "block", or "modify"), +optional findings (type, start, end, score), and an optional replacement text. Finding offsets are Unicode code point positions in text; end is exclusive. ##### Fields @@ -5319,10 +5319,10 @@ HTTPBodyMutation defines the mutation of HTTP request body JSON fields that will HTTPGuardrailProvider configures calls to a custom guardrail service. -The gateway sends a JSON request of the form -{"text": "...", "context": {"stage": "input"}} to endpoint+path and expects a -normalized response of the form -{"action": "allow|block|modify", "findings": [{"type": "PII", "start": 0, "end": 4, "score": 0.9}], "replacement": "..."}. +For each extracted text fragment, the gateway sends a JSON POST request with the +text and an evaluation context (stage "input" or "output") to endpoint+path. The +service returns a normalized response with an action ("allow", "block", or "modify"), +optional findings (type, start, end, score), and an optional replacement text. Finding offsets are Unicode code point positions in text; end is exclusive. ##### Fields From 78f889d7a66ae90c52ac0f7be6b8307904b8b71a Mon Sep 17 00:00:00 2001 From: Fernando Escolar Date: Wed, 30 Sep 2026 23:30:04 +0200 Subject: [PATCH 11/18] delete unused function: getSecretNameAndNamespace Signed-off-by: Fernando Escolar --- internal/controller/controller.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/internal/controller/controller.go b/internal/controller/controller.go index d70b5340f2..7f4b9a3561 100644 --- a/internal/controller/controller.go +++ b/internal/controller/controller.go @@ -611,13 +611,6 @@ func guardrailPolicySecretRefsIndexFunc(o client.Object) []string { return keys } -func getSecretNameAndNamespace(secretRef *gwapiv1.SecretObjectReference, namespace string) string { - if secretRef.Namespace != nil { - return fmt.Sprintf("%s.%s", secretRef.Name, *secretRef.Namespace) - } - return fmt.Sprintf("%s.%s", secretRef.Name, namespace) -} - func getReferenceGrantIndexKey(namespace, kind string) string { return fmt.Sprintf("%s.%s", namespace, kind) } From f9d802369cfb4e8729fcc4739fae651a3fa97f4a Mon Sep 17 00:00:00 2001 From: Fernando Escolar Date: Fri, 9 Oct 2026 23:37:01 +0200 Subject: [PATCH 12/18] adding missing guardrailMetrics Signed-off-by: Fernando Escolar --- cmd/extproc/mainlib/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/extproc/mainlib/main.go b/cmd/extproc/mainlib/main.go index d39ba37dc9..237e3b3f3a 100644 --- a/cmd/extproc/mainlib/main.go +++ b/cmd/extproc/mainlib/main.go @@ -336,7 +336,7 @@ func Main(ctx context.Context, args []string, stderr io.Writer) (err error) { server.Register(path.Join(flags.rootPrefix, endpointPrefixes.OpenAI, "/v1/responses"), extproc.NewFactory( responsesMetricsFactory, guardrailMetrics, tracing.ResponsesTracer(), endpointspec.ResponsesEndpointSpec{})) server.Register(path.Join(flags.rootPrefix, endpointPrefixes.OpenAI, "/v1/decisions"), extproc.NewFactory( - decisionsMetricsFactory, tracing.DecisionsTracer(), endpointspec.DecisionsEndpointSpec{})) + decisionsMetricsFactory, guardrailMetrics, tracing.DecisionsTracer(), endpointspec.DecisionsEndpointSpec{})) server.Register(path.Join(flags.rootPrefix, endpointPrefixes.OpenAI, "/v1/responses/input_tokens"), extproc.NewFactory( responsesInputTokensMetricsFactory, guardrailMetrics, tracing.ResponsesInputTokensTracer(), endpointspec.ResponsesInputTokensEndpointSpec{})) server.Register(path.Join(flags.rootPrefix, endpointPrefixes.OpenAI, "/v1/audio/speech"), extproc.NewFactory( From 039aa95fea18d84bf99767db08c810a07fa0a12a Mon Sep 17 00:00:00 2001 From: Fernando Escolar Date: Sat, 10 Oct 2026 10:36:39 +0200 Subject: [PATCH 13/18] Adding Google Model Armor as new guardrail provider Signed-off-by: Fernando Escolar --- api/v1alpha1/guardrail_policy.go | 44 ++++- api/v1alpha1/zz_generated.deepcopy.go | 25 +++ api/v1beta1/guardrail_policy.go | 44 ++++- api/v1beta1/zz_generated.deepcopy.go | 25 +++ docs/proposals/013-guardrails/proposal.md | 23 ++- examples/guardrails/README.md | 1 + examples/guardrails/providers.yaml | 11 ++ internal/controller/controller.go | 4 + internal/controller/gateway.go | 18 ++ internal/controller/guardrail_policy.go | 12 ++ internal/controller/guardrail_policy_test.go | 86 +++++++++ internal/filterapi/filterconfig.go | 12 ++ internal/guardrails/adapters.go | 2 + internal/guardrails/adapters_live_test.go | 21 +++ internal/guardrails/adapters_test.go | 128 +++++++++++++ internal/guardrails/modelarmor.go | 170 +++++++++++++++++ ...teway.envoyproxy.io_guardrailpolicies.yaml | 174 +++++++++++++++++- site/docs/api/api.mdx | 114 ++++++++++++ site/docs/capabilities/security/guardrails.md | 26 ++- tests/crdcel/main_test.go | 2 + .../testdata/guardrailpolicies/basic.yaml | 20 ++ .../model_armor_mismatch.yaml | 26 +++ .../model_armor_missing_template.yaml | 24 +++ 23 files changed, 984 insertions(+), 28 deletions(-) create mode 100644 internal/guardrails/modelarmor.go create mode 100644 tests/crdcel/testdata/guardrailpolicies/model_armor_mismatch.yaml create mode 100644 tests/crdcel/testdata/guardrailpolicies/model_armor_missing_template.yaml diff --git a/api/v1alpha1/guardrail_policy.go b/api/v1alpha1/guardrail_policy.go index 5433392670..2893cf7310 100644 --- a/api/v1alpha1/guardrail_policy.go +++ b/api/v1alpha1/guardrail_policy.go @@ -81,16 +81,17 @@ const ( ) // GuardrailProvider describes the implementation used to evaluate a rule. -// +kubebuilder:validation:XValidation:rule="self.type != 'Regex' || (has(self.pattern) && size(self.pattern) > 0 && !has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety) && !has(self.http))",message="Regex requires pattern and no external provider configuration" -// +kubebuilder:validation:XValidation:rule="self.type != 'Presidio' || (!has(self.pattern) && has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety) && !has(self.http))",message="Presidio requires only presidio provider configuration" -// +kubebuilder:validation:XValidation:rule="self.type != 'Bedrock' || (!has(self.pattern) && has(self.bedrock) && !has(self.presidio) && !has(self.azureContentSafety) && !has(self.http))",message="Bedrock requires only bedrock provider configuration" -// +kubebuilder:validation:XValidation:rule="self.type != 'AzureContentSafety' || (!has(self.pattern) && has(self.azureContentSafety) && !has(self.presidio) && !has(self.bedrock) && !has(self.http))",message="AzureContentSafety requires only azureContentSafety provider configuration" -// +kubebuilder:validation:XValidation:rule="self.type != 'HTTP' || (!has(self.pattern) && has(self.http) && !has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety))",message="HTTP requires only http provider configuration" +// +kubebuilder:validation:XValidation:rule="self.type != 'Regex' || (has(self.pattern) && size(self.pattern) > 0 && !has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety) && !has(self.http) && !has(self.modelArmor))",message="Regex requires pattern and no external provider configuration" +// +kubebuilder:validation:XValidation:rule="self.type != 'Presidio' || (!has(self.pattern) && has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety) && !has(self.http) && !has(self.modelArmor))",message="Presidio requires only presidio provider configuration" +// +kubebuilder:validation:XValidation:rule="self.type != 'Bedrock' || (!has(self.pattern) && has(self.bedrock) && !has(self.presidio) && !has(self.azureContentSafety) && !has(self.http) && !has(self.modelArmor))",message="Bedrock requires only bedrock provider configuration" +// +kubebuilder:validation:XValidation:rule="self.type != 'AzureContentSafety' || (!has(self.pattern) && has(self.azureContentSafety) && !has(self.presidio) && !has(self.bedrock) && !has(self.http) && !has(self.modelArmor))",message="AzureContentSafety requires only azureContentSafety provider configuration" +// +kubebuilder:validation:XValidation:rule="self.type != 'HTTP' || (!has(self.pattern) && has(self.http) && !has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety) && !has(self.modelArmor))",message="HTTP requires only http provider configuration" +// +kubebuilder:validation:XValidation:rule="self.type != 'ModelArmor' || (!has(self.pattern) && has(self.modelArmor) && !has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety) && !has(self.http))",message="ModelArmor requires only modelArmor provider configuration" // +kubebuilder:validation:XValidation:rule="self.action != 'Mask' || self.type != 'AzureContentSafety'",message="AzureContentSafety does not support Mask" type GuardrailProvider struct { // Type identifies the guardrail implementation. // - // +kubebuilder:validation:Enum=Regex;Presidio;Bedrock;AzureContentSafety;HTTP + // +kubebuilder:validation:Enum=Regex;Presidio;Bedrock;AzureContentSafety;HTTP;ModelArmor Type GuardrailProviderType `json:"type"` // Pattern is used for deterministic regex-based evaluations. // @@ -126,6 +127,10 @@ type GuardrailProvider struct { // // +optional HTTP *HTTPGuardrailProvider `json:"http,omitempty"` + // ModelArmor configures Google Cloud Model Armor. + // + // +optional + ModelArmor *ModelArmorGuardrailProvider `json:"modelArmor,omitempty"` // TimeoutSeconds limits each external provider evaluation. // +optional // +kubebuilder:default=10 @@ -208,6 +213,32 @@ type HTTPGuardrailProvider struct { APIKeySecretRef *gwapiv1.SecretObjectReference `json:"apiKeySecretRef,omitempty"` } +// ModelArmorGuardrailProvider configures calls to the Google Cloud Model Armor sanitize APIs. +// +// Request rules call sanitizeUserPrompt and Response rules call sanitizeModelResponse on the +// configured template. The filters (Responsible AI, prompt injection and jailbreak, malicious URLs, +// and Sensitive Data Protection) and their confidence levels are configured in the template. +// Mask requires a template with an advanced Sensitive Data Protection de-identify template. +type ModelArmorGuardrailProvider struct { + // Project is the Google Cloud project ID that owns the Model Armor template. + // +kubebuilder:validation:MinLength=1 + Project string `json:"project"` + // Location is the Google Cloud region of the Model Armor template, for example us-central1. + // +kubebuilder:validation:MinLength=1 + Location string `json:"location"` + // Template is the Model Armor template ID. + // +kubebuilder:validation:MinLength=1 + Template string `json:"template"` + // Endpoint overrides the regional Model Armor endpoint, primarily for private endpoints and testing. + // +optional + // +kubebuilder:validation:Format=uri + Endpoint string `json:"endpoint,omitempty"` + // CredentialsSecretRef optionally references a Secret whose credentials entry contains a Google Cloud + // service account key JSON. When omitted, Application Default Credentials are used. + // +optional + CredentialsSecretRef *gwapiv1.SecretObjectReference `json:"credentialsSecretRef,omitempty"` +} + // GuardrailProviderType is the guardrail implementation. type GuardrailProviderType string @@ -217,6 +248,7 @@ const ( GuardrailProviderTypeBedrockGuardrails GuardrailProviderType = "Bedrock" GuardrailProviderTypeAzureContentSafety GuardrailProviderType = "AzureContentSafety" GuardrailProviderTypeHTTP GuardrailProviderType = "HTTP" + GuardrailProviderTypeModelArmor GuardrailProviderType = "ModelArmor" ) // GuardrailAction defines the safeguard action. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 1b29aa1747..45351c57b9 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -1128,6 +1128,11 @@ func (in *GuardrailProvider) DeepCopyInto(out *GuardrailProvider) { *out = new(HTTPGuardrailProvider) (*in).DeepCopyInto(*out) } + if in.ModelArmor != nil { + in, out := &in.ModelArmor, &out.ModelArmor + *out = new(ModelArmorGuardrailProvider) + (*in).DeepCopyInto(*out) + } if in.TimeoutSeconds != nil { in, out := &in.TimeoutSeconds, &out.TimeoutSeconds *out = new(int32) @@ -1854,6 +1859,26 @@ func (in *MCPToolFilter) DeepCopy() *MCPToolFilter { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ModelArmorGuardrailProvider) DeepCopyInto(out *ModelArmorGuardrailProvider) { + *out = *in + if in.CredentialsSecretRef != nil { + in, out := &in.CredentialsSecretRef, &out.CredentialsSecretRef + *out = new(v1.SecretObjectReference) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ModelArmorGuardrailProvider. +func (in *ModelArmorGuardrailProvider) DeepCopy() *ModelArmorGuardrailProvider { + if in == nil { + return nil + } + out := new(ModelArmorGuardrailProvider) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PerModelQuota) DeepCopyInto(out *PerModelQuota) { *out = *in diff --git a/api/v1beta1/guardrail_policy.go b/api/v1beta1/guardrail_policy.go index 697c11f15a..b42d05bf81 100644 --- a/api/v1beta1/guardrail_policy.go +++ b/api/v1beta1/guardrail_policy.go @@ -81,16 +81,17 @@ const ( ) // GuardrailProvider describes the implementation used to evaluate a rule. -// +kubebuilder:validation:XValidation:rule="self.type != 'Regex' || (has(self.pattern) && size(self.pattern) > 0 && !has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety) && !has(self.http))",message="Regex requires pattern and no external provider configuration" -// +kubebuilder:validation:XValidation:rule="self.type != 'Presidio' || (!has(self.pattern) && has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety) && !has(self.http))",message="Presidio requires only presidio provider configuration" -// +kubebuilder:validation:XValidation:rule="self.type != 'Bedrock' || (!has(self.pattern) && has(self.bedrock) && !has(self.presidio) && !has(self.azureContentSafety) && !has(self.http))",message="Bedrock requires only bedrock provider configuration" -// +kubebuilder:validation:XValidation:rule="self.type != 'AzureContentSafety' || (!has(self.pattern) && has(self.azureContentSafety) && !has(self.presidio) && !has(self.bedrock) && !has(self.http))",message="AzureContentSafety requires only azureContentSafety provider configuration" -// +kubebuilder:validation:XValidation:rule="self.type != 'HTTP' || (!has(self.pattern) && has(self.http) && !has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety))",message="HTTP requires only http provider configuration" +// +kubebuilder:validation:XValidation:rule="self.type != 'Regex' || (has(self.pattern) && size(self.pattern) > 0 && !has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety) && !has(self.http) && !has(self.modelArmor))",message="Regex requires pattern and no external provider configuration" +// +kubebuilder:validation:XValidation:rule="self.type != 'Presidio' || (!has(self.pattern) && has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety) && !has(self.http) && !has(self.modelArmor))",message="Presidio requires only presidio provider configuration" +// +kubebuilder:validation:XValidation:rule="self.type != 'Bedrock' || (!has(self.pattern) && has(self.bedrock) && !has(self.presidio) && !has(self.azureContentSafety) && !has(self.http) && !has(self.modelArmor))",message="Bedrock requires only bedrock provider configuration" +// +kubebuilder:validation:XValidation:rule="self.type != 'AzureContentSafety' || (!has(self.pattern) && has(self.azureContentSafety) && !has(self.presidio) && !has(self.bedrock) && !has(self.http) && !has(self.modelArmor))",message="AzureContentSafety requires only azureContentSafety provider configuration" +// +kubebuilder:validation:XValidation:rule="self.type != 'HTTP' || (!has(self.pattern) && has(self.http) && !has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety) && !has(self.modelArmor))",message="HTTP requires only http provider configuration" +// +kubebuilder:validation:XValidation:rule="self.type != 'ModelArmor' || (!has(self.pattern) && has(self.modelArmor) && !has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety) && !has(self.http))",message="ModelArmor requires only modelArmor provider configuration" // +kubebuilder:validation:XValidation:rule="self.action != 'Mask' || self.type != 'AzureContentSafety'",message="AzureContentSafety does not support Mask" type GuardrailProvider struct { // Type identifies the guardrail implementation. // - // +kubebuilder:validation:Enum=Regex;Presidio;Bedrock;AzureContentSafety;HTTP + // +kubebuilder:validation:Enum=Regex;Presidio;Bedrock;AzureContentSafety;HTTP;ModelArmor Type GuardrailProviderType `json:"type"` // Pattern is used for deterministic regex-based evaluations. // @@ -126,6 +127,10 @@ type GuardrailProvider struct { // // +optional HTTP *HTTPGuardrailProvider `json:"http,omitempty"` + // ModelArmor configures Google Cloud Model Armor. + // + // +optional + ModelArmor *ModelArmorGuardrailProvider `json:"modelArmor,omitempty"` // TimeoutSeconds limits each external provider evaluation. // +optional // +kubebuilder:default=10 @@ -208,6 +213,32 @@ type HTTPGuardrailProvider struct { APIKeySecretRef *gwapiv1.SecretObjectReference `json:"apiKeySecretRef,omitempty"` } +// ModelArmorGuardrailProvider configures calls to the Google Cloud Model Armor sanitize APIs. +// +// Request rules call sanitizeUserPrompt and Response rules call sanitizeModelResponse on the +// configured template. The filters (Responsible AI, prompt injection and jailbreak, malicious URLs, +// and Sensitive Data Protection) and their confidence levels are configured in the template. +// Mask requires a template with an advanced Sensitive Data Protection de-identify template. +type ModelArmorGuardrailProvider struct { + // Project is the Google Cloud project ID that owns the Model Armor template. + // +kubebuilder:validation:MinLength=1 + Project string `json:"project"` + // Location is the Google Cloud region of the Model Armor template, for example us-central1. + // +kubebuilder:validation:MinLength=1 + Location string `json:"location"` + // Template is the Model Armor template ID. + // +kubebuilder:validation:MinLength=1 + Template string `json:"template"` + // Endpoint overrides the regional Model Armor endpoint, primarily for private endpoints and testing. + // +optional + // +kubebuilder:validation:Format=uri + Endpoint string `json:"endpoint,omitempty"` + // CredentialsSecretRef optionally references a Secret whose credentials entry contains a Google Cloud + // service account key JSON. When omitted, Application Default Credentials are used. + // +optional + CredentialsSecretRef *gwapiv1.SecretObjectReference `json:"credentialsSecretRef,omitempty"` +} + // GuardrailProviderType is the guardrail implementation. type GuardrailProviderType string @@ -217,6 +248,7 @@ const ( GuardrailProviderTypeBedrockGuardrails GuardrailProviderType = "Bedrock" GuardrailProviderTypeAzureContentSafety GuardrailProviderType = "AzureContentSafety" GuardrailProviderTypeHTTP GuardrailProviderType = "HTTP" + GuardrailProviderTypeModelArmor GuardrailProviderType = "ModelArmor" ) // GuardrailAction defines the safeguard action. diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index a3929fa067..226cf69df4 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -1210,6 +1210,11 @@ func (in *GuardrailProvider) DeepCopyInto(out *GuardrailProvider) { *out = new(HTTPGuardrailProvider) (*in).DeepCopyInto(*out) } + if in.ModelArmor != nil { + in, out := &in.ModelArmor, &out.ModelArmor + *out = new(ModelArmorGuardrailProvider) + (*in).DeepCopyInto(*out) + } if in.TimeoutSeconds != nil { in, out := &in.TimeoutSeconds, &out.TimeoutSeconds *out = new(int32) @@ -2011,6 +2016,26 @@ func (in *MCPToolFilter) DeepCopy() *MCPToolFilter { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ModelArmorGuardrailProvider) DeepCopyInto(out *ModelArmorGuardrailProvider) { + *out = *in + if in.CredentialsSecretRef != nil { + in, out := &in.CredentialsSecretRef, &out.CredentialsSecretRef + *out = new(v1.SecretObjectReference) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ModelArmorGuardrailProvider. +func (in *ModelArmorGuardrailProvider) DeepCopy() *ModelArmorGuardrailProvider { + if in == nil { + return nil + } + out := new(ModelArmorGuardrailProvider) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PresidioGuardrailProvider) DeepCopyInto(out *PresidioGuardrailProvider) { *out = *in diff --git a/docs/proposals/013-guardrails/proposal.md b/docs/proposals/013-guardrails/proposal.md index 30924cd7ba..96e738d2c2 100644 --- a/docs/proposals/013-guardrails/proposal.md +++ b/docs/proposals/013-guardrails/proposal.md @@ -27,6 +27,7 @@ - [Presidio](#presidio) - [AWS Bedrock Guardrails](#aws-bedrock-guardrails) - [Azure AI Content Safety](#azure-ai-content-safety) + - [Google Cloud Model Armor](#google-cloud-model-armor) - [Custom HTTP Guardrails](#custom-http-guardrails) - [Payload Extraction](#payload-extraction) - [Credentials and Security](#credentials-and-security) @@ -46,7 +47,7 @@ ## Summary -This proposal introduces `GuardrailPolicy`, a backend-attached policy for evaluating LLM request and response content before it is sent to an AI provider or returned to a client. The policy provides a common API for local regular-expression checks and external safety providers, initially Presidio, AWS Bedrock Guardrails, and Azure AI Content Safety. A generic HTTP provider defines a small, normalized contract that any custom guardrail service can implement, so providers without a native integration can be used without code changes to the gateway. +This proposal introduces `GuardrailPolicy`, a backend-attached policy for evaluating LLM request and response content before it is sent to an AI provider or returned to a client. The policy provides a common API for local regular-expression checks and external safety providers, initially Presidio, AWS Bedrock Guardrails, Azure AI Content Safety, and Google Cloud Model Armor. A generic HTTP provider defines a small, normalized contract that any custom guardrail service can implement, so providers without a native integration can be used without code changes to the gateway. The proposed implementation uses the existing Agent Router external processor. The controller resolves policies and credentials into the filter configuration, while ext-proc buffers the relevant body, extracts content, invokes the configured evaluator, and applies the selected action. @@ -72,7 +73,7 @@ A native policy should allow platform administrators to: - Scope policies to `AIServiceBackend` resources and preserve route/backend isolation. - Support request and response evaluation. - Provide a provider-neutral runtime evaluator interface. -- Initially support Regex, Presidio, AWS Bedrock Guardrails, and Azure AI Content Safety. +- Initially support Regex, Presidio, AWS Bedrock Guardrails, Azure AI Content Safety, and Google Cloud Model Armor. - Define a portable HTTP contract for custom guardrail services that are not natively supported. - Support explicit timeout and fail-open/fail-closed behavior. - Prevent partial delivery of blocked streaming responses. @@ -250,6 +251,7 @@ The provider is a tagged union. Admission validation requires exactly the config - `Bedrock` requires `bedrock`. - `AzureContentSafety` requires `azureContentSafety`. - `HTTP` requires `http`. +- `ModelArmor` requires `modelArmor`. Provider-specific configuration remains nested so new provider capabilities can be added without adding unrelated fields to every rule. @@ -268,6 +270,7 @@ Mask behavior depends on provider capability: - Bedrock uses transformed text returned by ApplyGuardrail. - Azure Text Analysis does not return transformed content, so admission rejects `Mask` for Azure rules. - HTTP uses the `replacement` text returned by the service, or replaces returned finding spans with `maskReplacement`. +- Model Armor uses the text de-identified by its Sensitive Data Protection filter, only when no other filter matched. Mask is applied only to extracted text fields and never replaces unrelated model or configuration fields. Masked streaming responses remain buffered until the complete body has been evaluated and rewritten. @@ -320,6 +323,17 @@ The Built on Envoy implementation highlights that Azure has multiple distinct sa A future shape could add `azureContentSafety.check` and check-specific configuration. Per-category thresholds should also replace a single threshold when Text Analysis is selected. Until then, the implementation should be documented as Text Analysis over the configured evaluation input, not as complete Azure Content Safety feature parity. +### Google Cloud Model Armor + +[Google Cloud Model Armor] screens content against a Model Armor template. Request rules call `sanitizeUserPrompt` and response rules call `sanitizeModelResponse` on the regional endpoint; the global endpoint does not serve these methods. The rule references the template by project, location, and template ID, so filter selection and confidence levels (Responsible AI, prompt injection and jailbreak, malicious URLs, Sensitive Data Protection) stay in Google Cloud, as with Bedrock guardrail versions. + +A rule matches when the sanitization result reports `MATCH_FOUND`. An invocation result of `FAILURE` is a provider error and follows the rule's failure mode. Authentication uses an OAuth2 access token from either: + +- Google Application Default Credentials, including GKE Workload Identity; or +- a service account key JSON read from a Kubernetes Secret. + +Mask requires a template with advanced Sensitive Data Protection and a de-identify template. The evaluator forwards the de-identified text only when Sensitive Data Protection is the sole matching filter; any other match (for example, a jailbreak) returns no replacement so Mask rules fail instead of forwarding flagged content. Model Armor file scanning and streaming sanitization are not used, because the gateway evaluates extracted text fields. + ### Custom HTTP Guardrails Native integrations cover common providers, but organizations frequently run in-house classifiers or services that the gateway does not support yet. The `HTTP` provider defines a small, provider-neutral interface, similar in spirit to the Presidio Analyzer API, that such services implement. It lets new guardrails be adopted without gateway changes and without the security and portability concerns of invoking arbitrary local executables. @@ -495,7 +509,7 @@ Running a user-supplied binary or script inside the ext-proc for each evaluation 1. Introduce the dual-version `GuardrailPolicy`, generated clients, CRD, and status. 2. Add target and Secret indexes, reconciliation, deletion propagation, and filter-config translation. 3. Add runtime compilation and backend-scoped request/response evaluation. -4. Add Regex, Presidio, Bedrock, and Azure Text Analysis evaluators. +4. Add Regex, Presidio, Bedrock, Azure Text Analysis, and Model Armor evaluators. 5. Add failure modes, timeouts, logs, metrics, traces, and guarded-response buffering. 6. Add CRD admission, controller, HTTP-stub, Testcontainers, live-provider, and dataplane tests. 7. Add semantic payload extraction and schema-aware Mask mutation. @@ -515,7 +529,7 @@ Steps 1 through 10 describe the current implementation. Steps 11 and 12 are prop - HTTP-stub tests for provider paths, payloads, headers, authentication, responses, malformed responses, and errors. - HTTP-stub tests for the custom HTTP contract: stages, paths, bearer authentication, each action, replacement and finding masking, and invalid responses. - Testcontainers integration with the pinned official Presidio analyzer image. -- Credential-gated live tests for Azure AI Content Safety and AWS Bedrock Guardrails. +- Credential-gated live tests for Azure AI Content Safety, AWS Bedrock Guardrails, and Google Cloud Model Armor. - Envoy dataplane tests for request blocking, response blocking, allowed traffic, and streaming mode behavior. - Staging tests for policy creation, update, Secret rotation, deletion, and multi-policy attachment. - Load tests for provider latency, concurrency, and buffered response limits. @@ -541,4 +555,5 @@ These gaps do not require changing the core policy-to-runtime architecture, but [Built on Envoy repository]: https://github.com/tetratelabs/built-on-envoy [AWS Bedrock ApplyGuardrail]: https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ApplyGuardrail.html [Azure AI Content Safety]: https://learn.microsoft.com/azure/ai-services/content-safety/ +[Google Cloud Model Armor]: https://docs.cloud.google.com/model-armor/overview [Presidio]: https://presidio.dataprivacystack.org/ diff --git a/examples/guardrails/README.md b/examples/guardrails/README.md index 27a9d88e1c..6db9a817a5 100644 --- a/examples/guardrails/README.md +++ b/examples/guardrails/README.md @@ -21,6 +21,7 @@ The expected Secret keys are: - Azure Content Safety: required `apiKey` - AWS Bedrock Guardrails: optional `credentials` containing an AWS shared credentials file; when omitted, the ext-proc uses the standard AWS credential chain. - Custom HTTP guardrail: optional `apiKey`, sent as a bearer token. +- Google Cloud Model Armor: optional `credentials` containing a service account key JSON; when omitted, the ext-proc uses Google Application Default Credentials, such as GKE Workload Identity. The `HTTP` provider calls any service that implements the generic guardrail contract: the gateway sends `{"text": "...", "context": {"stage": "input"}}` to `POST /analyze` and expects `{"action": "allow" | "block" | "modify", "findings": [...], "replacement": "..."}`. See the [guardrails documentation](../../site/docs/capabilities/security/guardrails.md#custom-http-guardrails) for the full contract. diff --git a/examples/guardrails/providers.yaml b/examples/guardrails/providers.yaml index 55ddeaa098..dfbcecdc9e 100644 --- a/examples/guardrails/providers.yaml +++ b/examples/guardrails/providers.yaml @@ -77,3 +77,14 @@ spec: path: /analyze apiKeySecretRef: name: custom-guardrail-key + - name: screen-with-model-armor + phase: Request + provider: + type: ModelArmor + timeoutSeconds: 10 + failureMode: FailClosed + message: Request was rejected by the configured Model Armor template + modelArmor: + project: replace-me + location: us-central1 + template: replace-me diff --git a/internal/controller/controller.go b/internal/controller/controller.go index a8cf4741a3..8c7ac82c49 100644 --- a/internal/controller/controller.go +++ b/internal/controller/controller.go @@ -606,6 +606,10 @@ func guardrailPolicySecretRefsIndexFunc(o client.Object) []string { if provider.HTTP != nil { ref = provider.HTTP.APIKeySecretRef } + case aigv1b1.GuardrailProviderTypeModelArmor: + if provider.ModelArmor != nil { + ref = provider.ModelArmor.CredentialsSecretRef + } } if ref != nil { namespace := policy.Namespace diff --git a/internal/controller/gateway.go b/internal/controller/gateway.go index cf4cd0778c..074b3223f0 100644 --- a/internal/controller/gateway.go +++ b/internal/controller/gateway.go @@ -820,6 +820,24 @@ func (c *GatewayController) guardrailProviderToFilterAPI(ctx context.Context, na } converted.HTTP.APIKey = apiKey } + case aigv1b1.GuardrailProviderTypeModelArmor: + if provider.ModelArmor == nil { + return converted, fmt.Errorf("model Armor configuration is required") + } + config := provider.ModelArmor + converted.ModelArmor = &filterapi.ModelArmorGuardrailProvider{ + Endpoint: config.Endpoint, + Project: config.Project, + Location: config.Location, + Template: config.Template, + } + if config.CredentialsSecretRef != nil { + credentials, err := c.getGuardrailSecretData(ctx, namespace, config.CredentialsSecretRef, "credentials") + if err != nil { + return converted, err + } + converted.ModelArmor.CredentialsJSON = credentials + } default: return converted, fmt.Errorf("unsupported provider type %q", provider.Type) } diff --git a/internal/controller/guardrail_policy.go b/internal/controller/guardrail_policy.go index 1ce52ea904..a5a088c8fd 100644 --- a/internal/controller/guardrail_policy.go +++ b/internal/controller/guardrail_policy.go @@ -173,6 +173,18 @@ func (c *GuardrailPolicyController) validateGuardrailProvider(ctx context.Contex if provider.HTTP.APIKeySecretRef != nil { return c.validateGuardrailSecret(ctx, namespace, provider.HTTP.APIKeySecretRef, "apiKey") } + case aigv1b1.GuardrailProviderTypeModelArmor: + if provider.ModelArmor == nil || provider.ModelArmor.Project == "" || provider.ModelArmor.Location == "" || provider.ModelArmor.Template == "" { + return fmt.Errorf("model Armor project, location, and template are required") + } + if provider.ModelArmor.Endpoint != "" { + if err := validateGuardrailEndpoint(provider.ModelArmor.Endpoint); err != nil { + return err + } + } + if provider.ModelArmor.CredentialsSecretRef != nil { + return c.validateGuardrailSecret(ctx, namespace, provider.ModelArmor.CredentialsSecretRef, "credentials") + } default: return fmt.Errorf("unsupported provider type %q", provider.Type) } diff --git a/internal/controller/guardrail_policy_test.go b/internal/controller/guardrail_policy_test.go index 84812a8ac0..12d7a99da0 100644 --- a/internal/controller/guardrail_policy_test.go +++ b/internal/controller/guardrail_policy_test.go @@ -437,3 +437,89 @@ func TestGuardrailPolicySecretRefsIndexIncludesHTTPProvider(t *testing.T) { } require.Equal(t, []string{"custom-guardrail-key.default"}, guardrailPolicySecretRefsIndexFunc(policy)) } + +func TestGuardrailModelArmorProviderToFilterAPI(t *testing.T) { + kube := fakekube.NewClientset(&corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "model-armor-sa", Namespace: "default"}, + Data: map[string][]byte{"credentials": []byte(`{"type":"service_account"}`)}, + }) + controller := &GatewayController{kube: kube} + converted, err := controller.guardrailProviderToFilterAPI(t.Context(), "default", &aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeModelArmor, + Action: aigv1b1.GuardrailActionMask, + ModelArmor: &aigv1b1.ModelArmorGuardrailProvider{ + Project: "project-id", + Location: "us-central1", + Template: "template-id", + Endpoint: "https://modelarmor.example.com", + CredentialsSecretRef: &gwapiv1.SecretObjectReference{Name: "model-armor-sa"}, + }, + }) + require.NoError(t, err) + require.Equal(t, filterapi.GuardrailActionMask, converted.Action) + require.Equal(t, &filterapi.ModelArmorGuardrailProvider{ + Endpoint: "https://modelarmor.example.com", + Project: "project-id", + Location: "us-central1", + Template: "template-id", + CredentialsJSON: `{"type":"service_account"}`, + }, converted.ModelArmor) + + _, err = controller.guardrailProviderToFilterAPI(t.Context(), "default", &aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeModelArmor, + }) + require.ErrorContains(t, err, "model Armor configuration is required") +} + +func TestGuardrailModelArmorProviderValidation(t *testing.T) { + kube := fakekube.NewClientset(&corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "model-armor-sa", Namespace: "default"}, + Data: map[string][]byte{"credentials": []byte(`{"type":"service_account"}`)}, + }) + controller := &GuardrailPolicyController{kube: kube} + require.ErrorContains(t, controller.validateGuardrailProvider(t.Context(), "default", &aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeModelArmor, + }), "model Armor project, location, and template are required") + require.ErrorContains(t, controller.validateGuardrailProvider(t.Context(), "default", &aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeModelArmor, + ModelArmor: &aigv1b1.ModelArmorGuardrailProvider{Project: "project-id", Location: "us-central1"}, + }), "model Armor project, location, and template are required") + require.ErrorContains(t, controller.validateGuardrailProvider(t.Context(), "default", &aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeModelArmor, + ModelArmor: &aigv1b1.ModelArmorGuardrailProvider{ + Project: "project-id", Location: "us-central1", Template: "template-id", Endpoint: "not-a-url", + }, + }), "valid provider endpoint is required") + require.ErrorContains(t, controller.validateGuardrailProvider(t.Context(), "default", &aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeModelArmor, + ModelArmor: &aigv1b1.ModelArmorGuardrailProvider{ + Project: "project-id", Location: "us-central1", Template: "template-id", + CredentialsSecretRef: &gwapiv1.SecretObjectReference{Name: "missing"}, + }, + }), "failed to get secret missing") + require.NoError(t, controller.validateGuardrailProvider(t.Context(), "default", &aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeModelArmor, + Action: aigv1b1.GuardrailActionMask, + ModelArmor: &aigv1b1.ModelArmorGuardrailProvider{ + Project: "project-id", Location: "us-central1", Template: "template-id", + CredentialsSecretRef: &gwapiv1.SecretObjectReference{Name: "model-armor-sa"}, + }, + })) +} + +func TestGuardrailPolicySecretRefsIndexIncludesModelArmorProvider(t *testing.T) { + policy := &aigv1b1.GuardrailPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: "model-armor", Namespace: "default"}, + Spec: aigv1b1.GuardrailPolicySpec{Rules: []aigv1b1.GuardrailRule{{ + Name: "model-armor", Phase: aigv1b1.GuardrailPhaseRequest, + Provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeModelArmor, + ModelArmor: &aigv1b1.ModelArmorGuardrailProvider{ + Project: "project-id", Location: "us-central1", Template: "template-id", + CredentialsSecretRef: &gwapiv1.SecretObjectReference{Name: "model-armor-sa"}, + }, + }, + }}}, + } + require.Equal(t, []string{"model-armor-sa.default"}, guardrailPolicySecretRefsIndexFunc(policy)) +} diff --git a/internal/filterapi/filterconfig.go b/internal/filterapi/filterconfig.go index c160f0f7d4..70f245aa6b 100644 --- a/internal/filterapi/filterconfig.go +++ b/internal/filterapi/filterconfig.go @@ -86,6 +86,7 @@ type GuardrailProvider struct { Bedrock *BedrockGuardrailProvider `json:"bedrock,omitempty"` AzureContentSafety *AzureContentSafetyGuardrailProvider `json:"azureContentSafety,omitempty"` HTTP *HTTPGuardrailProvider `json:"http,omitempty"` + ModelArmor *ModelArmorGuardrailProvider `json:"modelArmor,omitempty"` TimeoutSeconds int32 `json:"timeoutSeconds,omitempty"` FailureMode GuardrailFailureMode `json:"failureMode,omitempty"` } @@ -130,6 +131,16 @@ type HTTPGuardrailProvider struct { APIKey string `json:"apiKey,omitempty"` } +// ModelArmorGuardrailProvider configures a Google Cloud Model Armor sanitize request. +type ModelArmorGuardrailProvider struct { + Endpoint string `json:"endpoint,omitempty"` + Project string `json:"project"` + Location string `json:"location"` + Template string `json:"template"` + // CredentialsJSON is a service account key JSON. Empty means Application Default Credentials. + CredentialsJSON string `json:"credentialsJSON,omitempty"` +} + // GuardrailProviderType identifies a guardrail implementation. type GuardrailProviderType string @@ -139,6 +150,7 @@ const ( GuardrailProviderTypeBedrockGuardrails GuardrailProviderType = "Bedrock" GuardrailProviderTypeAzureContentSafety GuardrailProviderType = "AzureContentSafety" GuardrailProviderTypeHTTP GuardrailProviderType = "HTTP" + GuardrailProviderTypeModelArmor GuardrailProviderType = "ModelArmor" ) // GuardrailAction is the action taken when the rule matches. diff --git a/internal/guardrails/adapters.go b/internal/guardrails/adapters.go index 3040049d77..3f56764065 100644 --- a/internal/guardrails/adapters.go +++ b/internal/guardrails/adapters.go @@ -34,6 +34,8 @@ func NewEvaluator(ctx context.Context, provider *filterapi.GuardrailProvider) (f return newAzureContentSafetyEvaluator(provider.AzureContentSafety, client) case filterapi.GuardrailProviderTypeHTTP: return newHTTPEvaluator(provider.HTTP, provider.MaskReplacement, client) + case filterapi.GuardrailProviderTypeModelArmor: + return newModelArmorEvaluator(ctx, provider.ModelArmor, client) default: return nil, fmt.Errorf("unsupported external guardrail provider %q", provider.Type) } diff --git a/internal/guardrails/adapters_live_test.go b/internal/guardrails/adapters_live_test.go index 79a4d07ea9..94f47a58a7 100644 --- a/internal/guardrails/adapters_live_test.go +++ b/internal/guardrails/adapters_live_test.go @@ -74,3 +74,24 @@ func TestLiveBedrockGuardrail(t *testing.T) { require.NoError(t, err) require.True(t, evaluation.Matched) } + +func TestLiveModelArmor(t *testing.T) { + project := os.Getenv("TEST_GCP_MODEL_ARMOR_PROJECT") + location := os.Getenv("TEST_GCP_MODEL_ARMOR_LOCATION") + template := os.Getenv("TEST_GCP_MODEL_ARMOR_TEMPLATE") + blockedText := os.Getenv("TEST_GCP_MODEL_ARMOR_BLOCKED_TEXT") + if project == "" || location == "" || template == "" || blockedText == "" { + t.Skip("Model Armor template configuration and TEST_GCP_MODEL_ARMOR_BLOCKED_TEXT are not set") + } + provider := &filterapi.GuardrailProvider{ + Type: filterapi.GuardrailProviderTypeModelArmor, + ModelArmor: &filterapi.ModelArmorGuardrailProvider{ + Project: project, Location: location, Template: template, + }, + } + evaluator, err := NewEvaluator(t.Context(), provider) + require.NoError(t, err) + evaluation, err := evaluator.Evaluate(t.Context(), []byte(blockedText), filterapi.GuardrailPhaseRequest) + require.NoError(t, err) + require.True(t, evaluation.Matched) +} diff --git a/internal/guardrails/adapters_test.go b/internal/guardrails/adapters_test.go index 75341c5a27..4990bedda5 100644 --- a/internal/guardrails/adapters_test.go +++ b/internal/guardrails/adapters_test.go @@ -6,6 +6,11 @@ package guardrails import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "io" "net/http" "net/http/httptest" "strings" @@ -122,3 +127,126 @@ func TestNewEvaluatorConfiguresTimeout(t *testing.T) { require.NoError(t, err) require.Equal(t, 3*time.Second, evaluator.(*presidioEvaluator).client.Timeout) } + +// newModelArmorTestCredentials returns a service account key JSON whose token_uri points to a fake +// token server, so tests exercise the real OAuth2 JWT flow without contacting Google. +func newModelArmorTestCredentials(t *testing.T) string { + t.Helper() + tokenServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + require.NoError(t, req.ParseForm()) + require.Equal(t, "urn:ietf:params:oauth:grant-type:jwt-bearer", req.Form.Get("grant_type")) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"access_token":"gcp-token","token_type":"Bearer","expires_in":3600}`)) + })) + t.Cleanup(tokenServer.Close) + key, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + der, err := x509.MarshalPKCS8PrivateKey(key) + require.NoError(t, err) + credentials, err := json.Marshal(map[string]string{ + "type": "service_account", + "project_id": "project-id", + "private_key_id": "key-id", + "private_key": string(pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der})), + "client_email": "guardrails@project-id.iam.gserviceaccount.com", + "token_uri": tokenServer.URL, + }) + require.NoError(t, err) + return string(credentials) +} + +func TestModelArmorEvaluatorHTTP(t *testing.T) { + credentials := newModelArmorTestCredentials(t) + tests := []struct { + name string + phase filterapi.GuardrailPhase + wantPath string + wantBody string + response string + wantMatched bool + wantReplacement string + }{ + { + name: "request no match", + phase: filterapi.GuardrailPhaseRequest, + wantPath: "/v1/projects/project-id/locations/us-central1/templates/template-id:sanitizeUserPrompt", + wantBody: `{"userPromptData":{"text":"hello"}}`, + response: `{"sanitizationResult":{"filterMatchState":"NO_MATCH_FOUND","invocationResult":"SUCCESS"}}`, + }, + { + name: "request jailbreak match", + phase: filterapi.GuardrailPhaseRequest, + wantPath: "/v1/projects/project-id/locations/us-central1/templates/template-id:sanitizeUserPrompt", + wantBody: `{"userPromptData":{"text":"hello"}}`, + response: `{"sanitizationResult":{"filterMatchState":"MATCH_FOUND","invocationResult":"SUCCESS","filterResults":{ + "pi_and_jailbreak":{"piAndJailbreakFilterResult":{"executionState":"EXECUTION_SUCCESS","matchState":"MATCH_FOUND"}}}}}`, + wantMatched: true, + }, + { + name: "response sdp de-identify match", + phase: filterapi.GuardrailPhaseResponse, + wantPath: "/v1/projects/project-id/locations/us-central1/templates/template-id:sanitizeModelResponse", + wantBody: `{"modelResponseData":{"text":"hello"}}`, + response: `{"sanitizationResult":{"filterMatchState":"MATCH_FOUND","invocationResult":"SUCCESS","filterResults":{ + "rai":{"raiFilterResult":{"matchState":"NO_MATCH_FOUND"}}, + "sdp":{"sdpFilterResult":{"deidentifyResult":{"matchState":"MATCH_FOUND","data":{"text":"[EMAIL_ADDRESS]"}}}}}}}`, + wantMatched: true, + wantReplacement: "[EMAIL_ADDRESS]", + }, + { + name: "sdp de-identify with another filter match", + phase: filterapi.GuardrailPhaseResponse, + wantPath: "/v1/projects/project-id/locations/us-central1/templates/template-id:sanitizeModelResponse", + wantBody: `{"modelResponseData":{"text":"hello"}}`, + response: `{"sanitizationResult":{"filterMatchState":"MATCH_FOUND","invocationResult":"SUCCESS","filterResults":{ + "rai":{"raiFilterResult":{"matchState":"MATCH_FOUND"}}, + "sdp":{"sdpFilterResult":{"deidentifyResult":{"matchState":"MATCH_FOUND","data":{"text":"[EMAIL_ADDRESS]"}}}}}}}`, + wantMatched: true, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + require.Equal(t, test.wantPath, req.URL.Path) + require.Equal(t, "Bearer gcp-token", req.Header.Get("Authorization")) + body, err := io.ReadAll(req.Body) + require.NoError(t, err) + require.JSONEq(t, test.wantBody, string(body)) + _, _ = w.Write([]byte(test.response)) + })) + t.Cleanup(server.Close) + + evaluator, err := newModelArmorEvaluator(t.Context(), &filterapi.ModelArmorGuardrailProvider{ + Endpoint: server.URL, Project: "project-id", Location: "us-central1", Template: "template-id", + CredentialsJSON: credentials, + }, server.Client()) + require.NoError(t, err) + evaluation, err := evaluator.Evaluate(t.Context(), []byte("hello"), test.phase) + require.NoError(t, err) + require.Equal(t, test.wantMatched, evaluation.Matched) + require.Equal(t, test.wantReplacement, string(evaluation.Replacement)) + }) + } +} + +func TestModelArmorEvaluatorErrors(t *testing.T) { + _, err := newModelArmorEvaluator(t.Context(), &filterapi.ModelArmorGuardrailProvider{Project: "project-id"}, http.DefaultClient) + require.ErrorContains(t, err, "project, location, and template are required") + _, err = newModelArmorEvaluator(t.Context(), &filterapi.ModelArmorGuardrailProvider{ + Project: "project-id", Location: "us-central1", Template: "template-id", CredentialsJSON: `{"type":"authorized_user"}`, + }, http.DefaultClient) + require.ErrorContains(t, err, "cannot load GCP credentials") + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"sanitizationResult":{"filterMatchState":"NO_MATCH_FOUND","invocationResult":"FAILURE"}}`)) + })) + t.Cleanup(server.Close) + evaluator, err := newModelArmorEvaluator(t.Context(), &filterapi.ModelArmorGuardrailProvider{ + Endpoint: server.URL, Project: "project-id", Location: "us-central1", Template: "template-id", + CredentialsJSON: newModelArmorTestCredentials(t), + }, server.Client()) + require.NoError(t, err) + evaluation, err := evaluator.Evaluate(t.Context(), []byte("hello"), filterapi.GuardrailPhaseRequest) + require.False(t, evaluation.Matched) + require.ErrorContains(t, err, "sanitizeUserPrompt invocation failed") +} diff --git a/internal/guardrails/modelarmor.go b/internal/guardrails/modelarmor.go new file mode 100644 index 0000000000..ae359ada6a --- /dev/null +++ b/internal/guardrails/modelarmor.go @@ -0,0 +1,170 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package guardrails + +import ( + "context" + "fmt" + "net/http" + "net/url" + "strings" + + "golang.org/x/oauth2" + "golang.org/x/oauth2/google" + + "github.com/envoyproxy/ai-gateway/internal/filterapi" + "github.com/envoyproxy/ai-gateway/internal/gcpauth" +) + +const modelArmorScope = "https://www.googleapis.com/auth/cloud-platform" + +// Model Armor sanitization result values. +const ( + modelArmorMatchFound = "MATCH_FOUND" + modelArmorInvocationFailed = "FAILURE" +) + +type modelArmorEvaluator struct { + templateURL string + client *http.Client + tokenSource oauth2.TokenSource +} + +func newModelArmorEvaluator(ctx context.Context, config *filterapi.ModelArmorGuardrailProvider, client *http.Client) (filterapi.GuardrailEvaluator, error) { + if config == nil || config.Project == "" || config.Location == "" || config.Template == "" { + return nil, fmt.Errorf("model Armor project, location, and template are required") + } + tokenSource, err := loadGCPTokenSource(ctx, config, client) + if err != nil { + return nil, err + } + endpoint := config.Endpoint + if endpoint == "" { + endpoint = fmt.Sprintf("https://modelarmor.%s.rep.googleapis.com", config.Location) + } + templateURL := strings.TrimRight(endpoint, "/") + "/v1/projects/" + url.PathEscape(config.Project) + + "/locations/" + url.PathEscape(config.Location) + "/templates/" + url.PathEscape(config.Template) + return &modelArmorEvaluator{templateURL: templateURL, client: client, tokenSource: tokenSource}, nil +} + +// loadGCPTokenSource returns an auto-refreshing token source from the configured service account key, +// or from Application Default Credentials (e.g. GKE Workload Identity) when no key is configured. +func loadGCPTokenSource(ctx context.Context, config *filterapi.ModelArmorGuardrailProvider, client *http.Client) (oauth2.TokenSource, error) { + transport, err := gcpauth.NewTransport() + if err != nil { + return nil, err + } + ctx = context.WithValue(ctx, oauth2.HTTPClient, &http.Client{Transport: transport, Timeout: client.Timeout}) + var credentials *google.Credentials + if config.CredentialsJSON != "" { + credentials, err = google.CredentialsFromJSONWithType(ctx, []byte(config.CredentialsJSON), google.ServiceAccount, modelArmorScope) + } else { + credentials, err = google.FindDefaultCredentials(ctx, modelArmorScope) + } + if err != nil { + return nil, fmt.Errorf("cannot load GCP credentials: %w", err) + } + return credentials.TokenSource, nil +} + +// modelArmorMatch is the common shape of every Model Armor filter result. +type modelArmorMatch struct { + MatchState string `json:"matchState"` +} + +// modelArmorFilterResult holds the result of one filter. At most one field is set. +type modelArmorFilterResult struct { + RaiFilterResult *modelArmorMatch `json:"raiFilterResult,omitempty"` + PiAndJailbreakFilterResult *modelArmorMatch `json:"piAndJailbreakFilterResult,omitempty"` + MaliciousURIFilterResult *modelArmorMatch `json:"maliciousUriFilterResult,omitempty"` + CsamFilterFilterResult *modelArmorMatch `json:"csamFilterFilterResult,omitempty"` + VirusScanFilterResult *modelArmorMatch `json:"virusScanFilterResult,omitempty"` + SdpFilterResult *struct { + InspectResult *modelArmorMatch `json:"inspectResult,omitempty"` + DeidentifyResult *struct { + MatchState string `json:"matchState"` + Data *struct { + Text string `json:"text"` + } `json:"data,omitempty"` + } `json:"deidentifyResult,omitempty"` + } `json:"sdpFilterResult,omitempty"` +} + +type modelArmorResponse struct { + SanitizationResult struct { + FilterMatchState string `json:"filterMatchState"` + InvocationResult string `json:"invocationResult"` + FilterResults map[string]modelArmorFilterResult `json:"filterResults"` + } `json:"sanitizationResult"` +} + +func (e *modelArmorEvaluator) Evaluate(ctx context.Context, body []byte, phase filterapi.GuardrailPhase) (filterapi.GuardrailEvaluationResult, error) { + method := "sanitizeUserPrompt" + var payload any = struct { + UserPromptData struct { + Text string `json:"text"` + } `json:"userPromptData"` + }{UserPromptData: struct { + Text string `json:"text"` + }{Text: string(body)}} + if phase == filterapi.GuardrailPhaseResponse { + method = "sanitizeModelResponse" + payload = struct { + ModelResponseData struct { + Text string `json:"text"` + } `json:"modelResponseData"` + }{ModelResponseData: struct { + Text string `json:"text"` + }{Text: string(body)}} + } + token, err := e.tokenSource.Token() + if err != nil { + return filterapi.GuardrailEvaluationResult{}, fmt.Errorf("cannot retrieve GCP access token: %w", err) + } + + var result modelArmorResponse + if err = doJSON(ctx, e.client, http.MethodPost, e.templateURL+":"+method, payload, func(req *http.Request) { + token.SetAuthHeader(req) + }, &result); err != nil { + return filterapi.GuardrailEvaluationResult{}, fmt.Errorf("model Armor %s request failed: %w", method, err) + } + if result.SanitizationResult.InvocationResult == modelArmorInvocationFailed { + return filterapi.GuardrailEvaluationResult{}, fmt.Errorf("model Armor %s invocation failed", method) + } + if result.SanitizationResult.FilterMatchState != modelArmorMatchFound { + return filterapi.GuardrailEvaluationResult{}, nil + } + return filterapi.GuardrailEvaluationResult{ + Matched: true, + Replacement: modelArmorDeidentifiedText(result.SanitizationResult.FilterResults), + }, nil +} + +// modelArmorDeidentifiedText returns the Sensitive Data Protection de-identified text when it is the +// only filter that matched. It returns nil when any other filter matched, so Mask rules never forward +// content that was flagged for a reason masking cannot fix (e.g. a jailbreak attempt). +func modelArmorDeidentifiedText(results map[string]modelArmorFilterResult) []byte { + var replacement []byte + for _, result := range results { + for _, match := range []*modelArmorMatch{ + result.RaiFilterResult, result.PiAndJailbreakFilterResult, result.MaliciousURIFilterResult, + result.CsamFilterFilterResult, result.VirusScanFilterResult, + } { + if match != nil && match.MatchState == modelArmorMatchFound { + return nil + } + } + if sdp := result.SdpFilterResult; sdp != nil { + if sdp.InspectResult != nil && sdp.InspectResult.MatchState == modelArmorMatchFound { + return nil + } + if d := sdp.DeidentifyResult; d != nil && d.MatchState == modelArmorMatchFound && d.Data != nil { + replacement = []byte(d.Data.Text) + } + } + } + return replacement +} diff --git a/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_guardrailpolicies.yaml b/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_guardrailpolicies.yaml index c5c7ddfbf7..5485cc22cc 100644 --- a/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_guardrailpolicies.yaml +++ b/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_guardrailpolicies.yaml @@ -308,6 +308,77 @@ spec: description: Message is returned to the caller when the rule blocks a request or response. type: string + modelArmor: + description: ModelArmor configures Google Cloud Model Armor. + properties: + credentialsSecretRef: + description: |- + CredentialsSecretRef optionally references a Secret whose credentials entry contains a Google Cloud + service account key JSON. When omitted, Application Default Credentials are used. + properties: + group: + default: "" + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Secret + description: Kind is kind of the referent. For example + "Secret". + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the referenced object. When unspecified, the local + namespace is inferred. + + Note that when a namespace different than the local namespace is specified, + a ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - name + type: object + endpoint: + description: Endpoint overrides the regional Model Armor + endpoint, primarily for private endpoints and testing. + format: uri + type: string + location: + description: Location is the Google Cloud region of + the Model Armor template, for example us-central1. + minLength: 1 + type: string + project: + description: Project is the Google Cloud project ID + that owns the Model Armor template. + minLength: 1 + type: string + template: + description: Template is the Model Armor template ID. + minLength: 1 + type: string + required: + - location + - project + - template + type: object pattern: description: Pattern is used for deterministic regex-based evaluations. @@ -388,6 +459,7 @@ spec: - Bedrock - AzureContentSafety - HTTP + - ModelArmor type: string required: - type @@ -396,23 +468,28 @@ spec: - message: Regex requires pattern and no external provider configuration rule: self.type != 'Regex' || (has(self.pattern) && size(self.pattern) > 0 && !has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety) - && !has(self.http)) + && !has(self.http) && !has(self.modelArmor)) - message: Presidio requires only presidio provider configuration rule: self.type != 'Presidio' || (!has(self.pattern) && has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety) && - !has(self.http)) + !has(self.http) && !has(self.modelArmor)) - message: Bedrock requires only bedrock provider configuration rule: self.type != 'Bedrock' || (!has(self.pattern) && has(self.bedrock) && !has(self.presidio) && !has(self.azureContentSafety) - && !has(self.http)) + && !has(self.http) && !has(self.modelArmor)) - message: AzureContentSafety requires only azureContentSafety provider configuration rule: self.type != 'AzureContentSafety' || (!has(self.pattern) && has(self.azureContentSafety) && !has(self.presidio) && - !has(self.bedrock) && !has(self.http)) + !has(self.bedrock) && !has(self.http) && !has(self.modelArmor)) - message: HTTP requires only http provider configuration rule: self.type != 'HTTP' || (!has(self.pattern) && has(self.http) - && !has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety)) + && !has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety) + && !has(self.modelArmor)) + - message: ModelArmor requires only modelArmor provider configuration + rule: self.type != 'ModelArmor' || (!has(self.pattern) && + has(self.modelArmor) && !has(self.presidio) && !has(self.bedrock) + && !has(self.azureContentSafety) && !has(self.http)) - message: AzureContentSafety does not support Mask rule: self.action != 'Mask' || self.type != 'AzureContentSafety' required: @@ -820,6 +897,77 @@ spec: description: Message is returned to the caller when the rule blocks a request or response. type: string + modelArmor: + description: ModelArmor configures Google Cloud Model Armor. + properties: + credentialsSecretRef: + description: |- + CredentialsSecretRef optionally references a Secret whose credentials entry contains a Google Cloud + service account key JSON. When omitted, Application Default Credentials are used. + properties: + group: + default: "" + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Secret + description: Kind is kind of the referent. For example + "Secret". + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the referenced object. When unspecified, the local + namespace is inferred. + + Note that when a namespace different than the local namespace is specified, + a ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - name + type: object + endpoint: + description: Endpoint overrides the regional Model Armor + endpoint, primarily for private endpoints and testing. + format: uri + type: string + location: + description: Location is the Google Cloud region of + the Model Armor template, for example us-central1. + minLength: 1 + type: string + project: + description: Project is the Google Cloud project ID + that owns the Model Armor template. + minLength: 1 + type: string + template: + description: Template is the Model Armor template ID. + minLength: 1 + type: string + required: + - location + - project + - template + type: object pattern: description: Pattern is used for deterministic regex-based evaluations. @@ -900,6 +1048,7 @@ spec: - Bedrock - AzureContentSafety - HTTP + - ModelArmor type: string required: - type @@ -908,23 +1057,28 @@ spec: - message: Regex requires pattern and no external provider configuration rule: self.type != 'Regex' || (has(self.pattern) && size(self.pattern) > 0 && !has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety) - && !has(self.http)) + && !has(self.http) && !has(self.modelArmor)) - message: Presidio requires only presidio provider configuration rule: self.type != 'Presidio' || (!has(self.pattern) && has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety) && - !has(self.http)) + !has(self.http) && !has(self.modelArmor)) - message: Bedrock requires only bedrock provider configuration rule: self.type != 'Bedrock' || (!has(self.pattern) && has(self.bedrock) && !has(self.presidio) && !has(self.azureContentSafety) - && !has(self.http)) + && !has(self.http) && !has(self.modelArmor)) - message: AzureContentSafety requires only azureContentSafety provider configuration rule: self.type != 'AzureContentSafety' || (!has(self.pattern) && has(self.azureContentSafety) && !has(self.presidio) && - !has(self.bedrock) && !has(self.http)) + !has(self.bedrock) && !has(self.http) && !has(self.modelArmor)) - message: HTTP requires only http provider configuration rule: self.type != 'HTTP' || (!has(self.pattern) && has(self.http) - && !has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety)) + && !has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety) + && !has(self.modelArmor)) + - message: ModelArmor requires only modelArmor provider configuration + rule: self.type != 'ModelArmor' || (!has(self.pattern) && + has(self.modelArmor) && !has(self.presidio) && !has(self.bedrock) + && !has(self.azureContentSafety) && !has(self.http)) - message: AzureContentSafety does not support Mask rule: self.action != 'Mask' || self.type != 'AzureContentSafety' required: diff --git a/site/docs/api/api.mdx b/site/docs/api/api.mdx index 20dd4f1678..0a641c57d5 100644 --- a/site/docs/api/api.mdx +++ b/site/docs/api/api.mdx @@ -703,6 +703,7 @@ QuotaPolicyList contains a list of QuotaPolicy - [MCPRouteSpec](#github-com-envoyproxy-ai-gateway-api-v1alpha1-mcproutespec) - [MCPRouteStatus](#github-com-envoyproxy-ai-gateway-api-v1alpha1-mcproutestatus) - [MCPToolFilter](#github-com-envoyproxy-ai-gateway-api-v1alpha1-mcptoolfilter) +- [ModelArmorGuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1alpha1-modelarmorguardrailprovider) - [PerModelQuota](#github-com-envoyproxy-ai-gateway-api-v1alpha1-permodelquota) - [PresidioGuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1alpha1-presidioguardrailprovider) - [ProtectedResourceMetadata](#github-com-envoyproxy-ai-gateway-api-v1alpha1-protectedresourcemetadata) @@ -1957,6 +1958,11 @@ GuardrailProvider describes the implementation used to evaluate a rule. type="[HTTPGuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1alpha1-httpguardrailprovider)" required="false" description="HTTP configures a custom guardrail service that implements the generic guardrail HTTP contract." +/> #### GuardrailRule @@ -2830,6 +2841,52 @@ a tool must match an include rule AND not match any exclude rule to be allowed. /> +#### ModelArmorGuardrailProvider + + + +**Appears in:** +- [GuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailprovider) + +ModelArmorGuardrailProvider configures calls to the Google Cloud Model Armor sanitize APIs. + +Request rules call sanitizeUserPrompt and Response rules call sanitizeModelResponse on the +configured template. The filters (Responsible AI, prompt injection and jailbreak, malicious URLs, +and Sensitive Data Protection) and their confidence levels are configured in the template. +Mask requires a template with an advanced Sensitive Data Protection de-identify template. + +##### Fields + + + + + + #### PerModelQuota @@ -3818,6 +3875,7 @@ MCPRouteList contains a list of MCPRoute. - [MCPRouteSpec](#github-com-envoyproxy-ai-gateway-api-v1beta1-mcproutespec) - [MCPRouteStatus](#github-com-envoyproxy-ai-gateway-api-v1beta1-mcproutestatus) - [MCPToolFilter](#github-com-envoyproxy-ai-gateway-api-v1beta1-mcptoolfilter) +- [ModelArmorGuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1beta1-modelarmorguardrailprovider) - [PresidioGuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1beta1-presidioguardrailprovider) - [ProtectedResourceMetadata](#github-com-envoyproxy-ai-gateway-api-v1beta1-protectedresourcemetadata) - [ToolCall](#github-com-envoyproxy-ai-gateway-api-v1beta1-toolcall) @@ -5172,6 +5230,11 @@ GuardrailProvider describes the implementation used to evaluate a rule. type="[HTTPGuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1beta1-httpguardrailprovider)" required="false" description="HTTP configures a custom guardrail service that implements the generic guardrail HTTP contract." +/> #### GuardrailRule @@ -6225,6 +6293,52 @@ a tool must match an include rule AND not match any exclude rule to be allowed. /> +#### ModelArmorGuardrailProvider + + + +**Appears in:** +- [GuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailprovider) + +ModelArmorGuardrailProvider configures calls to the Google Cloud Model Armor sanitize APIs. + +Request rules call sanitizeUserPrompt and Response rules call sanitizeModelResponse on the +configured template. The filters (Responsible AI, prompt injection and jailbreak, malicious URLs, +and Sensitive Data Protection) and their confidence levels are configured in the template. +Mask requires a template with an advanced Sensitive Data Protection de-identify template. + +##### Fields + + + + + + #### PresidioGuardrailProvider diff --git a/site/docs/capabilities/security/guardrails.md b/site/docs/capabilities/security/guardrails.md index 9c7fea8b03..01842f7836 100644 --- a/site/docs/capabilities/security/guardrails.md +++ b/site/docs/capabilities/security/guardrails.md @@ -38,7 +38,7 @@ Rules support three actions: - `Block` rejects matching traffic. - `Monitor` records matching traffic without blocking or changing it. -- `Mask` replaces detected text. Regex and Presidio use `maskReplacement`; Bedrock uses transformed output returned by the provider; HTTP uses the returned `replacement` or masks returned findings with `maskReplacement`. Azure Text Analysis does not support Mask. +- `Mask` replaces detected text. Regex and Presidio use `maskReplacement`; Bedrock uses transformed output returned by the provider; HTTP uses the returned `replacement` or masks returned findings with `maskReplacement`; Model Armor uses the text de-identified by its Sensitive Data Protection filter. Azure Text Analysis does not support Mask. When multiple policies target one backend, policies are evaluated in namespace/name order and rules retain declaration order. The first Block result stops evaluation. @@ -93,6 +93,27 @@ provider: The referenced Secret must contain an `apiKey` entry. +### Google Cloud Model Armor + +Model Armor screens content with a [Model Armor template](https://docs.cloud.google.com/model-armor/overview). Request rules call `sanitizeUserPrompt` and Response rules call `sanitizeModelResponse` on the regional endpoint `https://modelarmor..rep.googleapis.com`. The filters (Responsible AI, prompt injection and jailbreak, malicious URLs, and Sensitive Data Protection) and their confidence levels are configured in the template, so the rule only references it: + +```yaml +provider: + type: ModelArmor + timeoutSeconds: 10 + failureMode: FailClosed + modelArmor: + project: my-project + location: us-central1 + template: my-template +``` + +A rule matches when Model Armor reports `MATCH_FOUND` for any filter in the template. An invocation result of `FAILURE` is treated as a provider error and follows the rule's failure mode. + +By default, the ext-proc uses Google Application Default Credentials, including GKE Workload Identity. The identity needs the `roles/modelarmor.user` role. For static credentials, set `credentialsSecretRef` to a Secret whose `credentials` entry contains a service account key JSON. `endpoint` can override the regional endpoint for a private endpoint. Token requests honor the `AI_GATEWAY_GCP_AUTH_PROXY_URL` proxy, like GCP backend authentication. + +`Mask` requires a template with advanced Sensitive Data Protection and a de-identify template; the gateway forwards the de-identified text returned by Model Armor. If any other filter also matches (for example, a jailbreak attempt), there is no safe replacement, so the rule fails instead of forwarding the content. + ### Custom HTTP guardrails Use the `HTTP` provider to integrate a guardrail service that has no native integration, such as an in-house classifier. The gateway calls the service over HTTP instead of running local executables. @@ -234,11 +255,12 @@ Presidio is tested against its official analyzer image with Testcontainers. The go test ./internal/guardrails -run '^TestPresidioEvaluatorContainer$' -v ``` -The Azure and Bedrock live tests are disabled unless all required environment variables for a provider are set: +The Azure, Bedrock, and Model Armor live tests are disabled unless all required environment variables for a provider are set: - Presidio managed/external deployment: `TEST_PRESIDIO_ENDPOINT`, `TEST_PRESIDIO_BLOCKED_TEXT`, and optionally `TEST_PRESIDIO_API_KEY`. - Azure: `TEST_AZURE_CONTENT_SAFETY_ENDPOINT`, `TEST_AZURE_CONTENT_SAFETY_API_KEY`, and `TEST_AZURE_CONTENT_SAFETY_BLOCKED_TEXT`. - Bedrock: `TEST_AWS_BEDROCK_GUARDRAIL_REGION`, `TEST_AWS_BEDROCK_GUARDRAIL_ID`, `TEST_AWS_BEDROCK_GUARDRAIL_VERSION`, and `TEST_AWS_BEDROCK_GUARDRAIL_BLOCKED_TEXT`. AWS credentials use the standard credential chain. +- Model Armor: `TEST_GCP_MODEL_ARMOR_PROJECT`, `TEST_GCP_MODEL_ARMOR_LOCATION`, `TEST_GCP_MODEL_ARMOR_TEMPLATE`, and `TEST_GCP_MODEL_ARMOR_BLOCKED_TEXT`. Google credentials use Application Default Credentials. Run them with: diff --git a/tests/crdcel/main_test.go b/tests/crdcel/main_test.go index 366aef8e10..8efa73e457 100644 --- a/tests/crdcel/main_test.go +++ b/tests/crdcel/main_test.go @@ -177,6 +177,8 @@ func TestGuardrailPolicies(t *testing.T) { {name: "azure_mask.yaml", expErr: "AzureContentSafety does not support Mask"}, {name: "http_mismatch.yaml", expErr: "HTTP requires only http provider configuration"}, {name: "http_invalid_path.yaml", expErr: "should match '^/.*$'"}, + {name: "model_armor_mismatch.yaml", expErr: "ModelArmor requires only modelArmor provider configuration"}, + {name: "model_armor_missing_template.yaml", expErr: "should be at least 1 chars long"}, } { t.Run(tc.name, func(t *testing.T) { data, err := testdata.ReadFile(path.Join("testdata/guardrailpolicies", tc.name)) diff --git a/tests/crdcel/testdata/guardrailpolicies/basic.yaml b/tests/crdcel/testdata/guardrailpolicies/basic.yaml index fb589a3db8..9ffbbd8f7e 100644 --- a/tests/crdcel/testdata/guardrailpolicies/basic.yaml +++ b/tests/crdcel/testdata/guardrailpolicies/basic.yaml @@ -62,3 +62,23 @@ spec: path: /v1/analyze apiKeySecretRef: name: custom-guardrail-key + - name: model-armor + phase: Request + provider: + type: ModelArmor + modelArmor: + project: my-project + location: us-central1 + template: my-template + - name: model-armor-mask + phase: Response + provider: + type: ModelArmor + action: Mask + modelArmor: + project: my-project + location: us-central1 + template: my-template + endpoint: https://modelarmor.example.com + credentialsSecretRef: + name: model-armor-sa diff --git a/tests/crdcel/testdata/guardrailpolicies/model_armor_mismatch.yaml b/tests/crdcel/testdata/guardrailpolicies/model_armor_mismatch.yaml new file mode 100644 index 0000000000..7720c9d9b8 --- /dev/null +++ b/tests/crdcel/testdata/guardrailpolicies/model_armor_mismatch.yaml @@ -0,0 +1,26 @@ +# Copyright Envoy AI Gateway Authors +# SPDX-License-Identifier: Apache-2.0 +# The full text of the Apache license is available in the LICENSE file at +# the root of the repo. + +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: GuardrailPolicy +metadata: + name: model-armor-mismatch + namespace: default +spec: + targetRefs: + - group: aigateway.envoyproxy.io + kind: AIServiceBackend + name: backend + rules: + - name: invalid + phase: Request + provider: + type: ModelArmor + modelArmor: + project: my-project + location: us-central1 + template: my-template + http: + endpoint: https://guardrail.example.com diff --git a/tests/crdcel/testdata/guardrailpolicies/model_armor_missing_template.yaml b/tests/crdcel/testdata/guardrailpolicies/model_armor_missing_template.yaml new file mode 100644 index 0000000000..0eed45daf9 --- /dev/null +++ b/tests/crdcel/testdata/guardrailpolicies/model_armor_missing_template.yaml @@ -0,0 +1,24 @@ +# Copyright Envoy AI Gateway Authors +# SPDX-License-Identifier: Apache-2.0 +# The full text of the Apache license is available in the LICENSE file at +# the root of the repo. + +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: GuardrailPolicy +metadata: + name: model-armor-missing-template + namespace: default +spec: + targetRefs: + - group: aigateway.envoyproxy.io + kind: AIServiceBackend + name: backend + rules: + - name: invalid + phase: Request + provider: + type: ModelArmor + modelArmor: + project: my-project + location: us-central1 + template: "" From ccf090ed82aa4728f9f9f09d20e6ed1327235809 Mon Sep 17 00:00:00 2001 From: Fernando Escolar Date: Sat, 10 Oct 2026 10:52:23 +0200 Subject: [PATCH 14/18] Fixing merge issue Signed-off-by: Fernando Escolar --- internal/extproc/processor_impl.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/extproc/processor_impl.go b/internal/extproc/processor_impl.go index cf664537e1..746997260e 100644 --- a/internal/extproc/processor_impl.go +++ b/internal/extproc/processor_impl.go @@ -787,7 +787,11 @@ func (u *upstreamProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) ProcessRespo if len(newBody) > 0 { guardrailBody = newBody } - outcome, guardrailErr := evaluateResponseGuardrails(ctx, u.parent.config.Guardrails, guardrailBody, u.backendName) + var configuredGuardrails []filterapi.RuntimeGuardrail + if u.parent.config != nil { + configuredGuardrails = u.parent.config.Guardrails + } + outcome, guardrailErr := evaluateResponseGuardrails(ctx, configuredGuardrails, guardrailBody, u.backendName) if guardrailErr != nil { u.parent.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseResponse, metrics.GuardrailResultError, u.backendName, true) u.parent.recordGuardrailTrace("", filterapi.GuardrailPhaseResponse, metrics.GuardrailResultError) From 3c520d81e09b063c81b7163360511104fa0ea5a0 Mon Sep 17 00:00:00 2001 From: Fernando Escolar Date: Sat, 10 Oct 2026 11:09:08 +0200 Subject: [PATCH 15/18] extproc: only record guardrail span events when guardrails apply Signed-off-by: Fernando Escolar --- internal/extproc/guardrails_test.go | 18 ++++++++- internal/extproc/mocks_test.go | 12 ++++++ internal/extproc/processor_impl.go | 57 +++++++++++------------------ 3 files changed, 51 insertions(+), 36 deletions(-) diff --git a/internal/extproc/guardrails_test.go b/internal/extproc/guardrails_test.go index d6883f86f8..69f0c078c2 100644 --- a/internal/extproc/guardrails_test.go +++ b/internal/extproc/guardrails_test.go @@ -157,15 +157,31 @@ func TestRequestGuardrailMaskMutatesBody(t *testing.T) { func TestRecordGuardrailEvaluationIgnoresUnconfiguredPhase(t *testing.T) { recorder := &recordingGuardrailMetrics{} + span := &mockGuardrailChatCompletionSpan{} processor := &chatCompletionProcessorRouterFilter{ config: &filterapi.RuntimeConfig{Guardrails: []filterapi.RuntimeGuardrail{{ Phase: filterapi.GuardrailPhaseResponse, }}}, guardrailMetrics: recorder, + span: span, } - processor.recordGuardrailEvaluation(t.Context(), filterapi.GuardrailPhaseRequest, metrics.GuardrailResultAllowed, "", true) + processor.recordGuardrailEvaluation(t.Context(), "", filterapi.GuardrailPhaseRequest, metrics.GuardrailResultAllowed, "", true) require.Zero(t, recorder.count) + require.Empty(t, span.guardrailEvents) + + processor.recordGuardrailEvaluation(t.Context(), "deny-pii", filterapi.GuardrailPhaseResponse, metrics.GuardrailResultBlocked, "", true) + require.Equal(t, 1, recorder.count) + require.Equal(t, []string{"deny-pii/Response/blocked"}, span.guardrailEvents) +} + +func TestRecordGuardrailEvaluationWithoutGuardrails(t *testing.T) { + span := &mockGuardrailChatCompletionSpan{} + processor := &chatCompletionProcessorRouterFilter{config: &filterapi.RuntimeConfig{}, span: span} + + processor.recordGuardrailEvaluation(t.Context(), "", filterapi.GuardrailPhaseRequest, metrics.GuardrailResultAllowed, "", true) + processor.recordGuardrailEvaluation(t.Context(), "", filterapi.GuardrailPhaseResponse, metrics.GuardrailResultAllowed, "backend", true) + require.Empty(t, span.guardrailEvents) } func TestBackendScopedGuardrail(t *testing.T) { diff --git a/internal/extproc/mocks_test.go b/internal/extproc/mocks_test.go index 7ccc14e62c..8e02763c12 100644 --- a/internal/extproc/mocks_test.go +++ b/internal/extproc/mocks_test.go @@ -325,6 +325,18 @@ func (m *mockChatCompletionSpan) EndSpan() { m.endedCount++ } var _ tracingapi.ChatCompletionSpan = &mockChatCompletionSpan{} +// mockGuardrailChatCompletionSpan additionally implements [tracingapi.GuardrailSpan]. +type mockGuardrailChatCompletionSpan struct { + mockChatCompletionSpan + guardrailEvents []string +} + +func (m *mockGuardrailChatCompletionSpan) RecordGuardrail(name, phase, result string) { + m.guardrailEvents = append(m.guardrailEvents, name+"/"+phase+"/"+result) +} + +var _ tracingapi.GuardrailSpan = &mockGuardrailChatCompletionSpan{} + // mockBackendChatCompletionSpan additionally implements [tracingapi.BackendSpan], // which only spans whose semantic convention records the resolved backend do. type mockBackendChatCompletionSpan struct { diff --git a/internal/extproc/processor_impl.go b/internal/extproc/processor_impl.go index 746997260e..f00a835eef 100644 --- a/internal/extproc/processor_impl.go +++ b/internal/extproc/processor_impl.go @@ -163,20 +163,22 @@ func newRouterProcessor[ReqT, RespT, RespChunkT any, EndpointSpecT endpointspec. } } +// recordGuardrailEvaluation records the guardrail metric and span event, only when guardrails are +// configured for the phase and backend so that unguarded traffic is not reported as evaluated. func (r *routerProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) recordGuardrailEvaluation( ctx context.Context, + name string, phase filterapi.GuardrailPhase, result metrics.GuardrailResult, backendName string, includeGlobal bool, ) { - if r.guardrailMetrics == nil || r.config == nil || !guardrailsConfiguredForPhase(r.config.Guardrails, phase, backendName, includeGlobal) { + if r.config == nil || !guardrailsConfiguredForPhase(r.config.Guardrails, phase, backendName, includeGlobal) { return } - r.guardrailMetrics.RecordEvaluation(ctx, string(phase), result) -} - -func (r *routerProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) recordGuardrailTrace(name string, phase filterapi.GuardrailPhase, result metrics.GuardrailResult) { + if r.guardrailMetrics != nil { + r.guardrailMetrics.RecordEvaluation(ctx, string(phase), result) + } if span, ok := r.span.(tracingapi.GuardrailSpan); ok { span.RecordGuardrail(name, string(phase), string(result)) } @@ -355,15 +357,13 @@ func (r *routerProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) ProcessRequest ) outcome, guardrailErr := evaluateRequestGuardrails(ctx, r.config.Guardrails, rawBody.Body) if guardrailErr != nil { - r.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultError, "", true) - r.recordGuardrailTrace("", filterapi.GuardrailPhaseRequest, metrics.GuardrailResultError) + r.recordGuardrailEvaluation(ctx, "", filterapi.GuardrailPhaseRequest, metrics.GuardrailResultError, "", true) if !isGuardrailFailOpenError(guardrailErr) { return nil, fmt.Errorf("failed to evaluate request guardrails: %w", guardrailErr) } r.logger.Warn("request guardrail provider failed open", slog.String("error", guardrailErr.Error())) } else if outcome.Violation != nil { - r.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultBlocked, "", true) - r.recordGuardrailTrace(outcome.Violation.Name, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultBlocked) + r.recordGuardrailEvaluation(ctx, outcome.Violation.Name, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultBlocked, "", true) r.logger.Warn("request blocked by guardrail", slog.String("guardrail.name", outcome.Violation.Name), slog.String("guardrail.phase", string(filterapi.GuardrailPhaseRequest))) @@ -378,16 +378,13 @@ func (r *routerProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) ProcessRequest if err = r.updateParsedRequestAfterGuardrailMask(outcome.Body, costConfigured); err != nil { return nil, err } - r.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultMasked, "", true) - r.recordGuardrailTrace(outcome.RuleName, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultMasked) + r.recordGuardrailEvaluation(ctx, outcome.RuleName, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultMasked, "", true) r.logger.Info("request masked by guardrail", slog.String("guardrail.name", outcome.RuleName)) case outcome.Monitored: - r.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultMonitored, "", true) - r.recordGuardrailTrace(outcome.RuleName, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultMonitored) + r.recordGuardrailEvaluation(ctx, outcome.RuleName, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultMonitored, "", true) r.logger.Info("request detected by monitor guardrail", slog.String("guardrail.name", outcome.RuleName)) case guardrailErr == nil: - r.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultAllowed, "", true) - r.recordGuardrailTrace("", filterapi.GuardrailPhaseRequest, metrics.GuardrailResultAllowed) + r.recordGuardrailEvaluation(ctx, "", filterapi.GuardrailPhaseRequest, metrics.GuardrailResultAllowed, "", true) } return &extprocv3.ProcessingResponse{ @@ -448,15 +445,13 @@ func (u *upstreamProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) ProcessReque } outcome, guardrailErr := evaluateBackendRequestGuardrails(ctx, configuredGuardrails, u.parent.originalRequestBodyRaw, u.backendName) if guardrailErr != nil { - u.parent.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultError, u.backendName, false) - u.parent.recordGuardrailTrace("", filterapi.GuardrailPhaseRequest, metrics.GuardrailResultError) + u.parent.recordGuardrailEvaluation(ctx, "", filterapi.GuardrailPhaseRequest, metrics.GuardrailResultError, u.backendName, false) if !isGuardrailFailOpenError(guardrailErr) { return nil, fmt.Errorf("failed to evaluate backend request guardrails: %w", guardrailErr) } u.logger.Warn("request guardrail provider failed open", slog.String("error", guardrailErr.Error())) } else if outcome.Violation != nil { - u.parent.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultBlocked, u.backendName, false) - u.parent.recordGuardrailTrace(outcome.Violation.Name, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultBlocked) + u.parent.recordGuardrailEvaluation(ctx, outcome.Violation.Name, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultBlocked, u.backendName, false) u.logger.Warn("request blocked by guardrail", slog.String("guardrail.name", outcome.Violation.Name), slog.String("guardrail.phase", string(filterapi.GuardrailPhaseRequest))) return u.respondLocally(ctx, http.StatusForbidden, "GuardrailViolation", outcome.Violation.Message), nil } @@ -466,16 +461,13 @@ func (u *upstreamProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) ProcessReque if err = u.parent.updateParsedRequestAfterGuardrailMask(outcome.Body, costConfigured); err != nil { return nil, err } - u.parent.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultMasked, u.backendName, false) - u.parent.recordGuardrailTrace(outcome.RuleName, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultMasked) + u.parent.recordGuardrailEvaluation(ctx, outcome.RuleName, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultMasked, u.backendName, false) u.logger.Info("request masked by guardrail", slog.String("guardrail.name", outcome.RuleName)) case outcome.Monitored: - u.parent.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultMonitored, u.backendName, false) - u.parent.recordGuardrailTrace(outcome.RuleName, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultMonitored) + u.parent.recordGuardrailEvaluation(ctx, outcome.RuleName, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultMonitored, u.backendName, false) u.logger.Info("request detected by monitor guardrail", slog.String("guardrail.name", outcome.RuleName)) case guardrailErr == nil: - u.parent.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultAllowed, u.backendName, false) - u.parent.recordGuardrailTrace("", filterapi.GuardrailPhaseRequest, metrics.GuardrailResultAllowed) + u.parent.recordGuardrailEvaluation(ctx, "", filterapi.GuardrailPhaseRequest, metrics.GuardrailResultAllowed, u.backendName, false) } if u.unsupportedBackendErr != nil { @@ -793,15 +785,13 @@ func (u *upstreamProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) ProcessRespo } outcome, guardrailErr := evaluateResponseGuardrails(ctx, configuredGuardrails, guardrailBody, u.backendName) if guardrailErr != nil { - u.parent.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseResponse, metrics.GuardrailResultError, u.backendName, true) - u.parent.recordGuardrailTrace("", filterapi.GuardrailPhaseResponse, metrics.GuardrailResultError) + u.parent.recordGuardrailEvaluation(ctx, "", filterapi.GuardrailPhaseResponse, metrics.GuardrailResultError, u.backendName, true) if !isGuardrailFailOpenError(guardrailErr) { return nil, fmt.Errorf("failed to evaluate response guardrails: %w", guardrailErr) } u.logger.Warn("response guardrail provider failed open", slog.String("error", guardrailErr.Error())) } else if outcome.Violation != nil { - u.parent.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseResponse, metrics.GuardrailResultBlocked, u.backendName, true) - u.parent.recordGuardrailTrace(outcome.Violation.Name, filterapi.GuardrailPhaseResponse, metrics.GuardrailResultBlocked) + u.parent.recordGuardrailEvaluation(ctx, outcome.Violation.Name, filterapi.GuardrailPhaseResponse, metrics.GuardrailResultBlocked, u.backendName, true) u.logger.Warn("response blocked by guardrail", slog.String("guardrail.name", outcome.Violation.Name), slog.String("guardrail.phase", string(filterapi.GuardrailPhaseResponse))) @@ -812,16 +802,13 @@ func (u *upstreamProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) ProcessRespo bodyMutation = &extprocv3.BodyMutation{Mutation: &extprocv3.BodyMutation_Body{Body: outcome.Body}} resp.GetResponseBody().Response.BodyMutation = bodyMutation setHeader(headerMutation, "content-length", strconv.Itoa(len(outcome.Body))) - u.parent.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseResponse, metrics.GuardrailResultMasked, u.backendName, true) - u.parent.recordGuardrailTrace(outcome.RuleName, filterapi.GuardrailPhaseResponse, metrics.GuardrailResultMasked) + u.parent.recordGuardrailEvaluation(ctx, outcome.RuleName, filterapi.GuardrailPhaseResponse, metrics.GuardrailResultMasked, u.backendName, true) u.logger.Info("response masked by guardrail", slog.String("guardrail.name", outcome.RuleName)) case outcome.Monitored: - u.parent.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseResponse, metrics.GuardrailResultMonitored, u.backendName, true) - u.parent.recordGuardrailTrace(outcome.RuleName, filterapi.GuardrailPhaseResponse, metrics.GuardrailResultMonitored) + u.parent.recordGuardrailEvaluation(ctx, outcome.RuleName, filterapi.GuardrailPhaseResponse, metrics.GuardrailResultMonitored, u.backendName, true) u.logger.Info("response detected by monitor guardrail", slog.String("guardrail.name", outcome.RuleName)) case guardrailErr == nil: - u.parent.recordGuardrailEvaluation(ctx, filterapi.GuardrailPhaseResponse, metrics.GuardrailResultAllowed, u.backendName, true) - u.parent.recordGuardrailTrace("", filterapi.GuardrailPhaseResponse, metrics.GuardrailResultAllowed) + u.parent.recordGuardrailEvaluation(ctx, "", filterapi.GuardrailPhaseResponse, metrics.GuardrailResultAllowed, u.backendName, true) } // Remove content-encoding when translation or guardrail masking produces an uncompressed body. From 2b002ff194013eb34be624c798b0b696f96c35dc Mon Sep 17 00:00:00 2001 From: Fernando Escolar Date: Sat, 10 Oct 2026 11:33:37 +0200 Subject: [PATCH 16/18] cover GuardrailPolicy validation and reconcile paths Signed-off-by: Fernando Escolar --- internal/controller/guardrail_policy_test.go | 234 +++++++++++++++++++ 1 file changed, 234 insertions(+) diff --git a/internal/controller/guardrail_policy_test.go b/internal/controller/guardrail_policy_test.go index 12d7a99da0..c940efca4b 100644 --- a/internal/controller/guardrail_policy_test.go +++ b/internal/controller/guardrail_policy_test.go @@ -523,3 +523,237 @@ func TestGuardrailPolicySecretRefsIndexIncludesModelArmorProvider(t *testing.T) } require.Equal(t, []string{"model-armor-sa.default"}, guardrailPolicySecretRefsIndexFunc(policy)) } + +func TestGuardrailProviderValidation(t *testing.T) { + const namespace = "default" + otherNamespace := gwapiv1.Namespace("other") + kube := fakekube.NewClientset( + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "api-key", Namespace: namespace}, + Data: map[string][]byte{"apiKey": []byte("secret")}, + }, + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "string-data", Namespace: namespace}, + StringData: map[string]string{"credentials": "[default]"}, + }, + ) + controller := &GuardrailPolicyController{kube: kube} + tests := []struct { + name string + provider aigv1b1.GuardrailProvider + wantErr string + }{ + { + name: "unsupported action", + provider: aigv1b1.GuardrailProvider{Type: aigv1b1.GuardrailProviderTypeRegex, Pattern: "x", Action: "Drop"}, + wantErr: `unsupported action "Drop"`, + }, + { + name: "azure mask", + provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeAzureContentSafety, Action: aigv1b1.GuardrailActionMask, + }, + wantErr: "azure Content Safety does not support mask", + }, + { + name: "unsupported failure mode", + provider: aigv1b1.GuardrailProvider{Type: aigv1b1.GuardrailProviderTypeRegex, Pattern: "x", FailureMode: "Retry"}, + wantErr: `unsupported failureMode "Retry"`, + }, + { + name: "regex", + provider: aigv1b1.GuardrailProvider{Type: aigv1b1.GuardrailProviderTypeRegex, Pattern: "secret", Action: aigv1b1.GuardrailActionMask}, + }, + { + name: "regex missing pattern", + provider: aigv1b1.GuardrailProvider{Type: aigv1b1.GuardrailProviderTypeRegex}, + wantErr: "regex pattern is required", + }, + { + name: "regex invalid pattern", + provider: aigv1b1.GuardrailProvider{Type: aigv1b1.GuardrailProviderTypeRegex, Pattern: "("}, + wantErr: "invalid regex pattern", + }, + { + name: "presidio", + provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypePresidio, + Presidio: &aigv1b1.PresidioGuardrailProvider{ + Endpoint: "https://presidio.example.com", + APIKeySecretRef: &gwapiv1.SecretObjectReference{Name: "api-key"}, + }, + }, + }, + { + name: "presidio missing configuration", + provider: aigv1b1.GuardrailProvider{Type: aigv1b1.GuardrailProviderTypePresidio}, + wantErr: "presidio configuration is required", + }, + { + name: "presidio invalid endpoint", + provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypePresidio, + Presidio: &aigv1b1.PresidioGuardrailProvider{Endpoint: "presidio"}, + }, + wantErr: "valid provider endpoint is required", + }, + { + name: "bedrock with secret in string data", + provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeBedrockGuardrails, + Bedrock: &aigv1b1.BedrockGuardrailProvider{ + Region: "us-east-1", GuardrailIdentifier: "guardrail-id", GuardrailVersion: "1", + Endpoint: "https://bedrock.example.com", + CredentialsSecretRef: &gwapiv1.SecretObjectReference{Name: "string-data"}, + }, + }, + }, + { + name: "bedrock missing identifier", + provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeBedrockGuardrails, + Bedrock: &aigv1b1.BedrockGuardrailProvider{Region: "us-east-1"}, + }, + wantErr: "bedrock region, guardrailIdentifier, and guardrailVersion are required", + }, + { + name: "bedrock invalid endpoint", + provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeBedrockGuardrails, + Bedrock: &aigv1b1.BedrockGuardrailProvider{ + Region: "us-east-1", GuardrailIdentifier: "guardrail-id", GuardrailVersion: "1", Endpoint: "bedrock", + }, + }, + wantErr: "valid provider endpoint is required", + }, + { + name: "azure", + provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeAzureContentSafety, + AzureContentSafety: &aigv1b1.AzureContentSafetyGuardrailProvider{ + Endpoint: "https://content-safety.example.com", + APIKeySecretRef: &gwapiv1.SecretObjectReference{Name: "api-key"}, + }, + }, + }, + { + name: "azure missing configuration", + provider: aigv1b1.GuardrailProvider{Type: aigv1b1.GuardrailProviderTypeAzureContentSafety}, + wantErr: "azure Content Safety configuration is required", + }, + { + name: "azure invalid endpoint", + provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeAzureContentSafety, + AzureContentSafety: &aigv1b1.AzureContentSafetyGuardrailProvider{Endpoint: "content-safety"}, + }, + wantErr: "valid provider endpoint is required", + }, + { + name: "azure missing secret reference", + provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeAzureContentSafety, + AzureContentSafety: &aigv1b1.AzureContentSafetyGuardrailProvider{Endpoint: "https://content-safety.example.com"}, + }, + wantErr: "secret reference is required", + }, + { + name: "cross-namespace secret", + provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeAzureContentSafety, + AzureContentSafety: &aigv1b1.AzureContentSafetyGuardrailProvider{ + Endpoint: "https://content-safety.example.com", + APIKeySecretRef: &gwapiv1.SecretObjectReference{Name: "api-key", Namespace: &otherNamespace}, + }, + }, + wantErr: "cross-namespace guardrail secret references are not supported", + }, + { + name: "secret missing key", + provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeBedrockGuardrails, + Bedrock: &aigv1b1.BedrockGuardrailProvider{ + Region: "us-east-1", GuardrailIdentifier: "guardrail-id", GuardrailVersion: "1", + CredentialsSecretRef: &gwapiv1.SecretObjectReference{Name: "api-key"}, + }, + }, + wantErr: "secret api-key does not contain key credentials", + }, + { + name: "unsupported provider", + provider: aigv1b1.GuardrailProvider{Type: "Unknown"}, + wantErr: `unsupported provider type "Unknown"`, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := controller.validateGuardrailProvider(t.Context(), namespace, &test.provider) + if test.wantErr == "" { + require.NoError(t, err) + } else { + require.ErrorContains(t, err, test.wantErr) + } + }) + } +} + +func TestBackendToGuardrailPolicy(t *testing.T) { + const namespace = "default" + controllerClient := newGuardrailPolicyTestClient(t) + for _, name := range []string{"targets-backend", "targets-other"} { + target := gwapiv1.ObjectName("backend") + if name == "targets-other" { + target = "other" + } + require.NoError(t, controllerClient.Create(t.Context(), &aigv1b1.GuardrailPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Spec: aigv1b1.GuardrailPolicySpec{ + TargetRefs: []gwapiv1a2.LocalPolicyTargetReference{{ + Group: "aigateway.envoyproxy.io", Kind: "AIServiceBackend", Name: target, + }}, + }, + })) + } + controller := NewGuardrailPolicyController(controllerClient, fakekube.NewClientset(), ctrl.Log, make(chan event.GenericEvent, 1)) + + requests := controller.BackendToGuardrailPolicy(t.Context(), &aigv1b1.AIServiceBackend{ + ObjectMeta: metav1.ObjectMeta{Name: "backend", Namespace: namespace}, + }) + require.Equal(t, []reconcile.Request{{NamespacedName: types.NamespacedName{ + Namespace: namespace, Name: "targets-backend", + }}}, requests) +} + +func TestGuardrailPolicyControllerReconcileNotFound(t *testing.T) { + controller := NewGuardrailPolicyController(newGuardrailPolicyTestClient(t), fakekube.NewClientset(), ctrl.Log, make(chan event.GenericEvent, 1)) + result, err := controller.Reconcile(t.Context(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: "default", Name: "missing"}}) + require.NoError(t, err) + require.Equal(t, ctrl.Result{}, result) +} + +func TestGuardrailPolicyControllerReconcileMissingTarget(t *testing.T) { + const namespace = "default" + controllerClient := newGuardrailPolicyTestClient(t) + policy := &aigv1b1.GuardrailPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: "policy", Namespace: namespace}, + Spec: aigv1b1.GuardrailPolicySpec{ + TargetRefs: []gwapiv1a2.LocalPolicyTargetReference{{ + Group: "aigateway.envoyproxy.io", Kind: "AIServiceBackend", Name: "missing", + }}, + Rules: []aigv1b1.GuardrailRule{{ + Name: "regex", Phase: aigv1b1.GuardrailPhaseRequest, + Provider: aigv1b1.GuardrailProvider{Type: aigv1b1.GuardrailProviderTypeRegex, Pattern: "secret"}, + }}, + }, + } + require.NoError(t, controllerClient.Create(t.Context(), policy)) + controller := NewGuardrailPolicyController(controllerClient, fakekube.NewClientset(), ctrl.Log, make(chan event.GenericEvent, 1)) + + _, err := controller.Reconcile(t.Context(), reconcile.Request{NamespacedName: client.ObjectKeyFromObject(policy)}) + require.ErrorContains(t, err, "target AIServiceBackend default/missing not found") + + var updated aigv1b1.GuardrailPolicy + require.NoError(t, controllerClient.Get(t.Context(), client.ObjectKeyFromObject(policy), &updated)) + require.Len(t, updated.Status.Conditions, 1) + require.Equal(t, aigv1b1.ConditionTypeNotAccepted, updated.Status.Conditions[0].Type) +} From 25861b80ceadf49275177dd4b1ff3356e36f288e Mon Sep 17 00:00:00 2001 From: Fernando Escolar Date: Sat, 10 Oct 2026 15:30:09 +0200 Subject: [PATCH 17/18] tests coverage: cover guardrail paths in extproc, controller, and filterapi Signed-off-by: Fernando Escolar --- internal/controller/guardrail_policy_test.go | 235 +++++++++++++++++++ internal/extproc/guardrails_test.go | 158 +++++++++++++ internal/filterapi/runtime_test.go | 54 +++++ 3 files changed, 447 insertions(+) diff --git a/internal/controller/guardrail_policy_test.go b/internal/controller/guardrail_policy_test.go index c940efca4b..ac77af7e3b 100644 --- a/internal/controller/guardrail_policy_test.go +++ b/internal/controller/guardrail_policy_test.go @@ -7,6 +7,7 @@ package controller import ( "context" + "errors" "net/http" "net/http/httptest" "testing" @@ -20,6 +21,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/reconcile" gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" @@ -757,3 +759,236 @@ func TestGuardrailPolicyControllerReconcileMissingTarget(t *testing.T) { require.Len(t, updated.Status.Conditions, 1) require.Equal(t, aigv1b1.ConditionTypeNotAccepted, updated.Status.Conditions[0].Type) } + +func TestGuardrailProviderToFilterAPI(t *testing.T) { + const namespace = "default" + otherNamespace := gwapiv1.Namespace("other") + timeout, threshold, severity := int32(3), int32(70), int32(2) + kube := fakekube.NewClientset( + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "api-key", Namespace: namespace}, + Data: map[string][]byte{"apiKey": []byte("api-secret")}, + }, + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "aws", Namespace: namespace}, + Data: map[string][]byte{"credentials": []byte("[default]")}, + }, + ) + controller := &GatewayController{kube: kube} + missing := &gwapiv1.SecretObjectReference{Name: "missing"} + tests := []struct { + name string + provider aigv1b1.GuardrailProvider + want filterapi.GuardrailProvider + wantErr string + }{ + { + name: "presidio", + provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypePresidio, TimeoutSeconds: &timeout, + Presidio: &aigv1b1.PresidioGuardrailProvider{ + Endpoint: "https://presidio.example.com", Language: "es", ScoreThresholdPercent: &threshold, + APIKeySecretRef: &gwapiv1.SecretObjectReference{Name: "api-key"}, + }, + }, + want: filterapi.GuardrailProvider{ + Type: filterapi.GuardrailProviderTypePresidio, TimeoutSeconds: 3, + Presidio: &filterapi.PresidioGuardrailProvider{ + Endpoint: "https://presidio.example.com", Language: "es", ScoreThresholdPercent: 70, APIKey: "api-secret", + }, + }, + }, + { + name: "bedrock", + provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeBedrockGuardrails, + Bedrock: &aigv1b1.BedrockGuardrailProvider{ + Region: "us-east-1", GuardrailIdentifier: "guardrail-id", GuardrailVersion: "1", + Endpoint: "https://bedrock.example.com", + CredentialsSecretRef: &gwapiv1.SecretObjectReference{Name: "aws"}, + }, + }, + want: filterapi.GuardrailProvider{ + Type: filterapi.GuardrailProviderTypeBedrockGuardrails, + Bedrock: &filterapi.BedrockGuardrailProvider{ + Endpoint: "https://bedrock.example.com", Region: "us-east-1", + GuardrailIdentifier: "guardrail-id", GuardrailVersion: "1", CredentialFileLiteral: "[default]", + }, + }, + }, + { + name: "azure", + provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeAzureContentSafety, + AzureContentSafety: &aigv1b1.AzureContentSafetyGuardrailProvider{ + Endpoint: "https://content-safety.example.com", APIVersion: "2024-09-01", SeverityThreshold: &severity, + APIKeySecretRef: &gwapiv1.SecretObjectReference{Name: "api-key"}, + }, + }, + want: filterapi.GuardrailProvider{ + Type: filterapi.GuardrailProviderTypeAzureContentSafety, + AzureContentSafety: &filterapi.AzureContentSafetyGuardrailProvider{ + Endpoint: "https://content-safety.example.com", APIVersion: "2024-09-01", SeverityThreshold: &severity, + APIKey: "api-secret", + }, + }, + }, + { + name: "regex", + provider: aigv1b1.GuardrailProvider{Type: aigv1b1.GuardrailProviderTypeRegex, Pattern: "secret", Action: aigv1b1.GuardrailActionMask}, + want: filterapi.GuardrailProvider{Type: filterapi.GuardrailProviderTypeRegex, Pattern: "secret", Action: filterapi.GuardrailActionMask}, + }, + { + name: "presidio missing configuration", + provider: aigv1b1.GuardrailProvider{Type: aigv1b1.GuardrailProviderTypePresidio}, + wantErr: "presidio configuration is required", + }, + { + name: "presidio missing secret", + provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypePresidio, + Presidio: &aigv1b1.PresidioGuardrailProvider{Endpoint: "https://presidio.example.com", APIKeySecretRef: missing}, + }, + wantErr: "missing", + }, + { + name: "bedrock missing configuration", + provider: aigv1b1.GuardrailProvider{Type: aigv1b1.GuardrailProviderTypeBedrockGuardrails}, + wantErr: "bedrock configuration is required", + }, + { + name: "bedrock missing secret", + provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeBedrockGuardrails, + Bedrock: &aigv1b1.BedrockGuardrailProvider{ + Region: "us-east-1", GuardrailIdentifier: "guardrail-id", GuardrailVersion: "1", CredentialsSecretRef: missing, + }, + }, + wantErr: "missing", + }, + { + name: "azure missing configuration", + provider: aigv1b1.GuardrailProvider{Type: aigv1b1.GuardrailProviderTypeAzureContentSafety}, + wantErr: "azure Content Safety configuration is required", + }, + { + name: "azure missing secret reference", + provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeAzureContentSafety, + AzureContentSafety: &aigv1b1.AzureContentSafetyGuardrailProvider{Endpoint: "https://content-safety.example.com"}, + }, + wantErr: "secret reference is required", + }, + { + name: "azure cross-namespace secret", + provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeAzureContentSafety, + AzureContentSafety: &aigv1b1.AzureContentSafetyGuardrailProvider{ + Endpoint: "https://content-safety.example.com", + APIKeySecretRef: &gwapiv1.SecretObjectReference{Name: "api-key", Namespace: &otherNamespace}, + }, + }, + wantErr: "cross-namespace guardrail secret references are not supported", + }, + { + name: "http missing configuration", + provider: aigv1b1.GuardrailProvider{Type: aigv1b1.GuardrailProviderTypeHTTP}, + wantErr: "http guardrail configuration is required", + }, + { + name: "http missing secret", + provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeHTTP, + HTTP: &aigv1b1.HTTPGuardrailProvider{Endpoint: "https://guardrail.example.com", APIKeySecretRef: missing}, + }, + wantErr: "missing", + }, + { + name: "model armor missing secret", + provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeModelArmor, + ModelArmor: &aigv1b1.ModelArmorGuardrailProvider{ + Project: "project-id", Location: "us-central1", Template: "template-id", CredentialsSecretRef: missing, + }, + }, + wantErr: "missing", + }, + { + name: "unsupported provider", + provider: aigv1b1.GuardrailProvider{Type: "Unknown"}, + wantErr: `unsupported provider type "Unknown"`, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + converted, err := controller.guardrailProviderToFilterAPI(t.Context(), namespace, &test.provider) + if test.wantErr != "" { + require.ErrorContains(t, err, test.wantErr) + return + } + require.NoError(t, err) + require.Equal(t, test.want, converted) + }) + } +} + +func TestGuardrailMaxPayloadBytes(t *testing.T) { + requestLimit, responseLimit := int64(2048), int64(4096) + policy := &aigv1b1.GuardrailPolicy{Spec: aigv1b1.GuardrailPolicySpec{ + MaxRequestBodyBytes: &requestLimit, MaxResponseBodyBytes: &responseLimit, + }} + require.Equal(t, requestLimit, guardrailMaxPayloadBytes(policy, aigv1b1.GuardrailPhaseRequest)) + require.Equal(t, responseLimit, guardrailMaxPayloadBytes(policy, aigv1b1.GuardrailPhaseResponse)) + require.Equal(t, defaultGuardrailMaxPayloadBytes, guardrailMaxPayloadBytes(&aigv1b1.GuardrailPolicy{}, aigv1b1.GuardrailPhaseResponse)) +} + +func TestInjectGuardrailsSkipsInjectedRulesAndForeignBackends(t *testing.T) { + const namespace = "default" + otherNamespace := gwapiv1.Namespace("other") + controllerClient := newGuardrailPolicyTestClient(t) + require.NoError(t, controllerClient.Create(t.Context(), &aigv1b1.GuardrailPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: "policy", Namespace: namespace}, + Spec: aigv1b1.GuardrailPolicySpec{ + TargetRefs: []gwapiv1a2.LocalPolicyTargetReference{{Name: "backend"}}, + Rules: []aigv1b1.GuardrailRule{ + { + Name: "injected", Phase: aigv1b1.GuardrailPhaseRequest, + Provider: aigv1b1.GuardrailProvider{Type: aigv1b1.GuardrailProviderTypeRegex, Pattern: "injected"}, + }, + { + Name: "new", Phase: aigv1b1.GuardrailPhaseResponse, + Provider: aigv1b1.GuardrailProvider{Type: aigv1b1.GuardrailProviderTypeRegex, Pattern: "new"}, + }, + }, + }, + })) + controller := &GatewayController{client: controllerClient, kube: fakekube.NewClientset(), logger: ctrl.Log} + config := &filterapi.Config{} + route := &aigv1b1.AIGatewayRoute{ + ObjectMeta: metav1.ObjectMeta{Name: "route", Namespace: namespace}, + Spec: aigv1b1.AIGatewayRouteSpec{Rules: []aigv1b1.AIGatewayRouteRule{{ + BackendRefs: []aigv1b1.AIGatewayRouteRuleBackendRef{ + {Name: "backend", Namespace: &otherNamespace}, + {Name: "other-backend"}, + {Name: "backend"}, + }, + }}}, + } + injected := map[string]struct{}{"default/policy/injected": {}} + require.NoError(t, controller.injectGuardrails(t.Context(), route, config, injected)) + require.Len(t, config.Guardrails, 1) + require.Equal(t, "default/policy/new", config.Guardrails[0].Name) + require.Equal(t, []string{"default/backend/route/route/rule/0/ref/2"}, config.Guardrails[0].Backends) + require.Contains(t, injected, "default/policy/new") +} + +func TestInjectGuardrailsListError(t *testing.T) { + controllerClient := fake.NewClientBuilder().WithScheme(Scheme).WithInterceptorFuncs(interceptor.Funcs{ + List: func(context.Context, client.WithWatch, client.ObjectList, ...client.ListOption) error { + return errors.New("list failed") + }, + }).Build() + controller := &GatewayController{client: controllerClient, kube: fakekube.NewClientset(), logger: ctrl.Log} + err := controller.injectGuardrails(t.Context(), &aigv1b1.AIGatewayRoute{}, &filterapi.Config{}, map[string]struct{}{}) + require.ErrorContains(t, err, "failed to list GuardrailPolicies: list failed") +} diff --git a/internal/extproc/guardrails_test.go b/internal/extproc/guardrails_test.go index 69f0c078c2..66fb728ae3 100644 --- a/internal/extproc/guardrails_test.go +++ b/internal/extproc/guardrails_test.go @@ -321,3 +321,161 @@ func TestGuardrailHTTPProviderActions(t *testing.T) { }) } } + +// guardrailProcessorCase describes one guardrail outcome exercised through a processor. +type guardrailProcessorCase struct { + name string + guardrail filterapi.RuntimeGuardrail + wantErr string + wantBlock bool + wantResult metrics.GuardrailResult +} + +func guardrailProcessorCases(phase filterapi.GuardrailPhase, backends []string) []guardrailProcessorCase { + regex := func(name string, action filterapi.GuardrailAction) filterapi.RuntimeGuardrail { + return filterapi.RuntimeGuardrail{ + Name: name, Phase: phase, Backends: backends, + Provider: filterapi.GuardrailProvider{ + Type: filterapi.GuardrailProviderTypeRegex, Action: action, MaskReplacement: "[MASKED]", Message: "blocked", + }, + Matcher: regexp.MustCompile("secret"), + } + } + failing := func(mode filterapi.GuardrailFailureMode) filterapi.RuntimeGuardrail { + return filterapi.RuntimeGuardrail{ + Name: "provider", Phase: phase, Backends: backends, + Provider: filterapi.GuardrailProvider{Type: filterapi.GuardrailProviderTypePresidio, FailureMode: mode}, + Evaluator: &failingGuardrailEvaluator{}, + } + } + return []guardrailProcessorCase{ + {name: "block", guardrail: regex("deny", filterapi.GuardrailActionBlock), wantBlock: true, wantResult: metrics.GuardrailResultBlocked}, + {name: "monitor", guardrail: regex("monitor", filterapi.GuardrailActionMonitor), wantResult: metrics.GuardrailResultMonitored}, + {name: "mask", guardrail: regex("mask", filterapi.GuardrailActionMask), wantResult: metrics.GuardrailResultMasked}, + {name: "fail open", guardrail: failing(filterapi.GuardrailFailureModeFailOpen), wantResult: metrics.GuardrailResultError}, + {name: "fail closed", guardrail: failing(filterapi.GuardrailFailureModeFailClosed), wantErr: "guardrails", wantResult: metrics.GuardrailResultError}, + } +} + +func TestRouterRequestGuardrailOutcomes(t *testing.T) { + for _, test := range guardrailProcessorCases(filterapi.GuardrailPhaseRequest, nil) { + t.Run(test.name, func(t *testing.T) { + recorder := &recordingGuardrailMetrics{} + span := &mockGuardrailChatCompletionSpan{} + factory := NewFactory(nil, recorder, &mockTracer{returnedSpan: span}, endpointspec.ChatCompletionsEndpointSpec{}) + processor, err := factory(&filterapi.RuntimeConfig{Guardrails: []filterapi.RuntimeGuardrail{test.guardrail}}, map[string]string{ + "content-type": "application/json", + ":path": "/v1/chat/completions", + }, slog.Default(), false, false) + require.NoError(t, err) + + response, err := processor.ProcessRequestBody(t.Context(), &extprocv3.HttpBody{ + Body: []byte(`{"model":"test","messages":[{"role":"user","content":"my secret"}]}`), + }) + require.Equal(t, test.wantResult, recorder.result) + require.Len(t, span.guardrailEvents, 1) + if test.wantErr != "" { + require.ErrorContains(t, err, "failed to evaluate request guardrails") + return + } + require.NoError(t, err) + require.Equal(t, test.wantBlock, response.GetImmediateResponse() != nil) + if test.wantBlock { + require.Equal(t, 1, span.endedOnErrorCount) + require.Equal(t, http.StatusForbidden, span.errorStatusCode) + return + } + if test.wantResult == metrics.GuardrailResultMasked { + require.Contains(t, string(processor.(*chatCompletionProcessorRouterFilter).originalRequestBodyRaw), "my [MASKED]") + } + }) + } +} + +func TestUpstreamRequestGuardrailOutcomes(t *testing.T) { + body := []byte(`{"model":"test","messages":[{"role":"user","content":"my secret"}]}`) + for _, test := range guardrailProcessorCases(filterapi.GuardrailPhaseRequest, []string{"backend-a"}) { + t.Run(test.name, func(t *testing.T) { + recorder := &recordingGuardrailMetrics{} + span := &mockGuardrailChatCompletionSpan{} + parent := &chatCompletionProcessorRouterFilter{ + config: &filterapi.RuntimeConfig{Guardrails: []filterapi.RuntimeGuardrail{test.guardrail}}, + logger: slog.Default(), + originalRequestBodyRaw: body, + originalModel: "test", + guardrailMetrics: recorder, + span: span, + } + p := &chatCompletionProcessorUpstreamFilter{ + parent: parent, + logger: slog.Default(), + requestHeaders: map[string]string{":path": "/v1/chat/completions"}, + metrics: &mockMetrics{}, + backendName: "backend-a", + // Stop right after the guardrail evaluation, before translation. + unsupportedBackendErr: errors.New("stop after guardrails"), + } + + response, err := p.ProcessRequestHeaders(t.Context(), nil) + require.Equal(t, test.wantResult, recorder.result) + require.Len(t, span.guardrailEvents, 1) + if test.wantErr != "" { + require.ErrorContains(t, err, "failed to evaluate backend request guardrails") + return + } + require.NoError(t, err) + if test.wantBlock { + require.Contains(t, string(response.GetImmediateResponse().GetBody()), "GuardrailViolation") + require.Equal(t, 1, span.endedOnErrorCount) + return + } + require.Contains(t, string(response.GetImmediateResponse().GetBody()), "stop after guardrails") + if test.wantResult == metrics.GuardrailResultMasked { + require.Contains(t, string(parent.originalRequestBodyRaw), "my [MASKED]") + require.True(t, parent.forceBodyMutation) + } + }) + } +} + +func TestUpstreamResponseGuardrailOutcomes(t *testing.T) { + responseBody := []byte(`{"choices":[{"index":0,"message":{"role":"assistant","content":"my secret"}}]}`) + for _, test := range guardrailProcessorCases(filterapi.GuardrailPhaseResponse, []string{"backend-a"}) { + t.Run(test.name, func(t *testing.T) { + recorder := &recordingGuardrailMetrics{} + span := &mockGuardrailChatCompletionSpan{} + p := &chatCompletionProcessorUpstreamFilter{ + parent: &chatCompletionProcessorRouterFilter{ + config: &filterapi.RuntimeConfig{Guardrails: []filterapi.RuntimeGuardrail{test.guardrail}}, + logger: slog.Default(), + guardrailMetrics: recorder, + span: span, + }, + logger: slog.Default(), + translator: &mockTranslator{t: t, retBodyMutation: responseBody}, + metrics: &mockMetrics{}, + backendName: "backend-a", + responseHeaders: map[string]string{":status": "200"}, + } + + response, err := p.ProcessResponseBody(t.Context(), &extprocv3.HttpBody{Body: responseBody, EndOfStream: true}) + require.Equal(t, test.wantResult, recorder.result) + require.Len(t, span.guardrailEvents, 1) + if test.wantErr != "" { + require.ErrorContains(t, err, "failed to evaluate response guardrails") + return + } + require.NoError(t, err) + if test.wantBlock { + require.Contains(t, string(response.GetImmediateResponse().GetBody()), "GuardrailViolation") + return + } + returnedBody := response.GetResponseBody().GetResponse().GetBodyMutation().GetBody() + if test.wantResult == metrics.GuardrailResultMasked { + require.Contains(t, string(returnedBody), "my [MASKED]") + return + } + require.Equal(t, responseBody, returnedBody) + }) + } +} diff --git a/internal/filterapi/runtime_test.go b/internal/filterapi/runtime_test.go index 8bb15d63af..981a608de8 100644 --- a/internal/filterapi/runtime_test.go +++ b/internal/filterapi/runtime_test.go @@ -7,6 +7,7 @@ package filterapi import ( "context" + "errors" "testing" "time" @@ -171,3 +172,56 @@ type testGuardrailEvaluator struct{} func (*testGuardrailEvaluator) Evaluate(context.Context, []byte, GuardrailPhase) (GuardrailEvaluationResult, error) { return GuardrailEvaluationResult{}, nil } + +func TestNewRuntimeConfigGuardrails(t *testing.T) { + noBackendAuth := func(context.Context, *BackendAuth) (BackendAuthHandler, error) { return nil, nil } + failingEvaluatorFactory := func(context.Context, *GuardrailProvider) (GuardrailEvaluator, error) { + return nil, errors.New("bad provider") + } + tests := []struct { + name string + guardrail Guardrail + evaluatorFactories []NewGuardrailEvaluatorFunc + wantErr string + }{ + { + name: "regex without pattern", + guardrail: Guardrail{Name: "regex", Provider: GuardrailProvider{Type: GuardrailProviderTypeRegex}}, + wantErr: `guardrail "regex" uses regex provider without a pattern`, + }, + { + name: "invalid regex pattern", + guardrail: Guardrail{Name: "regex", Provider: GuardrailProvider{Type: GuardrailProviderTypeRegex, Pattern: "("}}, + wantErr: `guardrail "regex" has an invalid regex pattern`, + }, + { + name: "external provider without evaluator factory", + guardrail: Guardrail{Name: "presidio", Provider: GuardrailProvider{Type: GuardrailProviderTypePresidio}}, + wantErr: `guardrail "presidio" uses provider "Presidio" but no evaluator factory is configured`, + }, + { + name: "evaluator factory error", + guardrail: Guardrail{Name: "presidio", Provider: GuardrailProvider{Type: GuardrailProviderTypePresidio}}, + evaluatorFactories: []NewGuardrailEvaluatorFunc{failingEvaluatorFactory}, + wantErr: `cannot create evaluator for guardrail "presidio": bad provider`, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + config := &Config{Guardrails: []Guardrail{test.guardrail}} + _, err := NewRuntimeConfig(t.Context(), config, noBackendAuth, test.evaluatorFactories...) + require.ErrorContains(t, err, test.wantErr) + }) + } + + t.Run("max payload bytes", func(t *testing.T) { + config := &Config{Guardrails: []Guardrail{ + {Name: "default", Provider: GuardrailProvider{Type: GuardrailProviderTypeRegex, Pattern: "a"}}, + {Name: "configured", MaxPayloadBytes: 2048, Provider: GuardrailProvider{Type: GuardrailProviderTypeRegex, Pattern: "a"}}, + }} + rc, err := NewRuntimeConfig(t.Context(), config, noBackendAuth) + require.NoError(t, err) + require.Equal(t, defaultGuardrailMaxPayloadBytes, rc.Guardrails[0].MaxPayloadBytes) + require.Equal(t, int64(2048), rc.Guardrails[1].MaxPayloadBytes) + }) +} From bad7a91385d26bbb23053c8379fd10681c1ea64e Mon Sep 17 00:00:00 2001 From: Fernando Escolar Date: Sat, 10 Oct 2026 18:01:24 +0200 Subject: [PATCH 18/18] cover guardrail adapter and error paths Codecov reported 64 uncovered lines in the guardrail changes, mostly in the provider adapters (Azure 76.92%, Bedrock 82.19%). Signed-off-by: Fernando Escolar --- internal/controller/guardrail_policy_test.go | 84 +++++++ internal/extproc/guardrails_test.go | 48 ++++ internal/guardrails/adapters_test.go | 227 +++++++++++++++++++ 3 files changed, 359 insertions(+) diff --git a/internal/controller/guardrail_policy_test.go b/internal/controller/guardrail_policy_test.go index ac77af7e3b..86297eb595 100644 --- a/internal/controller/guardrail_policy_test.go +++ b/internal/controller/guardrail_policy_test.go @@ -992,3 +992,87 @@ func TestInjectGuardrailsListError(t *testing.T) { err := controller.injectGuardrails(t.Context(), &aigv1b1.AIGatewayRoute{}, &filterapi.Config{}, map[string]struct{}{}) require.ErrorContains(t, err, "failed to list GuardrailPolicies: list failed") } + +func TestGuardrailPolicyControllerClientErrors(t *testing.T) { + const namespace = "default" + policy := &aigv1b1.GuardrailPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: "policy", Namespace: namespace}, + Spec: aigv1b1.GuardrailPolicySpec{ + TargetRefs: []gwapiv1a2.LocalPolicyTargetReference{{ + Group: "aigateway.envoyproxy.io", Kind: "AIServiceBackend", Name: "backend", + }}, + Rules: []aigv1b1.GuardrailRule{{ + Name: "regex", Phase: aigv1b1.GuardrailPhaseRequest, + Provider: aigv1b1.GuardrailProvider{Type: aigv1b1.GuardrailProviderTypeRegex, Pattern: "secret"}, + }}, + }, + } + errClient := errors.New("api server unavailable") + newController := func(t *testing.T, funcs interceptor.Funcs) *GuardrailPolicyController { + builder := fake.NewClientBuilder().WithScheme(Scheme). + WithStatusSubresource(&aigv1b1.GuardrailPolicy{}). + WithObjects(policy.DeepCopy()). + WithInterceptorFuncs(funcs) + require.NoError(t, ApplyIndexing(t.Context(), func(_ context.Context, obj client.Object, field string, extractValue client.IndexerFunc) error { + builder = builder.WithIndex(obj, field, extractValue) + return nil + })) + return NewGuardrailPolicyController(builder.Build(), fakekube.NewClientset(), ctrl.Log, make(chan event.GenericEvent, 1)) + } + failList := interceptor.Funcs{List: func(context.Context, client.WithWatch, client.ObjectList, ...client.ListOption) error { + return errClient + }} + + t.Run("get policy", func(t *testing.T) { + controller := newController(t, interceptor.Funcs{Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error { + return errClient + }}) + _, err := controller.Reconcile(t.Context(), reconcile.Request{NamespacedName: client.ObjectKeyFromObject(policy)}) + require.ErrorIs(t, err, errClient) + }) + + t.Run("get target backend", func(t *testing.T) { + controller := newController(t, interceptor.Funcs{Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*aigv1b1.AIServiceBackend); ok { + return errClient + } + return c.Get(ctx, key, obj, opts...) + }}) + _, err := controller.Reconcile(t.Context(), reconcile.Request{NamespacedName: client.ObjectKeyFromObject(policy)}) + require.ErrorContains(t, err, "failed to get AIServiceBackend default/backend: api server unavailable") + }) + + t.Run("list policies and routes", func(t *testing.T) { + controller := newController(t, failList) + require.Nil(t, controller.SecretToGuardrailPolicy(t.Context(), &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "secret", Namespace: namespace}, + })) + require.Nil(t, controller.BackendToGuardrailPolicy(t.Context(), &aigv1b1.AIServiceBackend{ + ObjectMeta: metav1.ObjectMeta{Name: "backend", Namespace: namespace}, + })) + controller.notifyAIGatewayRoutesForGuardrailPolicy(t.Context(), policy) + require.Empty(t, controller.aiGatewayRouteChan) + }) + + t.Run("status update", func(t *testing.T) { + controller := newController(t, interceptor.Funcs{SubResourceUpdate: func(context.Context, client.Client, string, client.Object, ...client.SubResourceUpdateOption) error { + return errClient + }}) + // The status update error is logged, not returned. + controller.updateGuardrailPolicyStatus(t.Context(), policy.DeepCopy(), aigv1b1.ConditionTypeAccepted, "ok") + }) + + t.Run("status update for deleted policy", func(t *testing.T) { + controller := newController(t, interceptor.Funcs{Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error { + return apierrors.NewNotFound(aigv1b1.SchemeGroupVersion.WithResource("guardrailpolicies").GroupResource(), policy.Name) + }}) + controller.updateGuardrailPolicyStatus(t.Context(), policy.DeepCopy(), aigv1b1.ConditionTypeAccepted, "ok") + }) + + t.Run("status get error", func(t *testing.T) { + controller := newController(t, interceptor.Funcs{Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error { + return errClient + }}) + controller.updateGuardrailPolicyStatus(t.Context(), policy.DeepCopy(), aigv1b1.ConditionTypeAccepted, "ok") + }) +} diff --git a/internal/extproc/guardrails_test.go b/internal/extproc/guardrails_test.go index 66fb728ae3..7697ebfccd 100644 --- a/internal/extproc/guardrails_test.go +++ b/internal/extproc/guardrails_test.go @@ -479,3 +479,51 @@ func TestUpstreamResponseGuardrailOutcomes(t *testing.T) { }) } } + +func TestEvaluateGuardrailsForPhaseEdgeCases(t *testing.T) { + body := []byte(`{"model":"test","messages":[{"role":"user","content":"my secret"}]}`) + + t.Run("regex block without match", func(t *testing.T) { + outcome, err := evaluateRequestGuardrails(t.Context(), []filterapi.RuntimeGuardrail{{ + Name: "deny", Phase: filterapi.GuardrailPhaseRequest, + Provider: filterapi.GuardrailProvider{Type: filterapi.GuardrailProviderTypeRegex}, + Matcher: regexp.MustCompile("forbidden"), + }}, body) + require.NoError(t, err) + require.Nil(t, outcome.Violation) + require.False(t, outcome.Monitored) + }) + + t.Run("regex mask uses default replacement", func(t *testing.T) { + outcome, err := evaluateRequestGuardrails(t.Context(), []filterapi.RuntimeGuardrail{{ + Name: "mask", Phase: filterapi.GuardrailPhaseRequest, + Provider: filterapi.GuardrailProvider{Type: filterapi.GuardrailProviderTypeRegex, Action: filterapi.GuardrailActionMask}, + Matcher: regexp.MustCompile("secret"), + }}, body) + require.NoError(t, err) + require.True(t, outcome.Masked) + require.Contains(t, string(outcome.Body), "my [REDACTED]") + }) + + t.Run("regex mask without compiled matcher", func(t *testing.T) { + _, err := evaluateRequestGuardrails(t.Context(), []filterapi.RuntimeGuardrail{{ + Name: "mask", Phase: filterapi.GuardrailPhaseRequest, + Provider: filterapi.GuardrailProvider{Type: filterapi.GuardrailProviderTypeRegex, Action: filterapi.GuardrailActionMask}, + }}, body) + require.ErrorContains(t, err, `guardrail "mask" uses regex provider without a compiled matcher`) + }) + + t.Run("external provider without evaluator", func(t *testing.T) { + _, err := evaluateRequestGuardrails(t.Context(), []filterapi.RuntimeGuardrail{{ + Name: "presidio", Phase: filterapi.GuardrailPhaseRequest, + Provider: filterapi.GuardrailProvider{Type: filterapi.GuardrailProviderTypePresidio}, + }}, body) + require.ErrorContains(t, err, `guardrail "presidio" uses provider "Presidio" without an evaluator`) + }) +} + +func TestReplaceGuardrailContentWholeBody(t *testing.T) { + replaced, err := replaceGuardrailContent([]byte("plain text"), guardrailContent{}, []byte("masked")) + require.NoError(t, err) + require.Equal(t, "masked", string(replaced)) +} diff --git a/internal/guardrails/adapters_test.go b/internal/guardrails/adapters_test.go index 4990bedda5..abeeeb8bac 100644 --- a/internal/guardrails/adapters_test.go +++ b/internal/guardrails/adapters_test.go @@ -13,6 +13,8 @@ import ( "io" "net/http" "net/http/httptest" + "os" + "path/filepath" "strings" "testing" "time" @@ -20,6 +22,7 @@ import ( "github.com/stretchr/testify/require" "github.com/envoyproxy/ai-gateway/internal/filterapi" + "github.com/envoyproxy/ai-gateway/internal/gcpauth" "github.com/envoyproxy/ai-gateway/internal/json" ) @@ -250,3 +253,227 @@ func TestModelArmorEvaluatorErrors(t *testing.T) { require.False(t, evaluation.Matched) require.ErrorContains(t, err, "sanitizeUserPrompt invocation failed") } + +// isolateAWSEnvironment keeps AWS credential resolution hermetic: no ambient credentials and no IMDS calls. +func isolateAWSEnvironment(t *testing.T) { + t.Helper() + t.Setenv("AWS_EC2_METADATA_DISABLED", "true") + t.Setenv("AWS_ACCESS_KEY_ID", "") + t.Setenv("AWS_SECRET_ACCESS_KEY", "") + t.Setenv("AWS_SESSION_TOKEN", "") + t.Setenv("AWS_PROFILE", "") + t.Setenv("AWS_CONFIG_FILE", filepath.Join(t.TempDir(), "config")) + t.Setenv("AWS_SHARED_CREDENTIALS_FILE", filepath.Join(t.TempDir(), "credentials")) +} + +func TestNewEvaluatorProviders(t *testing.T) { + isolateAWSEnvironment(t) + severity := int32(2) + for _, provider := range []*filterapi.GuardrailProvider{ + {Type: filterapi.GuardrailProviderTypeBedrockGuardrails, Bedrock: &filterapi.BedrockGuardrailProvider{ + Region: "us-east-1", GuardrailIdentifier: "guardrail-id", GuardrailVersion: "1", + }}, + {Type: filterapi.GuardrailProviderTypeAzureContentSafety, AzureContentSafety: &filterapi.AzureContentSafetyGuardrailProvider{ + Endpoint: "https://content-safety.example.com", APIKey: "key", SeverityThreshold: &severity, + }}, + {Type: filterapi.GuardrailProviderTypeHTTP, HTTP: &filterapi.HTTPGuardrailProvider{Endpoint: "https://guardrail.example.com"}}, + {Type: filterapi.GuardrailProviderTypeModelArmor, ModelArmor: &filterapi.ModelArmorGuardrailProvider{ + Project: "project-id", Location: "us-central1", Template: "template-id", CredentialsJSON: newModelArmorTestCredentials(t), + }}, + } { + t.Run(string(provider.Type), func(t *testing.T) { + evaluator, err := NewEvaluator(t.Context(), provider) + require.NoError(t, err) + require.NotNil(t, evaluator) + }) + } + + _, err := NewEvaluator(t.Context(), &filterapi.GuardrailProvider{Type: "Unknown"}) + require.ErrorContains(t, err, `unsupported external guardrail provider "Unknown"`) +} + +func TestTimeoutSecondsDefault(t *testing.T) { + require.Equal(t, defaultTimeoutSeconds, timeoutSeconds(0)) + require.Equal(t, int32(5), timeoutSeconds(5)) +} + +func TestDoJSONErrors(t *testing.T) { + err := doJSON(t.Context(), http.DefaultClient, http.MethodPost, "https://example.com", make(chan int), nil, nil) + require.ErrorContains(t, err, "unsupported type") + + err = doJSON(t.Context(), http.DefaultClient, http.MethodPost, "http://[::1", struct{}{}, nil, nil) + require.ErrorContains(t, err, "missing ']' in host") + + server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + server.Close() + err = doJSON(t.Context(), http.DefaultClient, http.MethodPost, server.URL, struct{}{}, nil, nil) + require.ErrorContains(t, err, "connection refused") + + malformed := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("not json")) + })) + t.Cleanup(malformed.Close) + var response struct{} + err = doJSON(t.Context(), malformed.Client(), http.MethodPost, malformed.URL, struct{}{}, nil, &response) + require.ErrorContains(t, err, "cannot decode provider response") +} + +func TestPresidioEvaluatorConfigurationAndNoFindings(t *testing.T) { + _, err := newPresidioEvaluator(&filterapi.PresidioGuardrailProvider{}, "", http.DefaultClient) + require.ErrorContains(t, err, "presidio endpoint is required") + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`[]`)) + })) + t.Cleanup(server.Close) + evaluator, err := newPresidioEvaluator(&filterapi.PresidioGuardrailProvider{Endpoint: server.URL}, "", server.Client()) + require.NoError(t, err) + evaluation, err := evaluator.Evaluate(t.Context(), []byte("nothing sensitive"), filterapi.GuardrailPhaseRequest) + require.NoError(t, err) + require.False(t, evaluation.Matched) +} + +func TestAzureContentSafetyEvaluatorConfiguration(t *testing.T) { + _, err := newAzureContentSafetyEvaluator(&filterapi.AzureContentSafetyGuardrailProvider{APIKey: "key"}, http.DefaultClient) + require.ErrorContains(t, err, "azure Content Safety endpoint is required") + _, err = newAzureContentSafetyEvaluator(&filterapi.AzureContentSafetyGuardrailProvider{Endpoint: "https://content-safety.example.com"}, http.DefaultClient) + require.ErrorContains(t, err, "azure Content Safety API key is required") + + evaluator, err := newAzureContentSafetyEvaluator(&filterapi.AzureContentSafetyGuardrailProvider{ + Endpoint: "https://content-safety.example.com", APIKey: "key", + }, http.DefaultClient) + require.NoError(t, err) + require.Equal(t, int32(4), *evaluator.(*azureContentSafetyEvaluator).config.SeverityThreshold) +} + +func TestAzureContentSafetyEvaluatorBelowThresholdAndError(t *testing.T) { + status := http.StatusOK + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(status) + _, _ = w.Write([]byte(`{"categoriesAnalysis":[{"category":"Violence","severity":2}]}`)) + })) + t.Cleanup(server.Close) + evaluator, err := newAzureContentSafetyEvaluator(&filterapi.AzureContentSafetyGuardrailProvider{ + Endpoint: server.URL, APIKey: "key", + }, server.Client()) + require.NoError(t, err) + + evaluation, err := evaluator.Evaluate(t.Context(), []byte("mild"), filterapi.GuardrailPhaseRequest) + require.NoError(t, err) + require.False(t, evaluation.Matched) + + status = http.StatusTooManyRequests + _, err = evaluator.Evaluate(t.Context(), []byte("mild"), filterapi.GuardrailPhaseRequest) + require.ErrorContains(t, err, "azure Content Safety analyze request failed: provider returned HTTP 429") +} + +func TestBedrockEvaluatorConfiguration(t *testing.T) { + isolateAWSEnvironment(t) + _, err := newBedrockEvaluator(t.Context(), &filterapi.BedrockGuardrailProvider{Region: "us-east-1"}, http.DefaultClient) + require.ErrorContains(t, err, "bedrock region, guardrailIdentifier, and guardrailVersion are required") + + _, err = newBedrockEvaluator(t.Context(), &filterapi.BedrockGuardrailProvider{ + Region: "us-east-1", GuardrailIdentifier: "guardrail-id", GuardrailVersion: "1", + CredentialFileLiteral: "[other]\naws_access_key_id = AKIDEXAMPLE\naws_secret_access_key = secret\n", + }, http.DefaultClient) + require.ErrorContains(t, err, "cannot load AWS credentials") + + t.Setenv("AWS_PROFILE", "missing") + _, err = newBedrockEvaluator(t.Context(), &filterapi.BedrockGuardrailProvider{ + Region: "us-east-1", GuardrailIdentifier: "guardrail-id", GuardrailVersion: "1", + }, http.DefaultClient) + require.ErrorContains(t, err, "cannot load AWS config") +} + +func TestBedrockEvaluatorErrors(t *testing.T) { + isolateAWSEnvironment(t) + // Without a credentials Secret, the default chain is used and only fails when credentials are retrieved. + evaluator, err := newBedrockEvaluator(t.Context(), &filterapi.BedrockGuardrailProvider{ + Region: "us-east-1", GuardrailIdentifier: "guardrail-id", GuardrailVersion: "1", + }, http.DefaultClient) + require.NoError(t, err) + require.Equal(t, "https://bedrock-runtime.us-east-1.amazonaws.com", evaluator.(*bedrockEvaluator).config.Endpoint) + _, err = evaluator.Evaluate(t.Context(), []byte("payload"), filterapi.GuardrailPhaseRequest) + require.ErrorContains(t, err, "cannot retrieve AWS credentials") + + credentials := "[default]\naws_access_key_id = AKIDEXAMPLE\naws_secret_access_key = secret\n" + evaluator, err = newBedrockEvaluator(t.Context(), &filterapi.BedrockGuardrailProvider{ + Endpoint: "http://[::1", Region: "us-east-1", GuardrailIdentifier: "guardrail-id", GuardrailVersion: "1", + CredentialFileLiteral: credentials, + }, http.DefaultClient) + require.NoError(t, err) + _, err = evaluator.Evaluate(t.Context(), []byte("payload"), filterapi.GuardrailPhaseRequest) + require.ErrorContains(t, err, "missing ']' in host") + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "throttled", http.StatusTooManyRequests) + })) + t.Cleanup(server.Close) + evaluator, err = newBedrockEvaluator(t.Context(), &filterapi.BedrockGuardrailProvider{ + Endpoint: server.URL, Region: "us-east-1", GuardrailIdentifier: "guardrail-id", GuardrailVersion: "1", + CredentialFileLiteral: credentials, + }, server.Client()) + require.NoError(t, err) + _, err = evaluator.Evaluate(t.Context(), []byte("payload"), filterapi.GuardrailPhaseRequest) + require.ErrorContains(t, err, "bedrock ApplyGuardrail request failed: provider returned HTTP 429") +} + +func TestModelArmorEvaluatorDefaults(t *testing.T) { + credentialsPath := filepath.Join(t.TempDir(), "credentials.json") + require.NoError(t, os.WriteFile(credentialsPath, []byte(newModelArmorTestCredentials(t)), 0o600)) + t.Setenv("GOOGLE_APPLICATION_CREDENTIALS", credentialsPath) + + evaluator, err := newModelArmorEvaluator(t.Context(), &filterapi.ModelArmorGuardrailProvider{ + Project: "project-id", Location: "europe-west4", Template: "template-id", + }, http.DefaultClient) + require.NoError(t, err) + require.Equal(t, "https://modelarmor.europe-west4.rep.googleapis.com/v1/projects/project-id/locations/europe-west4/templates/template-id", + evaluator.(*modelArmorEvaluator).templateURL) + + t.Setenv(gcpauth.ProxyEnvVar, "://invalid") + _, err = newModelArmorEvaluator(t.Context(), &filterapi.ModelArmorGuardrailProvider{ + Project: "project-id", Location: "europe-west4", Template: "template-id", + }, http.DefaultClient) + require.ErrorContains(t, err, "invalid "+gcpauth.ProxyEnvVar) +} + +func TestModelArmorEvaluatorRequestErrors(t *testing.T) { + tokenStatus := http.StatusOK + tokenServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(tokenStatus) + _, _ = w.Write([]byte(`{"access_token":"gcp-token","token_type":"Bearer","expires_in":3600}`)) + })) + t.Cleanup(tokenServer.Close) + var credentials map[string]string + require.NoError(t, json.Unmarshal([]byte(newModelArmorTestCredentials(t)), &credentials)) + credentials["token_uri"] = tokenServer.URL + credentialsJSON, err := json.Marshal(credentials) + require.NoError(t, err) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "unavailable", http.StatusServiceUnavailable) + })) + t.Cleanup(server.Close) + newEvaluator := func() filterapi.GuardrailEvaluator { + evaluator, newErr := newModelArmorEvaluator(t.Context(), &filterapi.ModelArmorGuardrailProvider{ + Endpoint: server.URL, Project: "project-id", Location: "us-central1", Template: "template-id", + CredentialsJSON: string(credentialsJSON), + }, server.Client()) + require.NoError(t, newErr) + return evaluator + } + + _, err = newEvaluator().Evaluate(t.Context(), []byte("hello"), filterapi.GuardrailPhaseResponse) + require.ErrorContains(t, err, "model Armor sanitizeModelResponse request failed: provider returned HTTP 503") + + tokenStatus = http.StatusUnauthorized + _, err = newEvaluator().Evaluate(t.Context(), []byte("hello"), filterapi.GuardrailPhaseRequest) + require.ErrorContains(t, err, "cannot retrieve GCP access token") +} + +func TestModelArmorDeidentifiedTextWithSDPInspectMatch(t *testing.T) { + var results map[string]modelArmorFilterResult + require.NoError(t, json.Unmarshal([]byte(`{"sdp":{"sdpFilterResult":{"inspectResult":{"matchState":"MATCH_FOUND"}}}}`), &results)) + require.Nil(t, modelArmorDeidentifiedText(results)) +}