diff --git a/api/v1alpha1/client/clientset/versioned/typed/api/v1alpha1/api_client.go b/api/v1alpha1/client/clientset/versioned/typed/api/v1alpha1/api_client.go index 5ee81af3c9..1f0d60a5ed 100644 --- a/api/v1alpha1/client/clientset/versioned/typed/api/v1alpha1/api_client.go +++ b/api/v1alpha1/client/clientset/versioned/typed/api/v1alpha1/api_client.go @@ -21,6 +21,7 @@ type AigatewayV1alpha1Interface interface { AIServiceBackendsGetter BackendSecurityPoliciesGetter GatewayConfigsGetter + GuardrailPoliciesGetter MCPRoutesGetter QuotaPoliciesGetter } @@ -46,6 +47,10 @@ func (c *AigatewayV1alpha1Client) GatewayConfigs(namespace string) GatewayConfig return newGatewayConfigs(c, namespace) } +func (c *AigatewayV1alpha1Client) GuardrailPolicies(namespace string) GuardrailPolicyInterface { + return newGuardrailPolicies(c, namespace) +} + func (c *AigatewayV1alpha1Client) MCPRoutes(namespace string) MCPRouteInterface { return newMCPRoutes(c, namespace) } diff --git a/api/v1alpha1/client/clientset/versioned/typed/api/v1alpha1/fake/fake_api_client.go b/api/v1alpha1/client/clientset/versioned/typed/api/v1alpha1/fake/fake_api_client.go index 5e14621080..b17694ba5d 100644 --- a/api/v1alpha1/client/clientset/versioned/typed/api/v1alpha1/fake/fake_api_client.go +++ b/api/v1alpha1/client/clientset/versioned/typed/api/v1alpha1/fake/fake_api_client.go @@ -33,6 +33,10 @@ func (c *FakeAigatewayV1alpha1) GatewayConfigs(namespace string) v1alpha1.Gatewa return newFakeGatewayConfigs(c, namespace) } +func (c *FakeAigatewayV1alpha1) GuardrailPolicies(namespace string) v1alpha1.GuardrailPolicyInterface { + return newFakeGuardrailPolicies(c, namespace) +} + func (c *FakeAigatewayV1alpha1) MCPRoutes(namespace string) v1alpha1.MCPRouteInterface { return newFakeMCPRoutes(c, namespace) } diff --git a/api/v1alpha1/client/clientset/versioned/typed/api/v1alpha1/fake/fake_guardrailpolicy.go b/api/v1alpha1/client/clientset/versioned/typed/api/v1alpha1/fake/fake_guardrailpolicy.go new file mode 100644 index 0000000000..78f1960461 --- /dev/null +++ b/api/v1alpha1/client/clientset/versioned/typed/api/v1alpha1/fake/fake_guardrailpolicy.go @@ -0,0 +1,41 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + v1alpha1 "github.com/envoyproxy/ai-gateway/api/v1alpha1" + apiv1alpha1 "github.com/envoyproxy/ai-gateway/api/v1alpha1/client/clientset/versioned/typed/api/v1alpha1" + gentype "k8s.io/client-go/gentype" +) + +// fakeGuardrailPolicies implements GuardrailPolicyInterface +type fakeGuardrailPolicies struct { + *gentype.FakeClientWithList[*v1alpha1.GuardrailPolicy, *v1alpha1.GuardrailPolicyList] + Fake *FakeAigatewayV1alpha1 +} + +func newFakeGuardrailPolicies(fake *FakeAigatewayV1alpha1, namespace string) apiv1alpha1.GuardrailPolicyInterface { + return &fakeGuardrailPolicies{ + gentype.NewFakeClientWithList[*v1alpha1.GuardrailPolicy, *v1alpha1.GuardrailPolicyList]( + fake.Fake, + namespace, + v1alpha1.SchemeGroupVersion.WithResource("guardrailpolicies"), + v1alpha1.SchemeGroupVersion.WithKind("GuardrailPolicy"), + func() *v1alpha1.GuardrailPolicy { return &v1alpha1.GuardrailPolicy{} }, + func() *v1alpha1.GuardrailPolicyList { return &v1alpha1.GuardrailPolicyList{} }, + func(dst, src *v1alpha1.GuardrailPolicyList) { dst.ListMeta = src.ListMeta }, + func(list *v1alpha1.GuardrailPolicyList) []*v1alpha1.GuardrailPolicy { + return gentype.ToPointerSlice(list.Items) + }, + func(list *v1alpha1.GuardrailPolicyList, items []*v1alpha1.GuardrailPolicy) { + list.Items = gentype.FromPointerSlice(items) + }, + ), + fake, + } +} diff --git a/api/v1alpha1/client/clientset/versioned/typed/api/v1alpha1/generated_expansion.go b/api/v1alpha1/client/clientset/versioned/typed/api/v1alpha1/generated_expansion.go index 3b0636bead..1c6e837fdf 100644 --- a/api/v1alpha1/client/clientset/versioned/typed/api/v1alpha1/generated_expansion.go +++ b/api/v1alpha1/client/clientset/versioned/typed/api/v1alpha1/generated_expansion.go @@ -15,6 +15,8 @@ type BackendSecurityPolicyExpansion interface{} type GatewayConfigExpansion interface{} +type GuardrailPolicyExpansion interface{} + type MCPRouteExpansion interface{} type QuotaPolicyExpansion interface{} diff --git a/api/v1alpha1/client/clientset/versioned/typed/api/v1alpha1/guardrailpolicy.go b/api/v1alpha1/client/clientset/versioned/typed/api/v1alpha1/guardrailpolicy.go new file mode 100644 index 0000000000..55bffbf124 --- /dev/null +++ b/api/v1alpha1/client/clientset/versioned/typed/api/v1alpha1/guardrailpolicy.go @@ -0,0 +1,59 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + context "context" + + apiv1alpha1 "github.com/envoyproxy/ai-gateway/api/v1alpha1" + scheme "github.com/envoyproxy/ai-gateway/api/v1alpha1/client/clientset/versioned/scheme" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + gentype "k8s.io/client-go/gentype" +) + +// GuardrailPoliciesGetter has a method to return a GuardrailPolicyInterface. +// A group's client should implement this interface. +type GuardrailPoliciesGetter interface { + GuardrailPolicies(namespace string) GuardrailPolicyInterface +} + +// GuardrailPolicyInterface has methods to work with GuardrailPolicy resources. +type GuardrailPolicyInterface interface { + Create(ctx context.Context, guardrailPolicy *apiv1alpha1.GuardrailPolicy, opts v1.CreateOptions) (*apiv1alpha1.GuardrailPolicy, error) + Update(ctx context.Context, guardrailPolicy *apiv1alpha1.GuardrailPolicy, opts v1.UpdateOptions) (*apiv1alpha1.GuardrailPolicy, error) + // Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). + UpdateStatus(ctx context.Context, guardrailPolicy *apiv1alpha1.GuardrailPolicy, opts v1.UpdateOptions) (*apiv1alpha1.GuardrailPolicy, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*apiv1alpha1.GuardrailPolicy, error) + List(ctx context.Context, opts v1.ListOptions) (*apiv1alpha1.GuardrailPolicyList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *apiv1alpha1.GuardrailPolicy, err error) + GuardrailPolicyExpansion +} + +// guardrailPolicies implements GuardrailPolicyInterface +type guardrailPolicies struct { + *gentype.ClientWithList[*apiv1alpha1.GuardrailPolicy, *apiv1alpha1.GuardrailPolicyList] +} + +// newGuardrailPolicies returns a GuardrailPolicies +func newGuardrailPolicies(c *AigatewayV1alpha1Client, namespace string) *guardrailPolicies { + return &guardrailPolicies{ + gentype.NewClientWithList[*apiv1alpha1.GuardrailPolicy, *apiv1alpha1.GuardrailPolicyList]( + "guardrailpolicies", + c.RESTClient(), + scheme.ParameterCodec, + namespace, + func() *apiv1alpha1.GuardrailPolicy { return &apiv1alpha1.GuardrailPolicy{} }, + func() *apiv1alpha1.GuardrailPolicyList { return &apiv1alpha1.GuardrailPolicyList{} }, + ), + } +} diff --git a/api/v1alpha1/client/informers/externalversions/api/v1alpha1/guardrailpolicy.go b/api/v1alpha1/client/informers/externalversions/api/v1alpha1/guardrailpolicy.go new file mode 100644 index 0000000000..8996cbdc5e --- /dev/null +++ b/api/v1alpha1/client/informers/externalversions/api/v1alpha1/guardrailpolicy.go @@ -0,0 +1,91 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +// Code generated by informer-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + context "context" + time "time" + + aigatewayapiv1alpha1 "github.com/envoyproxy/ai-gateway/api/v1alpha1" + versioned "github.com/envoyproxy/ai-gateway/api/v1alpha1/client/clientset/versioned" + internalinterfaces "github.com/envoyproxy/ai-gateway/api/v1alpha1/client/informers/externalversions/internalinterfaces" + apiv1alpha1 "github.com/envoyproxy/ai-gateway/api/v1alpha1/client/listers/api/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// GuardrailPolicyInformer provides access to a shared informer and lister for +// GuardrailPolicies. +type GuardrailPolicyInformer interface { + Informer() cache.SharedIndexInformer + Lister() apiv1alpha1.GuardrailPolicyLister +} + +type guardrailPolicyInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc + namespace string +} + +// NewGuardrailPolicyInformer constructs a new informer for GuardrailPolicy type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewGuardrailPolicyInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredGuardrailPolicyInformer(client, namespace, resyncPeriod, indexers, nil) +} + +// NewFilteredGuardrailPolicyInformer constructs a new informer for GuardrailPolicy type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredGuardrailPolicyInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + cache.ToListWatcherWithWatchListSemantics(&cache.ListWatch{ + ListFunc: func(options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.AigatewayV1alpha1().GuardrailPolicies(namespace).List(context.Background(), options) + }, + WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.AigatewayV1alpha1().GuardrailPolicies(namespace).Watch(context.Background(), options) + }, + ListWithContextFunc: func(ctx context.Context, options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.AigatewayV1alpha1().GuardrailPolicies(namespace).List(ctx, options) + }, + WatchFuncWithContext: func(ctx context.Context, options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.AigatewayV1alpha1().GuardrailPolicies(namespace).Watch(ctx, options) + }, + }, client), + &aigatewayapiv1alpha1.GuardrailPolicy{}, + resyncPeriod, + indexers, + ) +} + +func (f *guardrailPolicyInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredGuardrailPolicyInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *guardrailPolicyInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&aigatewayapiv1alpha1.GuardrailPolicy{}, f.defaultInformer) +} + +func (f *guardrailPolicyInformer) Lister() apiv1alpha1.GuardrailPolicyLister { + return apiv1alpha1.NewGuardrailPolicyLister(f.Informer().GetIndexer()) +} diff --git a/api/v1alpha1/client/informers/externalversions/api/v1alpha1/interface.go b/api/v1alpha1/client/informers/externalversions/api/v1alpha1/interface.go index a208a0db37..ee83a90986 100644 --- a/api/v1alpha1/client/informers/externalversions/api/v1alpha1/interface.go +++ b/api/v1alpha1/client/informers/externalversions/api/v1alpha1/interface.go @@ -21,6 +21,8 @@ type Interface interface { BackendSecurityPolicies() BackendSecurityPolicyInformer // GatewayConfigs returns a GatewayConfigInformer. GatewayConfigs() GatewayConfigInformer + // GuardrailPolicies returns a GuardrailPolicyInformer. + GuardrailPolicies() GuardrailPolicyInformer // MCPRoutes returns a MCPRouteInformer. MCPRoutes() MCPRouteInformer // QuotaPolicies returns a QuotaPolicyInformer. @@ -58,6 +60,11 @@ func (v *version) GatewayConfigs() GatewayConfigInformer { return &gatewayConfigInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} } +// GuardrailPolicies returns a GuardrailPolicyInformer. +func (v *version) GuardrailPolicies() GuardrailPolicyInformer { + return &guardrailPolicyInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} +} + // MCPRoutes returns a MCPRouteInformer. func (v *version) MCPRoutes() MCPRouteInformer { return &mCPRouteInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} diff --git a/api/v1alpha1/client/informers/externalversions/generic.go b/api/v1alpha1/client/informers/externalversions/generic.go index 171d834867..09c8c134a9 100644 --- a/api/v1alpha1/client/informers/externalversions/generic.go +++ b/api/v1alpha1/client/informers/externalversions/generic.go @@ -50,6 +50,8 @@ func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource return &genericInformer{resource: resource.GroupResource(), informer: f.Aigateway().V1alpha1().BackendSecurityPolicies().Informer()}, nil case v1alpha1.SchemeGroupVersion.WithResource("gatewayconfigs"): return &genericInformer{resource: resource.GroupResource(), informer: f.Aigateway().V1alpha1().GatewayConfigs().Informer()}, nil + case v1alpha1.SchemeGroupVersion.WithResource("guardrailpolicies"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Aigateway().V1alpha1().GuardrailPolicies().Informer()}, nil case v1alpha1.SchemeGroupVersion.WithResource("mcproutes"): return &genericInformer{resource: resource.GroupResource(), informer: f.Aigateway().V1alpha1().MCPRoutes().Informer()}, nil case v1alpha1.SchemeGroupVersion.WithResource("quotapolicies"): diff --git a/api/v1alpha1/client/listers/api/v1alpha1/expansion_generated.go b/api/v1alpha1/client/listers/api/v1alpha1/expansion_generated.go index ed502a96bb..ad81ab7c21 100644 --- a/api/v1alpha1/client/listers/api/v1alpha1/expansion_generated.go +++ b/api/v1alpha1/client/listers/api/v1alpha1/expansion_generated.go @@ -39,6 +39,14 @@ type GatewayConfigListerExpansion interface{} // GatewayConfigNamespaceLister. type GatewayConfigNamespaceListerExpansion interface{} +// GuardrailPolicyListerExpansion allows custom methods to be added to +// GuardrailPolicyLister. +type GuardrailPolicyListerExpansion interface{} + +// GuardrailPolicyNamespaceListerExpansion allows custom methods to be added to +// GuardrailPolicyNamespaceLister. +type GuardrailPolicyNamespaceListerExpansion interface{} + // MCPRouteListerExpansion allows custom methods to be added to // MCPRouteLister. type MCPRouteListerExpansion interface{} diff --git a/api/v1alpha1/client/listers/api/v1alpha1/guardrailpolicy.go b/api/v1alpha1/client/listers/api/v1alpha1/guardrailpolicy.go new file mode 100644 index 0000000000..b1ff880e1f --- /dev/null +++ b/api/v1alpha1/client/listers/api/v1alpha1/guardrailpolicy.go @@ -0,0 +1,59 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +// Code generated by lister-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + apiv1alpha1 "github.com/envoyproxy/ai-gateway/api/v1alpha1" + labels "k8s.io/apimachinery/pkg/labels" + listers "k8s.io/client-go/listers" + cache "k8s.io/client-go/tools/cache" +) + +// GuardrailPolicyLister helps list GuardrailPolicies. +// All objects returned here must be treated as read-only. +type GuardrailPolicyLister interface { + // List lists all GuardrailPolicies in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*apiv1alpha1.GuardrailPolicy, err error) + // GuardrailPolicies returns an object that can list and get GuardrailPolicies. + GuardrailPolicies(namespace string) GuardrailPolicyNamespaceLister + GuardrailPolicyListerExpansion +} + +// guardrailPolicyLister implements the GuardrailPolicyLister interface. +type guardrailPolicyLister struct { + listers.ResourceIndexer[*apiv1alpha1.GuardrailPolicy] +} + +// NewGuardrailPolicyLister returns a new GuardrailPolicyLister. +func NewGuardrailPolicyLister(indexer cache.Indexer) GuardrailPolicyLister { + return &guardrailPolicyLister{listers.New[*apiv1alpha1.GuardrailPolicy](indexer, apiv1alpha1.Resource("guardrailpolicy"))} +} + +// GuardrailPolicies returns an object that can list and get GuardrailPolicies. +func (s *guardrailPolicyLister) GuardrailPolicies(namespace string) GuardrailPolicyNamespaceLister { + return guardrailPolicyNamespaceLister{listers.NewNamespaced[*apiv1alpha1.GuardrailPolicy](s.ResourceIndexer, namespace)} +} + +// GuardrailPolicyNamespaceLister helps list and get GuardrailPolicies. +// All objects returned here must be treated as read-only. +type GuardrailPolicyNamespaceLister interface { + // List lists all GuardrailPolicies in the indexer for a given namespace. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*apiv1alpha1.GuardrailPolicy, err error) + // Get retrieves the GuardrailPolicy from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. + Get(name string) (*apiv1alpha1.GuardrailPolicy, error) + GuardrailPolicyNamespaceListerExpansion +} + +// guardrailPolicyNamespaceLister implements the GuardrailPolicyNamespaceLister +// interface. +type guardrailPolicyNamespaceLister struct { + listers.ResourceIndexer[*apiv1alpha1.GuardrailPolicy] +} diff --git a/api/v1alpha1/guardrail_policy.go b/api/v1alpha1/guardrail_policy.go new file mode 100644 index 0000000000..2893cf7310 --- /dev/null +++ b/api/v1alpha1/guardrail_policy.go @@ -0,0 +1,279 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" + gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" +) + +// GuardrailPolicy evaluates content safety checks for request and response payloads. +// +// +genclient +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.conditions[-1:].type` +// +kubebuilder:metadata:labels="gateway.networking.k8s.io/policy=direct" +// +kubebuilder:deprecatedversion:warning="aigateway.envoyproxy.io/v1alpha1 is deprecated; use aigateway.envoyproxy.io/v1beta1 instead" +type GuardrailPolicy struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + Spec GuardrailPolicySpec `json:"spec,omitempty"` + // Status defines the status details of the GuardrailPolicy. + Status GuardrailPolicyStatus `json:"status,omitempty"` +} + +// GuardrailPolicySpec contains the configured checks attached to an AIServiceBackend. +type GuardrailPolicySpec struct { + // TargetRefs identify the AIServiceBackend resources this GuardrailPolicy is attached to. + // + // +kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:MaxItems=16 + // +kubebuilder:validation:XValidation:rule="self.all(ref, ref.group == 'aigateway.envoyproxy.io' && ref.kind == 'AIServiceBackend')", message="targetRefs must reference AIServiceBackend resources" + TargetRefs []gwapiv1a2.LocalPolicyTargetReference `json:"targetRefs,omitempty"` + // Rules are executed in order and can evaluate request or response payloads. + // + // +kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:MaxItems=32 + // +kubebuilder:validation:XValidation:rule="self.all(rule, self.exists_one(other, other.name == rule.name))",message="rule name must be unique within the policy" + Rules []GuardrailRule `json:"rules,omitempty"` + // MaxRequestBodyBytes is the largest request body evaluated by this policy. + // +optional + // +kubebuilder:default=10485760 + // +kubebuilder:validation:Minimum=1024 + // +kubebuilder:validation:Maximum=52428800 + MaxRequestBodyBytes *int64 `json:"maxRequestBodyBytes,omitempty"` + // MaxResponseBodyBytes is the largest response body buffered and evaluated by this policy. + // +optional + // +kubebuilder:default=10485760 + // +kubebuilder:validation:Minimum=1024 + // +kubebuilder:validation:Maximum=52428800 + MaxResponseBodyBytes *int64 `json:"maxResponseBodyBytes,omitempty"` +} + +// GuardrailRule defines one content-safety check to apply to a request or response. +type GuardrailRule struct { + // Name is a stable identifier for the rule. + // + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=63 + Name string `json:"name"` + // Phase determines whether the rule runs against the request or the response payload. + // + // +kubebuilder:validation:Enum=Request;Response + Phase GuardrailPhase `json:"phase"` + // Provider configures how the rule is evaluated. + Provider GuardrailProvider `json:"provider"` +} + +// GuardrailPhase determines when a guardrail runs. +type GuardrailPhase string + +const ( + GuardrailPhaseRequest GuardrailPhase = "Request" + GuardrailPhaseResponse GuardrailPhase = "Response" +) + +// GuardrailProvider describes the implementation used to evaluate a rule. +// +kubebuilder:validation:XValidation:rule="self.type != 'Regex' || (has(self.pattern) && size(self.pattern) > 0 && !has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety) && !has(self.http) && !has(self.modelArmor))",message="Regex requires pattern and no external provider configuration" +// +kubebuilder:validation:XValidation:rule="self.type != 'Presidio' || (!has(self.pattern) && has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety) && !has(self.http) && !has(self.modelArmor))",message="Presidio requires only presidio provider configuration" +// +kubebuilder:validation:XValidation:rule="self.type != 'Bedrock' || (!has(self.pattern) && has(self.bedrock) && !has(self.presidio) && !has(self.azureContentSafety) && !has(self.http) && !has(self.modelArmor))",message="Bedrock requires only bedrock provider configuration" +// +kubebuilder:validation:XValidation:rule="self.type != 'AzureContentSafety' || (!has(self.pattern) && has(self.azureContentSafety) && !has(self.presidio) && !has(self.bedrock) && !has(self.http) && !has(self.modelArmor))",message="AzureContentSafety requires only azureContentSafety provider configuration" +// +kubebuilder:validation:XValidation:rule="self.type != 'HTTP' || (!has(self.pattern) && has(self.http) && !has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety) && !has(self.modelArmor))",message="HTTP requires only http provider configuration" +// +kubebuilder:validation:XValidation:rule="self.type != 'ModelArmor' || (!has(self.pattern) && has(self.modelArmor) && !has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety) && !has(self.http))",message="ModelArmor requires only modelArmor provider configuration" +// +kubebuilder:validation:XValidation:rule="self.action != 'Mask' || self.type != 'AzureContentSafety'",message="AzureContentSafety does not support Mask" +type GuardrailProvider struct { + // Type identifies the guardrail implementation. + // + // +kubebuilder:validation:Enum=Regex;Presidio;Bedrock;AzureContentSafety;HTTP;ModelArmor + Type GuardrailProviderType `json:"type"` + // Pattern is used for deterministic regex-based evaluations. + // + // +optional + Pattern string `json:"pattern,omitempty"` + // Action is the action taken when the rule is matched. + // + // +optional + // +kubebuilder:default=Block + // +kubebuilder:validation:Enum=Block;Monitor;Mask + Action GuardrailAction `json:"action,omitempty"` + // MaskReplacement is used by Regex, Presidio, and HTTP Mask actions. + // +optional + // +kubebuilder:default="[REDACTED]" + MaskReplacement string `json:"maskReplacement,omitempty"` + // Message is returned to the caller when the rule blocks a request or response. + // + // +optional + Message string `json:"message,omitempty"` + // Presidio configures the Presidio analyzer provider. + // + // +optional + Presidio *PresidioGuardrailProvider `json:"presidio,omitempty"` + // Bedrock configures AWS Bedrock Guardrails. + // + // +optional + Bedrock *BedrockGuardrailProvider `json:"bedrock,omitempty"` + // AzureContentSafety configures Azure AI Content Safety. + // + // +optional + AzureContentSafety *AzureContentSafetyGuardrailProvider `json:"azureContentSafety,omitempty"` + // HTTP configures a custom guardrail service that implements the generic guardrail HTTP contract. + // + // +optional + HTTP *HTTPGuardrailProvider `json:"http,omitempty"` + // ModelArmor configures Google Cloud Model Armor. + // + // +optional + ModelArmor *ModelArmorGuardrailProvider `json:"modelArmor,omitempty"` + // TimeoutSeconds limits each external provider evaluation. + // +optional + // +kubebuilder:default=10 + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=60 + TimeoutSeconds *int32 `json:"timeoutSeconds,omitempty"` + // FailureMode determines whether provider errors block or allow the request. + // +optional + // +kubebuilder:default=FailClosed + // +kubebuilder:validation:Enum=FailClosed;FailOpen + FailureMode GuardrailFailureMode `json:"failureMode,omitempty"` +} + +// PresidioGuardrailProvider configures calls to a Presidio analyzer service. +type PresidioGuardrailProvider struct { + // +kubebuilder:validation:Format=uri + Endpoint string `json:"endpoint"` + // +optional + // +kubebuilder:default=en + Language string `json:"language,omitempty"` + // +optional + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Maximum=100 + ScoreThresholdPercent *int32 `json:"scoreThresholdPercent,omitempty"` + // APIKeySecretRef optionally references a Secret whose apiKey entry is sent as a Bearer token. + // +optional + APIKeySecretRef *gwapiv1.SecretObjectReference `json:"apiKeySecretRef,omitempty"` +} + +// BedrockGuardrailProvider configures calls to the AWS Bedrock ApplyGuardrail API. +type BedrockGuardrailProvider struct { + // +kubebuilder:validation:MinLength=1 + Region string `json:"region"` + // +kubebuilder:validation:MinLength=1 + GuardrailIdentifier string `json:"guardrailIdentifier"` + // +kubebuilder:validation:MinLength=1 + GuardrailVersion string `json:"guardrailVersion"` + // Endpoint overrides the Bedrock runtime endpoint, primarily for private endpoints and testing. + // +optional + // +kubebuilder:validation:Format=uri + Endpoint string `json:"endpoint,omitempty"` + // CredentialsSecretRef optionally references a Secret whose credentials entry contains an AWS shared credentials file. + // +optional + CredentialsSecretRef *gwapiv1.SecretObjectReference `json:"credentialsSecretRef,omitempty"` +} + +// AzureContentSafetyGuardrailProvider configures calls to Azure AI Content Safety. +type AzureContentSafetyGuardrailProvider struct { + // +kubebuilder:validation:Format=uri + Endpoint string `json:"endpoint"` + // +optional + // +kubebuilder:default="2024-09-01" + APIVersion string `json:"apiVersion,omitempty"` + // +optional + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Maximum=7 + SeverityThreshold *int32 `json:"severityThreshold,omitempty"` + // APIKeySecretRef references a Secret whose apiKey entry is sent to Azure. + APIKeySecretRef *gwapiv1.SecretObjectReference `json:"apiKeySecretRef"` +} + +// HTTPGuardrailProvider configures calls to a custom guardrail service. +// +// For each extracted text fragment, the gateway sends a JSON POST request with the +// text and an evaluation context (stage "input" or "output") to endpoint+path. The +// service returns a normalized response with an action ("allow", "block", or "modify"), +// optional findings (type, start, end, score), and an optional replacement text. +// Finding offsets are Unicode code point positions in text; end is exclusive. +type HTTPGuardrailProvider struct { + // Endpoint is the base URL of the guardrail service. + // +kubebuilder:validation:Format=uri + Endpoint string `json:"endpoint"` + // Path is appended to the endpoint for each evaluation. + // +optional + // +kubebuilder:default="/analyze" + // +kubebuilder:validation:Pattern=`^/.*$` + Path string `json:"path,omitempty"` + // APIKeySecretRef optionally references a Secret whose apiKey entry is sent as a Bearer token. + // +optional + APIKeySecretRef *gwapiv1.SecretObjectReference `json:"apiKeySecretRef,omitempty"` +} + +// ModelArmorGuardrailProvider configures calls to the Google Cloud Model Armor sanitize APIs. +// +// Request rules call sanitizeUserPrompt and Response rules call sanitizeModelResponse on the +// configured template. The filters (Responsible AI, prompt injection and jailbreak, malicious URLs, +// and Sensitive Data Protection) and their confidence levels are configured in the template. +// Mask requires a template with an advanced Sensitive Data Protection de-identify template. +type ModelArmorGuardrailProvider struct { + // Project is the Google Cloud project ID that owns the Model Armor template. + // +kubebuilder:validation:MinLength=1 + Project string `json:"project"` + // Location is the Google Cloud region of the Model Armor template, for example us-central1. + // +kubebuilder:validation:MinLength=1 + Location string `json:"location"` + // Template is the Model Armor template ID. + // +kubebuilder:validation:MinLength=1 + Template string `json:"template"` + // Endpoint overrides the regional Model Armor endpoint, primarily for private endpoints and testing. + // +optional + // +kubebuilder:validation:Format=uri + Endpoint string `json:"endpoint,omitempty"` + // CredentialsSecretRef optionally references a Secret whose credentials entry contains a Google Cloud + // service account key JSON. When omitted, Application Default Credentials are used. + // +optional + CredentialsSecretRef *gwapiv1.SecretObjectReference `json:"credentialsSecretRef,omitempty"` +} + +// GuardrailProviderType is the guardrail implementation. +type GuardrailProviderType string + +const ( + GuardrailProviderTypeRegex GuardrailProviderType = "Regex" + GuardrailProviderTypePresidio GuardrailProviderType = "Presidio" + GuardrailProviderTypeBedrockGuardrails GuardrailProviderType = "Bedrock" + GuardrailProviderTypeAzureContentSafety GuardrailProviderType = "AzureContentSafety" + GuardrailProviderTypeHTTP GuardrailProviderType = "HTTP" + GuardrailProviderTypeModelArmor GuardrailProviderType = "ModelArmor" +) + +// GuardrailAction defines the safeguard action. +type GuardrailAction string + +const ( + GuardrailActionBlock GuardrailAction = "Block" + GuardrailActionMonitor GuardrailAction = "Monitor" + GuardrailActionMask GuardrailAction = "Mask" +) + +// GuardrailFailureMode defines behavior when an external provider cannot evaluate content. +type GuardrailFailureMode string + +const ( + GuardrailFailureModeFailClosed GuardrailFailureMode = "FailClosed" + GuardrailFailureModeFailOpen GuardrailFailureMode = "FailOpen" +) + +// GuardrailPolicyList contains a list of GuardrailPolicy resources. +// +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:object:root=true +type GuardrailPolicyList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []GuardrailPolicy `json:"items"` +} diff --git a/api/v1alpha1/registry.go b/api/v1alpha1/registry.go index 92bfd4e2cb..92f175cd12 100644 --- a/api/v1alpha1/registry.go +++ b/api/v1alpha1/registry.go @@ -19,6 +19,7 @@ func init() { SchemeBuilder.Register(&MCPRoute{}, &MCPRouteList{}) SchemeBuilder.Register(&GatewayConfig{}, &GatewayConfigList{}) SchemeBuilder.Register(&QuotaPolicy{}, &QuotaPolicyList{}) + SchemeBuilder.Register(&GuardrailPolicy{}, &GuardrailPolicyList{}) } const GroupName = "aigateway.envoyproxy.io" @@ -54,6 +55,8 @@ func AddKnownTypes(scheme *runtime.Scheme) error { &GatewayConfigList{}, &QuotaPolicy{}, &QuotaPolicyList{}, + &GuardrailPolicy{}, + &GuardrailPolicyList{}, ) metav1.AddToGroupVersion(scheme, SchemeGroupVersion) return nil diff --git a/api/v1alpha1/status.go b/api/v1alpha1/status.go index 6759f6f5bf..2a4d833051 100644 --- a/api/v1alpha1/status.go +++ b/api/v1alpha1/status.go @@ -55,6 +55,15 @@ type MCPRouteStatus struct { Conditions []metav1.Condition `json:"conditions,omitempty"` } +// GuardrailPolicyStatus contains the conditions by the reconciliation result. +type GuardrailPolicyStatus struct { + // Conditions is the list of conditions by the reconciliation result. + // Currently, at most one condition is set. + // + // Known .status.conditions.type are: "Accepted", "NotAccepted". + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + // QuotaPolicyStatus contains the conditions by the reconciliation result. type QuotaPolicyStatus struct { // Conditions is the list of conditions by the reconciliation result. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index b2cf9666f7..73dcdce3b1 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -408,6 +408,31 @@ func (in *AWSOIDCExchangeToken) DeepCopy() *AWSOIDCExchangeToken { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureContentSafetyGuardrailProvider) DeepCopyInto(out *AzureContentSafetyGuardrailProvider) { + *out = *in + if in.SeverityThreshold != nil { + in, out := &in.SeverityThreshold, &out.SeverityThreshold + *out = new(int32) + **out = **in + } + if in.APIKeySecretRef != nil { + in, out := &in.APIKeySecretRef, &out.APIKeySecretRef + *out = new(v1.SecretObjectReference) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureContentSafetyGuardrailProvider. +func (in *AzureContentSafetyGuardrailProvider) DeepCopy() *AzureContentSafetyGuardrailProvider { + if in == nil { + return nil + } + out := new(AzureContentSafetyGuardrailProvider) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AzureOIDCExchangeToken) DeepCopyInto(out *AzureOIDCExchangeToken) { *out = *in @@ -706,6 +731,26 @@ func (in *BackendSecurityPolicyStatus) DeepCopy() *BackendSecurityPolicyStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BedrockGuardrailProvider) DeepCopyInto(out *BedrockGuardrailProvider) { + *out = *in + if in.CredentialsSecretRef != nil { + in, out := &in.CredentialsSecretRef, &out.CredentialsSecretRef + *out = new(v1.SecretObjectReference) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BedrockGuardrailProvider. +func (in *BedrockGuardrailProvider) DeepCopy() *BedrockGuardrailProvider { + if in == nil { + return nil + } + out := new(BedrockGuardrailProvider) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GCPCredentialsFile) DeepCopyInto(out *GCPCredentialsFile) { *out = *in @@ -926,6 +971,185 @@ func (in *GatewayConfigStatus) DeepCopy() *GatewayConfigStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GuardrailPolicy) DeepCopyInto(out *GuardrailPolicy) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GuardrailPolicy. +func (in *GuardrailPolicy) DeepCopy() *GuardrailPolicy { + if in == nil { + return nil + } + out := new(GuardrailPolicy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *GuardrailPolicy) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GuardrailPolicyList) DeepCopyInto(out *GuardrailPolicyList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]GuardrailPolicy, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GuardrailPolicyList. +func (in *GuardrailPolicyList) DeepCopy() *GuardrailPolicyList { + if in == nil { + return nil + } + out := new(GuardrailPolicyList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *GuardrailPolicyList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GuardrailPolicySpec) DeepCopyInto(out *GuardrailPolicySpec) { + *out = *in + if in.TargetRefs != nil { + in, out := &in.TargetRefs, &out.TargetRefs + *out = make([]v1alpha2.LocalPolicyTargetReference, len(*in)) + copy(*out, *in) + } + if in.Rules != nil { + in, out := &in.Rules, &out.Rules + *out = make([]GuardrailRule, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.MaxRequestBodyBytes != nil { + in, out := &in.MaxRequestBodyBytes, &out.MaxRequestBodyBytes + *out = new(int64) + **out = **in + } + if in.MaxResponseBodyBytes != nil { + in, out := &in.MaxResponseBodyBytes, &out.MaxResponseBodyBytes + *out = new(int64) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GuardrailPolicySpec. +func (in *GuardrailPolicySpec) DeepCopy() *GuardrailPolicySpec { + if in == nil { + return nil + } + out := new(GuardrailPolicySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GuardrailPolicyStatus) DeepCopyInto(out *GuardrailPolicyStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]metav1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GuardrailPolicyStatus. +func (in *GuardrailPolicyStatus) DeepCopy() *GuardrailPolicyStatus { + if in == nil { + return nil + } + out := new(GuardrailPolicyStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GuardrailProvider) DeepCopyInto(out *GuardrailProvider) { + *out = *in + if in.Presidio != nil { + in, out := &in.Presidio, &out.Presidio + *out = new(PresidioGuardrailProvider) + (*in).DeepCopyInto(*out) + } + if in.Bedrock != nil { + in, out := &in.Bedrock, &out.Bedrock + *out = new(BedrockGuardrailProvider) + (*in).DeepCopyInto(*out) + } + if in.AzureContentSafety != nil { + in, out := &in.AzureContentSafety, &out.AzureContentSafety + *out = new(AzureContentSafetyGuardrailProvider) + (*in).DeepCopyInto(*out) + } + if in.HTTP != nil { + in, out := &in.HTTP, &out.HTTP + *out = new(HTTPGuardrailProvider) + (*in).DeepCopyInto(*out) + } + if in.ModelArmor != nil { + in, out := &in.ModelArmor, &out.ModelArmor + *out = new(ModelArmorGuardrailProvider) + (*in).DeepCopyInto(*out) + } + if in.TimeoutSeconds != nil { + in, out := &in.TimeoutSeconds, &out.TimeoutSeconds + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GuardrailProvider. +func (in *GuardrailProvider) DeepCopy() *GuardrailProvider { + if in == nil { + return nil + } + out := new(GuardrailProvider) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GuardrailRule) DeepCopyInto(out *GuardrailRule) { + *out = *in + in.Provider.DeepCopyInto(&out.Provider) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GuardrailRule. +func (in *GuardrailRule) DeepCopy() *GuardrailRule { + if in == nil { + return nil + } + out := new(GuardrailRule) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HTTPBodyField) DeepCopyInto(out *HTTPBodyField) { *out = *in @@ -966,6 +1190,26 @@ func (in *HTTPBodyMutation) DeepCopy() *HTTPBodyMutation { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HTTPGuardrailProvider) DeepCopyInto(out *HTTPGuardrailProvider) { + *out = *in + if in.APIKeySecretRef != nil { + in, out := &in.APIKeySecretRef, &out.APIKeySecretRef + *out = new(v1.SecretObjectReference) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HTTPGuardrailProvider. +func (in *HTTPGuardrailProvider) DeepCopy() *HTTPGuardrailProvider { + if in == nil { + return nil + } + out := new(HTTPGuardrailProvider) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HTTPHeaderMutation) DeepCopyInto(out *HTTPHeaderMutation) { *out = *in @@ -1599,6 +1843,26 @@ func (in *MCPToolFilter) DeepCopy() *MCPToolFilter { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ModelArmorGuardrailProvider) DeepCopyInto(out *ModelArmorGuardrailProvider) { + *out = *in + if in.CredentialsSecretRef != nil { + in, out := &in.CredentialsSecretRef, &out.CredentialsSecretRef + *out = new(v1.SecretObjectReference) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ModelArmorGuardrailProvider. +func (in *ModelArmorGuardrailProvider) DeepCopy() *ModelArmorGuardrailProvider { + if in == nil { + return nil + } + out := new(ModelArmorGuardrailProvider) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PerModelQuota) DeepCopyInto(out *PerModelQuota) { *out = *in @@ -1620,6 +1884,31 @@ func (in *PerModelQuota) DeepCopy() *PerModelQuota { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PresidioGuardrailProvider) DeepCopyInto(out *PresidioGuardrailProvider) { + *out = *in + if in.ScoreThresholdPercent != nil { + in, out := &in.ScoreThresholdPercent, &out.ScoreThresholdPercent + *out = new(int32) + **out = **in + } + if in.APIKeySecretRef != nil { + in, out := &in.APIKeySecretRef, &out.APIKeySecretRef + *out = new(v1.SecretObjectReference) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PresidioGuardrailProvider. +func (in *PresidioGuardrailProvider) DeepCopy() *PresidioGuardrailProvider { + if in == nil { + return nil + } + out := new(PresidioGuardrailProvider) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ProtectedResourceMetadata) DeepCopyInto(out *ProtectedResourceMetadata) { *out = *in diff --git a/api/v1beta1/client/clientset/versioned/typed/api/v1beta1/api_client.go b/api/v1beta1/client/clientset/versioned/typed/api/v1beta1/api_client.go index 6b6cc75409..21ef334fc4 100644 --- a/api/v1beta1/client/clientset/versioned/typed/api/v1beta1/api_client.go +++ b/api/v1beta1/client/clientset/versioned/typed/api/v1beta1/api_client.go @@ -21,6 +21,7 @@ type AigatewayV1beta1Interface interface { AIServiceBackendsGetter BackendSecurityPoliciesGetter GatewayConfigsGetter + GuardrailPoliciesGetter MCPRoutesGetter } @@ -45,6 +46,10 @@ func (c *AigatewayV1beta1Client) GatewayConfigs(namespace string) GatewayConfigI return newGatewayConfigs(c, namespace) } +func (c *AigatewayV1beta1Client) GuardrailPolicies(namespace string) GuardrailPolicyInterface { + return newGuardrailPolicies(c, namespace) +} + func (c *AigatewayV1beta1Client) MCPRoutes(namespace string) MCPRouteInterface { return newMCPRoutes(c, namespace) } diff --git a/api/v1beta1/client/clientset/versioned/typed/api/v1beta1/fake/fake_api_client.go b/api/v1beta1/client/clientset/versioned/typed/api/v1beta1/fake/fake_api_client.go index d05148a148..6deb1795df 100644 --- a/api/v1beta1/client/clientset/versioned/typed/api/v1beta1/fake/fake_api_client.go +++ b/api/v1beta1/client/clientset/versioned/typed/api/v1beta1/fake/fake_api_client.go @@ -33,6 +33,10 @@ func (c *FakeAigatewayV1beta1) GatewayConfigs(namespace string) v1beta1.GatewayC return newFakeGatewayConfigs(c, namespace) } +func (c *FakeAigatewayV1beta1) GuardrailPolicies(namespace string) v1beta1.GuardrailPolicyInterface { + return newFakeGuardrailPolicies(c, namespace) +} + func (c *FakeAigatewayV1beta1) MCPRoutes(namespace string) v1beta1.MCPRouteInterface { return newFakeMCPRoutes(c, namespace) } diff --git a/api/v1beta1/client/clientset/versioned/typed/api/v1beta1/fake/fake_guardrailpolicy.go b/api/v1beta1/client/clientset/versioned/typed/api/v1beta1/fake/fake_guardrailpolicy.go new file mode 100644 index 0000000000..e7ea5555df --- /dev/null +++ b/api/v1beta1/client/clientset/versioned/typed/api/v1beta1/fake/fake_guardrailpolicy.go @@ -0,0 +1,41 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + v1beta1 "github.com/envoyproxy/ai-gateway/api/v1beta1" + apiv1beta1 "github.com/envoyproxy/ai-gateway/api/v1beta1/client/clientset/versioned/typed/api/v1beta1" + gentype "k8s.io/client-go/gentype" +) + +// fakeGuardrailPolicies implements GuardrailPolicyInterface +type fakeGuardrailPolicies struct { + *gentype.FakeClientWithList[*v1beta1.GuardrailPolicy, *v1beta1.GuardrailPolicyList] + Fake *FakeAigatewayV1beta1 +} + +func newFakeGuardrailPolicies(fake *FakeAigatewayV1beta1, namespace string) apiv1beta1.GuardrailPolicyInterface { + return &fakeGuardrailPolicies{ + gentype.NewFakeClientWithList[*v1beta1.GuardrailPolicy, *v1beta1.GuardrailPolicyList]( + fake.Fake, + namespace, + v1beta1.SchemeGroupVersion.WithResource("guardrailpolicies"), + v1beta1.SchemeGroupVersion.WithKind("GuardrailPolicy"), + func() *v1beta1.GuardrailPolicy { return &v1beta1.GuardrailPolicy{} }, + func() *v1beta1.GuardrailPolicyList { return &v1beta1.GuardrailPolicyList{} }, + func(dst, src *v1beta1.GuardrailPolicyList) { dst.ListMeta = src.ListMeta }, + func(list *v1beta1.GuardrailPolicyList) []*v1beta1.GuardrailPolicy { + return gentype.ToPointerSlice(list.Items) + }, + func(list *v1beta1.GuardrailPolicyList, items []*v1beta1.GuardrailPolicy) { + list.Items = gentype.FromPointerSlice(items) + }, + ), + fake, + } +} diff --git a/api/v1beta1/client/clientset/versioned/typed/api/v1beta1/generated_expansion.go b/api/v1beta1/client/clientset/versioned/typed/api/v1beta1/generated_expansion.go index 57e2bb7675..9ec2c16cdc 100644 --- a/api/v1beta1/client/clientset/versioned/typed/api/v1beta1/generated_expansion.go +++ b/api/v1beta1/client/clientset/versioned/typed/api/v1beta1/generated_expansion.go @@ -15,4 +15,6 @@ type BackendSecurityPolicyExpansion interface{} type GatewayConfigExpansion interface{} +type GuardrailPolicyExpansion interface{} + type MCPRouteExpansion interface{} diff --git a/api/v1beta1/client/clientset/versioned/typed/api/v1beta1/guardrailpolicy.go b/api/v1beta1/client/clientset/versioned/typed/api/v1beta1/guardrailpolicy.go new file mode 100644 index 0000000000..b82deb93e2 --- /dev/null +++ b/api/v1beta1/client/clientset/versioned/typed/api/v1beta1/guardrailpolicy.go @@ -0,0 +1,59 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +// Code generated by client-gen. DO NOT EDIT. + +package v1beta1 + +import ( + context "context" + + apiv1beta1 "github.com/envoyproxy/ai-gateway/api/v1beta1" + scheme "github.com/envoyproxy/ai-gateway/api/v1beta1/client/clientset/versioned/scheme" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + gentype "k8s.io/client-go/gentype" +) + +// GuardrailPoliciesGetter has a method to return a GuardrailPolicyInterface. +// A group's client should implement this interface. +type GuardrailPoliciesGetter interface { + GuardrailPolicies(namespace string) GuardrailPolicyInterface +} + +// GuardrailPolicyInterface has methods to work with GuardrailPolicy resources. +type GuardrailPolicyInterface interface { + Create(ctx context.Context, guardrailPolicy *apiv1beta1.GuardrailPolicy, opts v1.CreateOptions) (*apiv1beta1.GuardrailPolicy, error) + Update(ctx context.Context, guardrailPolicy *apiv1beta1.GuardrailPolicy, opts v1.UpdateOptions) (*apiv1beta1.GuardrailPolicy, error) + // Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). + UpdateStatus(ctx context.Context, guardrailPolicy *apiv1beta1.GuardrailPolicy, opts v1.UpdateOptions) (*apiv1beta1.GuardrailPolicy, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*apiv1beta1.GuardrailPolicy, error) + List(ctx context.Context, opts v1.ListOptions) (*apiv1beta1.GuardrailPolicyList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *apiv1beta1.GuardrailPolicy, err error) + GuardrailPolicyExpansion +} + +// guardrailPolicies implements GuardrailPolicyInterface +type guardrailPolicies struct { + *gentype.ClientWithList[*apiv1beta1.GuardrailPolicy, *apiv1beta1.GuardrailPolicyList] +} + +// newGuardrailPolicies returns a GuardrailPolicies +func newGuardrailPolicies(c *AigatewayV1beta1Client, namespace string) *guardrailPolicies { + return &guardrailPolicies{ + gentype.NewClientWithList[*apiv1beta1.GuardrailPolicy, *apiv1beta1.GuardrailPolicyList]( + "guardrailpolicies", + c.RESTClient(), + scheme.ParameterCodec, + namespace, + func() *apiv1beta1.GuardrailPolicy { return &apiv1beta1.GuardrailPolicy{} }, + func() *apiv1beta1.GuardrailPolicyList { return &apiv1beta1.GuardrailPolicyList{} }, + ), + } +} diff --git a/api/v1beta1/client/informers/externalversions/api/v1beta1/guardrailpolicy.go b/api/v1beta1/client/informers/externalversions/api/v1beta1/guardrailpolicy.go new file mode 100644 index 0000000000..c3bbe20eb2 --- /dev/null +++ b/api/v1beta1/client/informers/externalversions/api/v1beta1/guardrailpolicy.go @@ -0,0 +1,91 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +// Code generated by informer-gen. DO NOT EDIT. + +package v1beta1 + +import ( + context "context" + time "time" + + aigatewayapiv1beta1 "github.com/envoyproxy/ai-gateway/api/v1beta1" + versioned "github.com/envoyproxy/ai-gateway/api/v1beta1/client/clientset/versioned" + internalinterfaces "github.com/envoyproxy/ai-gateway/api/v1beta1/client/informers/externalversions/internalinterfaces" + apiv1beta1 "github.com/envoyproxy/ai-gateway/api/v1beta1/client/listers/api/v1beta1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// GuardrailPolicyInformer provides access to a shared informer and lister for +// GuardrailPolicies. +type GuardrailPolicyInformer interface { + Informer() cache.SharedIndexInformer + Lister() apiv1beta1.GuardrailPolicyLister +} + +type guardrailPolicyInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc + namespace string +} + +// NewGuardrailPolicyInformer constructs a new informer for GuardrailPolicy type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewGuardrailPolicyInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredGuardrailPolicyInformer(client, namespace, resyncPeriod, indexers, nil) +} + +// NewFilteredGuardrailPolicyInformer constructs a new informer for GuardrailPolicy type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredGuardrailPolicyInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + cache.ToListWatcherWithWatchListSemantics(&cache.ListWatch{ + ListFunc: func(options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.AigatewayV1beta1().GuardrailPolicies(namespace).List(context.Background(), options) + }, + WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.AigatewayV1beta1().GuardrailPolicies(namespace).Watch(context.Background(), options) + }, + ListWithContextFunc: func(ctx context.Context, options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.AigatewayV1beta1().GuardrailPolicies(namespace).List(ctx, options) + }, + WatchFuncWithContext: func(ctx context.Context, options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.AigatewayV1beta1().GuardrailPolicies(namespace).Watch(ctx, options) + }, + }, client), + &aigatewayapiv1beta1.GuardrailPolicy{}, + resyncPeriod, + indexers, + ) +} + +func (f *guardrailPolicyInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredGuardrailPolicyInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *guardrailPolicyInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&aigatewayapiv1beta1.GuardrailPolicy{}, f.defaultInformer) +} + +func (f *guardrailPolicyInformer) Lister() apiv1beta1.GuardrailPolicyLister { + return apiv1beta1.NewGuardrailPolicyLister(f.Informer().GetIndexer()) +} diff --git a/api/v1beta1/client/informers/externalversions/api/v1beta1/interface.go b/api/v1beta1/client/informers/externalversions/api/v1beta1/interface.go index 587fca5078..3c87eb2bc5 100644 --- a/api/v1beta1/client/informers/externalversions/api/v1beta1/interface.go +++ b/api/v1beta1/client/informers/externalversions/api/v1beta1/interface.go @@ -21,6 +21,8 @@ type Interface interface { BackendSecurityPolicies() BackendSecurityPolicyInformer // GatewayConfigs returns a GatewayConfigInformer. GatewayConfigs() GatewayConfigInformer + // GuardrailPolicies returns a GuardrailPolicyInformer. + GuardrailPolicies() GuardrailPolicyInformer // MCPRoutes returns a MCPRouteInformer. MCPRoutes() MCPRouteInformer } @@ -56,6 +58,11 @@ func (v *version) GatewayConfigs() GatewayConfigInformer { return &gatewayConfigInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} } +// GuardrailPolicies returns a GuardrailPolicyInformer. +func (v *version) GuardrailPolicies() GuardrailPolicyInformer { + return &guardrailPolicyInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} +} + // MCPRoutes returns a MCPRouteInformer. func (v *version) MCPRoutes() MCPRouteInformer { return &mCPRouteInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} diff --git a/api/v1beta1/client/informers/externalversions/generic.go b/api/v1beta1/client/informers/externalversions/generic.go index 2393d2db0b..623b1c6c42 100644 --- a/api/v1beta1/client/informers/externalversions/generic.go +++ b/api/v1beta1/client/informers/externalversions/generic.go @@ -50,6 +50,8 @@ func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource return &genericInformer{resource: resource.GroupResource(), informer: f.Aigateway().V1beta1().BackendSecurityPolicies().Informer()}, nil case v1beta1.SchemeGroupVersion.WithResource("gatewayconfigs"): return &genericInformer{resource: resource.GroupResource(), informer: f.Aigateway().V1beta1().GatewayConfigs().Informer()}, nil + case v1beta1.SchemeGroupVersion.WithResource("guardrailpolicies"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Aigateway().V1beta1().GuardrailPolicies().Informer()}, nil case v1beta1.SchemeGroupVersion.WithResource("mcproutes"): return &genericInformer{resource: resource.GroupResource(), informer: f.Aigateway().V1beta1().MCPRoutes().Informer()}, nil diff --git a/api/v1beta1/client/listers/api/v1beta1/expansion_generated.go b/api/v1beta1/client/listers/api/v1beta1/expansion_generated.go index 7b135a7511..c3a85eac96 100644 --- a/api/v1beta1/client/listers/api/v1beta1/expansion_generated.go +++ b/api/v1beta1/client/listers/api/v1beta1/expansion_generated.go @@ -39,6 +39,14 @@ type GatewayConfigListerExpansion interface{} // GatewayConfigNamespaceLister. type GatewayConfigNamespaceListerExpansion interface{} +// GuardrailPolicyListerExpansion allows custom methods to be added to +// GuardrailPolicyLister. +type GuardrailPolicyListerExpansion interface{} + +// GuardrailPolicyNamespaceListerExpansion allows custom methods to be added to +// GuardrailPolicyNamespaceLister. +type GuardrailPolicyNamespaceListerExpansion interface{} + // MCPRouteListerExpansion allows custom methods to be added to // MCPRouteLister. type MCPRouteListerExpansion interface{} diff --git a/api/v1beta1/client/listers/api/v1beta1/guardrailpolicy.go b/api/v1beta1/client/listers/api/v1beta1/guardrailpolicy.go new file mode 100644 index 0000000000..ed3f50c682 --- /dev/null +++ b/api/v1beta1/client/listers/api/v1beta1/guardrailpolicy.go @@ -0,0 +1,59 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +// Code generated by lister-gen. DO NOT EDIT. + +package v1beta1 + +import ( + apiv1beta1 "github.com/envoyproxy/ai-gateway/api/v1beta1" + labels "k8s.io/apimachinery/pkg/labels" + listers "k8s.io/client-go/listers" + cache "k8s.io/client-go/tools/cache" +) + +// GuardrailPolicyLister helps list GuardrailPolicies. +// All objects returned here must be treated as read-only. +type GuardrailPolicyLister interface { + // List lists all GuardrailPolicies in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*apiv1beta1.GuardrailPolicy, err error) + // GuardrailPolicies returns an object that can list and get GuardrailPolicies. + GuardrailPolicies(namespace string) GuardrailPolicyNamespaceLister + GuardrailPolicyListerExpansion +} + +// guardrailPolicyLister implements the GuardrailPolicyLister interface. +type guardrailPolicyLister struct { + listers.ResourceIndexer[*apiv1beta1.GuardrailPolicy] +} + +// NewGuardrailPolicyLister returns a new GuardrailPolicyLister. +func NewGuardrailPolicyLister(indexer cache.Indexer) GuardrailPolicyLister { + return &guardrailPolicyLister{listers.New[*apiv1beta1.GuardrailPolicy](indexer, apiv1beta1.Resource("guardrailpolicy"))} +} + +// GuardrailPolicies returns an object that can list and get GuardrailPolicies. +func (s *guardrailPolicyLister) GuardrailPolicies(namespace string) GuardrailPolicyNamespaceLister { + return guardrailPolicyNamespaceLister{listers.NewNamespaced[*apiv1beta1.GuardrailPolicy](s.ResourceIndexer, namespace)} +} + +// GuardrailPolicyNamespaceLister helps list and get GuardrailPolicies. +// All objects returned here must be treated as read-only. +type GuardrailPolicyNamespaceLister interface { + // List lists all GuardrailPolicies in the indexer for a given namespace. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*apiv1beta1.GuardrailPolicy, err error) + // Get retrieves the GuardrailPolicy from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. + Get(name string) (*apiv1beta1.GuardrailPolicy, error) + GuardrailPolicyNamespaceListerExpansion +} + +// guardrailPolicyNamespaceLister implements the GuardrailPolicyNamespaceLister +// interface. +type guardrailPolicyNamespaceLister struct { + listers.ResourceIndexer[*apiv1beta1.GuardrailPolicy] +} diff --git a/api/v1beta1/guardrail_policy.go b/api/v1beta1/guardrail_policy.go new file mode 100644 index 0000000000..b42d05bf81 --- /dev/null +++ b/api/v1beta1/guardrail_policy.go @@ -0,0 +1,279 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package v1beta1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" + gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" +) + +// GuardrailPolicy evaluates content safety checks for request and response payloads. +// +// +genclient +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.conditions[-1:].type` +// +kubebuilder:metadata:labels="gateway.networking.k8s.io/policy=direct" +// +kubebuilder:storageversion +type GuardrailPolicy struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + Spec GuardrailPolicySpec `json:"spec,omitempty"` + // Status defines the status details of the GuardrailPolicy. + Status GuardrailPolicyStatus `json:"status,omitempty"` +} + +// GuardrailPolicySpec contains the configured checks attached to an AIServiceBackend. +type GuardrailPolicySpec struct { + // TargetRefs identify the AIServiceBackend resources this GuardrailPolicy is attached to. + // + // +kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:MaxItems=16 + // +kubebuilder:validation:XValidation:rule="self.all(ref, ref.group == 'aigateway.envoyproxy.io' && ref.kind == 'AIServiceBackend')", message="targetRefs must reference AIServiceBackend resources" + TargetRefs []gwapiv1a2.LocalPolicyTargetReference `json:"targetRefs,omitempty"` + // Rules are executed in order and can evaluate request or response payloads. + // + // +kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:MaxItems=32 + // +kubebuilder:validation:XValidation:rule="self.all(rule, self.exists_one(other, other.name == rule.name))",message="rule name must be unique within the policy" + Rules []GuardrailRule `json:"rules,omitempty"` + // MaxRequestBodyBytes is the largest request body evaluated by this policy. + // +optional + // +kubebuilder:default=10485760 + // +kubebuilder:validation:Minimum=1024 + // +kubebuilder:validation:Maximum=52428800 + MaxRequestBodyBytes *int64 `json:"maxRequestBodyBytes,omitempty"` + // MaxResponseBodyBytes is the largest response body buffered and evaluated by this policy. + // +optional + // +kubebuilder:default=10485760 + // +kubebuilder:validation:Minimum=1024 + // +kubebuilder:validation:Maximum=52428800 + MaxResponseBodyBytes *int64 `json:"maxResponseBodyBytes,omitempty"` +} + +// GuardrailRule defines one content-safety check to apply to a request or response. +type GuardrailRule struct { + // Name is a stable identifier for the rule. + // + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=63 + Name string `json:"name"` + // Phase determines whether the rule runs against the request or the response payload. + // + // +kubebuilder:validation:Enum=Request;Response + Phase GuardrailPhase `json:"phase"` + // Provider configures how the rule is evaluated. + Provider GuardrailProvider `json:"provider"` +} + +// GuardrailPhase determines when a guardrail runs. +type GuardrailPhase string + +const ( + GuardrailPhaseRequest GuardrailPhase = "Request" + GuardrailPhaseResponse GuardrailPhase = "Response" +) + +// GuardrailProvider describes the implementation used to evaluate a rule. +// +kubebuilder:validation:XValidation:rule="self.type != 'Regex' || (has(self.pattern) && size(self.pattern) > 0 && !has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety) && !has(self.http) && !has(self.modelArmor))",message="Regex requires pattern and no external provider configuration" +// +kubebuilder:validation:XValidation:rule="self.type != 'Presidio' || (!has(self.pattern) && has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety) && !has(self.http) && !has(self.modelArmor))",message="Presidio requires only presidio provider configuration" +// +kubebuilder:validation:XValidation:rule="self.type != 'Bedrock' || (!has(self.pattern) && has(self.bedrock) && !has(self.presidio) && !has(self.azureContentSafety) && !has(self.http) && !has(self.modelArmor))",message="Bedrock requires only bedrock provider configuration" +// +kubebuilder:validation:XValidation:rule="self.type != 'AzureContentSafety' || (!has(self.pattern) && has(self.azureContentSafety) && !has(self.presidio) && !has(self.bedrock) && !has(self.http) && !has(self.modelArmor))",message="AzureContentSafety requires only azureContentSafety provider configuration" +// +kubebuilder:validation:XValidation:rule="self.type != 'HTTP' || (!has(self.pattern) && has(self.http) && !has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety) && !has(self.modelArmor))",message="HTTP requires only http provider configuration" +// +kubebuilder:validation:XValidation:rule="self.type != 'ModelArmor' || (!has(self.pattern) && has(self.modelArmor) && !has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety) && !has(self.http))",message="ModelArmor requires only modelArmor provider configuration" +// +kubebuilder:validation:XValidation:rule="self.action != 'Mask' || self.type != 'AzureContentSafety'",message="AzureContentSafety does not support Mask" +type GuardrailProvider struct { + // Type identifies the guardrail implementation. + // + // +kubebuilder:validation:Enum=Regex;Presidio;Bedrock;AzureContentSafety;HTTP;ModelArmor + Type GuardrailProviderType `json:"type"` + // Pattern is used for deterministic regex-based evaluations. + // + // +optional + Pattern string `json:"pattern,omitempty"` + // Action is the action taken when the rule is matched. + // + // +optional + // +kubebuilder:default=Block + // +kubebuilder:validation:Enum=Block;Monitor;Mask + Action GuardrailAction `json:"action,omitempty"` + // MaskReplacement is used by Regex, Presidio, and HTTP Mask actions. + // +optional + // +kubebuilder:default="[REDACTED]" + MaskReplacement string `json:"maskReplacement,omitempty"` + // Message is returned to the caller when the rule blocks a request or response. + // + // +optional + Message string `json:"message,omitempty"` + // Presidio configures the Presidio analyzer provider. + // + // +optional + Presidio *PresidioGuardrailProvider `json:"presidio,omitempty"` + // Bedrock configures AWS Bedrock Guardrails. + // + // +optional + Bedrock *BedrockGuardrailProvider `json:"bedrock,omitempty"` + // AzureContentSafety configures Azure AI Content Safety. + // + // +optional + AzureContentSafety *AzureContentSafetyGuardrailProvider `json:"azureContentSafety,omitempty"` + // HTTP configures a custom guardrail service that implements the generic guardrail HTTP contract. + // + // +optional + HTTP *HTTPGuardrailProvider `json:"http,omitempty"` + // ModelArmor configures Google Cloud Model Armor. + // + // +optional + ModelArmor *ModelArmorGuardrailProvider `json:"modelArmor,omitempty"` + // TimeoutSeconds limits each external provider evaluation. + // +optional + // +kubebuilder:default=10 + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=60 + TimeoutSeconds *int32 `json:"timeoutSeconds,omitempty"` + // FailureMode determines whether provider errors block or allow the request. + // +optional + // +kubebuilder:default=FailClosed + // +kubebuilder:validation:Enum=FailClosed;FailOpen + FailureMode GuardrailFailureMode `json:"failureMode,omitempty"` +} + +// PresidioGuardrailProvider configures calls to a Presidio analyzer service. +type PresidioGuardrailProvider struct { + // +kubebuilder:validation:Format=uri + Endpoint string `json:"endpoint"` + // +optional + // +kubebuilder:default=en + Language string `json:"language,omitempty"` + // +optional + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Maximum=100 + ScoreThresholdPercent *int32 `json:"scoreThresholdPercent,omitempty"` + // APIKeySecretRef optionally references a Secret whose apiKey entry is sent as a Bearer token. + // +optional + APIKeySecretRef *gwapiv1.SecretObjectReference `json:"apiKeySecretRef,omitempty"` +} + +// BedrockGuardrailProvider configures calls to the AWS Bedrock ApplyGuardrail API. +type BedrockGuardrailProvider struct { + // +kubebuilder:validation:MinLength=1 + Region string `json:"region"` + // +kubebuilder:validation:MinLength=1 + GuardrailIdentifier string `json:"guardrailIdentifier"` + // +kubebuilder:validation:MinLength=1 + GuardrailVersion string `json:"guardrailVersion"` + // Endpoint overrides the Bedrock runtime endpoint, primarily for private endpoints and testing. + // +optional + // +kubebuilder:validation:Format=uri + Endpoint string `json:"endpoint,omitempty"` + // CredentialsSecretRef optionally references a Secret whose credentials entry contains an AWS shared credentials file. + // +optional + CredentialsSecretRef *gwapiv1.SecretObjectReference `json:"credentialsSecretRef,omitempty"` +} + +// AzureContentSafetyGuardrailProvider configures calls to Azure AI Content Safety. +type AzureContentSafetyGuardrailProvider struct { + // +kubebuilder:validation:Format=uri + Endpoint string `json:"endpoint"` + // +optional + // +kubebuilder:default="2024-09-01" + APIVersion string `json:"apiVersion,omitempty"` + // +optional + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Maximum=7 + SeverityThreshold *int32 `json:"severityThreshold,omitempty"` + // APIKeySecretRef references a Secret whose apiKey entry is sent to Azure. + APIKeySecretRef *gwapiv1.SecretObjectReference `json:"apiKeySecretRef"` +} + +// HTTPGuardrailProvider configures calls to a custom guardrail service. +// +// For each extracted text fragment, the gateway sends a JSON POST request with the +// text and an evaluation context (stage "input" or "output") to endpoint+path. The +// service returns a normalized response with an action ("allow", "block", or "modify"), +// optional findings (type, start, end, score), and an optional replacement text. +// Finding offsets are Unicode code point positions in text; end is exclusive. +type HTTPGuardrailProvider struct { + // Endpoint is the base URL of the guardrail service. + // +kubebuilder:validation:Format=uri + Endpoint string `json:"endpoint"` + // Path is appended to the endpoint for each evaluation. + // +optional + // +kubebuilder:default="/analyze" + // +kubebuilder:validation:Pattern=`^/.*$` + Path string `json:"path,omitempty"` + // APIKeySecretRef optionally references a Secret whose apiKey entry is sent as a Bearer token. + // +optional + APIKeySecretRef *gwapiv1.SecretObjectReference `json:"apiKeySecretRef,omitempty"` +} + +// ModelArmorGuardrailProvider configures calls to the Google Cloud Model Armor sanitize APIs. +// +// Request rules call sanitizeUserPrompt and Response rules call sanitizeModelResponse on the +// configured template. The filters (Responsible AI, prompt injection and jailbreak, malicious URLs, +// and Sensitive Data Protection) and their confidence levels are configured in the template. +// Mask requires a template with an advanced Sensitive Data Protection de-identify template. +type ModelArmorGuardrailProvider struct { + // Project is the Google Cloud project ID that owns the Model Armor template. + // +kubebuilder:validation:MinLength=1 + Project string `json:"project"` + // Location is the Google Cloud region of the Model Armor template, for example us-central1. + // +kubebuilder:validation:MinLength=1 + Location string `json:"location"` + // Template is the Model Armor template ID. + // +kubebuilder:validation:MinLength=1 + Template string `json:"template"` + // Endpoint overrides the regional Model Armor endpoint, primarily for private endpoints and testing. + // +optional + // +kubebuilder:validation:Format=uri + Endpoint string `json:"endpoint,omitempty"` + // CredentialsSecretRef optionally references a Secret whose credentials entry contains a Google Cloud + // service account key JSON. When omitted, Application Default Credentials are used. + // +optional + CredentialsSecretRef *gwapiv1.SecretObjectReference `json:"credentialsSecretRef,omitempty"` +} + +// GuardrailProviderType is the guardrail implementation. +type GuardrailProviderType string + +const ( + GuardrailProviderTypeRegex GuardrailProviderType = "Regex" + GuardrailProviderTypePresidio GuardrailProviderType = "Presidio" + GuardrailProviderTypeBedrockGuardrails GuardrailProviderType = "Bedrock" + GuardrailProviderTypeAzureContentSafety GuardrailProviderType = "AzureContentSafety" + GuardrailProviderTypeHTTP GuardrailProviderType = "HTTP" + GuardrailProviderTypeModelArmor GuardrailProviderType = "ModelArmor" +) + +// GuardrailAction defines the safeguard action. +type GuardrailAction string + +const ( + GuardrailActionBlock GuardrailAction = "Block" + GuardrailActionMonitor GuardrailAction = "Monitor" + GuardrailActionMask GuardrailAction = "Mask" +) + +// GuardrailFailureMode defines behavior when an external provider cannot evaluate content. +type GuardrailFailureMode string + +const ( + GuardrailFailureModeFailClosed GuardrailFailureMode = "FailClosed" + GuardrailFailureModeFailOpen GuardrailFailureMode = "FailOpen" +) + +// GuardrailPolicyList contains a list of GuardrailPolicy resources. +// +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:object:root=true +type GuardrailPolicyList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []GuardrailPolicy `json:"items"` +} diff --git a/api/v1beta1/registry.go b/api/v1beta1/registry.go index 01c2e169b6..d38dd9b9cd 100644 --- a/api/v1beta1/registry.go +++ b/api/v1beta1/registry.go @@ -18,6 +18,7 @@ func init() { SchemeBuilder.Register(&BackendSecurityPolicy{}, &BackendSecurityPolicyList{}) SchemeBuilder.Register(&MCPRoute{}, &MCPRouteList{}) SchemeBuilder.Register(&GatewayConfig{}, &GatewayConfigList{}) + SchemeBuilder.Register(&GuardrailPolicy{}, &GuardrailPolicyList{}) } const GroupName = "aigateway.envoyproxy.io" @@ -51,6 +52,8 @@ func AddKnownTypes(scheme *runtime.Scheme) error { &MCPRouteList{}, &GatewayConfig{}, &GatewayConfigList{}, + &GuardrailPolicy{}, + &GuardrailPolicyList{}, ) metav1.AddToGroupVersion(scheme, SchemeGroupVersion) return nil diff --git a/api/v1beta1/status.go b/api/v1beta1/status.go index 09dafe7376..162e6fecee 100644 --- a/api/v1beta1/status.go +++ b/api/v1beta1/status.go @@ -51,3 +51,12 @@ type MCPRouteStatus struct { // Known .status.conditions.type are: "Accepted", "NotAccepted". Conditions []metav1.Condition `json:"conditions,omitempty"` } + +// GuardrailPolicyStatus contains the conditions by the reconciliation result. +type GuardrailPolicyStatus struct { + // Conditions is the list of conditions by the reconciliation result. + // Currently, at most one condition is set. + // + // Known .status.conditions.type are: "Accepted", "NotAccepted". + Conditions []metav1.Condition `json:"conditions,omitempty"` +} diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 702b47c414..3b6c2e4b16 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -415,6 +415,31 @@ func (in *AWSOIDCExchangeToken) DeepCopy() *AWSOIDCExchangeToken { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureContentSafetyGuardrailProvider) DeepCopyInto(out *AzureContentSafetyGuardrailProvider) { + *out = *in + if in.SeverityThreshold != nil { + in, out := &in.SeverityThreshold, &out.SeverityThreshold + *out = new(int32) + **out = **in + } + if in.APIKeySecretRef != nil { + in, out := &in.APIKeySecretRef, &out.APIKeySecretRef + *out = new(v1.SecretObjectReference) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureContentSafetyGuardrailProvider. +func (in *AzureContentSafetyGuardrailProvider) DeepCopy() *AzureContentSafetyGuardrailProvider { + if in == nil { + return nil + } + out := new(AzureContentSafetyGuardrailProvider) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AzureOIDCExchangeToken) DeepCopyInto(out *AzureOIDCExchangeToken) { *out = *in @@ -743,6 +768,26 @@ func (in *BackendSecurityPolicyStatus) DeepCopy() *BackendSecurityPolicyStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BedrockGuardrailProvider) DeepCopyInto(out *BedrockGuardrailProvider) { + *out = *in + if in.CredentialsSecretRef != nil { + in, out := &in.CredentialsSecretRef, &out.CredentialsSecretRef + *out = new(v1.SecretObjectReference) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BedrockGuardrailProvider. +func (in *BedrockGuardrailProvider) DeepCopy() *BedrockGuardrailProvider { + if in == nil { + return nil + } + out := new(BedrockGuardrailProvider) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CredentialOverrideFromDynamicMetadata) DeepCopyInto(out *CredentialOverrideFromDynamicMetadata) { *out = *in @@ -1008,6 +1053,185 @@ func (in *GatewayConfigStatus) DeepCopy() *GatewayConfigStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GuardrailPolicy) DeepCopyInto(out *GuardrailPolicy) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GuardrailPolicy. +func (in *GuardrailPolicy) DeepCopy() *GuardrailPolicy { + if in == nil { + return nil + } + out := new(GuardrailPolicy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *GuardrailPolicy) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GuardrailPolicyList) DeepCopyInto(out *GuardrailPolicyList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]GuardrailPolicy, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GuardrailPolicyList. +func (in *GuardrailPolicyList) DeepCopy() *GuardrailPolicyList { + if in == nil { + return nil + } + out := new(GuardrailPolicyList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *GuardrailPolicyList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GuardrailPolicySpec) DeepCopyInto(out *GuardrailPolicySpec) { + *out = *in + if in.TargetRefs != nil { + in, out := &in.TargetRefs, &out.TargetRefs + *out = make([]v1alpha2.LocalPolicyTargetReference, len(*in)) + copy(*out, *in) + } + if in.Rules != nil { + in, out := &in.Rules, &out.Rules + *out = make([]GuardrailRule, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.MaxRequestBodyBytes != nil { + in, out := &in.MaxRequestBodyBytes, &out.MaxRequestBodyBytes + *out = new(int64) + **out = **in + } + if in.MaxResponseBodyBytes != nil { + in, out := &in.MaxResponseBodyBytes, &out.MaxResponseBodyBytes + *out = new(int64) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GuardrailPolicySpec. +func (in *GuardrailPolicySpec) DeepCopy() *GuardrailPolicySpec { + if in == nil { + return nil + } + out := new(GuardrailPolicySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GuardrailPolicyStatus) DeepCopyInto(out *GuardrailPolicyStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]metav1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GuardrailPolicyStatus. +func (in *GuardrailPolicyStatus) DeepCopy() *GuardrailPolicyStatus { + if in == nil { + return nil + } + out := new(GuardrailPolicyStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GuardrailProvider) DeepCopyInto(out *GuardrailProvider) { + *out = *in + if in.Presidio != nil { + in, out := &in.Presidio, &out.Presidio + *out = new(PresidioGuardrailProvider) + (*in).DeepCopyInto(*out) + } + if in.Bedrock != nil { + in, out := &in.Bedrock, &out.Bedrock + *out = new(BedrockGuardrailProvider) + (*in).DeepCopyInto(*out) + } + if in.AzureContentSafety != nil { + in, out := &in.AzureContentSafety, &out.AzureContentSafety + *out = new(AzureContentSafetyGuardrailProvider) + (*in).DeepCopyInto(*out) + } + if in.HTTP != nil { + in, out := &in.HTTP, &out.HTTP + *out = new(HTTPGuardrailProvider) + (*in).DeepCopyInto(*out) + } + if in.ModelArmor != nil { + in, out := &in.ModelArmor, &out.ModelArmor + *out = new(ModelArmorGuardrailProvider) + (*in).DeepCopyInto(*out) + } + if in.TimeoutSeconds != nil { + in, out := &in.TimeoutSeconds, &out.TimeoutSeconds + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GuardrailProvider. +func (in *GuardrailProvider) DeepCopy() *GuardrailProvider { + if in == nil { + return nil + } + out := new(GuardrailProvider) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GuardrailRule) DeepCopyInto(out *GuardrailRule) { + *out = *in + in.Provider.DeepCopyInto(&out.Provider) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GuardrailRule. +func (in *GuardrailRule) DeepCopy() *GuardrailRule { + if in == nil { + return nil + } + out := new(GuardrailRule) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HTTPBodyField) DeepCopyInto(out *HTTPBodyField) { *out = *in @@ -1048,6 +1272,26 @@ func (in *HTTPBodyMutation) DeepCopy() *HTTPBodyMutation { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HTTPGuardrailProvider) DeepCopyInto(out *HTTPGuardrailProvider) { + *out = *in + if in.APIKeySecretRef != nil { + in, out := &in.APIKeySecretRef, &out.APIKeySecretRef + *out = new(v1.SecretObjectReference) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HTTPGuardrailProvider. +func (in *HTTPGuardrailProvider) DeepCopy() *HTTPGuardrailProvider { + if in == nil { + return nil + } + out := new(HTTPGuardrailProvider) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HTTPHeaderMutation) DeepCopyInto(out *HTTPHeaderMutation) { *out = *in @@ -1756,6 +2000,51 @@ func (in *MCPToolFilter) DeepCopy() *MCPToolFilter { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ModelArmorGuardrailProvider) DeepCopyInto(out *ModelArmorGuardrailProvider) { + *out = *in + if in.CredentialsSecretRef != nil { + in, out := &in.CredentialsSecretRef, &out.CredentialsSecretRef + *out = new(v1.SecretObjectReference) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ModelArmorGuardrailProvider. +func (in *ModelArmorGuardrailProvider) DeepCopy() *ModelArmorGuardrailProvider { + if in == nil { + return nil + } + out := new(ModelArmorGuardrailProvider) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PresidioGuardrailProvider) DeepCopyInto(out *PresidioGuardrailProvider) { + *out = *in + if in.ScoreThresholdPercent != nil { + in, out := &in.ScoreThresholdPercent, &out.ScoreThresholdPercent + *out = new(int32) + **out = **in + } + if in.APIKeySecretRef != nil { + in, out := &in.APIKeySecretRef, &out.APIKeySecretRef + *out = new(v1.SecretObjectReference) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PresidioGuardrailProvider. +func (in *PresidioGuardrailProvider) DeepCopy() *PresidioGuardrailProvider { + if in == nil { + return nil + } + out := new(PresidioGuardrailProvider) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ProtectedResourceMetadata) DeepCopyInto(out *ProtectedResourceMetadata) { *out = *in diff --git a/cmd/extproc/mainlib/main.go b/cmd/extproc/mainlib/main.go index 9830c99b50..237e3b3f3a 100644 --- a/cmd/extproc/mainlib/main.go +++ b/cmd/extproc/mainlib/main.go @@ -319,6 +319,7 @@ func Main(ctx context.Context, args []string, stderr io.Writer) (err error) { responsesInputTokensMetricsFactory := metrics.NewMetricsFactory(meter, metricsRequestHeaderAttributes, metrics.GenAIOperationResponsesInputTokens) countTokensMetricsFactory := metrics.NewMetricsFactory(meter, metricsRequestHeaderAttributes, metrics.GenAIOperationCountTokens) mcpMetrics := metrics.NewMCP(meter, metricsRequestHeaderAttributes) + guardrailMetrics := metrics.NewGuardrailMetrics(meter) extproc.LogRequestHeaderAttributes = logRequestHeaderAttributes @@ -327,38 +328,38 @@ func Main(ctx context.Context, args []string, stderr io.Writer) (err error) { return fmt.Errorf("failed to create external processor server: %w", err) } server.Register(path.Join(flags.rootPrefix, endpointPrefixes.OpenAI, "/v1/chat/completions"), extproc.NewFactory( - chatCompletionMetricsFactory, tracing.ChatCompletionTracer(), endpointspec.ChatCompletionsEndpointSpec{})) + chatCompletionMetricsFactory, guardrailMetrics, tracing.ChatCompletionTracer(), endpointspec.ChatCompletionsEndpointSpec{})) server.Register(path.Join(flags.rootPrefix, endpointPrefixes.OpenAI, "/v1/completions"), extproc.NewFactory( - completionMetricsFactory, tracing.CompletionTracer(), endpointspec.CompletionsEndpointSpec{})) + completionMetricsFactory, guardrailMetrics, tracing.CompletionTracer(), endpointspec.CompletionsEndpointSpec{})) server.Register(path.Join(flags.rootPrefix, endpointPrefixes.OpenAI, "/v1/embeddings"), extproc.NewFactory( - embeddingsMetricsFactory, tracing.EmbeddingsTracer(), endpointspec.EmbeddingsEndpointSpec{})) + embeddingsMetricsFactory, guardrailMetrics, tracing.EmbeddingsTracer(), endpointspec.EmbeddingsEndpointSpec{})) server.Register(path.Join(flags.rootPrefix, endpointPrefixes.OpenAI, "/v1/responses"), extproc.NewFactory( - responsesMetricsFactory, tracing.ResponsesTracer(), endpointspec.ResponsesEndpointSpec{})) + responsesMetricsFactory, guardrailMetrics, tracing.ResponsesTracer(), endpointspec.ResponsesEndpointSpec{})) server.Register(path.Join(flags.rootPrefix, endpointPrefixes.OpenAI, "/v1/decisions"), extproc.NewFactory( - decisionsMetricsFactory, tracing.DecisionsTracer(), endpointspec.DecisionsEndpointSpec{})) + decisionsMetricsFactory, guardrailMetrics, tracing.DecisionsTracer(), endpointspec.DecisionsEndpointSpec{})) server.Register(path.Join(flags.rootPrefix, endpointPrefixes.OpenAI, "/v1/responses/input_tokens"), extproc.NewFactory( - responsesInputTokensMetricsFactory, tracing.ResponsesInputTokensTracer(), endpointspec.ResponsesInputTokensEndpointSpec{})) + responsesInputTokensMetricsFactory, guardrailMetrics, tracing.ResponsesInputTokensTracer(), endpointspec.ResponsesInputTokensEndpointSpec{})) server.Register(path.Join(flags.rootPrefix, endpointPrefixes.OpenAI, "/v1/audio/speech"), extproc.NewFactory( - speechMetricsFactory, tracing.SpeechTracer(), endpointspec.SpeechEndpointSpec{})) + speechMetricsFactory, guardrailMetrics, tracing.SpeechTracer(), endpointspec.SpeechEndpointSpec{})) server.Register(path.Join(flags.rootPrefix, endpointPrefixes.OpenAI, "/v1/audio/transcriptions"), extproc.NewFactory( - transcriptionMetricsFactory, tracing.TranscriptionTracer(), endpointspec.TranscriptionEndpointSpec{})) + transcriptionMetricsFactory, guardrailMetrics, tracing.TranscriptionTracer(), endpointspec.TranscriptionEndpointSpec{})) server.Register(path.Join(flags.rootPrefix, endpointPrefixes.OpenAI, "/v1/audio/translations"), extproc.NewFactory( - translationMetricsFactory, tracing.TranslationTracer(), endpointspec.TranslationEndpointSpec{})) + translationMetricsFactory, guardrailMetrics, tracing.TranslationTracer(), endpointspec.TranslationEndpointSpec{})) server.Register(path.Join(flags.rootPrefix, endpointPrefixes.OpenAI, "/v1/images/generations"), extproc.NewFactory( - imageGenerationMetricsFactory, tracing.ImageGenerationTracer(), endpointspec.ImageGenerationEndpointSpec{})) + imageGenerationMetricsFactory, guardrailMetrics, tracing.ImageGenerationTracer(), endpointspec.ImageGenerationEndpointSpec{})) server.Register(path.Join(flags.rootPrefix, endpointPrefixes.Cohere, "/v2/rerank"), extproc.NewFactory( - rerankMetricsFactory, tracing.RerankTracer(), endpointspec.RerankEndpointSpec{})) + rerankMetricsFactory, guardrailMetrics, tracing.RerankTracer(), endpointspec.RerankEndpointSpec{})) server.Register(path.Join(flags.rootPrefix, endpointPrefixes.TypeSafe, "/v1/systemone"), extproc.NewFactory( - systemOneMetricsFactory, tracing.SystemOneTracer(), endpointspec.SystemOneEndpointSpec{})) + systemOneMetricsFactory, guardrailMetrics, tracing.SystemOneTracer(), endpointspec.SystemOneEndpointSpec{})) server.Register(path.Join(flags.rootPrefix, endpointPrefixes.OpenAI, "/v1/models"), extproc.NewModelsProcessor) server.Register(path.Join(flags.rootPrefix, endpointPrefixes.Anthropic, "/v1/models"), extproc.NewAnthropicModelsProcessor) server.Register(path.Join(flags.rootPrefix, endpointPrefixes.Anthropic, "/v1/messages"), extproc.NewFactory( - messagesMetricsFactory, tracing.MessageTracer(), endpointspec.MessagesEndpointSpec{})) + messagesMetricsFactory, guardrailMetrics, tracing.MessageTracer(), endpointspec.MessagesEndpointSpec{})) // Use /tokenize to be consistent with vLLM: https://github.com/vllm-project/vllm/blob/344b50d5258d7cf3f136416e1dbcd9b5ee99bb00/vllm/entrypoints/serve/tokenize/api_router.py#L37 server.Register(path.Join(flags.rootPrefix, endpointPrefixes.OpenAI, "/tokenize"), extproc.NewFactory( - tokenizeMetricsFactory, tracing.TokenizeTracer(), endpointspec.TokenizeEndpointSpec{})) + tokenizeMetricsFactory, guardrailMetrics, tracing.TokenizeTracer(), endpointspec.TokenizeEndpointSpec{})) server.Register(path.Join(flags.rootPrefix, endpointPrefixes.Anthropic, "/v1/messages/count_tokens"), extproc.NewFactory( - countTokensMetricsFactory, tracing.CountTokensTracer(), endpointspec.MessagesCountTokensEndpointSpec{})) + countTokensMetricsFactory, guardrailMetrics, tracing.CountTokensTracer(), endpointspec.MessagesCountTokensEndpointSpec{})) // Create and register gRPC server with ExternalProcessorServer (the service Envoy calls). if err = filterapi.StartConfigBundleWatcher(ctx, flags.configBundlePath, server, l, time.Second*5); err != nil { diff --git a/docs/proposals/013-guardrails/proposal.md b/docs/proposals/013-guardrails/proposal.md new file mode 100644 index 0000000000..96e738d2c2 --- /dev/null +++ b/docs/proposals/013-guardrails/proposal.md @@ -0,0 +1,559 @@ +# Guardrails Proposal + +## Table of Contents + +- [Guardrails Proposal](#guardrails-proposal) + - [Table of Contents](#table-of-contents) + - [Summary](#summary) + - [Motivation](#motivation) + - [Goals](#goals) + - [Non-Goals](#non-goals) + - [Prior Art](#prior-art) + - [Built on Envoy Bedrock Guardrails](#built-on-envoy-bedrock-guardrails) + - [Built on Envoy Azure Content Safety](#built-on-envoy-azure-content-safety) + - [Lessons Applied to Agent Router](#lessons-applied-to-agent-router) + - [Proposed Architecture](#proposed-architecture) + - [Why ExtProc](#why-extproc) + - [Policy Attachment and Backend Scoping](#policy-attachment-and-backend-scoping) + - [Evaluation Lifecycle](#evaluation-lifecycle) + - [Streaming Responses](#streaming-responses) + - [API Design](#api-design) + - [GuardrailPolicy](#guardrailpolicy) + - [Provider Configuration](#provider-configuration) + - [Actions](#actions) + - [Failure Modes and Timeouts](#failure-modes-and-timeouts) + - [Provider Behavior](#provider-behavior) + - [Regex](#regex) + - [Presidio](#presidio) + - [AWS Bedrock Guardrails](#aws-bedrock-guardrails) + - [Azure AI Content Safety](#azure-ai-content-safety) + - [Google Cloud Model Armor](#google-cloud-model-armor) + - [Custom HTTP Guardrails](#custom-http-guardrails) + - [Payload Extraction](#payload-extraction) + - [Credentials and Security](#credentials-and-security) + - [Status and Reconciliation](#status-and-reconciliation) + - [Observability](#observability) + - [Performance and Reliability](#performance-and-reliability) + - [Alternatives Considered](#alternatives-considered) + - [Use Built on Envoy Dynamic Modules Directly](#use-built-on-envoy-dynamic-modules-directly) + - [Configure Guardrails Directly on AIGatewayRoute](#configure-guardrails-directly-on-aigatewayroute) + - [Use BackendSecurityPolicy](#use-backendsecuritypolicy) + - [Use Only Provider-Native Model Guardrails](#use-only-provider-native-model-guardrails) + - [Invoke Local Executables for Custom Guardrails](#invoke-local-executables-for-custom-guardrails) + - [Implementation Plan](#implementation-plan) + - [Testing Strategy](#testing-strategy) + - [Current Implementation and Gaps](#current-implementation-and-gaps) + - [References](#references) + +## Summary + +This proposal introduces `GuardrailPolicy`, a backend-attached policy for evaluating LLM request and response content before it is sent to an AI provider or returned to a client. The policy provides a common API for local regular-expression checks and external safety providers, initially Presidio, AWS Bedrock Guardrails, Azure AI Content Safety, and Google Cloud Model Armor. A generic HTTP provider defines a small, normalized contract that any custom guardrail service can implement, so providers without a native integration can be used without code changes to the gateway. + +The proposed implementation uses the existing Agent Router external processor. The controller resolves policies and credentials into the filter configuration, while ext-proc buffers the relevant body, extracts content, invokes the configured evaluator, and applies the selected action. + +This design is informed by the Built on Envoy Bedrock Guardrails and Azure Content Safety dynamic-module extensions. It adopts their strongest behavioral patterns while retaining Agent Router's Kubernetes policy model, backend scoping, provider-neutral configuration, existing ext-proc deployment, and observability conventions. + +## Motivation + +AI Gateway users need a consistent way to enforce content-safety requirements across different models and providers. Today, users must deploy an additional filter, application middleware, or provider-specific integration. Those approaches make policy attachment, credentials, observability, and failure behavior inconsistent across backends. + +A native policy should allow platform administrators to: + +- attach safety checks to one or more `AIServiceBackend` resources; +- inspect prompts before upstream routing and completions before downstream delivery; +- use deterministic local checks or managed safety services; +- choose blocking or observational behavior independently from provider failures; +- rotate credentials without restarting gateways; and +- observe allowed, blocked, and failed evaluations without recording sensitive payloads. + +## Goals + +- Define a Kubernetes-native `GuardrailPolicy` consistent with the repository's direct-policy conventions. +- Support both `v1alpha1` and `v1beta1`, with `v1beta1` as the storage version. +- Scope policies to `AIServiceBackend` resources and preserve route/backend isolation. +- Support request and response evaluation. +- Provide a provider-neutral runtime evaluator interface. +- Initially support Regex, Presidio, AWS Bedrock Guardrails, Azure AI Content Safety, and Google Cloud Model Armor. +- Define a portable HTTP contract for custom guardrail services that are not natively supported. +- Support explicit timeout and fail-open/fail-closed behavior. +- Prevent partial delivery of blocked streaming responses. +- Integrate with existing logs, metrics, traces, config distribution, Secret watches, and status conditions. +- Keep provider credentials out of the CRD and user-visible status. + +## Non-Goals + +- Replacing provider-native model safety configuration. +- Defining a general-purpose Web Application Firewall API. +- Supporting arbitrary executable plugins supplied by users. +- Performing image, audio, or video moderation in the initial API. +- Guaranteeing identical safety classifications across providers. +- Retrying external safety calls in the initial implementation. +- Exposing provider response details or sensitive matched content to downstream clients. + +## Prior Art + +### Built on Envoy Bedrock Guardrails + +The [Built on Envoy Bedrock Guardrails] extension is an Envoy dynamic-module HTTP filter. It recognizes OpenAI Chat Completions requests, extracts user prompts, invokes each configured Bedrock guardrail, and can apply block, mask, or no-op outcomes. + +Notable design choices are: + +- semantic extraction of user messages instead of sending the serialized request envelope; +- multiple Bedrock guardrails evaluated for one request; +- configurable request timeout; +- response outputs used to replace masked prompt content; and +- an Envoy cluster used for provider callouts. + +The extension authenticates with a Bedrock API key. Agent Router already supports AWS workload credentials and SigV4, so this proposal uses the standard AWS credential chain or a referenced Kubernetes Secret instead. + +### Built on Envoy Azure Content Safety + +The [Built on Envoy Azure Content Safety] extension is also an Envoy dynamic-module HTTP filter. It auto-detects OpenAI Chat Completions, OpenAI Responses, and Anthropic Messages payloads and uses phase-specific Azure APIs: + +- Prompt Shield for request prompt-injection detection; +- Task Adherence for request tool-use alignment; +- Text Analysis for harmful response content; and +- Protected Material Detection for response content. + +It supports block and monitor modes, fail-open behavior, selected categories, and independent severity thresholds for hate, self-harm, sexual, and violent content. + +The extension demonstrates that vendor integration is more useful when it understands the LLM API schema and chooses a provider operation based on the intended safety check, rather than treating every JSON body as undifferentiated text. + +### Lessons Applied to Agent Router + +The following ideas should be adopted: + +- buffer content before a blocking decision; +- extract semantically relevant text from known LLM schemas; +- distinguish enforcement mode from provider failure behavior; +- support provider-specific checks without leaking them into unrelated providers; +- allow per-category thresholds where the provider exposes them; +- short-circuit after a blocking result; and +- make provider latency and errors observable. + +At the time of this proposal, both extension pages identify the implementation as version `0.12.0-dev` with an Envoy compatibility range of 1.38 through 1.39. The following choices remain specific to Agent Router: + +- policy attachment through Kubernetes `targetRefs`; +- backend-specific scoping after route selection; +- credentials resolved through Kubernetes Secrets or cloud workload identity; +- configuration distributed through the existing ext-proc bundle; +- a provider-neutral evaluator interface; and +- no additional Envoy binary or dynamic-module deployment requirement. + +## Proposed Architecture + +```text +GuardrailPolicy + Secret + | + v +Gateway controller + - validates targets and provider configuration + - resolves credential references + - scopes rules to generated backend identities + | + v +Filter configuration bundle + | + v +Envoy ext_proc + - parses the known LLM request/response schema + - selects rules for phase and chosen backend + - invokes local or external evaluators + - records logs, metrics, and span events + | + +--> allow / monitor + | + +--> block with HTTP 403 +``` + +### Why ExtProc + +Agent Router already uses ext-proc for request parsing, provider translation, credential handling, token accounting, and response processing. Guardrails require the same parsed content and backend identity. + +Using the existing ext-proc has these advantages: + +- no second extension packaging or compatibility lifecycle; +- no dependency on a narrow Envoy dynamic-module version range; +- direct reuse of endpoint schemas and translators; +- one configuration and observability path; and +- consistent behavior in controller-managed and standalone deployments. + +A dynamic module may reduce cross-process overhead and can use Envoy clusters for callouts. However, adopting the [Built on Envoy repository] extensions directly would require reconciling their configuration, credentials, deployment, API parsing, and release compatibility with Agent Router. This proposal therefore uses them as prior art rather than as runtime dependencies. + +### Policy Attachment and Backend Scoping + +`GuardrailPolicy.spec.targetRefs` references one or more `AIServiceBackend` resources in the policy namespace. During gateway reconciliation, the controller maps each target to generated per-route backend identities. Runtime rules are evaluated only when the matching backend is selected. + +Request rules that do not depend on backend selection may run at router level. Backend-attached rules run after selection and before upstream forwarding. Response rules run against the selected backend's response. + +Multiple policies targeting the same backend compose additively. The controller sorts policies lexicographically by namespace and name, then preserves declaration order within each policy. Evaluation stops at the first blocking rule. Monitor and Mask rules continue to later rules after recording or applying their result. + +### Evaluation Lifecycle + +Rules execute in order for their phase and backend: + +1. Select rules matching the request or response phase. +2. Filter rules by the selected generated backend identity. +3. Extract provider-relevant content from the parsed LLM payload. +4. Invoke the evaluator with the request context and configured timeout. +5. Record the result. +6. Stop at the first blocking result. +7. Continue after an allowed result or a fail-open provider error. + +A provider error under `FailClosed` prevents unchecked traffic. A provider error under `FailOpen` records an error and allows evaluation to continue. Provider failures are not counted as allowed evaluations. + +### Streaming Responses + +A response cannot be safely blocked or rewritten after bytes have already reached the client. When any applicable response rule is configured, ext-proc must keep response-body processing buffered. Only routes without applicable response guardrails may switch to streamed response processing. + +This increases latency and memory use for guarded streaming responses. All response rules, including Monitor, use buffering so providers receive a complete semantic payload rather than partial SSE events. Each policy defaults to a 10 MiB response evaluation limit and may configure up to 50 MiB, matching the Envoy per-connection buffer ceiling. The limit is checked before provider evaluation; exceeding it follows the rule's failure mode. + +## API Design + +### GuardrailPolicy + +```yaml +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: GuardrailPolicy +metadata: + name: content-safety + namespace: ai-gateway +spec: + maxRequestBodyBytes: 10485760 + maxResponseBodyBytes: 10485760 + targetRefs: + - group: aigateway.envoyproxy.io + kind: AIServiceBackend + name: openai + rules: + - name: block-pii + phase: Request + provider: + type: Presidio + action: Block + failureMode: FailClosed + timeoutSeconds: 5 + message: Request contains sensitive information + presidio: + endpoint: http://presidio-analyzer.presidio.svc.cluster.local:3000 + language: en + scoreThresholdPercent: 70 +``` + +Rules have stable unique names and execute in order. The default action is `Block`; users may explicitly select `Monitor` or `Mask` when the provider supports it. The API reserves room for additional actions without forcing all providers to support identical capabilities. + +### Provider Configuration + +The provider is a tagged union. Admission validation requires exactly the configuration matching `provider.type`. + +- `Regex` requires `pattern`. +- `Presidio` requires `presidio`. +- `Bedrock` requires `bedrock`. +- `AzureContentSafety` requires `azureContentSafety`. +- `HTTP` requires `http`. +- `ModelArmor` requires `modelArmor`. + +Provider-specific configuration remains nested so new provider capabilities can be added without adding unrelated fields to every rule. + +### Actions + +The portable actions are: + +- `Block`: return HTTP 403 with error type `GuardrailViolation`. +- `Monitor`: record a detection and continue without modifying traffic. +- `Mask`: replace detected text and continue evaluation. + +Mask behavior depends on provider capability: + +- Regex replaces matching substrings with `maskReplacement`, defaulting to `[REDACTED]`. +- Presidio replaces detected spans with `maskReplacement`. +- Bedrock uses transformed text returned by ApplyGuardrail. +- Azure Text Analysis does not return transformed content, so admission rejects `Mask` for Azure rules. +- HTTP uses the `replacement` text returned by the service, or replaces returned finding spans with `maskReplacement`. +- Model Armor uses the text de-identified by its Sensitive Data Protection filter, only when no other filter matched. + +Mask is applied only to extracted text fields and never replaces unrelated model or configuration fields. Masked streaming responses remain buffered until the complete body has been evaluated and rewritten. + +### Failure Modes and Timeouts + +`failureMode` controls provider errors, not policy matches: + +- `FailClosed` is the default and prevents unchecked traffic. +- `FailOpen` records the provider error and continues processing. + +`timeoutSeconds` applies to each external evaluation. The initial implementation does not retry. This avoids multiplying tail latency and prevents ambiguous duplicate calls. If retries are added, they should be limited to clearly retryable transport and 5xx failures and remain inside the rule's total timeout budget. + +## Provider Behavior + +### Regex + +Regex rules compile when the runtime configuration is loaded. Invalid patterns reject the configuration. Evaluation is local and deterministic. + +Regex is useful for tests, organization-specific markers, and simple deny patterns. It is not a replacement for semantic safety classification. + +### Presidio + +[Presidio] calls the Analyzer `/analyze` endpoint with extracted text, language, and an optional score threshold. An optional Secret-provided API key can be sent as a bearer token for deployments that place authentication in front of Presidio. + +The official Presidio analyzer image is suitable for deterministic integration tests through Testcontainers. + +Future configuration may add entity allow/deny lists. Those should use Presidio entity names and avoid exposing matched text in status, logs, or metrics. + +### AWS Bedrock Guardrails + +Bedrock uses the [AWS Bedrock ApplyGuardrail] API. The request source is `INPUT` for requests and `OUTPUT` for responses. Authentication uses SigV4 with either: + +- the standard AWS credential chain, including workload identity; or +- an AWS shared credentials file read from a Kubernetes Secret. + +The Built on Envoy extension additionally supports Bedrock API-key authentication and mask results. API-key support can be added as another credential source if users require it. Mask support requires a separate action and schema-aware mutation contract. + +Each rule invokes one Bedrock guardrail identifier/version. Multiple rules may target the same backend, but each external call adds latency and cost. Administrators should prefer consolidating related checks into one managed Bedrock guardrail where possible. + +### Azure AI Content Safety + +The initial [Azure AI Content Safety] provider configuration includes endpoint, API version, API key Secret, and severity threshold. + +The Built on Envoy implementation highlights that Azure has multiple distinct safety operations. The API should eventually model the check explicitly instead of inferring it only from request/response phase: + +- request Prompt Shield; +- request Task Adherence; +- request or response Text Analysis; +- response Protected Material Detection. + +A future shape could add `azureContentSafety.check` and check-specific configuration. Per-category thresholds should also replace a single threshold when Text Analysis is selected. Until then, the implementation should be documented as Text Analysis over the configured evaluation input, not as complete Azure Content Safety feature parity. + +### Google Cloud Model Armor + +[Google Cloud Model Armor] screens content against a Model Armor template. Request rules call `sanitizeUserPrompt` and response rules call `sanitizeModelResponse` on the regional endpoint; the global endpoint does not serve these methods. The rule references the template by project, location, and template ID, so filter selection and confidence levels (Responsible AI, prompt injection and jailbreak, malicious URLs, Sensitive Data Protection) stay in Google Cloud, as with Bedrock guardrail versions. + +A rule matches when the sanitization result reports `MATCH_FOUND`. An invocation result of `FAILURE` is a provider error and follows the rule's failure mode. Authentication uses an OAuth2 access token from either: + +- Google Application Default Credentials, including GKE Workload Identity; or +- a service account key JSON read from a Kubernetes Secret. + +Mask requires a template with advanced Sensitive Data Protection and a de-identify template. The evaluator forwards the de-identified text only when Sensitive Data Protection is the sole matching filter; any other match (for example, a jailbreak) returns no replacement so Mask rules fail instead of forwarding flagged content. Model Armor file scanning and streaming sanitization are not used, because the gateway evaluates extracted text fields. + +### Custom HTTP Guardrails + +Native integrations cover common providers, but organizations frequently run in-house classifiers or services that the gateway does not support yet. The `HTTP` provider defines a small, provider-neutral interface, similar in spirit to the Presidio Analyzer API, that such services implement. It lets new guardrails be adopted without gateway changes and without the security and portability concerns of invoking arbitrary local executables. + +```yaml +provider: + type: HTTP + action: Mask + timeoutSeconds: 5 + failureMode: FailClosed + http: + endpoint: http://custom-guardrail.guardrails.svc.cluster.local:8080 + path: /analyze # default + apiKeySecretRef: # optional; sent as a bearer token + name: custom-guardrail-key +``` + +For each extracted text fragment, the gateway sends `POST {endpoint}{path}` with `Content-Type: application/json`: + +```json +{ + "text": "some user input", + "context": { + "stage": "input" + } +} +``` + +`context.stage` is `input` for `Request` rules and `output` for `Response` rules. The `context` object is extensible; services must ignore unknown fields so the gateway can add information such as the endpoint schema or message role later. + +The service returns HTTP 2xx with a normalized response: + +```json +{ + "action": "allow", + "findings": [ + { + "type": "PII", + "start": 10, + "end": 20, + "score": 0.92 + } + ] +} +``` + +| Field | Required | Description | +| ------------- | -------- | --------------------------------------------------------------------------------------------------------------------------------------- | +| `action` | yes | `allow`, `block`, or `modify` (case-insensitive). | +| `findings` | no | Detected spans. `start` and `end` are Unicode code point offsets into `text`; `end` is exclusive. `type` and `score` are informational. | +| `replacement` | no | Full replacement for `text`, used with `modify`. | + +The response is mapped to the provider-neutral evaluation result: + +- `allow`: the rule does not match. Findings are ignored. +- `block`: the rule matches. Findings, if present, are masked with `maskReplacement` so the result can also drive a `Mask` rule. +- `modify`: the rule matches. `replacement` is used when present; otherwise findings are masked with `maskReplacement`. A `modify` response with neither is an error. + +The service reports whether content violates its policy; the rule's `action` remains authoritative for enforcement. A `block` or `modify` result is therefore blocked by `Block`, recorded by `Monitor`, and rewritten by `Mask`. A `Mask` rule returns an error when the service matches content without providing a replacement or valid findings, so unmasked content is never forwarded. Invalid spans are ignored and overlapping spans are merged before masking. + +Non-2xx responses, malformed JSON, missing or unknown actions, and timeouts are provider errors governed by `failureMode`. Provider error bodies are truncated before they are logged. Findings, scores, and replacement text are never returned to downstream clients or recorded in metrics. + +Native integrations remain preferable when the provider exposes richer semantics, such as Bedrock transformed output or Azure check selection. Services can also implement this contract as a thin adapter in front of an existing provider. + +## Payload Extraction + +Provider callouts receive relevant user or assistant text, not an arbitrary serialized JSON envelope. Schema-aware extraction reduces false matches, removes model/configuration metadata, lowers provider cost, and avoids provider input-size limits. + +The implementation walks known LLM text and container fields in deterministic path order and records each JSON path so Mask can write transformed text back into the same field. It recognizes fields such as `messages`, `content`, `text`, `input`, `prompt`, `instructions`, `system`, `choices`, and `output` while ignoring unrelated values such as model names and image URLs. + +A future richer provider-neutral input may add structured fields such as: + +```go +type EvaluationInput struct { + Phase GuardrailPhase + Text []string + Documents []string + Tools []Tool + Messages []Message +} +``` + +External providers and Mask rules consume extracted fragments. Regex Block and Monitor preserve raw-body matching for backward compatibility. Unknown or malformed structured formats produce an evaluation error governed by `failureMode`; they are not silently interpreted as a known chat schema. + +## Credentials and Security + +Provider credentials are referenced from Kubernetes Secrets and resolved into the ext-proc configuration using the repository's existing bundle mechanism. The controller watches referenced Secrets so rotation or deletion triggers policy and route reconciliation. + +Security requirements are: + +- never include credentials in `GuardrailPolicy` status, logs, metrics, traces, examples, or error responses; +- never include evaluated payloads or matched text in default logs, metrics, or traces; +- restrict cross-namespace Secret references unless `ReferenceGrant` support is explicitly designed; +- preserve Secret file permissions in local tests; +- use workload identity instead of static AWS credentials where possible; and +- bound provider error bodies before logging or returning errors. + +A fail-closed policy whose credentials become unavailable must not leave a stale accepted configuration active. A fail-open policy may omit the unavailable evaluator while recording the configuration failure. + +## Status and Reconciliation + +The controller sets `Accepted` only when: + +- all target backends exist; +- all provider configuration is valid; +- regular expressions compile; and +- all required Secrets and keys are present. + +Backend changes, Secret changes, policy updates, and policy deletion trigger reconciliation. Deletion uses the finalizer to notify affected routes before the policy disappears, ensuring stale rules are removed from generated configuration. + +Status does not perform a live provider health check. Provider availability is a runtime concern represented by metrics, traces, logs, and failure mode. + +## Observability + +The runtime emits: + +- structured logs for blocks and provider failures without payload content; +- `aigateway.guardrail.evaluation.count` with phase and result attributes; and +- `guardrail.evaluation` span events with rule name, phase, and result. + +Metric attributes intentionally exclude rule names and error text to avoid unbounded cardinality. Result values are `allowed`, `blocked`, `monitored`, `masked`, and `error`. Provider type may be added because it is bounded, subject to consistency with the project's metric conventions. + +The design should also expose provider-call latency in a future histogram so operators can measure the cost of each external integration. + +## Performance and Reliability + +External rules add network latency and may add provider cost. Rules currently execute sequentially and short-circuit on block. Sequential execution preserves deterministic ordering but makes worst-case latency the sum of rule timeouts. + +The policy defines separate `maxRequestBodyBytes` and `maxResponseBodyBytes` values. Both default to 10 MiB and are capped at 50 MiB. The cap aligns with the configured Envoy per-connection buffer limit. These are evaluation limits rather than streaming truncation: fail-closed rules reject oversized payloads, while fail-open and Monitor rules record the error and continue. + +Before increasing rule limits or adding many external checks, the implementation should further define: + +- a total guardrail evaluation budget per phase; +- whether independent monitor-only rules can run concurrently; +- provider rate-limit handling. + +No automatic retries are proposed initially. Connection reuse is delegated to `http.Client`. Provider contexts are canceled with the ext-proc request. + +## Alternatives Considered + +### Use Built on Envoy Dynamic Modules Directly + +This provides mature provider-specific behavior close to Envoy and avoids ext-proc RPC overhead. It also provides Azure semantic parsing and Bedrock mask behavior today. + +It is not selected for the initial implementation because it would introduce: + +- a second extension deployment and release lifecycle; +- Envoy dynamic-module ABI/version compatibility constraints; +- separate configuration and credential models; +- duplicated endpoint parsing and observability; and +- additional integration work for controller-managed backend scoping. + +The extensions remain valuable prior art, and sharing provider-neutral parsing or client libraries upstream may be preferable to independently maintaining equivalent logic. + +### Configure Guardrails Directly on AIGatewayRoute + +This makes route-level differences easy but duplicates policy across routes and couples safety configuration to routing. Backend attachment better represents a requirement that should follow a provider endpoint wherever it is referenced. + +### Use BackendSecurityPolicy + +`BackendSecurityPolicy` controls upstream authentication. Mixing content evaluation into it would combine unrelated ownership and status semantics. + +### Use Only Provider-Native Model Guardrails + +Provider-native model configuration does not cover cross-provider policy, request checks independent of inference, local Presidio deployments, or consistent gateway observability. + +### Invoke Local Executables for Custom Guardrails + +Running a user-supplied binary or script inside the ext-proc for each evaluation would allow arbitrary custom logic. It is rejected because it expands the ext-proc attack surface, requires distributing binaries into gateway pods, couples guardrail code to the gateway image and platform, and makes resource isolation, timeouts, and upgrades harder to control. The HTTP contract provides the same extensibility while keeping custom logic in an independently deployed and secured service. + +## Implementation Plan + +1. Introduce the dual-version `GuardrailPolicy`, generated clients, CRD, and status. +2. Add target and Secret indexes, reconciliation, deletion propagation, and filter-config translation. +3. Add runtime compilation and backend-scoped request/response evaluation. +4. Add Regex, Presidio, Bedrock, Azure Text Analysis, and Model Armor evaluators. +5. Add failure modes, timeouts, logs, metrics, traces, and guarded-response buffering. +6. Add CRD admission, controller, HTTP-stub, Testcontainers, live-provider, and dataplane tests. +7. Add semantic payload extraction and schema-aware Mask mutation. +8. Add Monitor mode and deterministic multi-policy composition. +9. Add explicit request and response evaluation limits. +10. Add the generic HTTP provider and its normalized request/response contract. +11. Add provider-specific Azure check selection. +12. Evaluate Bedrock API-key authentication. + +Steps 1 through 10 describe the current implementation. Steps 11 and 12 are proposed follow-up work informed by the Built on Envoy extensions. + +## Testing Strategy + +- CRD admission tests for valid and invalid provider unions, endpoints, actions, limits, and unique rule names. +- Controller tests for target validation, Secret resolution and rotation, status, backend scoping, failure modes, and deletion. +- Runtime tests for regex compilation and evaluator construction. +- HTTP-stub tests for provider paths, payloads, headers, authentication, responses, malformed responses, and errors. +- HTTP-stub tests for the custom HTTP contract: stages, paths, bearer authentication, each action, replacement and finding masking, and invalid responses. +- Testcontainers integration with the pinned official Presidio analyzer image. +- Credential-gated live tests for Azure AI Content Safety, AWS Bedrock Guardrails, and Google Cloud Model Armor. +- Envoy dataplane tests for request blocking, response blocking, allowed traffic, and streaming mode behavior. +- Staging tests for policy creation, update, Secret rotation, deletion, and multi-policy attachment. +- Load tests for provider latency, concurrency, and buffered response limits. + +## Current Implementation and Gaps + +The current implementation includes the policy resource, backend scoping, lifecycle reconciliation, external adapters, blocking, failure modes, timeout, response buffering, observability, and layered tests. + +The following gaps remain intentionally visible for design review: + +- Azure currently uses Text Analysis and does not yet expose Prompt Shield, Task Adherence, Protected Material Detection, categories, or per-category thresholds; +- schema extraction is field-oriented and does not yet expose structured tools/documents to providers; +- Mask is unsupported for Azure Text Analysis and for non-JSON payloads; +- there is no total per-phase timeout budget or provider latency histogram; and +- load testing is still needed to validate the selected 10 MiB default under production concurrency. + +These gaps do not require changing the core policy-to-runtime architecture, but some require API additions before the feature is declared complete. + +## References + +[Built on Envoy Bedrock Guardrails]: https://builtonenvoy.io/extensions/bedrock-guardrails/ +[Built on Envoy Azure Content Safety]: https://builtonenvoy.io/extensions/azure-content-safety/ +[Built on Envoy repository]: https://github.com/tetratelabs/built-on-envoy +[AWS Bedrock ApplyGuardrail]: https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ApplyGuardrail.html +[Azure AI Content Safety]: https://learn.microsoft.com/azure/ai-services/content-safety/ +[Google Cloud Model Armor]: https://docs.cloud.google.com/model-armor/overview +[Presidio]: https://presidio.dataprivacystack.org/ diff --git a/examples/README.md b/examples/README.md index 5aa788de8f..5e5637d6a7 100644 --- a/examples/README.md +++ b/examples/README.md @@ -40,3 +40,7 @@ Shows how to use prompt caching with Claude models to reduce costs and improve p ### [Monitoring](./monitoring/) Example setup for comprehensive monitoring and observability with Prometheus and Grafana. + +### [Guardrails](./guardrails/) + +Shows how to enforce request and response content policies with regex, Presidio, AWS Bedrock Guardrails, Azure AI Content Safety, and custom HTTP guardrail services. diff --git a/examples/guardrails/README.md b/examples/guardrails/README.md new file mode 100644 index 0000000000..6db9a817a5 --- /dev/null +++ b/examples/guardrails/README.md @@ -0,0 +1,28 @@ +# Guardrails + +These examples attach content-safety rules to an existing `AIServiceBackend` named `openai`. +Update `spec.targetRefs[].name` when your backend has a different name. + +Apply the deterministic regex example: + +```shell +kubectl apply -f regex.yaml +``` + +External providers require their service endpoint and, where applicable, a Kubernetes Secret: + +```shell +kubectl apply -f providers.yaml +``` + +The expected Secret keys are: + +- Presidio: optional `apiKey` +- Azure Content Safety: required `apiKey` +- AWS Bedrock Guardrails: optional `credentials` containing an AWS shared credentials file; when omitted, the ext-proc uses the standard AWS credential chain. +- Custom HTTP guardrail: optional `apiKey`, sent as a bearer token. +- Google Cloud Model Armor: optional `credentials` containing a service account key JSON; when omitted, the ext-proc uses Google Application Default Credentials, such as GKE Workload Identity. + +The `HTTP` provider calls any service that implements the generic guardrail contract: the gateway sends `{"text": "...", "context": {"stage": "input"}}` to `POST /analyze` and expects `{"action": "allow" | "block" | "modify", "findings": [...], "replacement": "..."}`. See the [guardrails documentation](../../site/docs/capabilities/security/guardrails.md#custom-http-guardrails) for the full contract. + +Provider failures block traffic by default. Set `failureMode: FailOpen` on a rule to continue traffic when the external provider is unavailable. Response guardrails force buffered response processing so blocked content is not partially delivered. diff --git a/examples/guardrails/providers.yaml b/examples/guardrails/providers.yaml new file mode 100644 index 0000000000..dfbcecdc9e --- /dev/null +++ b/examples/guardrails/providers.yaml @@ -0,0 +1,90 @@ +# Copyright Envoy AI Gateway Authors +# SPDX-License-Identifier: Apache-2.0 +# The full text of the Apache license is available in the LICENSE file at +# the root of the repo. + +apiVersion: v1 +kind: Secret +metadata: + name: guardrail-provider-keys +type: Opaque +stringData: + apiKey: replace-me +--- +apiVersion: v1 +kind: Secret +metadata: + name: custom-guardrail-key +type: Opaque +stringData: + apiKey: replace-me +--- +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: GuardrailPolicy +metadata: + name: external-content-guardrails +spec: + targetRefs: + - group: aigateway.envoyproxy.io + kind: AIServiceBackend + name: openai + rules: + - name: detect-pii-with-presidio + phase: Request + provider: + type: Presidio + timeoutSeconds: 5 + failureMode: FailClosed + message: Request contains personally identifiable information + presidio: + endpoint: http://presidio-analyzer.presidio.svc.cluster.local:3000 + language: en + scoreThresholdPercent: 70 + - name: apply-bedrock-guardrail + phase: Response + provider: + type: Bedrock + timeoutSeconds: 10 + failureMode: FailClosed + message: Response was rejected by the configured Bedrock guardrail + bedrock: + region: us-east-1 + guardrailIdentifier: replace-me + guardrailVersion: "1" + - name: analyze-with-azure-content-safety + phase: Response + provider: + type: AzureContentSafety + timeoutSeconds: 10 + failureMode: FailOpen + message: Response violates the configured content-safety threshold + azureContentSafety: + endpoint: https://replace-me.cognitiveservices.azure.com + apiVersion: "2024-09-01" + severityThreshold: 4 + apiKeySecretRef: + name: guardrail-provider-keys + - name: mask-with-custom-http-guardrail + phase: Request + provider: + type: HTTP + action: Mask + maskReplacement: "[REDACTED]" + timeoutSeconds: 5 + failureMode: FailClosed + http: + endpoint: http://custom-guardrail.guardrails.svc.cluster.local:8080 + path: /analyze + apiKeySecretRef: + name: custom-guardrail-key + - name: screen-with-model-armor + phase: Request + provider: + type: ModelArmor + timeoutSeconds: 10 + failureMode: FailClosed + message: Request was rejected by the configured Model Armor template + modelArmor: + project: replace-me + location: us-central1 + template: replace-me diff --git a/examples/guardrails/regex.yaml b/examples/guardrails/regex.yaml new file mode 100644 index 0000000000..99877acc23 --- /dev/null +++ b/examples/guardrails/regex.yaml @@ -0,0 +1,38 @@ +# Copyright Envoy AI Gateway Authors +# SPDX-License-Identifier: Apache-2.0 +# The full text of the Apache license is available in the LICENSE file at +# the root of the repo. + +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: GuardrailPolicy +metadata: + name: local-content-guardrails +spec: + maxRequestBodyBytes: 10485760 + maxResponseBodyBytes: 10485760 + targetRefs: + - group: aigateway.envoyproxy.io + kind: AIServiceBackend + name: openai + rules: + - name: block-sensitive-request + phase: Request + provider: + type: Regex + pattern: "(?i)password|social security number" + action: Block + message: Request contains sensitive content + - name: block-sensitive-response + phase: Response + provider: + type: Regex + pattern: "(?i)internal[- ]only" + action: Block + message: Response contains restricted content + - name: mask-email-addresses + phase: Response + provider: + type: Regex + pattern: '[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}' + action: Mask + maskReplacement: "[EMAIL REDACTED]" diff --git a/internal/controller/controller.go b/internal/controller/controller.go index f882ee40b4..e9b82c4979 100644 --- a/internal/controller/controller.go +++ b/internal/controller/controller.go @@ -264,6 +264,14 @@ func StartControllers(ctx context.Context, mgr manager.Manager, config *rest.Con } } + guardrailPolicyC := NewGuardrailPolicyController(c, kube, logger.WithName("guardrail-policy"), aiGatewayRouteEventChan) + if err = TypedControllerBuilderForCRD(mgr, &aigv1b1.GuardrailPolicy{}). + Watches(&aigv1b1.AIServiceBackend{}, handler.EnqueueRequestsFromMapFunc(guardrailPolicyC.BackendToGuardrailPolicy)). + Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(guardrailPolicyC.SecretToGuardrailPolicy)). + Complete(guardrailPolicyC); err != nil { + return fmt.Errorf("failed to create controller for GuardrailPolicy: %w", err) + } + // ReferenceGrant changes can authorize or revoke AIGatewayRoute, // BackendSecurityPolicy, and QuotaPolicy references. referenceGrantC := NewReferenceGrantController(c, logger.WithName("reference-grant"), aiGatewayRouteEventChan, backendSecurityPolicyEventChan, quotaPolicyEventChan) @@ -319,6 +327,11 @@ const ( // k8sClientIndexAIServiceBackendToTargetingQuotaPolicy is the index name that maps from an AIServiceBackend // to the QuotaPolicy whose targetRefs contains the AIServiceBackend. k8sClientIndexAIServiceBackendToTargetingQuotaPolicy = "AIServiceBackendToTargetingQuotaPolicy" + // k8sClientIndexAIServiceBackendToTargetingGuardrailPolicy is the index name that maps from an AIServiceBackend + // to the GuardrailPolicy whose targetRefs contains the AIServiceBackend. + k8sClientIndexAIServiceBackendToTargetingGuardrailPolicy = "AIServiceBackendToTargetingGuardrailPolicy" + // k8sClientIndexSecretToReferencingGuardrailPolicy maps a Secret to GuardrailPolicies using it. + k8sClientIndexSecretToReferencingGuardrailPolicy = "SecretToReferencingGuardrailPolicy" // k8sClientIndexQuotaPolicyTargetNamespace maps a target namespace to // QuotaPolicies that contain a targetRef resolved to that namespace. k8sClientIndexQuotaPolicyTargetNamespace = "QuotaPolicyTargetNamespace" @@ -376,6 +389,17 @@ func ApplyIndexing(ctx context.Context, indexer func(ctx context.Context, obj cl return fmt.Errorf("failed to index QuotaPolicy target namespaces: %w", err) } + err = indexer(ctx, &aigv1b1.GuardrailPolicy{}, + k8sClientIndexAIServiceBackendToTargetingGuardrailPolicy, guardrailPolicyTargetRefsIndexFunc) + if err != nil { + return fmt.Errorf("failed to index field for GuardrailPolicy targetRefs: %w", err) + } + err = indexer(ctx, &aigv1b1.GuardrailPolicy{}, + k8sClientIndexSecretToReferencingGuardrailPolicy, guardrailPolicySecretRefsIndexFunc) + if err != nil { + return fmt.Errorf("failed to index field for GuardrailPolicy secretRefs: %w", err) + } + err = indexer(ctx, &gwapiv1.Gateway{}, k8sClientIndexGatewayToGatewayConfig, gatewayToGatewayConfigIndexFunc) if err != nil { @@ -571,6 +595,54 @@ func quotaPolicyTargetRefsIndexFunc(o client.Object) []string { return ret } +func guardrailPolicyTargetRefsIndexFunc(o client.Object) []string { + guardrailPolicy := o.(*aigv1b1.GuardrailPolicy) + var ret []string + for _, targetRef := range guardrailPolicy.Spec.TargetRefs { + ret = append(ret, fmt.Sprintf("%s.%s", targetRef.Name, guardrailPolicy.Namespace)) + } + return ret +} + +func guardrailPolicySecretRefsIndexFunc(o client.Object) []string { + policy := o.(*aigv1b1.GuardrailPolicy) + var keys []string + for i := range policy.Spec.Rules { + provider := &policy.Spec.Rules[i].Provider + var ref *gwapiv1.SecretObjectReference + switch provider.Type { + case aigv1b1.GuardrailProviderTypePresidio: + if provider.Presidio != nil { + ref = provider.Presidio.APIKeySecretRef + } + case aigv1b1.GuardrailProviderTypeBedrockGuardrails: + if provider.Bedrock != nil { + ref = provider.Bedrock.CredentialsSecretRef + } + case aigv1b1.GuardrailProviderTypeAzureContentSafety: + if provider.AzureContentSafety != nil { + ref = provider.AzureContentSafety.APIKeySecretRef + } + case aigv1b1.GuardrailProviderTypeHTTP: + if provider.HTTP != nil { + ref = provider.HTTP.APIKeySecretRef + } + case aigv1b1.GuardrailProviderTypeModelArmor: + if provider.ModelArmor != nil { + ref = provider.ModelArmor.CredentialsSecretRef + } + } + if ref != nil { + namespace := policy.Namespace + if ref.Namespace != nil { + namespace = string(*ref.Namespace) + } + keys = append(keys, fmt.Sprintf("%s.%s", ref.Name, namespace)) + } + } + return keys +} + func quotaPolicyTargetNamespaceIndexFunc(o client.Object) []string { quotaPolicy := o.(*aigv1a1.QuotaPolicy) namespaces := make([]string, 0, len(quotaPolicy.Spec.TargetRefs)) diff --git a/internal/controller/gateway.go b/internal/controller/gateway.go index df23e93a2e..4c7a31a319 100644 --- a/internal/controller/gateway.go +++ b/internal/controller/gateway.go @@ -438,6 +438,7 @@ func (c *GatewayController) reconcileFilterConfigSecret( routeBackendNamesSet := map[string]struct{}{} routeBackendNames := []string{} injectedQuotaCosts := make(map[string]struct{}) + injectedGuardrailKeys := make(map[string]struct{}) for ruleIndex := range spec.Rules { rule := &spec.Rules[ruleIndex] if rule.ExcludeFromModelsEndpoint { @@ -594,6 +595,9 @@ func (c *GatewayController) reconcileFilterConfigSecret( // Inject QuotaPolicy cost expressions as LLMRequestCost entries so ext_proc // computes and stores them in metadata for the HitsAddend to read. c.injectQuotaPolicyCostExpressions(ctx, ec, injectedQuotaCosts, routeName) + if guardrailErr := c.injectGuardrails(ctx, aiGatewayRoute, ec, injectedGuardrailKeys); guardrailErr != nil { + return false, fmt.Errorf("failed to inject guardrails for route %s: %w", aiGatewayRoute.Name, guardrailErr) + } for _, fc := range dedup { ec.LLMRequestCosts = append(ec.LLMRequestCosts, fc) @@ -630,6 +634,230 @@ func (c *GatewayController) reconcileFilterConfigSecret( return hasEffectiveRoute, nil } +func (c *GatewayController) injectGuardrails( + ctx context.Context, + route *aigv1b1.AIGatewayRoute, + ec *filterapi.Config, + injected map[string]struct{}, +) error { + var policies aigv1b1.GuardrailPolicyList + if err := c.client.List(ctx, &policies, client.InNamespace(route.Namespace)); err != nil { + return fmt.Errorf("failed to list GuardrailPolicies: %w", err) + } + sort.Slice(policies.Items, func(i, j int) bool { + if policies.Items[i].Namespace != policies.Items[j].Namespace { + return policies.Items[i].Namespace < policies.Items[j].Namespace + } + return policies.Items[i].Name < policies.Items[j].Name + }) + + routeBackends := make(map[string]struct{}) + for _, routeRule := range route.Spec.Rules { + for _, backendRef := range routeRule.BackendRefs { + backendNamespace := backendRef.GetNamespace(route.Namespace) + routeBackends[backendNamespace+"/"+backendRef.Name] = struct{}{} + } + } + + for i := range policies.Items { + policy := &policies.Items[i] + targetsRoute := false + for _, targetRef := range policy.Spec.TargetRefs { + if _, ok := routeBackends[policy.Namespace+"/"+string(targetRef.Name)]; ok { + targetsRoute = true + break + } + } + if !targetsRoute { + continue + } + + for ruleIndex := range policy.Spec.Rules { + rule := &policy.Spec.Rules[ruleIndex] + key := policy.Namespace + "/" + policy.Name + "/" + rule.Name + if _, ok := injected[key]; ok { + continue + } + backends := guardrailBackendNames(route, policy) + provider, err := c.guardrailProviderToFilterAPI(ctx, policy.Namespace, &rule.Provider) + if err != nil { + if rule.Provider.FailureMode == aigv1b1.GuardrailFailureModeFailOpen || rule.Provider.Action == aigv1b1.GuardrailActionMonitor { + c.logger.Error(err, "guardrail configuration failed open", "policy", policy.Name, "rule", rule.Name) + injected[key] = struct{}{} + continue + } + c.logger.Error(err, "guardrail configuration failed closed", "policy", policy.Name, "rule", rule.Name) + provider = filterapi.GuardrailProvider{ + Type: filterapi.GuardrailProviderTypeRegex, + Pattern: `(?s).*`, + Action: filterapi.GuardrailActionBlock, + Message: "request blocked because guardrail configuration is unavailable", + } + } + ec.Guardrails = append(ec.Guardrails, filterapi.Guardrail{ + Name: key, + Phase: filterapi.GuardrailPhase(rule.Phase), + Provider: provider, + Backends: backends, + MaxPayloadBytes: guardrailMaxPayloadBytes(policy, rule.Phase), + }) + injected[key] = struct{}{} + } + } + return nil +} + +const defaultGuardrailMaxPayloadBytes int64 = 10 * 1024 * 1024 + +func guardrailMaxPayloadBytes(policy *aigv1b1.GuardrailPolicy, phase aigv1b1.GuardrailPhase) int64 { + configured := policy.Spec.MaxRequestBodyBytes + if phase == aigv1b1.GuardrailPhaseResponse { + configured = policy.Spec.MaxResponseBodyBytes + } + if configured == nil { + return defaultGuardrailMaxPayloadBytes + } + return *configured +} + +func guardrailBackendNames(route *aigv1b1.AIGatewayRoute, policy *aigv1b1.GuardrailPolicy) []string { + targets := make(map[string]struct{}, len(policy.Spec.TargetRefs)) + for _, targetRef := range policy.Spec.TargetRefs { + targets[string(targetRef.Name)] = struct{}{} + } + var names []string + for ruleIndex := range route.Spec.Rules { + for backendRefIndex := range route.Spec.Rules[ruleIndex].BackendRefs { + backendRef := &route.Spec.Rules[ruleIndex].BackendRefs[backendRefIndex] + if backendRef.GetNamespace(route.Namespace) != policy.Namespace { + continue + } + if _, ok := targets[backendRef.Name]; !ok { + continue + } + names = append(names, internalapi.PerRouteRuleRefBackendName( + route.Namespace, backendRef.Name, route.Name, ruleIndex, backendRefIndex)) + } + } + return names +} + +func (c *GatewayController) guardrailProviderToFilterAPI(ctx context.Context, namespace string, provider *aigv1b1.GuardrailProvider) (filterapi.GuardrailProvider, error) { + converted := filterapi.GuardrailProvider{ + Type: filterapi.GuardrailProviderType(provider.Type), + Pattern: provider.Pattern, + Action: filterapi.GuardrailAction(provider.Action), + MaskReplacement: provider.MaskReplacement, + Message: provider.Message, + FailureMode: filterapi.GuardrailFailureMode(provider.FailureMode), + } + if provider.TimeoutSeconds != nil { + converted.TimeoutSeconds = *provider.TimeoutSeconds + } + switch provider.Type { + case aigv1b1.GuardrailProviderTypeRegex: + case aigv1b1.GuardrailProviderTypePresidio: + if provider.Presidio == nil { + return converted, fmt.Errorf("presidio configuration is required") + } + config := provider.Presidio + converted.Presidio = &filterapi.PresidioGuardrailProvider{ + Endpoint: config.Endpoint, + Language: config.Language, + } + if config.ScoreThresholdPercent != nil { + converted.Presidio.ScoreThresholdPercent = *config.ScoreThresholdPercent + } + if config.APIKeySecretRef != nil { + apiKey, err := c.getGuardrailSecretData(ctx, namespace, config.APIKeySecretRef, "apiKey") + if err != nil { + return converted, err + } + converted.Presidio.APIKey = apiKey + } + case aigv1b1.GuardrailProviderTypeBedrockGuardrails: + if provider.Bedrock == nil { + return converted, fmt.Errorf("bedrock configuration is required") + } + config := provider.Bedrock + converted.Bedrock = &filterapi.BedrockGuardrailProvider{ + Endpoint: config.Endpoint, + Region: config.Region, + GuardrailIdentifier: config.GuardrailIdentifier, + GuardrailVersion: config.GuardrailVersion, + } + if config.CredentialsSecretRef != nil { + credentials, err := c.getGuardrailSecretData(ctx, namespace, config.CredentialsSecretRef, "credentials") + if err != nil { + return converted, err + } + converted.Bedrock.CredentialFileLiteral = credentials + } + case aigv1b1.GuardrailProviderTypeAzureContentSafety: + if provider.AzureContentSafety == nil { + return converted, fmt.Errorf("azure Content Safety configuration is required") + } + config := provider.AzureContentSafety + apiKey, err := c.getGuardrailSecretData(ctx, namespace, config.APIKeySecretRef, "apiKey") + if err != nil { + return converted, err + } + converted.AzureContentSafety = &filterapi.AzureContentSafetyGuardrailProvider{ + Endpoint: config.Endpoint, + APIVersion: config.APIVersion, + SeverityThreshold: config.SeverityThreshold, + APIKey: apiKey, + } + case aigv1b1.GuardrailProviderTypeHTTP: + if provider.HTTP == nil { + return converted, fmt.Errorf("http guardrail configuration is required") + } + config := provider.HTTP + converted.HTTP = &filterapi.HTTPGuardrailProvider{ + Endpoint: config.Endpoint, + Path: config.Path, + } + if config.APIKeySecretRef != nil { + apiKey, err := c.getGuardrailSecretData(ctx, namespace, config.APIKeySecretRef, "apiKey") + if err != nil { + return converted, err + } + converted.HTTP.APIKey = apiKey + } + case aigv1b1.GuardrailProviderTypeModelArmor: + if provider.ModelArmor == nil { + return converted, fmt.Errorf("model Armor configuration is required") + } + config := provider.ModelArmor + converted.ModelArmor = &filterapi.ModelArmorGuardrailProvider{ + Endpoint: config.Endpoint, + Project: config.Project, + Location: config.Location, + Template: config.Template, + } + if config.CredentialsSecretRef != nil { + credentials, err := c.getGuardrailSecretData(ctx, namespace, config.CredentialsSecretRef, "credentials") + if err != nil { + return converted, err + } + converted.ModelArmor.CredentialsJSON = credentials + } + default: + return converted, fmt.Errorf("unsupported provider type %q", provider.Type) + } + return converted, nil +} + +func (c *GatewayController) getGuardrailSecretData(ctx context.Context, namespace string, ref *gwapiv1.SecretObjectReference, key string) (string, error) { + if ref == nil { + return "", fmt.Errorf("secret reference is required") + } + if ref.Namespace != nil && string(*ref.Namespace) != namespace { + return "", fmt.Errorf("cross-namespace guardrail secret references are not supported") + } + return c.getSecretData(ctx, namespace, string(ref.Name), key) +} + // reconcileFilterConfigSecretForMCPGateway updates the filter config secret for the external processor. func mcpConfig(mcpRoutes []aigv1b1.MCPRoute) (_ *filterapi.MCPConfig, hasEffectiveRoute bool) { if len(mcpRoutes) == 0 { diff --git a/internal/controller/guardrail_policy.go b/internal/controller/guardrail_policy.go new file mode 100644 index 0000000000..a5a088c8fd --- /dev/null +++ b/internal/controller/guardrail_policy.go @@ -0,0 +1,275 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package controller + +import ( + "context" + "fmt" + "net/url" + "regexp" + "strings" + + "github.com/go-logr/logr" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/util/retry" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" + + aigv1b1 "github.com/envoyproxy/ai-gateway/api/v1beta1" +) + +// GuardrailPolicyController implements [reconcile.TypedReconciler] for [aigv1b1.GuardrailPolicy]. +type GuardrailPolicyController struct { + client client.Client + kube kubernetes.Interface + logger logr.Logger + aiGatewayRouteChan chan event.GenericEvent +} + +// NewGuardrailPolicyController creates a new reconciler for GuardrailPolicy resources. +func NewGuardrailPolicyController(client client.Client, kube kubernetes.Interface, logger logr.Logger, aiGatewayRouteChan chan event.GenericEvent) *GuardrailPolicyController { + return &GuardrailPolicyController{ + client: client, + kube: kube, + logger: logger, + aiGatewayRouteChan: aiGatewayRouteChan, + } +} + +// Reconcile implements [reconcile.TypedReconciler] for [aigv1b1.GuardrailPolicy]. +func (c *GuardrailPolicyController) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { + var policy aigv1b1.GuardrailPolicy + if err := c.client.Get(ctx, req.NamespacedName, &policy); err != nil { + if client.IgnoreNotFound(err) == nil { + c.logger.Info("Deleting GuardrailPolicy", + "namespace", req.Namespace, "name", req.Name) + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + } + + c.logger.Info("Reconciling GuardrailPolicy", "namespace", req.Namespace, "name", req.Name) + if handleFinalizer(ctx, c.client, c.logger, &policy, func(ctx context.Context, policy *aigv1b1.GuardrailPolicy) error { + c.notifyAIGatewayRoutesForGuardrailPolicy(ctx, policy) + return nil + }) { + return ctrl.Result{}, nil + } + if err := c.syncGuardrailPolicy(ctx, &policy); err != nil { + c.logger.Error(err, "failed to sync GuardrailPolicy") + c.updateGuardrailPolicyStatus(ctx, &policy, aigv1b1.ConditionTypeNotAccepted, err.Error()) + c.notifyAIGatewayRoutesForGuardrailPolicy(ctx, &policy) + return ctrl.Result{}, err + } + + c.updateGuardrailPolicyStatus(ctx, &policy, aigv1b1.ConditionTypeAccepted, "GuardrailPolicy reconciled successfully") + c.notifyAIGatewayRoutesForGuardrailPolicy(ctx, &policy) + return ctrl.Result{}, nil +} + +// SecretToGuardrailPolicy maps Secret changes to GuardrailPolicy reconcile requests. +func (c *GuardrailPolicyController) SecretToGuardrailPolicy(ctx context.Context, obj client.Object) []reconcile.Request { + var policies aigv1b1.GuardrailPolicyList + key := fmt.Sprintf("%s.%s", obj.GetName(), obj.GetNamespace()) + if err := c.client.List(ctx, &policies, + client.MatchingFields{k8sClientIndexSecretToReferencingGuardrailPolicy: key}); err != nil { + c.logger.Error(err, "failed to list GuardrailPolicies for secret", "secret", key) + return nil + } + requests := make([]reconcile.Request, 0, len(policies.Items)) + for i := range policies.Items { + requests = append(requests, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&policies.Items[i])}) + } + return requests +} + +func (c *GuardrailPolicyController) syncGuardrailPolicy(ctx context.Context, policy *aigv1b1.GuardrailPolicy) error { + for i := range policy.Spec.Rules { + if err := c.validateGuardrailProvider(ctx, policy.Namespace, &policy.Spec.Rules[i].Provider); err != nil { + return fmt.Errorf("rule %q: %w", policy.Spec.Rules[i].Name, err) + } + } + + for _, ref := range policy.Spec.TargetRefs { + var backend aigv1b1.AIServiceBackend + key := client.ObjectKey{Namespace: policy.Namespace, Name: string(ref.Name)} + if err := c.client.Get(ctx, key, &backend); err != nil { + if apierrors.IsNotFound(err) { + return fmt.Errorf("target AIServiceBackend %s not found", key) + } + return fmt.Errorf("failed to get AIServiceBackend %s: %w", key, err) + } + } + + return nil +} + +func (c *GuardrailPolicyController) validateGuardrailProvider(ctx context.Context, namespace string, provider *aigv1b1.GuardrailProvider) error { + if provider.Action != "" && provider.Action != aigv1b1.GuardrailActionBlock && provider.Action != aigv1b1.GuardrailActionMonitor && provider.Action != aigv1b1.GuardrailActionMask { + return fmt.Errorf("unsupported action %q", provider.Action) + } + if provider.Action == aigv1b1.GuardrailActionMask && provider.Type == aigv1b1.GuardrailProviderTypeAzureContentSafety { + return fmt.Errorf("azure Content Safety does not support mask") + } + if provider.FailureMode != "" && provider.FailureMode != aigv1b1.GuardrailFailureModeFailClosed && provider.FailureMode != aigv1b1.GuardrailFailureModeFailOpen { + return fmt.Errorf("unsupported failureMode %q", provider.FailureMode) + } + switch provider.Type { + case aigv1b1.GuardrailProviderTypeRegex: + if provider.Pattern == "" { + return fmt.Errorf("regex pattern is required") + } + if _, err := regexp.Compile(provider.Pattern); err != nil { + return fmt.Errorf("invalid regex pattern: %w", err) + } + case aigv1b1.GuardrailProviderTypePresidio: + if provider.Presidio == nil { + return fmt.Errorf("presidio configuration is required") + } + if err := validateGuardrailEndpoint(provider.Presidio.Endpoint); err != nil { + return err + } + if provider.Presidio.APIKeySecretRef != nil { + return c.validateGuardrailSecret(ctx, namespace, provider.Presidio.APIKeySecretRef, "apiKey") + } + case aigv1b1.GuardrailProviderTypeBedrockGuardrails: + if provider.Bedrock == nil || provider.Bedrock.Region == "" || provider.Bedrock.GuardrailIdentifier == "" || provider.Bedrock.GuardrailVersion == "" { + return fmt.Errorf("bedrock region, guardrailIdentifier, and guardrailVersion are required") + } + if provider.Bedrock.Endpoint != "" { + if err := validateGuardrailEndpoint(provider.Bedrock.Endpoint); err != nil { + return err + } + } + if provider.Bedrock.CredentialsSecretRef != nil { + return c.validateGuardrailSecret(ctx, namespace, provider.Bedrock.CredentialsSecretRef, "credentials") + } + case aigv1b1.GuardrailProviderTypeAzureContentSafety: + if provider.AzureContentSafety == nil { + return fmt.Errorf("azure Content Safety configuration is required") + } + if err := validateGuardrailEndpoint(provider.AzureContentSafety.Endpoint); err != nil { + return err + } + return c.validateGuardrailSecret(ctx, namespace, provider.AzureContentSafety.APIKeySecretRef, "apiKey") + case aigv1b1.GuardrailProviderTypeHTTP: + if provider.HTTP == nil { + return fmt.Errorf("http guardrail configuration is required") + } + if err := validateGuardrailEndpoint(provider.HTTP.Endpoint); err != nil { + return err + } + if provider.HTTP.Path != "" && !strings.HasPrefix(provider.HTTP.Path, "/") { + return fmt.Errorf("http guardrail path must start with /") + } + if provider.HTTP.APIKeySecretRef != nil { + return c.validateGuardrailSecret(ctx, namespace, provider.HTTP.APIKeySecretRef, "apiKey") + } + case aigv1b1.GuardrailProviderTypeModelArmor: + if provider.ModelArmor == nil || provider.ModelArmor.Project == "" || provider.ModelArmor.Location == "" || provider.ModelArmor.Template == "" { + return fmt.Errorf("model Armor project, location, and template are required") + } + if provider.ModelArmor.Endpoint != "" { + if err := validateGuardrailEndpoint(provider.ModelArmor.Endpoint); err != nil { + return err + } + } + if provider.ModelArmor.CredentialsSecretRef != nil { + return c.validateGuardrailSecret(ctx, namespace, provider.ModelArmor.CredentialsSecretRef, "credentials") + } + default: + return fmt.Errorf("unsupported provider type %q", provider.Type) + } + return nil +} + +func validateGuardrailEndpoint(endpoint string) error { + parsed, err := url.ParseRequestURI(endpoint) + if err != nil || parsed.Scheme == "" || parsed.Host == "" { + return fmt.Errorf("valid provider endpoint is required") + } + return nil +} + +func (c *GuardrailPolicyController) validateGuardrailSecret(ctx context.Context, namespace string, ref *gwapiv1.SecretObjectReference, key string) error { + if ref == nil { + return fmt.Errorf("secret reference is required") + } + if ref.Namespace != nil && string(*ref.Namespace) != namespace { + return fmt.Errorf("cross-namespace guardrail secret references are not supported") + } + secret, err := c.kube.CoreV1().Secrets(namespace).Get(ctx, string(ref.Name), metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get secret %s: %w", ref.Name, err) + } + if _, ok := secret.Data[key]; !ok { + if _, ok = secret.StringData[key]; !ok { + return fmt.Errorf("secret %s does not contain key %s", ref.Name, key) + } + } + return nil +} + +// BackendToGuardrailPolicy maps AIServiceBackend changes to GuardrailPolicy reconcile requests. +func (c *GuardrailPolicyController) BackendToGuardrailPolicy(ctx context.Context, obj client.Object) []reconcile.Request { + var policies aigv1b1.GuardrailPolicyList + key := fmt.Sprintf("%s.%s", obj.GetName(), obj.GetNamespace()) + if err := c.client.List(ctx, &policies, + client.MatchingFields{k8sClientIndexAIServiceBackendToTargetingGuardrailPolicy: key}); err != nil { + c.logger.Error(err, "failed to list GuardrailPolicies for backend", "backend", key) + return nil + } + + var requests []reconcile.Request + for i := range policies.Items { + policy := &policies.Items[i] + requests = append(requests, reconcile.Request{ + NamespacedName: client.ObjectKeyFromObject(policy), + }) + } + return requests +} + +func (c *GuardrailPolicyController) notifyAIGatewayRoutesForGuardrailPolicy(ctx context.Context, policy *aigv1b1.GuardrailPolicy) { + for _, ref := range policy.Spec.TargetRefs { + key := fmt.Sprintf("%s.%s", ref.Name, policy.Namespace) + var routes aigv1b1.AIGatewayRouteList + if err := c.client.List(ctx, &routes, + client.MatchingFields{k8sClientIndexBackendToReferencingAIGatewayRoute: key}); err != nil { + c.logger.Error(err, "failed to list AIGatewayRoutes for guardrail policy", "backend", key) + continue + } + for i := range routes.Items { + route := &routes.Items[i] + c.logger.Info("notifying AIGatewayRoute of GuardrailPolicy change", + "route", route.Name, "namespace", route.Namespace, + "guardrailPolicy", policy.Name) + c.aiGatewayRouteChan <- event.GenericEvent{Object: route} + } + } +} + +func (c *GuardrailPolicyController) updateGuardrailPolicyStatus(ctx context.Context, policy *aigv1b1.GuardrailPolicy, conditionType string, message string) { + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + if err := c.client.Get(ctx, client.ObjectKey{Name: policy.Name, Namespace: policy.Namespace}, policy); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return err + } + policy.Status.Conditions = newConditions(conditionType, message) + return c.client.Status().Update(ctx, policy) + }) + if err != nil { + c.logger.Error(err, "failed to update GuardrailPolicy status", + "namespace", policy.Namespace, "name", policy.Name) + } +} diff --git a/internal/controller/guardrail_policy_test.go b/internal/controller/guardrail_policy_test.go new file mode 100644 index 0000000000..86297eb595 --- /dev/null +++ b/internal/controller/guardrail_policy_test.go @@ -0,0 +1,1078 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package controller + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + fakekube "k8s.io/client-go/kubernetes/fake" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" + gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + + aigv1b1 "github.com/envoyproxy/ai-gateway/api/v1beta1" + "github.com/envoyproxy/ai-gateway/internal/filterapi" + "github.com/envoyproxy/ai-gateway/internal/guardrails" +) + +func newGuardrailPolicyTestClient(t *testing.T) client.Client { + t.Helper() + builder := fake.NewClientBuilder().WithScheme(Scheme). + WithStatusSubresource(&aigv1b1.GuardrailPolicy{}) + require.NoError(t, ApplyIndexing(t.Context(), func(_ context.Context, obj client.Object, field string, extractValue client.IndexerFunc) error { + builder = builder.WithIndex(obj, field, extractValue) + return nil + })) + return builder.Build() +} + +func TestGuardrailPolicyControllerReconcileProviderReadiness(t *testing.T) { + const namespace = "default" + tests := []struct { + name string + withSecret bool + wantCondition string + wantError bool + }{ + {name: "accepted with provider secret", withSecret: true, wantCondition: aigv1b1.ConditionTypeAccepted}, + {name: "not accepted without provider secret", wantCondition: aigv1b1.ConditionTypeNotAccepted, wantError: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + kube := fakekube.NewClientset() + if test.withSecret { + _, err := kube.CoreV1().Secrets(namespace).Create(t.Context(), &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "azure-key", Namespace: namespace}, + Data: map[string][]byte{"apiKey": []byte("secret")}, + }, metav1.CreateOptions{}) + require.NoError(t, err) + } + controllerClient := newGuardrailPolicyTestClient(t) + require.NoError(t, controllerClient.Create(t.Context(), &aigv1b1.AIServiceBackend{ + ObjectMeta: metav1.ObjectMeta{Name: "backend", Namespace: namespace}, + })) + policy := &aigv1b1.GuardrailPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: "policy", Namespace: namespace}, + Spec: aigv1b1.GuardrailPolicySpec{ + TargetRefs: []gwapiv1a2.LocalPolicyTargetReference{{ + Group: "aigateway.envoyproxy.io", Kind: "AIServiceBackend", Name: "backend", + }}, + Rules: []aigv1b1.GuardrailRule{{ + Name: "azure", Phase: aigv1b1.GuardrailPhaseRequest, + Provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeAzureContentSafety, + AzureContentSafety: &aigv1b1.AzureContentSafetyGuardrailProvider{ + Endpoint: "https://content-safety.example.com", + APIKeySecretRef: &gwapiv1.SecretObjectReference{Name: "azure-key"}, + }, + }, + }}, + }, + } + require.NoError(t, controllerClient.Create(t.Context(), policy)) + controller := NewGuardrailPolicyController(controllerClient, kube, ctrl.Log, make(chan event.GenericEvent, 1)) + _, err := controller.Reconcile(t.Context(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: namespace, Name: policy.Name}}) + if test.wantError { + require.Error(t, err) + } else { + require.NoError(t, err) + } + + var updated aigv1b1.GuardrailPolicy + require.NoError(t, controllerClient.Get(t.Context(), client.ObjectKeyFromObject(policy), &updated)) + require.Len(t, updated.Status.Conditions, 1) + require.Equal(t, test.wantCondition, updated.Status.Conditions[0].Type) + }) + } +} + +func TestGuardrailPolicyControllerDeletionNotifiesRoutes(t *testing.T) { + const namespace = "default" + controllerClient := newGuardrailPolicyTestClient(t) + route := &aigv1b1.AIGatewayRoute{ + ObjectMeta: metav1.ObjectMeta{Name: "route", Namespace: namespace}, + Spec: aigv1b1.AIGatewayRouteSpec{Rules: []aigv1b1.AIGatewayRouteRule{{ + BackendRefs: []aigv1b1.AIGatewayRouteRuleBackendRef{{Name: "backend"}}, + }}}, + } + require.NoError(t, controllerClient.Create(t.Context(), route)) + require.NoError(t, controllerClient.Create(t.Context(), &aigv1b1.AIServiceBackend{ + ObjectMeta: metav1.ObjectMeta{Name: "backend", Namespace: namespace}, + })) + policy := &aigv1b1.GuardrailPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "policy", + Namespace: namespace, + }, + Spec: aigv1b1.GuardrailPolicySpec{ + TargetRefs: []gwapiv1a2.LocalPolicyTargetReference{{Name: "backend"}}, + Rules: []aigv1b1.GuardrailRule{{ + Name: "deny", Phase: aigv1b1.GuardrailPhaseRequest, + Provider: aigv1b1.GuardrailProvider{Type: aigv1b1.GuardrailProviderTypeRegex, Pattern: "secret"}, + }}, + }, + } + require.NoError(t, controllerClient.Create(t.Context(), policy)) + routeEvents := make(chan event.GenericEvent, 1) + controller := NewGuardrailPolicyController(controllerClient, fakekube.NewClientset(), ctrl.Log, routeEvents) + request := reconcile.Request{NamespacedName: client.ObjectKeyFromObject(policy)} + _, err := controller.Reconcile(t.Context(), request) + require.NoError(t, err) + <-routeEvents + require.NoError(t, controllerClient.Delete(t.Context(), policy)) + + _, err = controller.Reconcile(t.Context(), request) + require.NoError(t, err) + select { + case got := <-routeEvents: + require.Equal(t, route.Name, got.Object.GetName()) + require.Equal(t, route.Namespace, got.Object.GetNamespace()) + default: + t.Fatal("expected route notification when GuardrailPolicy is deleted") + } + + var updated aigv1b1.GuardrailPolicy + err = controllerClient.Get(t.Context(), client.ObjectKeyFromObject(policy), &updated) + if err == nil { + require.NotContains(t, updated.Finalizers, aiGatewayControllerFinalizer) + } else { + require.True(t, apierrors.IsNotFound(err), "unexpected get error: %v", err) + } +} + +func TestGuardrailProviderToFilterAPIResolvesSecret(t *testing.T) { + const namespace = "default" + kube := fakekube.NewClientset(&corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "presidio-key", Namespace: namespace}, + Data: map[string][]byte{"apiKey": []byte("secret-value")}, + }) + controller := &GatewayController{kube: kube} + provider, err := controller.guardrailProviderToFilterAPI(t.Context(), namespace, &aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypePresidio, + Presidio: &aigv1b1.PresidioGuardrailProvider{ + Endpoint: "https://presidio.example.com", + APIKeySecretRef: &gwapiv1.SecretObjectReference{Name: "presidio-key"}, + }, + }) + require.NoError(t, err) + require.Equal(t, "secret-value", provider.Presidio.APIKey) +} + +func TestInjectGuardrailsTargetsRouteBackends(t *testing.T) { + const namespace = "default" + controllerClient := newGuardrailPolicyTestClient(t) + for _, policy := range []*aigv1b1.GuardrailPolicy{ + { + ObjectMeta: metav1.ObjectMeta{Name: "selected", Namespace: namespace}, + Spec: aigv1b1.GuardrailPolicySpec{ + TargetRefs: []gwapiv1a2.LocalPolicyTargetReference{{Name: "backend"}}, + Rules: []aigv1b1.GuardrailRule{{ + Name: "deny", Phase: aigv1b1.GuardrailPhaseRequest, + Provider: aigv1b1.GuardrailProvider{Type: aigv1b1.GuardrailProviderTypeRegex, Pattern: "secret"}, + }}, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "ignored", Namespace: namespace}, + Spec: aigv1b1.GuardrailPolicySpec{ + TargetRefs: []gwapiv1a2.LocalPolicyTargetReference{{Name: "other-backend"}}, + Rules: []aigv1b1.GuardrailRule{{ + Name: "ignored", Phase: aigv1b1.GuardrailPhaseRequest, + Provider: aigv1b1.GuardrailProvider{Type: aigv1b1.GuardrailProviderTypeRegex, Pattern: "ignored"}, + }}, + }, + }, + } { + require.NoError(t, controllerClient.Create(t.Context(), policy)) + } + + controller := &GatewayController{client: controllerClient, kube: fakekube.NewClientset(), logger: ctrl.Log} + config := &filterapi.Config{} + route := &aigv1b1.AIGatewayRoute{ + ObjectMeta: metav1.ObjectMeta{Name: "route", Namespace: namespace}, + Spec: aigv1b1.AIGatewayRouteSpec{Rules: []aigv1b1.AIGatewayRouteRule{{ + BackendRefs: []aigv1b1.AIGatewayRouteRuleBackendRef{{Name: "backend"}}, + }}}, + } + require.NoError(t, controller.injectGuardrails(t.Context(), route, config, map[string]struct{}{})) + require.Len(t, config.Guardrails, 1) + require.Equal(t, "default/selected/deny", config.Guardrails[0].Name) + require.Equal(t, "secret", config.Guardrails[0].Provider.Pattern) + require.Equal(t, []string{"default/backend/route/route/rule/0/ref/0"}, config.Guardrails[0].Backends) +} + +func TestInjectGuardrailsUsesDeterministicPolicyOrder(t *testing.T) { + const namespace = "default" + controllerClient := newGuardrailPolicyTestClient(t) + for _, name := range []string{"zeta", "alpha"} { + require.NoError(t, controllerClient.Create(t.Context(), &aigv1b1.GuardrailPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Spec: aigv1b1.GuardrailPolicySpec{ + TargetRefs: []gwapiv1a2.LocalPolicyTargetReference{{Name: "backend"}}, + Rules: []aigv1b1.GuardrailRule{{ + Name: "rule", Phase: aigv1b1.GuardrailPhaseRequest, + Provider: aigv1b1.GuardrailProvider{Type: aigv1b1.GuardrailProviderTypeRegex, Pattern: name}, + }}, + }, + })) + } + controller := &GatewayController{client: controllerClient, kube: fakekube.NewClientset(), logger: ctrl.Log} + config := &filterapi.Config{} + route := &aigv1b1.AIGatewayRoute{ + ObjectMeta: metav1.ObjectMeta{Name: "route", Namespace: namespace}, + Spec: aigv1b1.AIGatewayRouteSpec{Rules: []aigv1b1.AIGatewayRouteRule{{ + BackendRefs: []aigv1b1.AIGatewayRouteRuleBackendRef{{Name: "backend"}}, + }}}, + } + require.NoError(t, controller.injectGuardrails(t.Context(), route, config, map[string]struct{}{})) + require.Len(t, config.Guardrails, 2) + require.Equal(t, "default/alpha/rule", config.Guardrails[0].Name) + require.Equal(t, "default/zeta/rule", config.Guardrails[1].Name) + require.Equal(t, defaultGuardrailMaxPayloadBytes, config.Guardrails[0].MaxPayloadBytes) +} + +func TestSecretToGuardrailPolicy(t *testing.T) { + const namespace = "default" + controllerClient := newGuardrailPolicyTestClient(t) + policy := &aigv1b1.GuardrailPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: "policy", Namespace: namespace}, + Spec: aigv1b1.GuardrailPolicySpec{Rules: []aigv1b1.GuardrailRule{{ + Name: "presidio", Phase: aigv1b1.GuardrailPhaseRequest, + Provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypePresidio, + Presidio: &aigv1b1.PresidioGuardrailProvider{ + Endpoint: "https://presidio.example.com", + APIKeySecretRef: &gwapiv1.SecretObjectReference{Name: "provider-key"}, + }, + }, + }}}, + } + require.NoError(t, controllerClient.Create(t.Context(), policy)) + controller := NewGuardrailPolicyController(controllerClient, fakekube.NewClientset(), ctrl.Log, make(chan event.GenericEvent, 1)) + + requests := controller.SecretToGuardrailPolicy(t.Context(), &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "provider-key", Namespace: namespace}, + }) + require.Equal(t, []reconcile.Request{{NamespacedName: types.NamespacedName{ + Namespace: namespace, Name: policy.Name, + }}}, requests) +} + +func TestInjectGuardrailsConfigurationFailureModes(t *testing.T) { + const namespace = "default" + for _, test := range []struct { + name string + failureMode aigv1b1.GuardrailFailureMode + action aigv1b1.GuardrailAction + wantGuardrail bool + }{ + {name: "fail closed publishes blocking fallback", wantGuardrail: true}, + {name: "fail open omits unavailable rule", failureMode: aigv1b1.GuardrailFailureModeFailOpen}, + {name: "monitor omits unavailable rule", action: aigv1b1.GuardrailActionMonitor}, + } { + t.Run(test.name, func(t *testing.T) { + controllerClient := newGuardrailPolicyTestClient(t) + policy := &aigv1b1.GuardrailPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: "policy", Namespace: namespace}, + Spec: aigv1b1.GuardrailPolicySpec{ + TargetRefs: []gwapiv1a2.LocalPolicyTargetReference{{Name: "backend"}}, + Rules: []aigv1b1.GuardrailRule{{ + Name: "azure", Phase: aigv1b1.GuardrailPhaseRequest, + Provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeAzureContentSafety, + Action: test.action, + FailureMode: test.failureMode, + AzureContentSafety: &aigv1b1.AzureContentSafetyGuardrailProvider{ + Endpoint: "https://content-safety.example.com", + APIKeySecretRef: &gwapiv1.SecretObjectReference{Name: "missing"}, + }, + }, + }}, + }, + } + require.NoError(t, controllerClient.Create(t.Context(), policy)) + controller := &GatewayController{client: controllerClient, kube: fakekube.NewClientset(), logger: ctrl.Log} + config := &filterapi.Config{} + route := &aigv1b1.AIGatewayRoute{ + ObjectMeta: metav1.ObjectMeta{Name: "route", Namespace: namespace}, + Spec: aigv1b1.AIGatewayRouteSpec{Rules: []aigv1b1.AIGatewayRouteRule{{ + BackendRefs: []aigv1b1.AIGatewayRouteRuleBackendRef{{Name: "backend"}}, + }}}, + } + require.NoError(t, controller.injectGuardrails(t.Context(), route, config, map[string]struct{}{})) + if !test.wantGuardrail { + require.Empty(t, config.Guardrails) + return + } + require.Len(t, config.Guardrails, 1) + require.Equal(t, filterapi.GuardrailProviderTypeRegex, config.Guardrails[0].Provider.Type) + require.Equal(t, `(?s).*`, config.Guardrails[0].Provider.Pattern) + }) + } +} + +func TestGuardrailPolicyToRuntimeIntegration(t *testing.T) { + providerServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`[{"entity_type":"EMAIL_ADDRESS","score":0.99}]`)) + })) + t.Cleanup(providerServer.Close) + + kube := fakekube.NewClientset(&corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "presidio-key", Namespace: "default"}, + Data: map[string][]byte{"apiKey": []byte("secret")}, + }) + controller := &GatewayController{kube: kube} + converted, err := controller.guardrailProviderToFilterAPI(t.Context(), "default", &aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypePresidio, + Presidio: &aigv1b1.PresidioGuardrailProvider{ + Endpoint: providerServer.URL, + APIKeySecretRef: &gwapiv1.SecretObjectReference{Name: "presidio-key"}, + }, + }) + require.NoError(t, err) + runtimeConfig, err := filterapi.NewRuntimeConfig(t.Context(), &filterapi.Config{ + Guardrails: []filterapi.Guardrail{{Name: "pii", Phase: filterapi.GuardrailPhaseRequest, Provider: converted}}, + }, func(context.Context, *filterapi.BackendAuth) (filterapi.BackendAuthHandler, error) { + return nil, nil + }, guardrails.NewEvaluator) + require.NoError(t, err) + require.Len(t, runtimeConfig.Guardrails, 1) + evaluation, err := runtimeConfig.Guardrails[0].Evaluator.Evaluate(t.Context(), []byte("user@example.com"), filterapi.GuardrailPhaseRequest) + require.NoError(t, err) + require.True(t, evaluation.Matched) +} + +func TestGuardrailHTTPProviderToRuntimeEvaluator(t *testing.T) { + providerServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + require.Equal(t, "/v1/check", req.URL.Path) + require.Equal(t, "Bearer custom-secret", req.Header.Get("Authorization")) + _, _ = w.Write([]byte(`{"action":"block","findings":[{"type":"PII","start":0,"end":16,"score":0.92}]}`)) + })) + t.Cleanup(providerServer.Close) + + kube := fakekube.NewClientset(&corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "custom-guardrail-key", Namespace: "default"}, + Data: map[string][]byte{"apiKey": []byte("custom-secret")}, + }) + controller := &GatewayController{kube: kube} + converted, err := controller.guardrailProviderToFilterAPI(t.Context(), "default", &aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeHTTP, + HTTP: &aigv1b1.HTTPGuardrailProvider{ + Endpoint: providerServer.URL, + Path: "/v1/check", + APIKeySecretRef: &gwapiv1.SecretObjectReference{Name: "custom-guardrail-key"}, + }, + }) + require.NoError(t, err) + require.Equal(t, "custom-secret", converted.HTTP.APIKey) + runtimeConfig, err := filterapi.NewRuntimeConfig(t.Context(), &filterapi.Config{ + Guardrails: []filterapi.Guardrail{{Name: "custom", Phase: filterapi.GuardrailPhaseRequest, Provider: converted}}, + }, func(context.Context, *filterapi.BackendAuth) (filterapi.BackendAuthHandler, error) { + return nil, nil + }, guardrails.NewEvaluator) + require.NoError(t, err) + require.Len(t, runtimeConfig.Guardrails, 1) + evaluation, err := runtimeConfig.Guardrails[0].Evaluator.Evaluate(t.Context(), []byte("user@example.com"), filterapi.GuardrailPhaseRequest) + require.NoError(t, err) + require.True(t, evaluation.Matched) + require.Equal(t, "[REDACTED]", string(evaluation.Replacement)) +} + +func TestGuardrailHTTPProviderValidation(t *testing.T) { + kube := fakekube.NewClientset() + controller := &GuardrailPolicyController{kube: kube} + require.ErrorContains(t, controller.validateGuardrailProvider(t.Context(), "default", &aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeHTTP, + }), "http guardrail configuration is required") + require.ErrorContains(t, controller.validateGuardrailProvider(t.Context(), "default", &aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeHTTP, + HTTP: &aigv1b1.HTTPGuardrailProvider{Endpoint: "not-a-url"}, + }), "valid provider endpoint is required") + require.ErrorContains(t, controller.validateGuardrailProvider(t.Context(), "default", &aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeHTTP, + HTTP: &aigv1b1.HTTPGuardrailProvider{Endpoint: "https://guardrail.example.com", Path: "analyze"}, + }), "path must start with /") + require.ErrorContains(t, controller.validateGuardrailProvider(t.Context(), "default", &aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeHTTP, + HTTP: &aigv1b1.HTTPGuardrailProvider{ + Endpoint: "https://guardrail.example.com", + APIKeySecretRef: &gwapiv1.SecretObjectReference{Name: "missing"}, + }, + }), "failed to get secret missing") + require.NoError(t, controller.validateGuardrailProvider(t.Context(), "default", &aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeHTTP, + Action: aigv1b1.GuardrailActionMask, + HTTP: &aigv1b1.HTTPGuardrailProvider{Endpoint: "https://guardrail.example.com"}, + })) +} + +func TestGuardrailPolicySecretRefsIndexIncludesHTTPProvider(t *testing.T) { + policy := &aigv1b1.GuardrailPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: "custom", Namespace: "default"}, + Spec: aigv1b1.GuardrailPolicySpec{Rules: []aigv1b1.GuardrailRule{{ + Name: "custom", Phase: aigv1b1.GuardrailPhaseRequest, + Provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeHTTP, + HTTP: &aigv1b1.HTTPGuardrailProvider{ + Endpoint: "https://guardrail.example.com", + APIKeySecretRef: &gwapiv1.SecretObjectReference{Name: "custom-guardrail-key"}, + }, + }, + }}}, + } + require.Equal(t, []string{"custom-guardrail-key.default"}, guardrailPolicySecretRefsIndexFunc(policy)) +} + +func TestGuardrailModelArmorProviderToFilterAPI(t *testing.T) { + kube := fakekube.NewClientset(&corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "model-armor-sa", Namespace: "default"}, + Data: map[string][]byte{"credentials": []byte(`{"type":"service_account"}`)}, + }) + controller := &GatewayController{kube: kube} + converted, err := controller.guardrailProviderToFilterAPI(t.Context(), "default", &aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeModelArmor, + Action: aigv1b1.GuardrailActionMask, + ModelArmor: &aigv1b1.ModelArmorGuardrailProvider{ + Project: "project-id", + Location: "us-central1", + Template: "template-id", + Endpoint: "https://modelarmor.example.com", + CredentialsSecretRef: &gwapiv1.SecretObjectReference{Name: "model-armor-sa"}, + }, + }) + require.NoError(t, err) + require.Equal(t, filterapi.GuardrailActionMask, converted.Action) + require.Equal(t, &filterapi.ModelArmorGuardrailProvider{ + Endpoint: "https://modelarmor.example.com", + Project: "project-id", + Location: "us-central1", + Template: "template-id", + CredentialsJSON: `{"type":"service_account"}`, + }, converted.ModelArmor) + + _, err = controller.guardrailProviderToFilterAPI(t.Context(), "default", &aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeModelArmor, + }) + require.ErrorContains(t, err, "model Armor configuration is required") +} + +func TestGuardrailModelArmorProviderValidation(t *testing.T) { + kube := fakekube.NewClientset(&corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "model-armor-sa", Namespace: "default"}, + Data: map[string][]byte{"credentials": []byte(`{"type":"service_account"}`)}, + }) + controller := &GuardrailPolicyController{kube: kube} + require.ErrorContains(t, controller.validateGuardrailProvider(t.Context(), "default", &aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeModelArmor, + }), "model Armor project, location, and template are required") + require.ErrorContains(t, controller.validateGuardrailProvider(t.Context(), "default", &aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeModelArmor, + ModelArmor: &aigv1b1.ModelArmorGuardrailProvider{Project: "project-id", Location: "us-central1"}, + }), "model Armor project, location, and template are required") + require.ErrorContains(t, controller.validateGuardrailProvider(t.Context(), "default", &aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeModelArmor, + ModelArmor: &aigv1b1.ModelArmorGuardrailProvider{ + Project: "project-id", Location: "us-central1", Template: "template-id", Endpoint: "not-a-url", + }, + }), "valid provider endpoint is required") + require.ErrorContains(t, controller.validateGuardrailProvider(t.Context(), "default", &aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeModelArmor, + ModelArmor: &aigv1b1.ModelArmorGuardrailProvider{ + Project: "project-id", Location: "us-central1", Template: "template-id", + CredentialsSecretRef: &gwapiv1.SecretObjectReference{Name: "missing"}, + }, + }), "failed to get secret missing") + require.NoError(t, controller.validateGuardrailProvider(t.Context(), "default", &aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeModelArmor, + Action: aigv1b1.GuardrailActionMask, + ModelArmor: &aigv1b1.ModelArmorGuardrailProvider{ + Project: "project-id", Location: "us-central1", Template: "template-id", + CredentialsSecretRef: &gwapiv1.SecretObjectReference{Name: "model-armor-sa"}, + }, + })) +} + +func TestGuardrailPolicySecretRefsIndexIncludesModelArmorProvider(t *testing.T) { + policy := &aigv1b1.GuardrailPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: "model-armor", Namespace: "default"}, + Spec: aigv1b1.GuardrailPolicySpec{Rules: []aigv1b1.GuardrailRule{{ + Name: "model-armor", Phase: aigv1b1.GuardrailPhaseRequest, + Provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeModelArmor, + ModelArmor: &aigv1b1.ModelArmorGuardrailProvider{ + Project: "project-id", Location: "us-central1", Template: "template-id", + CredentialsSecretRef: &gwapiv1.SecretObjectReference{Name: "model-armor-sa"}, + }, + }, + }}}, + } + require.Equal(t, []string{"model-armor-sa.default"}, guardrailPolicySecretRefsIndexFunc(policy)) +} + +func TestGuardrailProviderValidation(t *testing.T) { + const namespace = "default" + otherNamespace := gwapiv1.Namespace("other") + kube := fakekube.NewClientset( + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "api-key", Namespace: namespace}, + Data: map[string][]byte{"apiKey": []byte("secret")}, + }, + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "string-data", Namespace: namespace}, + StringData: map[string]string{"credentials": "[default]"}, + }, + ) + controller := &GuardrailPolicyController{kube: kube} + tests := []struct { + name string + provider aigv1b1.GuardrailProvider + wantErr string + }{ + { + name: "unsupported action", + provider: aigv1b1.GuardrailProvider{Type: aigv1b1.GuardrailProviderTypeRegex, Pattern: "x", Action: "Drop"}, + wantErr: `unsupported action "Drop"`, + }, + { + name: "azure mask", + provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeAzureContentSafety, Action: aigv1b1.GuardrailActionMask, + }, + wantErr: "azure Content Safety does not support mask", + }, + { + name: "unsupported failure mode", + provider: aigv1b1.GuardrailProvider{Type: aigv1b1.GuardrailProviderTypeRegex, Pattern: "x", FailureMode: "Retry"}, + wantErr: `unsupported failureMode "Retry"`, + }, + { + name: "regex", + provider: aigv1b1.GuardrailProvider{Type: aigv1b1.GuardrailProviderTypeRegex, Pattern: "secret", Action: aigv1b1.GuardrailActionMask}, + }, + { + name: "regex missing pattern", + provider: aigv1b1.GuardrailProvider{Type: aigv1b1.GuardrailProviderTypeRegex}, + wantErr: "regex pattern is required", + }, + { + name: "regex invalid pattern", + provider: aigv1b1.GuardrailProvider{Type: aigv1b1.GuardrailProviderTypeRegex, Pattern: "("}, + wantErr: "invalid regex pattern", + }, + { + name: "presidio", + provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypePresidio, + Presidio: &aigv1b1.PresidioGuardrailProvider{ + Endpoint: "https://presidio.example.com", + APIKeySecretRef: &gwapiv1.SecretObjectReference{Name: "api-key"}, + }, + }, + }, + { + name: "presidio missing configuration", + provider: aigv1b1.GuardrailProvider{Type: aigv1b1.GuardrailProviderTypePresidio}, + wantErr: "presidio configuration is required", + }, + { + name: "presidio invalid endpoint", + provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypePresidio, + Presidio: &aigv1b1.PresidioGuardrailProvider{Endpoint: "presidio"}, + }, + wantErr: "valid provider endpoint is required", + }, + { + name: "bedrock with secret in string data", + provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeBedrockGuardrails, + Bedrock: &aigv1b1.BedrockGuardrailProvider{ + Region: "us-east-1", GuardrailIdentifier: "guardrail-id", GuardrailVersion: "1", + Endpoint: "https://bedrock.example.com", + CredentialsSecretRef: &gwapiv1.SecretObjectReference{Name: "string-data"}, + }, + }, + }, + { + name: "bedrock missing identifier", + provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeBedrockGuardrails, + Bedrock: &aigv1b1.BedrockGuardrailProvider{Region: "us-east-1"}, + }, + wantErr: "bedrock region, guardrailIdentifier, and guardrailVersion are required", + }, + { + name: "bedrock invalid endpoint", + provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeBedrockGuardrails, + Bedrock: &aigv1b1.BedrockGuardrailProvider{ + Region: "us-east-1", GuardrailIdentifier: "guardrail-id", GuardrailVersion: "1", Endpoint: "bedrock", + }, + }, + wantErr: "valid provider endpoint is required", + }, + { + name: "azure", + provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeAzureContentSafety, + AzureContentSafety: &aigv1b1.AzureContentSafetyGuardrailProvider{ + Endpoint: "https://content-safety.example.com", + APIKeySecretRef: &gwapiv1.SecretObjectReference{Name: "api-key"}, + }, + }, + }, + { + name: "azure missing configuration", + provider: aigv1b1.GuardrailProvider{Type: aigv1b1.GuardrailProviderTypeAzureContentSafety}, + wantErr: "azure Content Safety configuration is required", + }, + { + name: "azure invalid endpoint", + provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeAzureContentSafety, + AzureContentSafety: &aigv1b1.AzureContentSafetyGuardrailProvider{Endpoint: "content-safety"}, + }, + wantErr: "valid provider endpoint is required", + }, + { + name: "azure missing secret reference", + provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeAzureContentSafety, + AzureContentSafety: &aigv1b1.AzureContentSafetyGuardrailProvider{Endpoint: "https://content-safety.example.com"}, + }, + wantErr: "secret reference is required", + }, + { + name: "cross-namespace secret", + provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeAzureContentSafety, + AzureContentSafety: &aigv1b1.AzureContentSafetyGuardrailProvider{ + Endpoint: "https://content-safety.example.com", + APIKeySecretRef: &gwapiv1.SecretObjectReference{Name: "api-key", Namespace: &otherNamespace}, + }, + }, + wantErr: "cross-namespace guardrail secret references are not supported", + }, + { + name: "secret missing key", + provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeBedrockGuardrails, + Bedrock: &aigv1b1.BedrockGuardrailProvider{ + Region: "us-east-1", GuardrailIdentifier: "guardrail-id", GuardrailVersion: "1", + CredentialsSecretRef: &gwapiv1.SecretObjectReference{Name: "api-key"}, + }, + }, + wantErr: "secret api-key does not contain key credentials", + }, + { + name: "unsupported provider", + provider: aigv1b1.GuardrailProvider{Type: "Unknown"}, + wantErr: `unsupported provider type "Unknown"`, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := controller.validateGuardrailProvider(t.Context(), namespace, &test.provider) + if test.wantErr == "" { + require.NoError(t, err) + } else { + require.ErrorContains(t, err, test.wantErr) + } + }) + } +} + +func TestBackendToGuardrailPolicy(t *testing.T) { + const namespace = "default" + controllerClient := newGuardrailPolicyTestClient(t) + for _, name := range []string{"targets-backend", "targets-other"} { + target := gwapiv1.ObjectName("backend") + if name == "targets-other" { + target = "other" + } + require.NoError(t, controllerClient.Create(t.Context(), &aigv1b1.GuardrailPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Spec: aigv1b1.GuardrailPolicySpec{ + TargetRefs: []gwapiv1a2.LocalPolicyTargetReference{{ + Group: "aigateway.envoyproxy.io", Kind: "AIServiceBackend", Name: target, + }}, + }, + })) + } + controller := NewGuardrailPolicyController(controllerClient, fakekube.NewClientset(), ctrl.Log, make(chan event.GenericEvent, 1)) + + requests := controller.BackendToGuardrailPolicy(t.Context(), &aigv1b1.AIServiceBackend{ + ObjectMeta: metav1.ObjectMeta{Name: "backend", Namespace: namespace}, + }) + require.Equal(t, []reconcile.Request{{NamespacedName: types.NamespacedName{ + Namespace: namespace, Name: "targets-backend", + }}}, requests) +} + +func TestGuardrailPolicyControllerReconcileNotFound(t *testing.T) { + controller := NewGuardrailPolicyController(newGuardrailPolicyTestClient(t), fakekube.NewClientset(), ctrl.Log, make(chan event.GenericEvent, 1)) + result, err := controller.Reconcile(t.Context(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: "default", Name: "missing"}}) + require.NoError(t, err) + require.Equal(t, ctrl.Result{}, result) +} + +func TestGuardrailPolicyControllerReconcileMissingTarget(t *testing.T) { + const namespace = "default" + controllerClient := newGuardrailPolicyTestClient(t) + policy := &aigv1b1.GuardrailPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: "policy", Namespace: namespace}, + Spec: aigv1b1.GuardrailPolicySpec{ + TargetRefs: []gwapiv1a2.LocalPolicyTargetReference{{ + Group: "aigateway.envoyproxy.io", Kind: "AIServiceBackend", Name: "missing", + }}, + Rules: []aigv1b1.GuardrailRule{{ + Name: "regex", Phase: aigv1b1.GuardrailPhaseRequest, + Provider: aigv1b1.GuardrailProvider{Type: aigv1b1.GuardrailProviderTypeRegex, Pattern: "secret"}, + }}, + }, + } + require.NoError(t, controllerClient.Create(t.Context(), policy)) + controller := NewGuardrailPolicyController(controllerClient, fakekube.NewClientset(), ctrl.Log, make(chan event.GenericEvent, 1)) + + _, err := controller.Reconcile(t.Context(), reconcile.Request{NamespacedName: client.ObjectKeyFromObject(policy)}) + require.ErrorContains(t, err, "target AIServiceBackend default/missing not found") + + var updated aigv1b1.GuardrailPolicy + require.NoError(t, controllerClient.Get(t.Context(), client.ObjectKeyFromObject(policy), &updated)) + require.Len(t, updated.Status.Conditions, 1) + require.Equal(t, aigv1b1.ConditionTypeNotAccepted, updated.Status.Conditions[0].Type) +} + +func TestGuardrailProviderToFilterAPI(t *testing.T) { + const namespace = "default" + otherNamespace := gwapiv1.Namespace("other") + timeout, threshold, severity := int32(3), int32(70), int32(2) + kube := fakekube.NewClientset( + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "api-key", Namespace: namespace}, + Data: map[string][]byte{"apiKey": []byte("api-secret")}, + }, + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "aws", Namespace: namespace}, + Data: map[string][]byte{"credentials": []byte("[default]")}, + }, + ) + controller := &GatewayController{kube: kube} + missing := &gwapiv1.SecretObjectReference{Name: "missing"} + tests := []struct { + name string + provider aigv1b1.GuardrailProvider + want filterapi.GuardrailProvider + wantErr string + }{ + { + name: "presidio", + provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypePresidio, TimeoutSeconds: &timeout, + Presidio: &aigv1b1.PresidioGuardrailProvider{ + Endpoint: "https://presidio.example.com", Language: "es", ScoreThresholdPercent: &threshold, + APIKeySecretRef: &gwapiv1.SecretObjectReference{Name: "api-key"}, + }, + }, + want: filterapi.GuardrailProvider{ + Type: filterapi.GuardrailProviderTypePresidio, TimeoutSeconds: 3, + Presidio: &filterapi.PresidioGuardrailProvider{ + Endpoint: "https://presidio.example.com", Language: "es", ScoreThresholdPercent: 70, APIKey: "api-secret", + }, + }, + }, + { + name: "bedrock", + provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeBedrockGuardrails, + Bedrock: &aigv1b1.BedrockGuardrailProvider{ + Region: "us-east-1", GuardrailIdentifier: "guardrail-id", GuardrailVersion: "1", + Endpoint: "https://bedrock.example.com", + CredentialsSecretRef: &gwapiv1.SecretObjectReference{Name: "aws"}, + }, + }, + want: filterapi.GuardrailProvider{ + Type: filterapi.GuardrailProviderTypeBedrockGuardrails, + Bedrock: &filterapi.BedrockGuardrailProvider{ + Endpoint: "https://bedrock.example.com", Region: "us-east-1", + GuardrailIdentifier: "guardrail-id", GuardrailVersion: "1", CredentialFileLiteral: "[default]", + }, + }, + }, + { + name: "azure", + provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeAzureContentSafety, + AzureContentSafety: &aigv1b1.AzureContentSafetyGuardrailProvider{ + Endpoint: "https://content-safety.example.com", APIVersion: "2024-09-01", SeverityThreshold: &severity, + APIKeySecretRef: &gwapiv1.SecretObjectReference{Name: "api-key"}, + }, + }, + want: filterapi.GuardrailProvider{ + Type: filterapi.GuardrailProviderTypeAzureContentSafety, + AzureContentSafety: &filterapi.AzureContentSafetyGuardrailProvider{ + Endpoint: "https://content-safety.example.com", APIVersion: "2024-09-01", SeverityThreshold: &severity, + APIKey: "api-secret", + }, + }, + }, + { + name: "regex", + provider: aigv1b1.GuardrailProvider{Type: aigv1b1.GuardrailProviderTypeRegex, Pattern: "secret", Action: aigv1b1.GuardrailActionMask}, + want: filterapi.GuardrailProvider{Type: filterapi.GuardrailProviderTypeRegex, Pattern: "secret", Action: filterapi.GuardrailActionMask}, + }, + { + name: "presidio missing configuration", + provider: aigv1b1.GuardrailProvider{Type: aigv1b1.GuardrailProviderTypePresidio}, + wantErr: "presidio configuration is required", + }, + { + name: "presidio missing secret", + provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypePresidio, + Presidio: &aigv1b1.PresidioGuardrailProvider{Endpoint: "https://presidio.example.com", APIKeySecretRef: missing}, + }, + wantErr: "missing", + }, + { + name: "bedrock missing configuration", + provider: aigv1b1.GuardrailProvider{Type: aigv1b1.GuardrailProviderTypeBedrockGuardrails}, + wantErr: "bedrock configuration is required", + }, + { + name: "bedrock missing secret", + provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeBedrockGuardrails, + Bedrock: &aigv1b1.BedrockGuardrailProvider{ + Region: "us-east-1", GuardrailIdentifier: "guardrail-id", GuardrailVersion: "1", CredentialsSecretRef: missing, + }, + }, + wantErr: "missing", + }, + { + name: "azure missing configuration", + provider: aigv1b1.GuardrailProvider{Type: aigv1b1.GuardrailProviderTypeAzureContentSafety}, + wantErr: "azure Content Safety configuration is required", + }, + { + name: "azure missing secret reference", + provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeAzureContentSafety, + AzureContentSafety: &aigv1b1.AzureContentSafetyGuardrailProvider{Endpoint: "https://content-safety.example.com"}, + }, + wantErr: "secret reference is required", + }, + { + name: "azure cross-namespace secret", + provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeAzureContentSafety, + AzureContentSafety: &aigv1b1.AzureContentSafetyGuardrailProvider{ + Endpoint: "https://content-safety.example.com", + APIKeySecretRef: &gwapiv1.SecretObjectReference{Name: "api-key", Namespace: &otherNamespace}, + }, + }, + wantErr: "cross-namespace guardrail secret references are not supported", + }, + { + name: "http missing configuration", + provider: aigv1b1.GuardrailProvider{Type: aigv1b1.GuardrailProviderTypeHTTP}, + wantErr: "http guardrail configuration is required", + }, + { + name: "http missing secret", + provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeHTTP, + HTTP: &aigv1b1.HTTPGuardrailProvider{Endpoint: "https://guardrail.example.com", APIKeySecretRef: missing}, + }, + wantErr: "missing", + }, + { + name: "model armor missing secret", + provider: aigv1b1.GuardrailProvider{ + Type: aigv1b1.GuardrailProviderTypeModelArmor, + ModelArmor: &aigv1b1.ModelArmorGuardrailProvider{ + Project: "project-id", Location: "us-central1", Template: "template-id", CredentialsSecretRef: missing, + }, + }, + wantErr: "missing", + }, + { + name: "unsupported provider", + provider: aigv1b1.GuardrailProvider{Type: "Unknown"}, + wantErr: `unsupported provider type "Unknown"`, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + converted, err := controller.guardrailProviderToFilterAPI(t.Context(), namespace, &test.provider) + if test.wantErr != "" { + require.ErrorContains(t, err, test.wantErr) + return + } + require.NoError(t, err) + require.Equal(t, test.want, converted) + }) + } +} + +func TestGuardrailMaxPayloadBytes(t *testing.T) { + requestLimit, responseLimit := int64(2048), int64(4096) + policy := &aigv1b1.GuardrailPolicy{Spec: aigv1b1.GuardrailPolicySpec{ + MaxRequestBodyBytes: &requestLimit, MaxResponseBodyBytes: &responseLimit, + }} + require.Equal(t, requestLimit, guardrailMaxPayloadBytes(policy, aigv1b1.GuardrailPhaseRequest)) + require.Equal(t, responseLimit, guardrailMaxPayloadBytes(policy, aigv1b1.GuardrailPhaseResponse)) + require.Equal(t, defaultGuardrailMaxPayloadBytes, guardrailMaxPayloadBytes(&aigv1b1.GuardrailPolicy{}, aigv1b1.GuardrailPhaseResponse)) +} + +func TestInjectGuardrailsSkipsInjectedRulesAndForeignBackends(t *testing.T) { + const namespace = "default" + otherNamespace := gwapiv1.Namespace("other") + controllerClient := newGuardrailPolicyTestClient(t) + require.NoError(t, controllerClient.Create(t.Context(), &aigv1b1.GuardrailPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: "policy", Namespace: namespace}, + Spec: aigv1b1.GuardrailPolicySpec{ + TargetRefs: []gwapiv1a2.LocalPolicyTargetReference{{Name: "backend"}}, + Rules: []aigv1b1.GuardrailRule{ + { + Name: "injected", Phase: aigv1b1.GuardrailPhaseRequest, + Provider: aigv1b1.GuardrailProvider{Type: aigv1b1.GuardrailProviderTypeRegex, Pattern: "injected"}, + }, + { + Name: "new", Phase: aigv1b1.GuardrailPhaseResponse, + Provider: aigv1b1.GuardrailProvider{Type: aigv1b1.GuardrailProviderTypeRegex, Pattern: "new"}, + }, + }, + }, + })) + controller := &GatewayController{client: controllerClient, kube: fakekube.NewClientset(), logger: ctrl.Log} + config := &filterapi.Config{} + route := &aigv1b1.AIGatewayRoute{ + ObjectMeta: metav1.ObjectMeta{Name: "route", Namespace: namespace}, + Spec: aigv1b1.AIGatewayRouteSpec{Rules: []aigv1b1.AIGatewayRouteRule{{ + BackendRefs: []aigv1b1.AIGatewayRouteRuleBackendRef{ + {Name: "backend", Namespace: &otherNamespace}, + {Name: "other-backend"}, + {Name: "backend"}, + }, + }}}, + } + injected := map[string]struct{}{"default/policy/injected": {}} + require.NoError(t, controller.injectGuardrails(t.Context(), route, config, injected)) + require.Len(t, config.Guardrails, 1) + require.Equal(t, "default/policy/new", config.Guardrails[0].Name) + require.Equal(t, []string{"default/backend/route/route/rule/0/ref/2"}, config.Guardrails[0].Backends) + require.Contains(t, injected, "default/policy/new") +} + +func TestInjectGuardrailsListError(t *testing.T) { + controllerClient := fake.NewClientBuilder().WithScheme(Scheme).WithInterceptorFuncs(interceptor.Funcs{ + List: func(context.Context, client.WithWatch, client.ObjectList, ...client.ListOption) error { + return errors.New("list failed") + }, + }).Build() + controller := &GatewayController{client: controllerClient, kube: fakekube.NewClientset(), logger: ctrl.Log} + err := controller.injectGuardrails(t.Context(), &aigv1b1.AIGatewayRoute{}, &filterapi.Config{}, map[string]struct{}{}) + require.ErrorContains(t, err, "failed to list GuardrailPolicies: list failed") +} + +func TestGuardrailPolicyControllerClientErrors(t *testing.T) { + const namespace = "default" + policy := &aigv1b1.GuardrailPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: "policy", Namespace: namespace}, + Spec: aigv1b1.GuardrailPolicySpec{ + TargetRefs: []gwapiv1a2.LocalPolicyTargetReference{{ + Group: "aigateway.envoyproxy.io", Kind: "AIServiceBackend", Name: "backend", + }}, + Rules: []aigv1b1.GuardrailRule{{ + Name: "regex", Phase: aigv1b1.GuardrailPhaseRequest, + Provider: aigv1b1.GuardrailProvider{Type: aigv1b1.GuardrailProviderTypeRegex, Pattern: "secret"}, + }}, + }, + } + errClient := errors.New("api server unavailable") + newController := func(t *testing.T, funcs interceptor.Funcs) *GuardrailPolicyController { + builder := fake.NewClientBuilder().WithScheme(Scheme). + WithStatusSubresource(&aigv1b1.GuardrailPolicy{}). + WithObjects(policy.DeepCopy()). + WithInterceptorFuncs(funcs) + require.NoError(t, ApplyIndexing(t.Context(), func(_ context.Context, obj client.Object, field string, extractValue client.IndexerFunc) error { + builder = builder.WithIndex(obj, field, extractValue) + return nil + })) + return NewGuardrailPolicyController(builder.Build(), fakekube.NewClientset(), ctrl.Log, make(chan event.GenericEvent, 1)) + } + failList := interceptor.Funcs{List: func(context.Context, client.WithWatch, client.ObjectList, ...client.ListOption) error { + return errClient + }} + + t.Run("get policy", func(t *testing.T) { + controller := newController(t, interceptor.Funcs{Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error { + return errClient + }}) + _, err := controller.Reconcile(t.Context(), reconcile.Request{NamespacedName: client.ObjectKeyFromObject(policy)}) + require.ErrorIs(t, err, errClient) + }) + + t.Run("get target backend", func(t *testing.T) { + controller := newController(t, interceptor.Funcs{Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*aigv1b1.AIServiceBackend); ok { + return errClient + } + return c.Get(ctx, key, obj, opts...) + }}) + _, err := controller.Reconcile(t.Context(), reconcile.Request{NamespacedName: client.ObjectKeyFromObject(policy)}) + require.ErrorContains(t, err, "failed to get AIServiceBackend default/backend: api server unavailable") + }) + + t.Run("list policies and routes", func(t *testing.T) { + controller := newController(t, failList) + require.Nil(t, controller.SecretToGuardrailPolicy(t.Context(), &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "secret", Namespace: namespace}, + })) + require.Nil(t, controller.BackendToGuardrailPolicy(t.Context(), &aigv1b1.AIServiceBackend{ + ObjectMeta: metav1.ObjectMeta{Name: "backend", Namespace: namespace}, + })) + controller.notifyAIGatewayRoutesForGuardrailPolicy(t.Context(), policy) + require.Empty(t, controller.aiGatewayRouteChan) + }) + + t.Run("status update", func(t *testing.T) { + controller := newController(t, interceptor.Funcs{SubResourceUpdate: func(context.Context, client.Client, string, client.Object, ...client.SubResourceUpdateOption) error { + return errClient + }}) + // The status update error is logged, not returned. + controller.updateGuardrailPolicyStatus(t.Context(), policy.DeepCopy(), aigv1b1.ConditionTypeAccepted, "ok") + }) + + t.Run("status update for deleted policy", func(t *testing.T) { + controller := newController(t, interceptor.Funcs{Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error { + return apierrors.NewNotFound(aigv1b1.SchemeGroupVersion.WithResource("guardrailpolicies").GroupResource(), policy.Name) + }}) + controller.updateGuardrailPolicyStatus(t.Context(), policy.DeepCopy(), aigv1b1.ConditionTypeAccepted, "ok") + }) + + t.Run("status get error", func(t *testing.T) { + controller := newController(t, interceptor.Funcs{Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error { + return errClient + }}) + controller.updateGuardrailPolicyStatus(t.Context(), policy.DeepCopy(), aigv1b1.ConditionTypeAccepted, "ok") + }) +} diff --git a/internal/extproc/guardrail_content.go b/internal/extproc/guardrail_content.go new file mode 100644 index 0000000000..1e5d21067a --- /dev/null +++ b/internal/extproc/guardrail_content.go @@ -0,0 +1,90 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package extproc + +import ( + "fmt" + "sort" + "strconv" + "strings" + + "github.com/tidwall/sjson" + + "github.com/envoyproxy/ai-gateway/internal/json" +) + +type guardrailContent struct { + text []byte + path string +} + +var guardrailTextFields = map[string]struct{}{ + "content": {}, "input": {}, "instructions": {}, "output_text": {}, + "prompt": {}, "refusal": {}, "system": {}, "text": {}, +} + +var guardrailContainerFields = map[string]struct{}{ + "choices": {}, "contents": {}, "delta": {}, "message": {}, "messages": {}, + "output": {}, "parts": {}, +} + +func extractGuardrailContents(body []byte) ([]guardrailContent, error) { + var value any + if err := json.Unmarshal(body, &value); err != nil { + return nil, fmt.Errorf("cannot extract guardrail text from JSON body: %w", err) + } + var contents []guardrailContent + walkGuardrailContent(value, "", false, &contents) + return contents, nil +} + +func walkGuardrailContent(value any, path string, capture bool, contents *[]guardrailContent) { + switch typed := value.(type) { + case string: + if capture && typed != "" { + *contents = append(*contents, guardrailContent{text: []byte(typed), path: path}) + } + case []any: + for i := range typed { + walkGuardrailContent(typed[i], appendGuardrailPath(path, strconv.Itoa(i)), capture, contents) + } + case map[string]any: + keys := make([]string, 0, len(typed)) + for key := range typed { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + childPath := appendGuardrailPath(path, escapeGuardrailPathKey(key)) + if _, ok := guardrailTextFields[key]; ok { + walkGuardrailContent(typed[key], childPath, true, contents) + continue + } + if _, ok := guardrailContainerFields[key]; ok { + walkGuardrailContent(typed[key], childPath, false, contents) + } + } + } +} + +func appendGuardrailPath(path, component string) string { + if path == "" { + return component + } + return path + "." + component +} + +func escapeGuardrailPathKey(key string) string { + replacer := strings.NewReplacer("\\", "\\\\", ".", "\\.", "*", "\\*", "?", "\\?", "#", "\\#") + return replacer.Replace(key) +} + +func replaceGuardrailContent(body []byte, content guardrailContent, replacement []byte) ([]byte, error) { + if content.path == "" { + return replacement, nil + } + return sjson.SetBytesOptions(body, content.path, string(replacement), &sjson.Options{ReplaceInPlace: true}) +} diff --git a/internal/extproc/guardrail_content_test.go b/internal/extproc/guardrail_content_test.go new file mode 100644 index 0000000000..e18428793d --- /dev/null +++ b/internal/extproc/guardrail_content_test.go @@ -0,0 +1,27 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package extproc + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestExtractGuardrailContents(t *testing.T) { + contents, err := extractGuardrailContents([]byte(`{ + "model":"must-not-be-scanned", + "messages":[ + {"role":"system","content":"system text"}, + {"role":"user","content":[{"type":"text","text":"user text"},{"type":"image_url","image_url":{"url":"https://example.com"}}]} + ] + }`)) + require.NoError(t, err) + require.Equal(t, []guardrailContent{ + {text: []byte("system text"), path: "messages.0.content"}, + {text: []byte("user text"), path: "messages.1.content.0.text"}, + }, contents) +} diff --git a/internal/extproc/guardrails.go b/internal/extproc/guardrails.go new file mode 100644 index 0000000000..eefd797e51 --- /dev/null +++ b/internal/extproc/guardrails.go @@ -0,0 +1,198 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package extproc + +import ( + "context" + "errors" + "fmt" + "slices" + + "github.com/envoyproxy/ai-gateway/internal/filterapi" +) + +// guardrailViolation describes a trigger that should block the request/response. +type guardrailViolation struct { + Name string + Message string +} + +type guardrailOutcome struct { + Violation *guardrailViolation + Body []byte + Masked bool + Monitored bool + RuleName string +} + +type guardrailFailOpenError struct { + errors []error +} + +func (e *guardrailFailOpenError) Error() string { + return fmt.Sprintf("%d guardrail provider evaluation(s) failed open: %v", len(e.errors), errors.Join(e.errors...)) +} + +func isGuardrailFailOpenError(err error) bool { + var failOpenError *guardrailFailOpenError + return errors.As(err, &failOpenError) +} + +func guardrailsConfiguredForPhase(guardrails []filterapi.RuntimeGuardrail, phase filterapi.GuardrailPhase, backendName string, includeGlobal bool) bool { + for i := range guardrails { + if guardrails[i].Phase == phase && guardrailAppliesToBackend(&guardrails[i], backendName, includeGlobal) { + return true + } + } + return false +} + +func guardrailsRequireBufferedResponse(guardrails []filterapi.RuntimeGuardrail, backendName string) bool { + for i := range guardrails { + guardrail := &guardrails[i] + if guardrail.Phase == filterapi.GuardrailPhaseResponse && + guardrailAppliesToBackend(guardrail, backendName, true) { + return true + } + } + return false +} + +func evaluateGuardrailsForPhase(ctx context.Context, guardrails []filterapi.RuntimeGuardrail, phase filterapi.GuardrailPhase, body []byte, backendName string, includeGlobal bool) (guardrailOutcome, error) { + outcome := guardrailOutcome{Body: body} + var failOpenErrors []error + for i := range guardrails { + g := &guardrails[i] + if g.Phase != phase || !guardrailAppliesToBackend(g, backendName, includeGlobal) { + continue + } + maxPayloadBytes := g.MaxPayloadBytes + if maxPayloadBytes <= 0 { + maxPayloadBytes = 10 * 1024 * 1024 + } + if int64(len(outcome.Body)) > maxPayloadBytes { + err := fmt.Errorf("guardrail %q payload is %d bytes, exceeding the %d-byte limit", g.Name, len(outcome.Body), maxPayloadBytes) + if g.Provider.FailureMode == filterapi.GuardrailFailureModeFailOpen || guardrailAction(g.Provider.Action) == filterapi.GuardrailActionMonitor { + failOpenErrors = append(failOpenErrors, err) + continue + } + return outcome, err + } + + action := guardrailAction(g.Provider.Action) + if g.Provider.Type == filterapi.GuardrailProviderTypeRegex && action != filterapi.GuardrailActionMask { + if g.Matcher == nil { + return outcome, fmt.Errorf("guardrail %q uses regex provider without a compiled matcher", g.Name) + } + if !g.Matcher.Match(outcome.Body) { + continue + } + if action == filterapi.GuardrailActionMonitor { + outcome.Monitored = true + outcome.RuleName = g.Name + continue + } + outcome.Violation = newGuardrailViolation(g) + return outcome, nil + } + + contents, err := extractGuardrailContents(outcome.Body) + if err != nil { + if g.Provider.FailureMode == filterapi.GuardrailFailureModeFailOpen || action == filterapi.GuardrailActionMonitor { + failOpenErrors = append(failOpenErrors, fmt.Errorf("guardrail %q text extraction failed: %w", g.Name, err)) + continue + } + return outcome, fmt.Errorf("guardrail %q text extraction failed: %w", g.Name, err) + } + for _, content := range contents { + var evaluation filterapi.GuardrailEvaluationResult + if g.Provider.Type == filterapi.GuardrailProviderTypeRegex { + if g.Matcher == nil { + return outcome, fmt.Errorf("guardrail %q uses regex provider without a compiled matcher", g.Name) + } + evaluation.Matched = g.Matcher.Match(content.text) + if evaluation.Matched { + replacement := g.Provider.MaskReplacement + if replacement == "" { + replacement = "[REDACTED]" + } + evaluation.Replacement = g.Matcher.ReplaceAll(content.text, []byte(replacement)) + } + } else { + if g.Evaluator == nil { + return outcome, fmt.Errorf("guardrail %q uses provider %q without an evaluator", g.Name, g.Provider.Type) + } + evaluation, err = g.Evaluator.Evaluate(ctx, content.text, phase) + if err != nil { + if g.Provider.FailureMode == filterapi.GuardrailFailureModeFailOpen || action == filterapi.GuardrailActionMonitor { + failOpenErrors = append(failOpenErrors, fmt.Errorf("guardrail %q evaluation failed: %w", g.Name, err)) + continue + } + return outcome, fmt.Errorf("guardrail %q evaluation failed: %w", g.Name, err) + } + } + if !evaluation.Matched { + continue + } + switch action { + case filterapi.GuardrailActionMonitor: + outcome.Monitored = true + outcome.RuleName = g.Name + case filterapi.GuardrailActionMask: + if len(evaluation.Replacement) == 0 { + return outcome, fmt.Errorf("guardrail %q matched but provider %q returned no masked content", g.Name, g.Provider.Type) + } + outcome.Body, err = replaceGuardrailContent(outcome.Body, content, evaluation.Replacement) + if err != nil { + return outcome, fmt.Errorf("guardrail %q failed to mask content: %w", g.Name, err) + } + outcome.Masked = true + outcome.RuleName = g.Name + default: + outcome.Violation = newGuardrailViolation(g) + return outcome, nil + } + } + } + if len(failOpenErrors) > 0 { + return outcome, &guardrailFailOpenError{errors: failOpenErrors} + } + return outcome, nil +} + +func guardrailAction(action filterapi.GuardrailAction) filterapi.GuardrailAction { + if action == "" { + return filterapi.GuardrailActionBlock + } + return action +} + +func newGuardrailViolation(guardrail *filterapi.RuntimeGuardrail) *guardrailViolation { + message := guardrail.Provider.Message + if message == "" { + message = fmt.Sprintf("request blocked by guardrail %q", guardrail.Name) + } + return &guardrailViolation{Name: guardrail.Name, Message: message} +} + +func guardrailAppliesToBackend(guardrail *filterapi.RuntimeGuardrail, backendName string, includeGlobal bool) bool { + if len(guardrail.Backends) == 0 { + return includeGlobal + } + return backendName != "" && slices.Contains(guardrail.Backends, backendName) +} + +func evaluateRequestGuardrails(ctx context.Context, guardrails []filterapi.RuntimeGuardrail, body []byte) (guardrailOutcome, error) { + return evaluateGuardrailsForPhase(ctx, guardrails, filterapi.GuardrailPhaseRequest, body, "", true) +} + +func evaluateBackendRequestGuardrails(ctx context.Context, guardrails []filterapi.RuntimeGuardrail, body []byte, backendName string) (guardrailOutcome, error) { + return evaluateGuardrailsForPhase(ctx, guardrails, filterapi.GuardrailPhaseRequest, body, backendName, false) +} + +func evaluateResponseGuardrails(ctx context.Context, guardrails []filterapi.RuntimeGuardrail, body []byte, backendName string) (guardrailOutcome, error) { + return evaluateGuardrailsForPhase(ctx, guardrails, filterapi.GuardrailPhaseResponse, body, backendName, true) +} diff --git a/internal/extproc/guardrails_test.go b/internal/extproc/guardrails_test.go new file mode 100644 index 0000000000..7697ebfccd --- /dev/null +++ b/internal/extproc/guardrails_test.go @@ -0,0 +1,529 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package extproc + +import ( + "context" + "errors" + "log/slog" + "net/http" + "net/http/httptest" + "regexp" + "testing" + + extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" + "github.com/stretchr/testify/require" + + "github.com/envoyproxy/ai-gateway/internal/endpointspec" + "github.com/envoyproxy/ai-gateway/internal/filterapi" + "github.com/envoyproxy/ai-gateway/internal/guardrails" + "github.com/envoyproxy/ai-gateway/internal/metrics" + "github.com/envoyproxy/ai-gateway/internal/tracing/tracingapi" +) + +type recordingGuardrailMetrics struct { + phase string + result metrics.GuardrailResult + count int +} + +type failingGuardrailEvaluator struct{} + +func (*failingGuardrailEvaluator) Evaluate(context.Context, []byte, filterapi.GuardrailPhase) (filterapi.GuardrailEvaluationResult, error) { + return filterapi.GuardrailEvaluationResult{}, errors.New("provider unavailable") +} + +type maskingGuardrailEvaluator struct{} + +func (*maskingGuardrailEvaluator) Evaluate(_ context.Context, body []byte, _ filterapi.GuardrailPhase) (filterapi.GuardrailEvaluationResult, error) { + return filterapi.GuardrailEvaluationResult{Matched: true, Replacement: []byte("masked:" + string(body))}, nil +} + +func (m *recordingGuardrailMetrics) RecordEvaluation(_ context.Context, phase string, result metrics.GuardrailResult) { + m.phase = phase + m.result = result + m.count++ +} + +func TestEvaluateGuardrailsForPhase(t *testing.T) { + t.Run("request guardrail matches and returns violation", func(t *testing.T) { + guardrails := []filterapi.RuntimeGuardrail{{ + Name: "deny-pii", + Phase: filterapi.GuardrailPhaseRequest, + Provider: filterapi.GuardrailProvider{ + Type: filterapi.GuardrailProviderTypeRegex, + Message: "PII detected in request", + }, + Matcher: regexp.MustCompile(`\bSSN\b`), + }} + + outcome, err := evaluateRequestGuardrails(t.Context(), guardrails, []byte("customer SSN is present")) + require.NoError(t, err) + require.NotNil(t, outcome.Violation) + require.Equal(t, "deny-pii", outcome.Violation.Name) + require.Equal(t, "PII detected in request", outcome.Violation.Message) + }) + + t.Run("response guardrail ignores different phase", func(t *testing.T) { + guardrails := []filterapi.RuntimeGuardrail{{ + Name: "block-sensitive-response", + Phase: filterapi.GuardrailPhaseResponse, + Provider: filterapi.GuardrailProvider{ + Type: filterapi.GuardrailProviderTypeRegex, + }, + Matcher: regexp.MustCompile(`forbidden`), + }} + + outcome, err := evaluateRequestGuardrails(t.Context(), guardrails, []byte("forbidden")) + require.NoError(t, err) + require.Nil(t, outcome.Violation) + }) + + t.Run("regex guardrail without compiled matcher returns error on matching phase", func(t *testing.T) { + guardrails := []filterapi.RuntimeGuardrail{{ + Name: "missing-matcher", + Phase: filterapi.GuardrailPhaseRequest, + Provider: filterapi.GuardrailProvider{ + Type: filterapi.GuardrailProviderTypeRegex, + }, + }} + + outcome, err := evaluateRequestGuardrails(t.Context(), guardrails, []byte("forbidden")) + require.Error(t, err) + require.Nil(t, outcome.Violation) + require.Contains(t, err.Error(), "uses regex provider without a compiled matcher") + }) +} + +func TestRequestGuardrailBlockRecordsMetric(t *testing.T) { + recorder := &recordingGuardrailMetrics{} + config := &filterapi.RuntimeConfig{ + Guardrails: []filterapi.RuntimeGuardrail{{ + Name: "deny-pii", + Phase: filterapi.GuardrailPhaseRequest, + Provider: filterapi.GuardrailProvider{ + Type: filterapi.GuardrailProviderTypeRegex, + }, + Matcher: regexp.MustCompile(`SSN`), + }}, + } + factory := NewFactory(nil, recorder, tracingapi.NoopChatCompletionTracer{}, endpointspec.ChatCompletionsEndpointSpec{}) + processor, err := factory(config, map[string]string{ + "content-type": "application/json", + ":path": "/v1/chat/completions", + }, slog.Default(), false, false) + require.NoError(t, err) + + response, err := processor.ProcessRequestBody(t.Context(), &extprocv3.HttpBody{ + Body: []byte(`{"model":"test","messages":[{"role":"user","content":"customer SSN"}]}`), + }) + require.NoError(t, err) + require.NotNil(t, response.GetImmediateResponse()) + require.Equal(t, 1, recorder.count) + require.Equal(t, string(filterapi.GuardrailPhaseRequest), recorder.phase) + require.Equal(t, metrics.GuardrailResultBlocked, recorder.result) +} + +func TestRequestGuardrailMaskMutatesBody(t *testing.T) { + recorder := &recordingGuardrailMetrics{} + config := &filterapi.RuntimeConfig{Guardrails: []filterapi.RuntimeGuardrail{{ + Name: "mask-email", Phase: filterapi.GuardrailPhaseRequest, + Provider: filterapi.GuardrailProvider{ + Type: filterapi.GuardrailProviderTypeRegex, Action: filterapi.GuardrailActionMask, + MaskReplacement: "[EMAIL]", + }, + Matcher: regexp.MustCompile(`alice@example\.com`), + }}} + factory := NewFactory(nil, recorder, tracingapi.NoopChatCompletionTracer{}, endpointspec.ChatCompletionsEndpointSpec{}) + processor, err := factory(config, map[string]string{ + "content-type": "application/json", + ":path": "/v1/chat/completions", + }, slog.Default(), false, false) + require.NoError(t, err) + + response, err := processor.ProcessRequestBody(t.Context(), &extprocv3.HttpBody{ + Body: []byte(`{"model":"test","messages":[{"role":"user","content":"email alice@example.com"}]}`), + }) + require.NoError(t, err) + require.Nil(t, response.GetImmediateResponse()) + require.Nil(t, response.GetRequestBody().Response.BodyMutation) + processorImpl := processor.(*chatCompletionProcessorRouterFilter) + require.JSONEq(t, `{"model":"test","messages":[{"role":"user","content":"email [EMAIL]"}]}`, string(processorImpl.originalRequestBodyRaw)) + require.Equal(t, metrics.GuardrailResultMasked, recorder.result) +} + +func TestRecordGuardrailEvaluationIgnoresUnconfiguredPhase(t *testing.T) { + recorder := &recordingGuardrailMetrics{} + span := &mockGuardrailChatCompletionSpan{} + processor := &chatCompletionProcessorRouterFilter{ + config: &filterapi.RuntimeConfig{Guardrails: []filterapi.RuntimeGuardrail{{ + Phase: filterapi.GuardrailPhaseResponse, + }}}, + guardrailMetrics: recorder, + span: span, + } + + processor.recordGuardrailEvaluation(t.Context(), "", filterapi.GuardrailPhaseRequest, metrics.GuardrailResultAllowed, "", true) + require.Zero(t, recorder.count) + require.Empty(t, span.guardrailEvents) + + processor.recordGuardrailEvaluation(t.Context(), "deny-pii", filterapi.GuardrailPhaseResponse, metrics.GuardrailResultBlocked, "", true) + require.Equal(t, 1, recorder.count) + require.Equal(t, []string{"deny-pii/Response/blocked"}, span.guardrailEvents) +} + +func TestRecordGuardrailEvaluationWithoutGuardrails(t *testing.T) { + span := &mockGuardrailChatCompletionSpan{} + processor := &chatCompletionProcessorRouterFilter{config: &filterapi.RuntimeConfig{}, span: span} + + processor.recordGuardrailEvaluation(t.Context(), "", filterapi.GuardrailPhaseRequest, metrics.GuardrailResultAllowed, "", true) + processor.recordGuardrailEvaluation(t.Context(), "", filterapi.GuardrailPhaseResponse, metrics.GuardrailResultAllowed, "backend", true) + require.Empty(t, span.guardrailEvents) +} + +func TestBackendScopedGuardrail(t *testing.T) { + guardrails := []filterapi.RuntimeGuardrail{{ + Name: "backend-only", Phase: filterapi.GuardrailPhaseRequest, + Backends: []string{"selected-backend"}, + Provider: filterapi.GuardrailProvider{Type: filterapi.GuardrailProviderTypeRegex}, + Matcher: regexp.MustCompile("blocked"), + }} + + outcome, err := evaluateBackendRequestGuardrails(t.Context(), guardrails, []byte("blocked"), "other-backend") + require.NoError(t, err) + require.Nil(t, outcome.Violation) + + outcome, err = evaluateBackendRequestGuardrails(t.Context(), guardrails, []byte("blocked"), "selected-backend") + require.NoError(t, err) + require.NotNil(t, outcome.Violation) +} + +func TestGuardrailFailureModes(t *testing.T) { + guardrail := filterapi.RuntimeGuardrail{ + Name: "external", Phase: filterapi.GuardrailPhaseRequest, + Provider: filterapi.GuardrailProvider{Type: filterapi.GuardrailProviderTypePresidio}, + Evaluator: &failingGuardrailEvaluator{}, + } + + _, err := evaluateRequestGuardrails(t.Context(), []filterapi.RuntimeGuardrail{guardrail}, []byte("payload")) + require.Error(t, err) + require.False(t, isGuardrailFailOpenError(err)) + + guardrail.Provider.FailureMode = filterapi.GuardrailFailureModeFailOpen + outcome, err := evaluateRequestGuardrails(t.Context(), []filterapi.RuntimeGuardrail{guardrail}, []byte("payload")) + require.Nil(t, outcome.Violation) + require.Error(t, err) + require.True(t, isGuardrailFailOpenError(err)) +} + +func TestGuardrailMonitorAndMask(t *testing.T) { + body := []byte(`{"model":"safe-model","messages":[{"role":"user","content":"secret"}]}`) + + t.Run("monitor detects without changing body", func(t *testing.T) { + outcome, err := evaluateRequestGuardrails(t.Context(), []filterapi.RuntimeGuardrail{{ + Name: "monitor", Phase: filterapi.GuardrailPhaseRequest, + Provider: filterapi.GuardrailProvider{Type: filterapi.GuardrailProviderTypeRegex, Action: filterapi.GuardrailActionMonitor}, + Matcher: regexp.MustCompile("secret"), + }}, body) + require.NoError(t, err) + require.True(t, outcome.Monitored) + require.False(t, outcome.Masked) + require.Nil(t, outcome.Violation) + require.Equal(t, body, outcome.Body) + }) + + t.Run("mask replaces extracted content only", func(t *testing.T) { + outcome, err := evaluateRequestGuardrails(t.Context(), []filterapi.RuntimeGuardrail{{ + Name: "mask", Phase: filterapi.GuardrailPhaseRequest, + Provider: filterapi.GuardrailProvider{Type: filterapi.GuardrailProviderTypePresidio, Action: filterapi.GuardrailActionMask}, + Evaluator: &maskingGuardrailEvaluator{}, + }}, body) + require.NoError(t, err) + require.True(t, outcome.Masked) + require.JSONEq(t, `{"model":"safe-model","messages":[{"role":"user","content":"masked:secret"}]}`, string(outcome.Body)) + require.NotContains(t, string(outcome.Body), "masked:safe-model") + }) +} + +func TestGuardrailPayloadLimit(t *testing.T) { + guardrail := filterapi.RuntimeGuardrail{ + Name: "small", Phase: filterapi.GuardrailPhaseRequest, MaxPayloadBytes: 8, + Provider: filterapi.GuardrailProvider{Type: filterapi.GuardrailProviderTypeRegex}, + Matcher: regexp.MustCompile("secret"), + } + _, err := evaluateRequestGuardrails(t.Context(), []filterapi.RuntimeGuardrail{guardrail}, []byte(`{"content":"secret"}`)) + require.ErrorContains(t, err, "exceeding the 8-byte limit") + + guardrail.Provider.FailureMode = filterapi.GuardrailFailureModeFailOpen + _, err = evaluateRequestGuardrails(t.Context(), []filterapi.RuntimeGuardrail{guardrail}, []byte(`{"content":"secret"}`)) + require.True(t, isGuardrailFailOpenError(err)) +} + +func TestGuardrailHTTPProviderActions(t *testing.T) { + body := []byte(`{"model":"safe-model","messages":[{"role":"user","content":"call me at 555-0100"}]}`) + for _, tc := range []struct { + name string + response string + action filterapi.GuardrailAction + expViolation bool + expMonitored bool + expMasked bool + expBody string + expErr string + }{ + {name: "allow passes through", response: `{"action":"allow"}`, expBody: string(body)}, + {name: "block blocks", response: `{"action":"block"}`, expViolation: true, expBody: string(body)}, + {name: "block with monitor records", response: `{"action":"block"}`, action: filterapi.GuardrailActionMonitor, expMonitored: true, expBody: string(body)}, + { + name: "findings masked", + response: `{"action":"modify","findings":[{"type":"PHONE","start":11,"end":19,"score":0.9}]}`, + action: filterapi.GuardrailActionMask, + expMasked: true, + expBody: `{"model":"safe-model","messages":[{"role":"user","content":"call me at [REDACTED]"}]}`, + }, + { + name: "replacement applied", + response: `{"action":"modify","replacement":"call me later"}`, + action: filterapi.GuardrailActionMask, + expMasked: true, + expBody: `{"model":"safe-model","messages":[{"role":"user","content":"call me later"}]}`, + }, + {name: "block without findings cannot mask", response: `{"action":"block"}`, action: filterapi.GuardrailActionMask, expErr: "returned no masked content"}, + } { + t.Run(tc.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(tc.response)) + })) + t.Cleanup(server.Close) + provider := filterapi.GuardrailProvider{ + Type: filterapi.GuardrailProviderTypeHTTP, + Action: tc.action, + HTTP: &filterapi.HTTPGuardrailProvider{Endpoint: server.URL}, + } + evaluator, err := guardrails.NewEvaluator(t.Context(), &provider) + require.NoError(t, err) + + outcome, err := evaluateRequestGuardrails(t.Context(), []filterapi.RuntimeGuardrail{{ + Name: "custom", Phase: filterapi.GuardrailPhaseRequest, Provider: provider, Evaluator: evaluator, + }}, body) + if tc.expErr != "" { + require.ErrorContains(t, err, tc.expErr) + return + } + require.NoError(t, err) + require.Equal(t, tc.expViolation, outcome.Violation != nil) + require.Equal(t, tc.expMonitored, outcome.Monitored) + require.Equal(t, tc.expMasked, outcome.Masked) + require.JSONEq(t, tc.expBody, string(outcome.Body)) + }) + } +} + +// guardrailProcessorCase describes one guardrail outcome exercised through a processor. +type guardrailProcessorCase struct { + name string + guardrail filterapi.RuntimeGuardrail + wantErr string + wantBlock bool + wantResult metrics.GuardrailResult +} + +func guardrailProcessorCases(phase filterapi.GuardrailPhase, backends []string) []guardrailProcessorCase { + regex := func(name string, action filterapi.GuardrailAction) filterapi.RuntimeGuardrail { + return filterapi.RuntimeGuardrail{ + Name: name, Phase: phase, Backends: backends, + Provider: filterapi.GuardrailProvider{ + Type: filterapi.GuardrailProviderTypeRegex, Action: action, MaskReplacement: "[MASKED]", Message: "blocked", + }, + Matcher: regexp.MustCompile("secret"), + } + } + failing := func(mode filterapi.GuardrailFailureMode) filterapi.RuntimeGuardrail { + return filterapi.RuntimeGuardrail{ + Name: "provider", Phase: phase, Backends: backends, + Provider: filterapi.GuardrailProvider{Type: filterapi.GuardrailProviderTypePresidio, FailureMode: mode}, + Evaluator: &failingGuardrailEvaluator{}, + } + } + return []guardrailProcessorCase{ + {name: "block", guardrail: regex("deny", filterapi.GuardrailActionBlock), wantBlock: true, wantResult: metrics.GuardrailResultBlocked}, + {name: "monitor", guardrail: regex("monitor", filterapi.GuardrailActionMonitor), wantResult: metrics.GuardrailResultMonitored}, + {name: "mask", guardrail: regex("mask", filterapi.GuardrailActionMask), wantResult: metrics.GuardrailResultMasked}, + {name: "fail open", guardrail: failing(filterapi.GuardrailFailureModeFailOpen), wantResult: metrics.GuardrailResultError}, + {name: "fail closed", guardrail: failing(filterapi.GuardrailFailureModeFailClosed), wantErr: "guardrails", wantResult: metrics.GuardrailResultError}, + } +} + +func TestRouterRequestGuardrailOutcomes(t *testing.T) { + for _, test := range guardrailProcessorCases(filterapi.GuardrailPhaseRequest, nil) { + t.Run(test.name, func(t *testing.T) { + recorder := &recordingGuardrailMetrics{} + span := &mockGuardrailChatCompletionSpan{} + factory := NewFactory(nil, recorder, &mockTracer{returnedSpan: span}, endpointspec.ChatCompletionsEndpointSpec{}) + processor, err := factory(&filterapi.RuntimeConfig{Guardrails: []filterapi.RuntimeGuardrail{test.guardrail}}, map[string]string{ + "content-type": "application/json", + ":path": "/v1/chat/completions", + }, slog.Default(), false, false) + require.NoError(t, err) + + response, err := processor.ProcessRequestBody(t.Context(), &extprocv3.HttpBody{ + Body: []byte(`{"model":"test","messages":[{"role":"user","content":"my secret"}]}`), + }) + require.Equal(t, test.wantResult, recorder.result) + require.Len(t, span.guardrailEvents, 1) + if test.wantErr != "" { + require.ErrorContains(t, err, "failed to evaluate request guardrails") + return + } + require.NoError(t, err) + require.Equal(t, test.wantBlock, response.GetImmediateResponse() != nil) + if test.wantBlock { + require.Equal(t, 1, span.endedOnErrorCount) + require.Equal(t, http.StatusForbidden, span.errorStatusCode) + return + } + if test.wantResult == metrics.GuardrailResultMasked { + require.Contains(t, string(processor.(*chatCompletionProcessorRouterFilter).originalRequestBodyRaw), "my [MASKED]") + } + }) + } +} + +func TestUpstreamRequestGuardrailOutcomes(t *testing.T) { + body := []byte(`{"model":"test","messages":[{"role":"user","content":"my secret"}]}`) + for _, test := range guardrailProcessorCases(filterapi.GuardrailPhaseRequest, []string{"backend-a"}) { + t.Run(test.name, func(t *testing.T) { + recorder := &recordingGuardrailMetrics{} + span := &mockGuardrailChatCompletionSpan{} + parent := &chatCompletionProcessorRouterFilter{ + config: &filterapi.RuntimeConfig{Guardrails: []filterapi.RuntimeGuardrail{test.guardrail}}, + logger: slog.Default(), + originalRequestBodyRaw: body, + originalModel: "test", + guardrailMetrics: recorder, + span: span, + } + p := &chatCompletionProcessorUpstreamFilter{ + parent: parent, + logger: slog.Default(), + requestHeaders: map[string]string{":path": "/v1/chat/completions"}, + metrics: &mockMetrics{}, + backendName: "backend-a", + // Stop right after the guardrail evaluation, before translation. + unsupportedBackendErr: errors.New("stop after guardrails"), + } + + response, err := p.ProcessRequestHeaders(t.Context(), nil) + require.Equal(t, test.wantResult, recorder.result) + require.Len(t, span.guardrailEvents, 1) + if test.wantErr != "" { + require.ErrorContains(t, err, "failed to evaluate backend request guardrails") + return + } + require.NoError(t, err) + if test.wantBlock { + require.Contains(t, string(response.GetImmediateResponse().GetBody()), "GuardrailViolation") + require.Equal(t, 1, span.endedOnErrorCount) + return + } + require.Contains(t, string(response.GetImmediateResponse().GetBody()), "stop after guardrails") + if test.wantResult == metrics.GuardrailResultMasked { + require.Contains(t, string(parent.originalRequestBodyRaw), "my [MASKED]") + require.True(t, parent.forceBodyMutation) + } + }) + } +} + +func TestUpstreamResponseGuardrailOutcomes(t *testing.T) { + responseBody := []byte(`{"choices":[{"index":0,"message":{"role":"assistant","content":"my secret"}}]}`) + for _, test := range guardrailProcessorCases(filterapi.GuardrailPhaseResponse, []string{"backend-a"}) { + t.Run(test.name, func(t *testing.T) { + recorder := &recordingGuardrailMetrics{} + span := &mockGuardrailChatCompletionSpan{} + p := &chatCompletionProcessorUpstreamFilter{ + parent: &chatCompletionProcessorRouterFilter{ + config: &filterapi.RuntimeConfig{Guardrails: []filterapi.RuntimeGuardrail{test.guardrail}}, + logger: slog.Default(), + guardrailMetrics: recorder, + span: span, + }, + logger: slog.Default(), + translator: &mockTranslator{t: t, retBodyMutation: responseBody}, + metrics: &mockMetrics{}, + backendName: "backend-a", + responseHeaders: map[string]string{":status": "200"}, + } + + response, err := p.ProcessResponseBody(t.Context(), &extprocv3.HttpBody{Body: responseBody, EndOfStream: true}) + require.Equal(t, test.wantResult, recorder.result) + require.Len(t, span.guardrailEvents, 1) + if test.wantErr != "" { + require.ErrorContains(t, err, "failed to evaluate response guardrails") + return + } + require.NoError(t, err) + if test.wantBlock { + require.Contains(t, string(response.GetImmediateResponse().GetBody()), "GuardrailViolation") + return + } + returnedBody := response.GetResponseBody().GetResponse().GetBodyMutation().GetBody() + if test.wantResult == metrics.GuardrailResultMasked { + require.Contains(t, string(returnedBody), "my [MASKED]") + return + } + require.Equal(t, responseBody, returnedBody) + }) + } +} + +func TestEvaluateGuardrailsForPhaseEdgeCases(t *testing.T) { + body := []byte(`{"model":"test","messages":[{"role":"user","content":"my secret"}]}`) + + t.Run("regex block without match", func(t *testing.T) { + outcome, err := evaluateRequestGuardrails(t.Context(), []filterapi.RuntimeGuardrail{{ + Name: "deny", Phase: filterapi.GuardrailPhaseRequest, + Provider: filterapi.GuardrailProvider{Type: filterapi.GuardrailProviderTypeRegex}, + Matcher: regexp.MustCompile("forbidden"), + }}, body) + require.NoError(t, err) + require.Nil(t, outcome.Violation) + require.False(t, outcome.Monitored) + }) + + t.Run("regex mask uses default replacement", func(t *testing.T) { + outcome, err := evaluateRequestGuardrails(t.Context(), []filterapi.RuntimeGuardrail{{ + Name: "mask", Phase: filterapi.GuardrailPhaseRequest, + Provider: filterapi.GuardrailProvider{Type: filterapi.GuardrailProviderTypeRegex, Action: filterapi.GuardrailActionMask}, + Matcher: regexp.MustCompile("secret"), + }}, body) + require.NoError(t, err) + require.True(t, outcome.Masked) + require.Contains(t, string(outcome.Body), "my [REDACTED]") + }) + + t.Run("regex mask without compiled matcher", func(t *testing.T) { + _, err := evaluateRequestGuardrails(t.Context(), []filterapi.RuntimeGuardrail{{ + Name: "mask", Phase: filterapi.GuardrailPhaseRequest, + Provider: filterapi.GuardrailProvider{Type: filterapi.GuardrailProviderTypeRegex, Action: filterapi.GuardrailActionMask}, + }}, body) + require.ErrorContains(t, err, `guardrail "mask" uses regex provider without a compiled matcher`) + }) + + t.Run("external provider without evaluator", func(t *testing.T) { + _, err := evaluateRequestGuardrails(t.Context(), []filterapi.RuntimeGuardrail{{ + Name: "presidio", Phase: filterapi.GuardrailPhaseRequest, + Provider: filterapi.GuardrailProvider{Type: filterapi.GuardrailProviderTypePresidio}, + }}, body) + require.ErrorContains(t, err, `guardrail "presidio" uses provider "Presidio" without an evaluator`) + }) +} + +func TestReplaceGuardrailContentWholeBody(t *testing.T) { + replaced, err := replaceGuardrailContent([]byte("plain text"), guardrailContent{}, []byte("masked")) + require.NoError(t, err) + require.Equal(t, "masked", string(replaced)) +} diff --git a/internal/extproc/mocks_test.go b/internal/extproc/mocks_test.go index 7ccc14e62c..8e02763c12 100644 --- a/internal/extproc/mocks_test.go +++ b/internal/extproc/mocks_test.go @@ -325,6 +325,18 @@ func (m *mockChatCompletionSpan) EndSpan() { m.endedCount++ } var _ tracingapi.ChatCompletionSpan = &mockChatCompletionSpan{} +// mockGuardrailChatCompletionSpan additionally implements [tracingapi.GuardrailSpan]. +type mockGuardrailChatCompletionSpan struct { + mockChatCompletionSpan + guardrailEvents []string +} + +func (m *mockGuardrailChatCompletionSpan) RecordGuardrail(name, phase, result string) { + m.guardrailEvents = append(m.guardrailEvents, name+"/"+phase+"/"+result) +} + +var _ tracingapi.GuardrailSpan = &mockGuardrailChatCompletionSpan{} + // mockBackendChatCompletionSpan additionally implements [tracingapi.BackendSpan], // which only spans whose semantic convention records the resolved backend do. type mockBackendChatCompletionSpan struct { diff --git a/internal/extproc/processor_impl.go b/internal/extproc/processor_impl.go index d6f9929b2c..f9903a57c8 100644 --- a/internal/extproc/processor_impl.go +++ b/internal/extproc/processor_impl.go @@ -13,6 +13,7 @@ import ( "fmt" "io" "log/slog" + "net/http" "strconv" "strings" @@ -58,13 +59,14 @@ var LogRequestHeaderAttributes map[string]string // * ProcessorFactory: A factory function to create processors based on the configuration. func NewFactory[ReqT any, RespT any, RespChunkT any, EndpointSpecT endpointspec.Spec[ReqT, RespT, RespChunkT]]( f metrics.Factory, + guardrailMetrics metrics.GuardrailMetrics, tracer tracingapi.RequestTracer[ReqT, RespT, RespChunkT], _ EndpointSpecT, // This is a type marker to bind EndpointSpecT without specifying ReqT, RespT, RespChunkT explicitly. ) ProcessorFactory { return func(config *filterapi.RuntimeConfig, requestHeaders map[string]string, logger *slog.Logger, isUpstreamFilter bool, enableRedaction bool) (Processor, error) { logger = logger.With("isUpstreamFilter", fmt.Sprintf("%v", isUpstreamFilter)) if !isUpstreamFilter { - return newRouterProcessor[ReqT, RespT, RespChunkT, EndpointSpecT](config, requestHeaders, logger, tracer, enableRedaction), nil + return newRouterProcessor[ReqT, RespT, RespChunkT, EndpointSpecT](config, requestHeaders, logger, guardrailMetrics, tracer, enableRedaction), nil } return newUpstreamProcessor[ReqT, RespT, RespChunkT, EndpointSpecT](requestHeaders, f.NewMetrics(), logger), nil } @@ -96,6 +98,8 @@ type ( forceBodyMutation bool // tracer is the tracer used for requests. tracer tracingapi.RequestTracer[ReqT, RespT, RespChunkT] + // guardrailMetrics records guardrail evaluation outcomes. + guardrailMetrics metrics.GuardrailMetrics // span is the tracing span for this request, created in ProcessRequestBody. span tracingapi.Span[RespT, RespChunkT] // upstreamFilterCount is the number of upstream filters that have been processed. @@ -143,6 +147,7 @@ func newRouterProcessor[ReqT, RespT, RespChunkT any, EndpointSpecT endpointspec. config *filterapi.RuntimeConfig, requestHeaders map[string]string, logger *slog.Logger, + guardrailMetrics metrics.GuardrailMetrics, tracer tracingapi.RequestTracer[ReqT, RespT, RespChunkT], enableRedaction bool, ) *routerProcessor[ReqT, RespT, RespChunkT, EndpointSpecT] { @@ -151,6 +156,7 @@ func newRouterProcessor[ReqT, RespT, RespChunkT any, EndpointSpecT endpointspec. config: config, requestHeaders: requestHeaders, logger: logger, + guardrailMetrics: guardrailMetrics, tracer: tracer, forceBodyMutation: false, debugLogEnabled: debugLogEnabled, @@ -158,6 +164,27 @@ func newRouterProcessor[ReqT, RespT, RespChunkT any, EndpointSpecT endpointspec. } } +// recordGuardrailEvaluation records the guardrail metric and span event, only when guardrails are +// configured for the phase and backend so that unguarded traffic is not reported as evaluated. +func (r *routerProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) recordGuardrailEvaluation( + ctx context.Context, + name string, + phase filterapi.GuardrailPhase, + result metrics.GuardrailResult, + backendName string, + includeGlobal bool, +) { + if r.config == nil || !guardrailsConfiguredForPhase(r.config.Guardrails, phase, backendName, includeGlobal) { + return + } + if r.guardrailMetrics != nil { + r.guardrailMetrics.RecordEvaluation(ctx, string(phase), result) + } + if span, ok := r.span.(tracingapi.GuardrailSpan); ok { + span.RecordGuardrail(name, string(phase), string(result)) + } +} + func newUpstreamProcessor[ReqT, RespT, RespChunkT any, EndpointSpecT endpointspec.Spec[ReqT, RespT, RespChunkT]]( reqHeader map[string]string, metrics metrics.Metrics, logger *slog.Logger, @@ -216,6 +243,10 @@ func (u *upstreamProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) respondLocal // createUserFacingErrorResponse creates an ImmediateResponse for user-facing errors with JSON body. func createUserFacingErrorResponse(statusCode int, errorType string, message string) *extprocv3.ProcessingResponse { body := formatUserFacingErrorJSON(errorType, statusCode, message) + grpcStatus := codes.InvalidArgument + if statusCode == http.StatusForbidden { + grpcStatus = codes.PermissionDenied + } headerMutation := &extprocv3.HeaderMutation{} setHeader(headerMutation, "content-type", "application/json") setHeader(headerMutation, "content-length", strconv.Itoa(len(body))) @@ -226,7 +257,7 @@ func createUserFacingErrorResponse(statusCode int, errorType string, message str Status: &typev3.HttpStatus{Code: typev3.StatusCode(statusCode)}, // #nosec G115 - HTTP status codes are always in valid int32 range Headers: headerMutation, Body: body, - GrpcStatus: &extprocv3.GrpcStatus{Status: uint32(codes.InvalidArgument)}, + GrpcStatus: &extprocv3.GrpcStatus{Status: uint32(grpcStatus)}, }, }, } @@ -268,11 +299,11 @@ func (r *routerProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) ProcessRequest // Only log parsed request body when redaction is enabled if r.debugLogEnabled && r.enableRedaction { - if redactedBody, err := r.eh.RedactSensitiveInfoFromRequest(body); err != nil { - logger.Warn("failed to redact sensitive info from request, ignoring and continuing", slog.Any("error", err)) + if redactedBody, redactionErr := r.eh.RedactSensitiveInfoFromRequest(body); redactionErr != nil { + logger.Warn("failed to redact sensitive info from request, ignoring and continuing", slog.Any("error", redactionErr)) } else { - if jsonBody, err := json.Marshal(redactedBody); err != nil { - logger.Error("failed to marshal redacted request for logging, ignoring and continuing", slog.Any("error", err)) + if jsonBody, marshalErr := json.Marshal(redactedBody); marshalErr != nil { + logger.Error("failed to marshal redacted request for logging, ignoring and continuing", slog.Any("error", marshalErr)) } else { logger.Debug("request body processing", slog.Any("request", string(jsonBody))) } @@ -325,6 +356,37 @@ func (r *routerProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) ProcessRequest body, rawBody.Body, ) + outcome, guardrailErr := evaluateRequestGuardrails(ctx, r.config.Guardrails, rawBody.Body) + if guardrailErr != nil { + r.recordGuardrailEvaluation(ctx, "", filterapi.GuardrailPhaseRequest, metrics.GuardrailResultError, "", true) + if !isGuardrailFailOpenError(guardrailErr) { + return nil, fmt.Errorf("failed to evaluate request guardrails: %w", guardrailErr) + } + r.logger.Warn("request guardrail provider failed open", slog.String("error", guardrailErr.Error())) + } else if outcome.Violation != nil { + r.recordGuardrailEvaluation(ctx, outcome.Violation.Name, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultBlocked, "", true) + r.logger.Warn("request blocked by guardrail", + slog.String("guardrail.name", outcome.Violation.Name), + slog.String("guardrail.phase", string(filterapi.GuardrailPhaseRequest))) + response := createUserFacingErrorResponse(http.StatusForbidden, "GuardrailViolation", outcome.Violation.Message) + if r.span != nil { + r.span.EndSpanOnError(http.StatusForbidden, response.GetImmediateResponse().GetBody()) + } + return response, nil + } + switch { + case outcome.Masked: + if err = r.updateParsedRequestAfterGuardrailMask(outcome.Body, costConfigured); err != nil { + return nil, err + } + r.recordGuardrailEvaluation(ctx, outcome.RuleName, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultMasked, "", true) + r.logger.Info("request masked by guardrail", slog.String("guardrail.name", outcome.RuleName)) + case outcome.Monitored: + r.recordGuardrailEvaluation(ctx, outcome.RuleName, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultMonitored, "", true) + r.logger.Info("request detected by monitor guardrail", slog.String("guardrail.name", outcome.RuleName)) + case guardrailErr == nil: + r.recordGuardrailEvaluation(ctx, "", filterapi.GuardrailPhaseRequest, metrics.GuardrailResultAllowed, "", true) + } return &extprocv3.ProcessingResponse{ Response: &extprocv3.ProcessingResponse_RequestBody{ @@ -338,6 +400,22 @@ func (r *routerProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) ProcessRequest }, nil } +func (r *routerProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) updateParsedRequestAfterGuardrailMask(maskedBody []byte, costConfigured bool) error { + originalModel, parsedBody, stream, mutatedBody, err := r.eh.ParseBody(maskedBody, costConfigured) + if err != nil { + return fmt.Errorf("failed to parse guardrail-masked request body: %w", err) + } + r.originalModel = originalModel + r.originalRequestBody = parsedBody + r.stream = stream + r.originalRequestBodyRaw = maskedBody + if mutatedBody != nil { + r.originalRequestBodyRaw = mutatedBody + } + r.forceBodyMutation = true + return nil +} + func (u *upstreamProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) onRetry() bool { return u.parent.upstreamFilterCount > 1 } @@ -362,6 +440,36 @@ func (u *upstreamProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) ProcessReque // Set the request model for metrics from the original model or override if applied. reqModel := cmp.Or(u.requestHeaders[internalapi.ModelNameHeaderKeyDefault], u.parent.originalModel) u.metrics.SetRequestModel(reqModel) + var configuredGuardrails []filterapi.RuntimeGuardrail + if u.parent.config != nil { + configuredGuardrails = u.parent.config.Guardrails + } + outcome, guardrailErr := evaluateBackendRequestGuardrails(ctx, configuredGuardrails, u.parent.originalRequestBodyRaw, u.backendName) + if guardrailErr != nil { + u.parent.recordGuardrailEvaluation(ctx, "", filterapi.GuardrailPhaseRequest, metrics.GuardrailResultError, u.backendName, false) + if !isGuardrailFailOpenError(guardrailErr) { + return nil, fmt.Errorf("failed to evaluate backend request guardrails: %w", guardrailErr) + } + u.logger.Warn("request guardrail provider failed open", slog.String("error", guardrailErr.Error())) + } else if outcome.Violation != nil { + u.parent.recordGuardrailEvaluation(ctx, outcome.Violation.Name, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultBlocked, u.backendName, false) + u.logger.Warn("request blocked by guardrail", slog.String("guardrail.name", outcome.Violation.Name), slog.String("guardrail.phase", string(filterapi.GuardrailPhaseRequest))) + return u.respondLocally(ctx, http.StatusForbidden, "GuardrailViolation", outcome.Violation.Message), nil + } + switch { + case outcome.Masked: + costConfigured := len(u.parent.config.RequestCosts) > 0 || len(u.parent.config.GlobalRequestCosts) > 0 + if err = u.parent.updateParsedRequestAfterGuardrailMask(outcome.Body, costConfigured); err != nil { + return nil, err + } + u.parent.recordGuardrailEvaluation(ctx, outcome.RuleName, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultMasked, u.backendName, false) + u.logger.Info("request masked by guardrail", slog.String("guardrail.name", outcome.RuleName)) + case outcome.Monitored: + u.parent.recordGuardrailEvaluation(ctx, outcome.RuleName, filterapi.GuardrailPhaseRequest, metrics.GuardrailResultMonitored, u.backendName, false) + u.logger.Info("request detected by monitor guardrail", slog.String("guardrail.name", outcome.RuleName)) + case guardrailErr == nil: + u.parent.recordGuardrailEvaluation(ctx, "", filterapi.GuardrailPhaseRequest, metrics.GuardrailResultAllowed, u.backendName, false) + } if u.unsupportedBackendErr != nil { return u.respondLocally(ctx, 422, "UnprocessableEntity", u.unsupportedBackendErr.Error()), nil @@ -518,7 +626,8 @@ func (u *upstreamProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) ProcessRespo return nil, fmt.Errorf("failed to transform response headers: %w", err) } var mode *extprocv3http.ProcessingMode - if u.parent.stream && u.responseHeaders[":status"] == "200" { + hasResponseGuardrails := u.parent.config != nil && guardrailsRequireBufferedResponse(u.parent.config.Guardrails, u.backendName) + if u.parent.stream && u.responseHeaders[":status"] == "200" && !hasResponseGuardrails { // We only stream the response if the status code is 200 and the response is a stream. mode = &extprocv3http.ProcessingMode{ResponseBodyMode: extprocv3http.ProcessingMode_STREAMED} } @@ -619,7 +728,11 @@ func (u *upstreamProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) ProcessRespo }, nil } - newHeaders, newBody, tokenUsage, responseModel, err := u.translator.ResponseBody(u.responseHeaders, decodingResult.reader, body.EndOfStream, u.parent.span) + decodedBody, err := io.ReadAll(decodingResult.reader) + if err != nil { + return nil, fmt.Errorf("failed to read decoded response body: %w", err) + } + newHeaders, newBody, tokenUsage, responseModel, err := u.translator.ResponseBody(u.responseHeaders, bytes.NewReader(decodedBody), body.EndOfStream, u.parent.span) if err != nil { var streamErr *translator.StreamOverloadedError if u.parent.stream && errors.As(err, &streamErr) && len(newBody) > 0 { @@ -649,9 +762,6 @@ func (u *upstreamProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) ProcessRespo } headerMutation, bodyMutation := mutationsFromTranslationResult(newHeaders, newBody) - // Remove content-encoding header if original body encoded but was mutated in the processor. - headerMutation = removeContentEncodingIfNeeded(headerMutation, bodyMutation, decodingResult.isEncoded) - resp := &extprocv3.ProcessingResponse{ Response: &extprocv3.ProcessingResponse_ResponseBody{ ResponseBody: &extprocv3.BodyResponse{ @@ -666,6 +776,46 @@ func (u *upstreamProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) ProcessRespo // Translator reports the latest cumulative token usage which we use to override existing costs. u.costs.Override(tokenUsage) + guardrailBody := decodedBody + if len(newBody) > 0 { + guardrailBody = newBody + } + var configuredGuardrails []filterapi.RuntimeGuardrail + if u.parent.config != nil { + configuredGuardrails = u.parent.config.Guardrails + } + outcome, guardrailErr := evaluateResponseGuardrails(ctx, configuredGuardrails, guardrailBody, u.backendName) + if guardrailErr != nil { + u.parent.recordGuardrailEvaluation(ctx, "", filterapi.GuardrailPhaseResponse, metrics.GuardrailResultError, u.backendName, true) + if !isGuardrailFailOpenError(guardrailErr) { + return nil, fmt.Errorf("failed to evaluate response guardrails: %w", guardrailErr) + } + u.logger.Warn("response guardrail provider failed open", slog.String("error", guardrailErr.Error())) + } else if outcome.Violation != nil { + u.parent.recordGuardrailEvaluation(ctx, outcome.Violation.Name, filterapi.GuardrailPhaseResponse, metrics.GuardrailResultBlocked, u.backendName, true) + u.logger.Warn("response blocked by guardrail", + slog.String("guardrail.name", outcome.Violation.Name), + slog.String("guardrail.phase", string(filterapi.GuardrailPhaseResponse))) + return u.respondLocally(ctx, http.StatusForbidden, "GuardrailViolation", outcome.Violation.Message), nil + } + switch { + case outcome.Masked: + bodyMutation = &extprocv3.BodyMutation{Mutation: &extprocv3.BodyMutation_Body{Body: outcome.Body}} + resp.GetResponseBody().Response.BodyMutation = bodyMutation + setHeader(headerMutation, "content-length", strconv.Itoa(len(outcome.Body))) + u.parent.recordGuardrailEvaluation(ctx, outcome.RuleName, filterapi.GuardrailPhaseResponse, metrics.GuardrailResultMasked, u.backendName, true) + u.logger.Info("response masked by guardrail", slog.String("guardrail.name", outcome.RuleName)) + case outcome.Monitored: + u.parent.recordGuardrailEvaluation(ctx, outcome.RuleName, filterapi.GuardrailPhaseResponse, metrics.GuardrailResultMonitored, u.backendName, true) + u.logger.Info("response detected by monitor guardrail", slog.String("guardrail.name", outcome.RuleName)) + case guardrailErr == nil: + u.parent.recordGuardrailEvaluation(ctx, "", filterapi.GuardrailPhaseResponse, metrics.GuardrailResultAllowed, u.backendName, true) + } + + // Remove content-encoding when translation or guardrail masking produces an uncompressed body. + headerMutation = removeContentEncodingIfNeeded(headerMutation, bodyMutation, decodingResult.isEncoded) + resp.GetResponseBody().Response.HeaderMutation = headerMutation + // Set the response model for metrics u.metrics.SetResponseModel(responseModel) diff --git a/internal/extproc/processor_impl_test.go b/internal/extproc/processor_impl_test.go index 441edae767..c9fd0b95b9 100644 --- a/internal/extproc/processor_impl_test.go +++ b/internal/extproc/processor_impl_test.go @@ -49,7 +49,7 @@ func TestNewFactory(t *testing.T) { t.Run("router", func(t *testing.T) { t.Parallel() - factory := NewFactory(nil, tracingapi.NoopChatCompletionTracer{}, endpointspec.ChatCompletionsEndpointSpec{}) + factory := NewFactory(nil, nil, tracingapi.NoopChatCompletionTracer{}, endpointspec.ChatCompletionsEndpointSpec{}) proc, err := factory(cfg, headers, slog.Default(), false, false) require.NoError(t, err) require.IsType(t, &chatCompletionProcessorRouterFilter{}, proc) @@ -64,7 +64,7 @@ func TestNewFactory(t *testing.T) { t.Run("upstream", func(t *testing.T) { t.Parallel() - factory := NewFactory(&mockMetricsFactory{}, tracingapi.NoopChatCompletionTracer{}, endpointspec.ChatCompletionsEndpointSpec{}) + factory := NewFactory(&mockMetricsFactory{}, nil, tracingapi.NoopChatCompletionTracer{}, endpointspec.ChatCompletionsEndpointSpec{}) proc, err := factory(cfg, headers, slog.Default(), true, false) require.NoError(t, err) require.IsType(t, &chatCompletionProcessorUpstreamFilter{}, proc) @@ -352,6 +352,41 @@ func Test_chatCompletionProcessorUpstreamFilter_ProcessResponseHeaders(t *testin require.Equal(t, []string{"content-length"}, commonRes.HeaderMutation.RemoveHeaders) require.Equal(t, &extprocv3http.ProcessingMode{ResponseBodyMode: extprocv3http.ProcessingMode_STREAMED}, res.ModeOverride) }) + t.Run("streaming with response guardrail remains buffered", func(t *testing.T) { + inHeaders := &corev3.HeaderMap{Headers: []*corev3.HeaderValue{{Key: ":status", Value: "200"}}} + mt := &mockTranslator{t: t, expHeaders: map[string]string{":status": "200"}} + p := &chatCompletionProcessorUpstreamFilter{ + translator: mt, + metrics: &mockMetrics{}, + parent: &chatCompletionProcessorRouterFilter{ + stream: true, + config: &filterapi.RuntimeConfig{Guardrails: []filterapi.RuntimeGuardrail{{ + Phase: filterapi.GuardrailPhaseResponse, + }}}, + }, + } + res, err := p.ProcessResponseHeaders(t.Context(), inHeaders) + require.NoError(t, err) + require.Nil(t, res.ModeOverride) + }) + t.Run("streaming with monitor-only response guardrail remains buffered", func(t *testing.T) { + inHeaders := &corev3.HeaderMap{Headers: []*corev3.HeaderValue{{Key: ":status", Value: "200"}}} + mt := &mockTranslator{t: t, expHeaders: map[string]string{":status": "200"}} + p := &chatCompletionProcessorUpstreamFilter{ + translator: mt, + metrics: &mockMetrics{}, + parent: &chatCompletionProcessorRouterFilter{ + stream: true, + config: &filterapi.RuntimeConfig{Guardrails: []filterapi.RuntimeGuardrail{{ + Phase: filterapi.GuardrailPhaseResponse, + Provider: filterapi.GuardrailProvider{Action: filterapi.GuardrailActionMonitor}, + }}}, + }, + } + res, err := p.ProcessResponseHeaders(t.Context(), inHeaders) + require.NoError(t, err) + require.Nil(t, res.ModeOverride) + }) t.Run("error/streaming", func(t *testing.T) { inHeaders := &corev3.HeaderMap{ Headers: []*corev3.HeaderValue{{Key: ":status", Value: "500"}, {Key: "dog", RawValue: []byte("cat")}}, diff --git a/internal/extproc/server.go b/internal/extproc/server.go index 5ee1ba4789..c44a080db6 100644 --- a/internal/extproc/server.go +++ b/internal/extproc/server.go @@ -27,6 +27,7 @@ import ( "github.com/envoyproxy/ai-gateway/internal/backendauth" "github.com/envoyproxy/ai-gateway/internal/filterapi" + "github.com/envoyproxy/ai-gateway/internal/guardrails" "github.com/envoyproxy/ai-gateway/internal/internalapi" "github.com/envoyproxy/ai-gateway/internal/redaction" ) @@ -83,7 +84,7 @@ func NewServer(logger *slog.Logger, enableRedaction bool) (*Server, error) { // LoadConfig updates the configuration of the external processor. func (s *Server) LoadConfig(ctx context.Context, config *filterapi.Config) error { - newConfig, err := filterapi.NewRuntimeConfig(ctx, config, backendauth.NewHandler) + newConfig, err := filterapi.NewRuntimeConfig(ctx, config, backendauth.NewHandler, guardrails.NewEvaluator) if err != nil { return fmt.Errorf("cannot create runtime filter config: %w", err) } diff --git a/internal/filterapi/filterconfig.go b/internal/filterapi/filterconfig.go index 73d16ec9b1..419187312d 100644 --- a/internal/filterapi/filterconfig.go +++ b/internal/filterapi/filterconfig.go @@ -52,8 +52,116 @@ type Config struct { UnscopedModels []Model `json:"unscopedModels,omitempty"` // MCPConfig is the configuration for the MCPRoute implementations. MCPConfig *MCPConfig `json:"mcpConfig,omitempty"` + // Guardrails are content-safety checks evaluated before the request is forwarded upstream or before the response is returned to the client. + Guardrails []Guardrail `json:"guardrails,omitempty"` } +// Guardrail specifies a content-safety rule to evaluate. +type Guardrail struct { + Name string `json:"name"` + Phase GuardrailPhase `json:"phase"` + Provider GuardrailProvider `json:"provider"` + // Backends scopes this guardrail to generated backend names. Empty means global. + Backends []string `json:"backends,omitempty"` + // MaxPayloadBytes bounds evaluation and response buffering for this rule. + MaxPayloadBytes int64 `json:"maxPayloadBytes,omitempty"` +} + +// GuardrailPhase determines when the rule is evaluated. +type GuardrailPhase string + +const ( + GuardrailPhaseRequest GuardrailPhase = "Request" + GuardrailPhaseResponse GuardrailPhase = "Response" +) + +// GuardrailProvider describes the implementation used to evaluate a guardrail. +type GuardrailProvider struct { + Type GuardrailProviderType `json:"type"` + Pattern string `json:"pattern,omitempty"` + Action GuardrailAction `json:"action,omitempty"` + MaskReplacement string `json:"maskReplacement,omitempty"` + Message string `json:"message,omitempty"` + Presidio *PresidioGuardrailProvider `json:"presidio,omitempty"` + Bedrock *BedrockGuardrailProvider `json:"bedrock,omitempty"` + AzureContentSafety *AzureContentSafetyGuardrailProvider `json:"azureContentSafety,omitempty"` + HTTP *HTTPGuardrailProvider `json:"http,omitempty"` + ModelArmor *ModelArmorGuardrailProvider `json:"modelArmor,omitempty"` + TimeoutSeconds int32 `json:"timeoutSeconds,omitempty"` + FailureMode GuardrailFailureMode `json:"failureMode,omitempty"` +} + +// GuardrailFailureMode determines how external provider failures are handled. +type GuardrailFailureMode string + +const ( + GuardrailFailureModeFailClosed GuardrailFailureMode = "FailClosed" + GuardrailFailureModeFailOpen GuardrailFailureMode = "FailOpen" +) + +// PresidioGuardrailProvider configures a Presidio analyzer request. +type PresidioGuardrailProvider struct { + Endpoint string `json:"endpoint"` + Language string `json:"language,omitempty"` + ScoreThresholdPercent int32 `json:"scoreThresholdPercent,omitempty"` + APIKey string `json:"apiKey,omitempty"` +} + +// BedrockGuardrailProvider configures an AWS Bedrock ApplyGuardrail request. +type BedrockGuardrailProvider struct { + Endpoint string `json:"endpoint,omitempty"` + Region string `json:"region"` + GuardrailIdentifier string `json:"guardrailIdentifier"` + GuardrailVersion string `json:"guardrailVersion"` + CredentialFileLiteral string `json:"credentialFileLiteral,omitempty"` +} + +// AzureContentSafetyGuardrailProvider configures an Azure AI Content Safety request. +type AzureContentSafetyGuardrailProvider struct { + Endpoint string `json:"endpoint"` + APIVersion string `json:"apiVersion,omitempty"` + SeverityThreshold *int32 `json:"severityThreshold,omitempty"` + APIKey string `json:"apiKey"` +} + +// HTTPGuardrailProvider configures a request to a custom guardrail service implementing the generic HTTP contract. +type HTTPGuardrailProvider struct { + Endpoint string `json:"endpoint"` + Path string `json:"path,omitempty"` + APIKey string `json:"apiKey,omitempty"` +} + +// ModelArmorGuardrailProvider configures a Google Cloud Model Armor sanitize request. +type ModelArmorGuardrailProvider struct { + Endpoint string `json:"endpoint,omitempty"` + Project string `json:"project"` + Location string `json:"location"` + Template string `json:"template"` + // CredentialsJSON is a service account key JSON. Empty means Application Default Credentials. + CredentialsJSON string `json:"credentialsJSON,omitempty"` +} + +// GuardrailProviderType identifies a guardrail implementation. +type GuardrailProviderType string + +const ( + GuardrailProviderTypeRegex GuardrailProviderType = "Regex" + GuardrailProviderTypePresidio GuardrailProviderType = "Presidio" + GuardrailProviderTypeBedrockGuardrails GuardrailProviderType = "Bedrock" + GuardrailProviderTypeAzureContentSafety GuardrailProviderType = "AzureContentSafety" + GuardrailProviderTypeHTTP GuardrailProviderType = "HTTP" + GuardrailProviderTypeModelArmor GuardrailProviderType = "ModelArmor" +) + +// GuardrailAction is the action taken when the rule matches. +type GuardrailAction string + +const ( + GuardrailActionBlock GuardrailAction = "Block" + GuardrailActionMonitor GuardrailAction = "Monitor" + GuardrailActionMask GuardrailAction = "Mask" +) + // Model corresponds to the OpenAI model object in the OpenAI-compatible APIs // and is used to populate the "/models" endpoint in OpenAI-compatible APIs. type Model struct { diff --git a/internal/filterapi/runtime.go b/internal/filterapi/runtime.go index 3885fb71f9..9f60e208db 100644 --- a/internal/filterapi/runtime.go +++ b/internal/filterapi/runtime.go @@ -8,6 +8,7 @@ package filterapi import ( "context" "fmt" + "regexp" "github.com/google/cel-go/cel" @@ -25,6 +26,20 @@ type BackendAuthHandler interface { // NewBackendAuthHandlerFunc is a function type that creates a new BackendAuthHandler for a given BackendAuth configuration. type NewBackendAuthHandlerFunc func(ctx context.Context, auth *BackendAuth) (BackendAuthHandler, error) +// GuardrailEvaluator evaluates a request or response payload using a configured provider. +type GuardrailEvaluator interface { + Evaluate(ctx context.Context, body []byte, phase GuardrailPhase) (GuardrailEvaluationResult, error) +} + +// GuardrailEvaluationResult is the provider-neutral outcome for one text fragment. +type GuardrailEvaluationResult struct { + Matched bool + Replacement []byte +} + +// NewGuardrailEvaluatorFunc creates an evaluator for an external guardrail provider. +type NewGuardrailEvaluatorFunc func(ctx context.Context, provider *GuardrailProvider) (GuardrailEvaluator, error) + // RuntimeConfig is the runtime filter configuration that is derived from the filterapi.Config. type RuntimeConfig struct { // UUID is the unique identifier of the filter configuration, inherited from filterapi.Config. @@ -45,6 +60,8 @@ type RuntimeConfig struct { UnscopedModels []Model // Backends is the map of backends by name. Backends map[string]*RuntimeBackend + // Guardrails is the list of compiled runtime guardrails. + Guardrails []RuntimeGuardrail } // RuntimeBackend is a filter backend with its auth handler that is derived from the filterapi.Backend configuration. @@ -69,8 +86,23 @@ type RuntimeRequestCost struct { CELProg cel.Program } +// RuntimeGuardrail is a compiled guardrail rule for runtime evaluation. +type RuntimeGuardrail struct { + Name string + Phase GuardrailPhase + Provider GuardrailProvider + Backends []string + MaxPayloadBytes int64 + Matcher *regexp.Regexp + Evaluator GuardrailEvaluator +} + // NewRuntimeConfig creates a new runtime filter configuration from the given filterapi.Config and a function to create backend auth handlers. -func NewRuntimeConfig(ctx context.Context, config *Config, fn NewBackendAuthHandlerFunc) (*RuntimeConfig, error) { +func NewRuntimeConfig(ctx context.Context, config *Config, fn NewBackendAuthHandlerFunc, guardrailEvaluatorFactories ...NewGuardrailEvaluatorFunc) (*RuntimeConfig, error) { + var newGuardrailEvaluator NewGuardrailEvaluatorFunc + if len(guardrailEvaluatorFactories) > 0 { + newGuardrailEvaluator = guardrailEvaluatorFactories[0] + } backends := make(map[string]*RuntimeBackend, len(config.Backends)) for i := range config.Backends { b := &config.Backends[i] @@ -123,6 +155,44 @@ func NewRuntimeConfig(ctx context.Context, config *Config, fn NewBackendAuthHand costs = append(costs, RuntimeRequestCost{LLMRequestCost: c, CELProg: prog}) } + guardrails := make([]RuntimeGuardrail, 0, len(config.Guardrails)) + for i := range config.Guardrails { + g := &config.Guardrails[i] + if g.Provider.Type == GuardrailProviderTypeRegex { + if g.Provider.Pattern == "" { + return nil, fmt.Errorf("guardrail %q uses regex provider without a pattern", g.Name) + } + re, err := regexp.Compile(g.Provider.Pattern) + if err != nil { + return nil, fmt.Errorf("guardrail %q has an invalid regex pattern: %w", g.Name, err) + } + guardrails = append(guardrails, RuntimeGuardrail{ + Name: g.Name, + Phase: g.Phase, + Provider: g.Provider, + Backends: g.Backends, + MaxPayloadBytes: guardrailMaxPayloadBytes(g.MaxPayloadBytes), + Matcher: re, + }) + continue + } + if newGuardrailEvaluator == nil { + return nil, fmt.Errorf("guardrail %q uses provider %q but no evaluator factory is configured", g.Name, g.Provider.Type) + } + evaluator, err := newGuardrailEvaluator(ctx, &g.Provider) + if err != nil { + return nil, fmt.Errorf("cannot create evaluator for guardrail %q: %w", g.Name, err) + } + guardrails = append(guardrails, RuntimeGuardrail{ + Name: g.Name, + Phase: g.Phase, + Provider: g.Provider, + Backends: g.Backends, + MaxPayloadBytes: guardrailMaxPayloadBytes(g.MaxPayloadBytes), + Evaluator: evaluator, + }) + } + return &RuntimeConfig{ UUID: config.UUID, Backends: backends, @@ -131,5 +201,15 @@ func NewRuntimeConfig(ctx context.Context, config *Config, fn NewBackendAuthHand DeclaredModels: config.Models, ModelsByHost: config.ModelsByHost, UnscopedModels: config.UnscopedModels, + Guardrails: guardrails, }, nil } + +const defaultGuardrailMaxPayloadBytes int64 = 10 * 1024 * 1024 + +func guardrailMaxPayloadBytes(configured int64) int64 { + if configured <= 0 { + return defaultGuardrailMaxPayloadBytes + } + return configured +} diff --git a/internal/filterapi/runtime_test.go b/internal/filterapi/runtime_test.go index 1d58be6fd2..981a608de8 100644 --- a/internal/filterapi/runtime_test.go +++ b/internal/filterapi/runtime_test.go @@ -7,6 +7,7 @@ package filterapi import ( "context" + "errors" "testing" "time" @@ -118,6 +119,39 @@ func TestServer_LoadConfig(t *testing.T) { require.Contains(t, err.Error(), "cannot create CEL program for cost") }) + t.Run("guardrail regex config compiles", func(t *testing.T) { + config := &Config{ + Guardrails: []Guardrail{{ + Name: "deny-pii", + Phase: GuardrailPhaseRequest, + Provider: GuardrailProvider{ + Type: GuardrailProviderTypeRegex, + Pattern: `\bSSN\b`, + }, + }}, + } + rc, err := NewRuntimeConfig(t.Context(), config, func(_ context.Context, _ *BackendAuth) (BackendAuthHandler, error) { + return nil, nil + }) + require.NoError(t, err) + require.Len(t, rc.Guardrails, 1) + require.NotNil(t, rc.Guardrails[0].Matcher) + require.True(t, rc.Guardrails[0].Matcher.MatchString("SSN")) + }) + + t.Run("external guardrail creates evaluator", func(t *testing.T) { + config := &Config{Guardrails: []Guardrail{{ + Name: "presidio", Phase: GuardrailPhaseRequest, + Provider: GuardrailProvider{Type: GuardrailProviderTypePresidio}, + }}} + evaluator := &testGuardrailEvaluator{} + rc, err := NewRuntimeConfig(t.Context(), config, + func(_ context.Context, _ *BackendAuth) (BackendAuthHandler, error) { return nil, nil }, + func(_ context.Context, _ *GuardrailProvider) (GuardrailEvaluator, error) { return evaluator, nil }) + require.NoError(t, err) + require.Same(t, evaluator, rc.Guardrails[0].Evaluator) + }) + t.Run("error - route cost with empty RouteName", func(t *testing.T) { config := &Config{ LLMRequestCosts: []LLMRequestCost{ @@ -132,3 +166,62 @@ func TestServer_LoadConfig(t *testing.T) { require.Contains(t, err.Error(), "missing_route") }) } + +type testGuardrailEvaluator struct{} + +func (*testGuardrailEvaluator) Evaluate(context.Context, []byte, GuardrailPhase) (GuardrailEvaluationResult, error) { + return GuardrailEvaluationResult{}, nil +} + +func TestNewRuntimeConfigGuardrails(t *testing.T) { + noBackendAuth := func(context.Context, *BackendAuth) (BackendAuthHandler, error) { return nil, nil } + failingEvaluatorFactory := func(context.Context, *GuardrailProvider) (GuardrailEvaluator, error) { + return nil, errors.New("bad provider") + } + tests := []struct { + name string + guardrail Guardrail + evaluatorFactories []NewGuardrailEvaluatorFunc + wantErr string + }{ + { + name: "regex without pattern", + guardrail: Guardrail{Name: "regex", Provider: GuardrailProvider{Type: GuardrailProviderTypeRegex}}, + wantErr: `guardrail "regex" uses regex provider without a pattern`, + }, + { + name: "invalid regex pattern", + guardrail: Guardrail{Name: "regex", Provider: GuardrailProvider{Type: GuardrailProviderTypeRegex, Pattern: "("}}, + wantErr: `guardrail "regex" has an invalid regex pattern`, + }, + { + name: "external provider without evaluator factory", + guardrail: Guardrail{Name: "presidio", Provider: GuardrailProvider{Type: GuardrailProviderTypePresidio}}, + wantErr: `guardrail "presidio" uses provider "Presidio" but no evaluator factory is configured`, + }, + { + name: "evaluator factory error", + guardrail: Guardrail{Name: "presidio", Provider: GuardrailProvider{Type: GuardrailProviderTypePresidio}}, + evaluatorFactories: []NewGuardrailEvaluatorFunc{failingEvaluatorFactory}, + wantErr: `cannot create evaluator for guardrail "presidio": bad provider`, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + config := &Config{Guardrails: []Guardrail{test.guardrail}} + _, err := NewRuntimeConfig(t.Context(), config, noBackendAuth, test.evaluatorFactories...) + require.ErrorContains(t, err, test.wantErr) + }) + } + + t.Run("max payload bytes", func(t *testing.T) { + config := &Config{Guardrails: []Guardrail{ + {Name: "default", Provider: GuardrailProvider{Type: GuardrailProviderTypeRegex, Pattern: "a"}}, + {Name: "configured", MaxPayloadBytes: 2048, Provider: GuardrailProvider{Type: GuardrailProviderTypeRegex, Pattern: "a"}}, + }} + rc, err := NewRuntimeConfig(t.Context(), config, noBackendAuth) + require.NoError(t, err) + require.Equal(t, defaultGuardrailMaxPayloadBytes, rc.Guardrails[0].MaxPayloadBytes) + require.Equal(t, int64(2048), rc.Guardrails[1].MaxPayloadBytes) + }) +} diff --git a/internal/guardrails/adapters.go b/internal/guardrails/adapters.go new file mode 100644 index 0000000000..3f56764065 --- /dev/null +++ b/internal/guardrails/adapters.go @@ -0,0 +1,120 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +// Package guardrails implements external content-safety provider adapters. +package guardrails + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "sort" + "strings" + "time" + + "github.com/envoyproxy/ai-gateway/internal/filterapi" + "github.com/envoyproxy/ai-gateway/internal/json" +) + +const defaultTimeoutSeconds int32 = 10 + +// NewEvaluator creates an evaluator for the configured external provider. +func NewEvaluator(ctx context.Context, provider *filterapi.GuardrailProvider) (filterapi.GuardrailEvaluator, error) { + client := &http.Client{Timeout: time.Duration(timeoutSeconds(provider.TimeoutSeconds)) * time.Second} + switch provider.Type { + case filterapi.GuardrailProviderTypePresidio: + return newPresidioEvaluator(provider.Presidio, provider.MaskReplacement, client) + case filterapi.GuardrailProviderTypeBedrockGuardrails: + return newBedrockEvaluator(ctx, provider.Bedrock, client) + case filterapi.GuardrailProviderTypeAzureContentSafety: + return newAzureContentSafetyEvaluator(provider.AzureContentSafety, client) + case filterapi.GuardrailProviderTypeHTTP: + return newHTTPEvaluator(provider.HTTP, provider.MaskReplacement, client) + case filterapi.GuardrailProviderTypeModelArmor: + return newModelArmorEvaluator(ctx, provider.ModelArmor, client) + default: + return nil, fmt.Errorf("unsupported external guardrail provider %q", provider.Type) + } +} + +func timeoutSeconds(configured int32) int32 { + if configured <= 0 { + return defaultTimeoutSeconds + } + return configured +} + +func doJSON(ctx context.Context, client *http.Client, method, endpoint string, body any, mutate func(*http.Request), response any) error { + payload, err := json.Marshal(body) + if err != nil { + return err + } + req, err := http.NewRequestWithContext(ctx, method, endpoint, bytes.NewReader(payload)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + if mutate != nil { + mutate(req) + } + return sendJSON(req, client, response) +} + +func sendJSON(req *http.Request, client *http.Client, response any) error { + resp, err := client.Do(req) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + return fmt.Errorf("provider returned HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(body))) + } + if err = json.NewDecoder(resp.Body).Decode(response); err != nil { + return fmt.Errorf("cannot decode provider response: %w", err) + } + return nil +} + +// textSpan is a detected region of evaluated text. Offsets are Unicode code point +// positions and End is exclusive. +type textSpan struct { + Start int `json:"start"` + End int `json:"end"` + Score float64 `json:"score"` +} + +// maskSpans replaces each valid span in body with replacement. Invalid spans are +// ignored and overlapping or adjacent spans are merged into a single replacement. +// It returns nil when no span is valid so Mask rules never forward unmasked content. +func maskSpans(body []byte, spans []textSpan, replacement string) []byte { + runes := []rune(string(body)) + valid := make([]textSpan, 0, len(spans)) + for _, span := range spans { + if span.Start < 0 || span.End > len(runes) || span.Start >= span.End { + continue + } + valid = append(valid, span) + } + if len(valid) == 0 { + return nil + } + sort.Slice(valid, func(i, j int) bool { return valid[i].Start < valid[j].Start }) + merged := valid[:0] + for _, span := range valid { + if n := len(merged); n > 0 && span.Start <= merged[n-1].End { + merged[n-1].End = max(merged[n-1].End, span.End) + continue + } + merged = append(merged, span) + } + for i := len(merged) - 1; i >= 0; i-- { + span := merged[i] + runes = append(runes[:span.Start], append([]rune(replacement), runes[span.End:]...)...) + } + return []byte(string(runes)) +} diff --git a/internal/guardrails/adapters_live_test.go b/internal/guardrails/adapters_live_test.go new file mode 100644 index 0000000000..94f47a58a7 --- /dev/null +++ b/internal/guardrails/adapters_live_test.go @@ -0,0 +1,97 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package guardrails + +import ( + "os" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/envoyproxy/ai-gateway/internal/filterapi" +) + +func TestLivePresidio(t *testing.T) { + endpoint := os.Getenv("TEST_PRESIDIO_ENDPOINT") + blockedText := os.Getenv("TEST_PRESIDIO_BLOCKED_TEXT") + if endpoint == "" || blockedText == "" { + t.Skip("TEST_PRESIDIO_ENDPOINT and TEST_PRESIDIO_BLOCKED_TEXT are not set") + } + provider := &filterapi.GuardrailProvider{ + Type: filterapi.GuardrailProviderTypePresidio, + Presidio: &filterapi.PresidioGuardrailProvider{ + Endpoint: endpoint, + APIKey: os.Getenv("TEST_PRESIDIO_API_KEY"), + }, + } + evaluator, err := NewEvaluator(t.Context(), provider) + require.NoError(t, err) + evaluation, err := evaluator.Evaluate(t.Context(), []byte(blockedText), filterapi.GuardrailPhaseRequest) + require.NoError(t, err) + require.True(t, evaluation.Matched) +} + +func TestLiveAzureContentSafety(t *testing.T) { + endpoint, apiKey := os.Getenv("TEST_AZURE_CONTENT_SAFETY_ENDPOINT"), os.Getenv("TEST_AZURE_CONTENT_SAFETY_API_KEY") + blockedText := os.Getenv("TEST_AZURE_CONTENT_SAFETY_BLOCKED_TEXT") + if endpoint == "" || apiKey == "" || blockedText == "" { + t.Skip("TEST_AZURE_CONTENT_SAFETY_ENDPOINT, TEST_AZURE_CONTENT_SAFETY_API_KEY, and TEST_AZURE_CONTENT_SAFETY_BLOCKED_TEXT are not set") + } + threshold := int32(4) + provider := &filterapi.GuardrailProvider{ + Type: filterapi.GuardrailProviderTypeAzureContentSafety, + AzureContentSafety: &filterapi.AzureContentSafetyGuardrailProvider{ + Endpoint: endpoint, APIKey: apiKey, SeverityThreshold: &threshold, + }, + } + evaluator, err := NewEvaluator(t.Context(), provider) + require.NoError(t, err) + evaluation, err := evaluator.Evaluate(t.Context(), []byte(blockedText), filterapi.GuardrailPhaseRequest) + require.NoError(t, err) + require.True(t, evaluation.Matched) +} + +func TestLiveBedrockGuardrail(t *testing.T) { + region := os.Getenv("TEST_AWS_BEDROCK_GUARDRAIL_REGION") + identifier := os.Getenv("TEST_AWS_BEDROCK_GUARDRAIL_ID") + version := os.Getenv("TEST_AWS_BEDROCK_GUARDRAIL_VERSION") + blockedText := os.Getenv("TEST_AWS_BEDROCK_GUARDRAIL_BLOCKED_TEXT") + if region == "" || identifier == "" || version == "" || blockedText == "" { + t.Skip("Bedrock guardrail configuration and TEST_AWS_BEDROCK_GUARDRAIL_BLOCKED_TEXT are not set") + } + provider := &filterapi.GuardrailProvider{ + Type: filterapi.GuardrailProviderTypeBedrockGuardrails, + Bedrock: &filterapi.BedrockGuardrailProvider{ + Region: region, GuardrailIdentifier: identifier, GuardrailVersion: version, + }, + } + evaluator, err := NewEvaluator(t.Context(), provider) + require.NoError(t, err) + evaluation, err := evaluator.Evaluate(t.Context(), []byte(blockedText), filterapi.GuardrailPhaseRequest) + require.NoError(t, err) + require.True(t, evaluation.Matched) +} + +func TestLiveModelArmor(t *testing.T) { + project := os.Getenv("TEST_GCP_MODEL_ARMOR_PROJECT") + location := os.Getenv("TEST_GCP_MODEL_ARMOR_LOCATION") + template := os.Getenv("TEST_GCP_MODEL_ARMOR_TEMPLATE") + blockedText := os.Getenv("TEST_GCP_MODEL_ARMOR_BLOCKED_TEXT") + if project == "" || location == "" || template == "" || blockedText == "" { + t.Skip("Model Armor template configuration and TEST_GCP_MODEL_ARMOR_BLOCKED_TEXT are not set") + } + provider := &filterapi.GuardrailProvider{ + Type: filterapi.GuardrailProviderTypeModelArmor, + ModelArmor: &filterapi.ModelArmorGuardrailProvider{ + Project: project, Location: location, Template: template, + }, + } + evaluator, err := NewEvaluator(t.Context(), provider) + require.NoError(t, err) + evaluation, err := evaluator.Evaluate(t.Context(), []byte(blockedText), filterapi.GuardrailPhaseRequest) + require.NoError(t, err) + require.True(t, evaluation.Matched) +} diff --git a/internal/guardrails/adapters_test.go b/internal/guardrails/adapters_test.go new file mode 100644 index 0000000000..abeeeb8bac --- /dev/null +++ b/internal/guardrails/adapters_test.go @@ -0,0 +1,479 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package guardrails + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/envoyproxy/ai-gateway/internal/filterapi" + "github.com/envoyproxy/ai-gateway/internal/gcpauth" + "github.com/envoyproxy/ai-gateway/internal/json" +) + +func TestPresidioEvaluatorHTTP(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + require.Equal(t, "/analyze", req.URL.Path) + require.Equal(t, "Bearer secret", req.Header.Get("Authorization")) + var body struct { + Text string `json:"text"` + Language string `json:"language"` + ScoreThreshold float64 `json:"score_threshold"` + } + require.NoError(t, json.NewDecoder(req.Body).Decode(&body)) + require.Equal(t, "customer SSN", body.Text) + require.Equal(t, "es", body.Language) + require.Equal(t, 0.75, body.ScoreThreshold) + _, _ = w.Write([]byte(`[{"entity_type":"US_SSN","start":9,"end":12,"score":0.98}]`)) + })) + t.Cleanup(server.Close) + + evaluator, err := newPresidioEvaluator(&filterapi.PresidioGuardrailProvider{ + Endpoint: server.URL, Language: "es", ScoreThresholdPercent: 75, APIKey: "secret", + }, "[REDACTED]", server.Client()) + require.NoError(t, err) + evaluation, err := evaluator.Evaluate(t.Context(), []byte("customer SSN"), filterapi.GuardrailPhaseRequest) + require.NoError(t, err) + require.True(t, evaluation.Matched) + require.Equal(t, "customer [REDACTED]", string(evaluation.Replacement)) +} + +func TestAzureContentSafetyEvaluatorHTTP(t *testing.T) { + severityThreshold := int32(4) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + require.Equal(t, "/contentsafety/text:analyze", req.URL.Path) + require.Equal(t, "2024-09-01", req.URL.Query().Get("api-version")) + require.Equal(t, "azure-secret", req.Header.Get("Ocp-Apim-Subscription-Key")) + _, _ = w.Write([]byte(`{"categoriesAnalysis":[{"category":"Violence","severity":6}]}`)) + })) + t.Cleanup(server.Close) + + evaluator, err := newAzureContentSafetyEvaluator(&filterapi.AzureContentSafetyGuardrailProvider{ + Endpoint: server.URL, APIKey: "azure-secret", SeverityThreshold: &severityThreshold, + }, server.Client()) + require.NoError(t, err) + evaluation, err := evaluator.Evaluate(t.Context(), []byte("unsafe response"), filterapi.GuardrailPhaseResponse) + require.NoError(t, err) + require.True(t, evaluation.Matched) +} + +func TestBedrockEvaluatorHTTP(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + require.Equal(t, "/guardrail/guardrail-id/version/1/apply", req.URL.Path) + require.Contains(t, req.Header.Get("Authorization"), "Credential=AKIDEXAMPLE/") + require.NotEmpty(t, req.Header.Get("X-Amz-Date")) + var body struct { + Source string `json:"source"` + Content []struct { + Text struct { + Text string `json:"text"` + } `json:"text"` + } `json:"content"` + } + require.NoError(t, json.NewDecoder(req.Body).Decode(&body)) + require.Equal(t, "OUTPUT", body.Source) + require.Equal(t, "unsafe response", body.Content[0].Text.Text) + _, _ = w.Write([]byte(`{"action":"GUARDRAIL_INTERVENED","outputs":[{"text":"safe response"}]}`)) + })) + t.Cleanup(server.Close) + + evaluator, err := newBedrockEvaluator(t.Context(), &filterapi.BedrockGuardrailProvider{ + Endpoint: server.URL, Region: "us-east-1", GuardrailIdentifier: "guardrail-id", GuardrailVersion: "1", + CredentialFileLiteral: strings.TrimSpace(` +[default] +aws_access_key_id = AKIDEXAMPLE +aws_secret_access_key = secret +`), + }, server.Client()) + require.NoError(t, err) + evaluation, err := evaluator.Evaluate(t.Context(), []byte("unsafe response"), filterapi.GuardrailPhaseResponse) + require.NoError(t, err) + require.True(t, evaluation.Matched) + require.Equal(t, "safe response", string(evaluation.Replacement)) +} + +func TestEvaluatorProviderError(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "provider unavailable", http.StatusServiceUnavailable) + })) + t.Cleanup(server.Close) + + evaluator, err := newPresidioEvaluator(&filterapi.PresidioGuardrailProvider{Endpoint: server.URL}, "", server.Client()) + require.NoError(t, err) + evaluation, err := evaluator.Evaluate(t.Context(), []byte("payload"), filterapi.GuardrailPhaseRequest) + require.False(t, evaluation.Matched) + require.ErrorContains(t, err, "HTTP 503") + require.ErrorContains(t, err, "provider unavailable") +} + +func TestNewEvaluatorConfiguresTimeout(t *testing.T) { + evaluator, err := NewEvaluator(t.Context(), &filterapi.GuardrailProvider{ + Type: filterapi.GuardrailProviderTypePresidio, + TimeoutSeconds: 3, + Presidio: &filterapi.PresidioGuardrailProvider{Endpoint: "https://presidio.example.com"}, + }) + require.NoError(t, err) + require.Equal(t, 3*time.Second, evaluator.(*presidioEvaluator).client.Timeout) +} + +// newModelArmorTestCredentials returns a service account key JSON whose token_uri points to a fake +// token server, so tests exercise the real OAuth2 JWT flow without contacting Google. +func newModelArmorTestCredentials(t *testing.T) string { + t.Helper() + tokenServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + require.NoError(t, req.ParseForm()) + require.Equal(t, "urn:ietf:params:oauth:grant-type:jwt-bearer", req.Form.Get("grant_type")) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"access_token":"gcp-token","token_type":"Bearer","expires_in":3600}`)) + })) + t.Cleanup(tokenServer.Close) + key, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + der, err := x509.MarshalPKCS8PrivateKey(key) + require.NoError(t, err) + credentials, err := json.Marshal(map[string]string{ + "type": "service_account", + "project_id": "project-id", + "private_key_id": "key-id", + "private_key": string(pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der})), + "client_email": "guardrails@project-id.iam.gserviceaccount.com", + "token_uri": tokenServer.URL, + }) + require.NoError(t, err) + return string(credentials) +} + +func TestModelArmorEvaluatorHTTP(t *testing.T) { + credentials := newModelArmorTestCredentials(t) + tests := []struct { + name string + phase filterapi.GuardrailPhase + wantPath string + wantBody string + response string + wantMatched bool + wantReplacement string + }{ + { + name: "request no match", + phase: filterapi.GuardrailPhaseRequest, + wantPath: "/v1/projects/project-id/locations/us-central1/templates/template-id:sanitizeUserPrompt", + wantBody: `{"userPromptData":{"text":"hello"}}`, + response: `{"sanitizationResult":{"filterMatchState":"NO_MATCH_FOUND","invocationResult":"SUCCESS"}}`, + }, + { + name: "request jailbreak match", + phase: filterapi.GuardrailPhaseRequest, + wantPath: "/v1/projects/project-id/locations/us-central1/templates/template-id:sanitizeUserPrompt", + wantBody: `{"userPromptData":{"text":"hello"}}`, + response: `{"sanitizationResult":{"filterMatchState":"MATCH_FOUND","invocationResult":"SUCCESS","filterResults":{ + "pi_and_jailbreak":{"piAndJailbreakFilterResult":{"executionState":"EXECUTION_SUCCESS","matchState":"MATCH_FOUND"}}}}}`, + wantMatched: true, + }, + { + name: "response sdp de-identify match", + phase: filterapi.GuardrailPhaseResponse, + wantPath: "/v1/projects/project-id/locations/us-central1/templates/template-id:sanitizeModelResponse", + wantBody: `{"modelResponseData":{"text":"hello"}}`, + response: `{"sanitizationResult":{"filterMatchState":"MATCH_FOUND","invocationResult":"SUCCESS","filterResults":{ + "rai":{"raiFilterResult":{"matchState":"NO_MATCH_FOUND"}}, + "sdp":{"sdpFilterResult":{"deidentifyResult":{"matchState":"MATCH_FOUND","data":{"text":"[EMAIL_ADDRESS]"}}}}}}}`, + wantMatched: true, + wantReplacement: "[EMAIL_ADDRESS]", + }, + { + name: "sdp de-identify with another filter match", + phase: filterapi.GuardrailPhaseResponse, + wantPath: "/v1/projects/project-id/locations/us-central1/templates/template-id:sanitizeModelResponse", + wantBody: `{"modelResponseData":{"text":"hello"}}`, + response: `{"sanitizationResult":{"filterMatchState":"MATCH_FOUND","invocationResult":"SUCCESS","filterResults":{ + "rai":{"raiFilterResult":{"matchState":"MATCH_FOUND"}}, + "sdp":{"sdpFilterResult":{"deidentifyResult":{"matchState":"MATCH_FOUND","data":{"text":"[EMAIL_ADDRESS]"}}}}}}}`, + wantMatched: true, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + require.Equal(t, test.wantPath, req.URL.Path) + require.Equal(t, "Bearer gcp-token", req.Header.Get("Authorization")) + body, err := io.ReadAll(req.Body) + require.NoError(t, err) + require.JSONEq(t, test.wantBody, string(body)) + _, _ = w.Write([]byte(test.response)) + })) + t.Cleanup(server.Close) + + evaluator, err := newModelArmorEvaluator(t.Context(), &filterapi.ModelArmorGuardrailProvider{ + Endpoint: server.URL, Project: "project-id", Location: "us-central1", Template: "template-id", + CredentialsJSON: credentials, + }, server.Client()) + require.NoError(t, err) + evaluation, err := evaluator.Evaluate(t.Context(), []byte("hello"), test.phase) + require.NoError(t, err) + require.Equal(t, test.wantMatched, evaluation.Matched) + require.Equal(t, test.wantReplacement, string(evaluation.Replacement)) + }) + } +} + +func TestModelArmorEvaluatorErrors(t *testing.T) { + _, err := newModelArmorEvaluator(t.Context(), &filterapi.ModelArmorGuardrailProvider{Project: "project-id"}, http.DefaultClient) + require.ErrorContains(t, err, "project, location, and template are required") + _, err = newModelArmorEvaluator(t.Context(), &filterapi.ModelArmorGuardrailProvider{ + Project: "project-id", Location: "us-central1", Template: "template-id", CredentialsJSON: `{"type":"authorized_user"}`, + }, http.DefaultClient) + require.ErrorContains(t, err, "cannot load GCP credentials") + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"sanitizationResult":{"filterMatchState":"NO_MATCH_FOUND","invocationResult":"FAILURE"}}`)) + })) + t.Cleanup(server.Close) + evaluator, err := newModelArmorEvaluator(t.Context(), &filterapi.ModelArmorGuardrailProvider{ + Endpoint: server.URL, Project: "project-id", Location: "us-central1", Template: "template-id", + CredentialsJSON: newModelArmorTestCredentials(t), + }, server.Client()) + require.NoError(t, err) + evaluation, err := evaluator.Evaluate(t.Context(), []byte("hello"), filterapi.GuardrailPhaseRequest) + require.False(t, evaluation.Matched) + require.ErrorContains(t, err, "sanitizeUserPrompt invocation failed") +} + +// isolateAWSEnvironment keeps AWS credential resolution hermetic: no ambient credentials and no IMDS calls. +func isolateAWSEnvironment(t *testing.T) { + t.Helper() + t.Setenv("AWS_EC2_METADATA_DISABLED", "true") + t.Setenv("AWS_ACCESS_KEY_ID", "") + t.Setenv("AWS_SECRET_ACCESS_KEY", "") + t.Setenv("AWS_SESSION_TOKEN", "") + t.Setenv("AWS_PROFILE", "") + t.Setenv("AWS_CONFIG_FILE", filepath.Join(t.TempDir(), "config")) + t.Setenv("AWS_SHARED_CREDENTIALS_FILE", filepath.Join(t.TempDir(), "credentials")) +} + +func TestNewEvaluatorProviders(t *testing.T) { + isolateAWSEnvironment(t) + severity := int32(2) + for _, provider := range []*filterapi.GuardrailProvider{ + {Type: filterapi.GuardrailProviderTypeBedrockGuardrails, Bedrock: &filterapi.BedrockGuardrailProvider{ + Region: "us-east-1", GuardrailIdentifier: "guardrail-id", GuardrailVersion: "1", + }}, + {Type: filterapi.GuardrailProviderTypeAzureContentSafety, AzureContentSafety: &filterapi.AzureContentSafetyGuardrailProvider{ + Endpoint: "https://content-safety.example.com", APIKey: "key", SeverityThreshold: &severity, + }}, + {Type: filterapi.GuardrailProviderTypeHTTP, HTTP: &filterapi.HTTPGuardrailProvider{Endpoint: "https://guardrail.example.com"}}, + {Type: filterapi.GuardrailProviderTypeModelArmor, ModelArmor: &filterapi.ModelArmorGuardrailProvider{ + Project: "project-id", Location: "us-central1", Template: "template-id", CredentialsJSON: newModelArmorTestCredentials(t), + }}, + } { + t.Run(string(provider.Type), func(t *testing.T) { + evaluator, err := NewEvaluator(t.Context(), provider) + require.NoError(t, err) + require.NotNil(t, evaluator) + }) + } + + _, err := NewEvaluator(t.Context(), &filterapi.GuardrailProvider{Type: "Unknown"}) + require.ErrorContains(t, err, `unsupported external guardrail provider "Unknown"`) +} + +func TestTimeoutSecondsDefault(t *testing.T) { + require.Equal(t, defaultTimeoutSeconds, timeoutSeconds(0)) + require.Equal(t, int32(5), timeoutSeconds(5)) +} + +func TestDoJSONErrors(t *testing.T) { + err := doJSON(t.Context(), http.DefaultClient, http.MethodPost, "https://example.com", make(chan int), nil, nil) + require.ErrorContains(t, err, "unsupported type") + + err = doJSON(t.Context(), http.DefaultClient, http.MethodPost, "http://[::1", struct{}{}, nil, nil) + require.ErrorContains(t, err, "missing ']' in host") + + server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + server.Close() + err = doJSON(t.Context(), http.DefaultClient, http.MethodPost, server.URL, struct{}{}, nil, nil) + require.ErrorContains(t, err, "connection refused") + + malformed := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("not json")) + })) + t.Cleanup(malformed.Close) + var response struct{} + err = doJSON(t.Context(), malformed.Client(), http.MethodPost, malformed.URL, struct{}{}, nil, &response) + require.ErrorContains(t, err, "cannot decode provider response") +} + +func TestPresidioEvaluatorConfigurationAndNoFindings(t *testing.T) { + _, err := newPresidioEvaluator(&filterapi.PresidioGuardrailProvider{}, "", http.DefaultClient) + require.ErrorContains(t, err, "presidio endpoint is required") + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`[]`)) + })) + t.Cleanup(server.Close) + evaluator, err := newPresidioEvaluator(&filterapi.PresidioGuardrailProvider{Endpoint: server.URL}, "", server.Client()) + require.NoError(t, err) + evaluation, err := evaluator.Evaluate(t.Context(), []byte("nothing sensitive"), filterapi.GuardrailPhaseRequest) + require.NoError(t, err) + require.False(t, evaluation.Matched) +} + +func TestAzureContentSafetyEvaluatorConfiguration(t *testing.T) { + _, err := newAzureContentSafetyEvaluator(&filterapi.AzureContentSafetyGuardrailProvider{APIKey: "key"}, http.DefaultClient) + require.ErrorContains(t, err, "azure Content Safety endpoint is required") + _, err = newAzureContentSafetyEvaluator(&filterapi.AzureContentSafetyGuardrailProvider{Endpoint: "https://content-safety.example.com"}, http.DefaultClient) + require.ErrorContains(t, err, "azure Content Safety API key is required") + + evaluator, err := newAzureContentSafetyEvaluator(&filterapi.AzureContentSafetyGuardrailProvider{ + Endpoint: "https://content-safety.example.com", APIKey: "key", + }, http.DefaultClient) + require.NoError(t, err) + require.Equal(t, int32(4), *evaluator.(*azureContentSafetyEvaluator).config.SeverityThreshold) +} + +func TestAzureContentSafetyEvaluatorBelowThresholdAndError(t *testing.T) { + status := http.StatusOK + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(status) + _, _ = w.Write([]byte(`{"categoriesAnalysis":[{"category":"Violence","severity":2}]}`)) + })) + t.Cleanup(server.Close) + evaluator, err := newAzureContentSafetyEvaluator(&filterapi.AzureContentSafetyGuardrailProvider{ + Endpoint: server.URL, APIKey: "key", + }, server.Client()) + require.NoError(t, err) + + evaluation, err := evaluator.Evaluate(t.Context(), []byte("mild"), filterapi.GuardrailPhaseRequest) + require.NoError(t, err) + require.False(t, evaluation.Matched) + + status = http.StatusTooManyRequests + _, err = evaluator.Evaluate(t.Context(), []byte("mild"), filterapi.GuardrailPhaseRequest) + require.ErrorContains(t, err, "azure Content Safety analyze request failed: provider returned HTTP 429") +} + +func TestBedrockEvaluatorConfiguration(t *testing.T) { + isolateAWSEnvironment(t) + _, err := newBedrockEvaluator(t.Context(), &filterapi.BedrockGuardrailProvider{Region: "us-east-1"}, http.DefaultClient) + require.ErrorContains(t, err, "bedrock region, guardrailIdentifier, and guardrailVersion are required") + + _, err = newBedrockEvaluator(t.Context(), &filterapi.BedrockGuardrailProvider{ + Region: "us-east-1", GuardrailIdentifier: "guardrail-id", GuardrailVersion: "1", + CredentialFileLiteral: "[other]\naws_access_key_id = AKIDEXAMPLE\naws_secret_access_key = secret\n", + }, http.DefaultClient) + require.ErrorContains(t, err, "cannot load AWS credentials") + + t.Setenv("AWS_PROFILE", "missing") + _, err = newBedrockEvaluator(t.Context(), &filterapi.BedrockGuardrailProvider{ + Region: "us-east-1", GuardrailIdentifier: "guardrail-id", GuardrailVersion: "1", + }, http.DefaultClient) + require.ErrorContains(t, err, "cannot load AWS config") +} + +func TestBedrockEvaluatorErrors(t *testing.T) { + isolateAWSEnvironment(t) + // Without a credentials Secret, the default chain is used and only fails when credentials are retrieved. + evaluator, err := newBedrockEvaluator(t.Context(), &filterapi.BedrockGuardrailProvider{ + Region: "us-east-1", GuardrailIdentifier: "guardrail-id", GuardrailVersion: "1", + }, http.DefaultClient) + require.NoError(t, err) + require.Equal(t, "https://bedrock-runtime.us-east-1.amazonaws.com", evaluator.(*bedrockEvaluator).config.Endpoint) + _, err = evaluator.Evaluate(t.Context(), []byte("payload"), filterapi.GuardrailPhaseRequest) + require.ErrorContains(t, err, "cannot retrieve AWS credentials") + + credentials := "[default]\naws_access_key_id = AKIDEXAMPLE\naws_secret_access_key = secret\n" + evaluator, err = newBedrockEvaluator(t.Context(), &filterapi.BedrockGuardrailProvider{ + Endpoint: "http://[::1", Region: "us-east-1", GuardrailIdentifier: "guardrail-id", GuardrailVersion: "1", + CredentialFileLiteral: credentials, + }, http.DefaultClient) + require.NoError(t, err) + _, err = evaluator.Evaluate(t.Context(), []byte("payload"), filterapi.GuardrailPhaseRequest) + require.ErrorContains(t, err, "missing ']' in host") + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "throttled", http.StatusTooManyRequests) + })) + t.Cleanup(server.Close) + evaluator, err = newBedrockEvaluator(t.Context(), &filterapi.BedrockGuardrailProvider{ + Endpoint: server.URL, Region: "us-east-1", GuardrailIdentifier: "guardrail-id", GuardrailVersion: "1", + CredentialFileLiteral: credentials, + }, server.Client()) + require.NoError(t, err) + _, err = evaluator.Evaluate(t.Context(), []byte("payload"), filterapi.GuardrailPhaseRequest) + require.ErrorContains(t, err, "bedrock ApplyGuardrail request failed: provider returned HTTP 429") +} + +func TestModelArmorEvaluatorDefaults(t *testing.T) { + credentialsPath := filepath.Join(t.TempDir(), "credentials.json") + require.NoError(t, os.WriteFile(credentialsPath, []byte(newModelArmorTestCredentials(t)), 0o600)) + t.Setenv("GOOGLE_APPLICATION_CREDENTIALS", credentialsPath) + + evaluator, err := newModelArmorEvaluator(t.Context(), &filterapi.ModelArmorGuardrailProvider{ + Project: "project-id", Location: "europe-west4", Template: "template-id", + }, http.DefaultClient) + require.NoError(t, err) + require.Equal(t, "https://modelarmor.europe-west4.rep.googleapis.com/v1/projects/project-id/locations/europe-west4/templates/template-id", + evaluator.(*modelArmorEvaluator).templateURL) + + t.Setenv(gcpauth.ProxyEnvVar, "://invalid") + _, err = newModelArmorEvaluator(t.Context(), &filterapi.ModelArmorGuardrailProvider{ + Project: "project-id", Location: "europe-west4", Template: "template-id", + }, http.DefaultClient) + require.ErrorContains(t, err, "invalid "+gcpauth.ProxyEnvVar) +} + +func TestModelArmorEvaluatorRequestErrors(t *testing.T) { + tokenStatus := http.StatusOK + tokenServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(tokenStatus) + _, _ = w.Write([]byte(`{"access_token":"gcp-token","token_type":"Bearer","expires_in":3600}`)) + })) + t.Cleanup(tokenServer.Close) + var credentials map[string]string + require.NoError(t, json.Unmarshal([]byte(newModelArmorTestCredentials(t)), &credentials)) + credentials["token_uri"] = tokenServer.URL + credentialsJSON, err := json.Marshal(credentials) + require.NoError(t, err) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "unavailable", http.StatusServiceUnavailable) + })) + t.Cleanup(server.Close) + newEvaluator := func() filterapi.GuardrailEvaluator { + evaluator, newErr := newModelArmorEvaluator(t.Context(), &filterapi.ModelArmorGuardrailProvider{ + Endpoint: server.URL, Project: "project-id", Location: "us-central1", Template: "template-id", + CredentialsJSON: string(credentialsJSON), + }, server.Client()) + require.NoError(t, newErr) + return evaluator + } + + _, err = newEvaluator().Evaluate(t.Context(), []byte("hello"), filterapi.GuardrailPhaseResponse) + require.ErrorContains(t, err, "model Armor sanitizeModelResponse request failed: provider returned HTTP 503") + + tokenStatus = http.StatusUnauthorized + _, err = newEvaluator().Evaluate(t.Context(), []byte("hello"), filterapi.GuardrailPhaseRequest) + require.ErrorContains(t, err, "cannot retrieve GCP access token") +} + +func TestModelArmorDeidentifiedTextWithSDPInspectMatch(t *testing.T) { + var results map[string]modelArmorFilterResult + require.NoError(t, json.Unmarshal([]byte(`{"sdp":{"sdpFilterResult":{"inspectResult":{"matchState":"MATCH_FOUND"}}}}`), &results)) + require.Nil(t, modelArmorDeidentifiedText(results)) +} diff --git a/internal/guardrails/azure.go b/internal/guardrails/azure.go new file mode 100644 index 0000000000..8c8b03d1e9 --- /dev/null +++ b/internal/guardrails/azure.go @@ -0,0 +1,62 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package guardrails + +import ( + "context" + "fmt" + "net/http" + "net/url" + "strings" + + "github.com/envoyproxy/ai-gateway/internal/filterapi" +) + +type azureContentSafetyEvaluator struct { + config *filterapi.AzureContentSafetyGuardrailProvider + client *http.Client +} + +func newAzureContentSafetyEvaluator(config *filterapi.AzureContentSafetyGuardrailProvider, client *http.Client) (filterapi.GuardrailEvaluator, error) { + if config == nil || config.Endpoint == "" { + return nil, fmt.Errorf("azure Content Safety endpoint is required") + } + if config.APIKey == "" { + return nil, fmt.Errorf("azure Content Safety API key is required") + } + configCopy := *config + if configCopy.APIVersion == "" { + configCopy.APIVersion = "2024-09-01" + } + if configCopy.SeverityThreshold == nil { + threshold := int32(4) + configCopy.SeverityThreshold = &threshold + } + return &azureContentSafetyEvaluator{config: &configCopy, client: client}, nil +} + +func (e *azureContentSafetyEvaluator) Evaluate(ctx context.Context, body []byte, _ filterapi.GuardrailPhase) (filterapi.GuardrailEvaluationResult, error) { + payload := struct { + Text string `json:"text"` + }{Text: string(body)} + result := struct { + CategoriesAnalysis []struct { + Severity int32 `json:"severity"` + } `json:"categoriesAnalysis"` + }{} + endpoint := strings.TrimRight(e.config.Endpoint, "/") + "/contentsafety/text:analyze?api-version=" + url.QueryEscape(e.config.APIVersion) + if err := doJSON(ctx, e.client, http.MethodPost, endpoint, payload, func(req *http.Request) { + req.Header.Set("Ocp-Apim-Subscription-Key", e.config.APIKey) + }, &result); err != nil { + return filterapi.GuardrailEvaluationResult{}, fmt.Errorf("azure Content Safety analyze request failed: %w", err) + } + for _, category := range result.CategoriesAnalysis { + if category.Severity >= *e.config.SeverityThreshold { + return filterapi.GuardrailEvaluationResult{Matched: true}, nil + } + } + return filterapi.GuardrailEvaluationResult{}, nil +} diff --git a/internal/guardrails/bedrock.go b/internal/guardrails/bedrock.go new file mode 100644 index 0000000000..af43810bec --- /dev/null +++ b/internal/guardrails/bedrock.go @@ -0,0 +1,134 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package guardrails + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "net/http" + "net/url" + "os" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + awsv4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + + "github.com/envoyproxy/ai-gateway/internal/filterapi" + "github.com/envoyproxy/ai-gateway/internal/json" +) + +type bedrockEvaluator struct { + config *filterapi.BedrockGuardrailProvider + client *http.Client + credentials aws.CredentialsProvider + signer *awsv4.Signer +} + +func newBedrockEvaluator(ctx context.Context, config *filterapi.BedrockGuardrailProvider, client *http.Client) (filterapi.GuardrailEvaluator, error) { + if config == nil || config.Region == "" || config.GuardrailIdentifier == "" || config.GuardrailVersion == "" { + return nil, fmt.Errorf("bedrock region, guardrailIdentifier, and guardrailVersion are required") + } + credentials, err := loadAWSCredentials(ctx, config) + if err != nil { + return nil, err + } + configCopy := *config + if configCopy.Endpoint == "" { + configCopy.Endpoint = fmt.Sprintf("https://bedrock-runtime.%s.amazonaws.com", configCopy.Region) + } + return &bedrockEvaluator{config: &configCopy, client: client, credentials: credentials, signer: awsv4.NewSigner()}, nil +} + +func loadAWSCredentials(ctx context.Context, guardrailConfig *filterapi.BedrockGuardrailProvider) (aws.CredentialsProvider, error) { + options := []func(*awsconfig.LoadOptions) error{awsconfig.WithRegion(guardrailConfig.Region)} + var credentialsFile *os.File + if guardrailConfig.CredentialFileLiteral != "" { + var err error + credentialsFile, err = os.CreateTemp("", "guardrail-aws-credentials") + if err != nil { + return nil, fmt.Errorf("cannot create temporary AWS credentials file: %w", err) + } + defer func() { + _ = credentialsFile.Close() + _ = os.Remove(credentialsFile.Name()) + }() + if _, err = credentialsFile.WriteString(guardrailConfig.CredentialFileLiteral); err != nil { + return nil, fmt.Errorf("cannot write AWS credentials file: %w", err) + } + options = append(options, awsconfig.WithSharedCredentialsFiles([]string{credentialsFile.Name()})) + } + cfg, err := awsconfig.LoadDefaultConfig(ctx, options...) + if err != nil { + return nil, fmt.Errorf("cannot load AWS config: %w", err) + } + if credentialsFile == nil { + return cfg.Credentials, nil + } + credentials, err := cfg.Credentials.Retrieve(ctx) + if err != nil { + return nil, fmt.Errorf("cannot load AWS credentials: %w", err) + } + return aws.CredentialsProviderFunc(func(context.Context) (aws.Credentials, error) { + return credentials, nil + }), nil +} + +func (e *bedrockEvaluator) Evaluate(ctx context.Context, body []byte, phase filterapi.GuardrailPhase) (filterapi.GuardrailEvaluationResult, error) { + source := "INPUT" + if phase == filterapi.GuardrailPhaseResponse { + source = "OUTPUT" + } + payload, err := json.Marshal(struct { + Source string `json:"source"` + Content []struct { + Text struct { + Text string `json:"text"` + } `json:"text"` + } `json:"content"` + }{Source: source, Content: []struct { + Text struct { + Text string `json:"text"` + } `json:"text"` + }{{Text: struct { + Text string `json:"text"` + }{Text: string(body)}}}}) + if err != nil { + return filterapi.GuardrailEvaluationResult{}, err + } + endpoint := strings.TrimRight(e.config.Endpoint, "/") + "/guardrail/" + url.PathEscape(e.config.GuardrailIdentifier) + "/version/" + url.PathEscape(e.config.GuardrailVersion) + "/apply" + req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(payload)) + if err != nil { + return filterapi.GuardrailEvaluationResult{}, err + } + req.Header.Set("Content-Type", "application/json") + credentials, err := e.credentials.Retrieve(ctx) + if err != nil { + return filterapi.GuardrailEvaluationResult{}, fmt.Errorf("cannot retrieve AWS credentials: %w", err) + } + payloadHash := sha256.Sum256(payload) + if err = e.signer.SignHTTP(ctx, credentials, req, hex.EncodeToString(payloadHash[:]), "bedrock", e.config.Region, time.Now()); err != nil { + return filterapi.GuardrailEvaluationResult{}, fmt.Errorf("cannot sign Bedrock request: %w", err) + } + result := struct { + Action string `json:"action"` + Outputs []struct { + Text string `json:"text"` + } `json:"outputs"` + }{} + if err = sendJSON(req, e.client, &result); err != nil { + return filterapi.GuardrailEvaluationResult{}, fmt.Errorf("bedrock ApplyGuardrail request failed: %w", err) + } + evaluation := filterapi.GuardrailEvaluationResult{Matched: result.Action == "GUARDRAIL_INTERVENED"} + if len(result.Outputs) > 0 { + evaluation.Replacement = []byte(result.Outputs[0].Text) + } + return evaluation, nil +} diff --git a/internal/guardrails/http.go b/internal/guardrails/http.go new file mode 100644 index 0000000000..2d8b8753be --- /dev/null +++ b/internal/guardrails/http.go @@ -0,0 +1,128 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package guardrails + +import ( + "context" + "fmt" + "net/http" + "strings" + + "github.com/envoyproxy/ai-gateway/internal/filterapi" +) + +const defaultHTTPGuardrailPath = "/analyze" + +// HTTP guardrail contract stages sent in the request context. +const ( + httpGuardrailStageInput = "input" + httpGuardrailStageOutput = "output" +) + +// HTTP guardrail contract actions returned by the custom service. +const ( + httpGuardrailActionAllow = "allow" + httpGuardrailActionBlock = "block" + httpGuardrailActionModify = "modify" +) + +// httpGuardrailRequest is the payload sent to a custom guardrail service. +type httpGuardrailRequest struct { + Text string `json:"text"` + Context httpGuardrailRequestContext `json:"context"` +} + +type httpGuardrailRequestContext struct { + Stage string `json:"stage"` +} + +// httpGuardrailResponse is the normalized response returned by a custom guardrail service. +type httpGuardrailResponse struct { + Action string `json:"action"` + Findings []httpGuardrailFinding `json:"findings,omitempty"` + Replacement *string `json:"replacement,omitempty"` +} + +type httpGuardrailFinding struct { + Type string `json:"type,omitempty"` + textSpan +} + +type httpEvaluator struct { + url string + apiKey string + maskReplacement string + client *http.Client +} + +func newHTTPEvaluator(config *filterapi.HTTPGuardrailProvider, maskReplacement string, client *http.Client) (filterapi.GuardrailEvaluator, error) { + if config == nil || config.Endpoint == "" { + return nil, fmt.Errorf("http guardrail endpoint is required") + } + path := config.Path + if path == "" { + path = defaultHTTPGuardrailPath + } + if !strings.HasPrefix(path, "/") { + return nil, fmt.Errorf("http guardrail path must start with /") + } + if maskReplacement == "" { + maskReplacement = "[REDACTED]" + } + return &httpEvaluator{ + url: strings.TrimRight(config.Endpoint, "/") + path, + apiKey: config.APIKey, + maskReplacement: maskReplacement, + client: client, + }, nil +} + +func (e *httpEvaluator) Evaluate(ctx context.Context, body []byte, phase filterapi.GuardrailPhase) (filterapi.GuardrailEvaluationResult, error) { + stage := httpGuardrailStageInput + if phase == filterapi.GuardrailPhaseResponse { + stage = httpGuardrailStageOutput + } + payload := httpGuardrailRequest{Text: string(body), Context: httpGuardrailRequestContext{Stage: stage}} + + var result httpGuardrailResponse + if err := doJSON(ctx, e.client, http.MethodPost, e.url, payload, func(req *http.Request) { + if e.apiKey != "" { + req.Header.Set("Authorization", "Bearer "+e.apiKey) + } + }, &result); err != nil { + return filterapi.GuardrailEvaluationResult{}, fmt.Errorf("http guardrail analyze request failed: %w", err) + } + + switch strings.ToLower(result.Action) { + case httpGuardrailActionAllow: + return filterapi.GuardrailEvaluationResult{}, nil + case httpGuardrailActionBlock: + return filterapi.GuardrailEvaluationResult{Matched: true, Replacement: e.maskFindings(body, result.Findings)}, nil + case httpGuardrailActionModify: + if result.Replacement != nil { + return filterapi.GuardrailEvaluationResult{Matched: true, Replacement: []byte(*result.Replacement)}, nil + } + if len(result.Findings) == 0 { + return filterapi.GuardrailEvaluationResult{}, fmt.Errorf("http guardrail returned modify without replacement or findings") + } + return filterapi.GuardrailEvaluationResult{Matched: true, Replacement: e.maskFindings(body, result.Findings)}, nil + default: + return filterapi.GuardrailEvaluationResult{}, fmt.Errorf("http guardrail returned unsupported action %q", result.Action) + } +} + +// maskFindings masks the finding spans in body. It returns nil when there are no findings, +// so Mask rules fail instead of forwarding unchanged content. +func (e *httpEvaluator) maskFindings(body []byte, findings []httpGuardrailFinding) []byte { + if len(findings) == 0 { + return nil + } + spans := make([]textSpan, len(findings)) + for i := range findings { + spans[i] = findings[i].textSpan + } + return maskSpans(body, spans, e.maskReplacement) +} diff --git a/internal/guardrails/http_test.go b/internal/guardrails/http_test.go new file mode 100644 index 0000000000..cdb8685a23 --- /dev/null +++ b/internal/guardrails/http_test.go @@ -0,0 +1,202 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package guardrails + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/envoyproxy/ai-gateway/internal/filterapi" + "github.com/envoyproxy/ai-gateway/internal/json" +) + +func newHTTPGuardrailTestServer(t *testing.T, response string, check func(*http.Request, httpGuardrailRequest)) *httptest.Server { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + require.Equal(t, http.MethodPost, req.Method) + require.Equal(t, "application/json", req.Header.Get("Content-Type")) + var body httpGuardrailRequest + require.NoError(t, json.NewDecoder(req.Body).Decode(&body)) + if check != nil { + check(req, body) + } + _, _ = w.Write([]byte(response)) + })) + t.Cleanup(server.Close) + return server +} + +func TestHTTPEvaluatorRequestContract(t *testing.T) { + for _, tc := range []struct { + name string + path string + apiKey string + phase filterapi.GuardrailPhase + expPath string + expStage string + expHeader string + }{ + {name: "default path request", phase: filterapi.GuardrailPhaseRequest, expPath: "/analyze", expStage: "input"}, + {name: "custom path response with api key", path: "/v1/check", apiKey: "secret", phase: filterapi.GuardrailPhaseResponse, expPath: "/v1/check", expStage: "output", expHeader: "Bearer secret"}, + } { + t.Run(tc.name, func(t *testing.T) { + server := newHTTPGuardrailTestServer(t, `{"action":"allow"}`, func(req *http.Request, body httpGuardrailRequest) { + require.Equal(t, tc.expPath, req.URL.Path) + require.Equal(t, tc.expHeader, req.Header.Get("Authorization")) + require.Equal(t, "some user input", body.Text) + require.Equal(t, tc.expStage, body.Context.Stage) + }) + evaluator, err := newHTTPEvaluator(&filterapi.HTTPGuardrailProvider{ + Endpoint: server.URL + "/", Path: tc.path, APIKey: tc.apiKey, + }, "", server.Client()) + require.NoError(t, err) + evaluation, err := evaluator.Evaluate(t.Context(), []byte("some user input"), tc.phase) + require.NoError(t, err) + require.False(t, evaluation.Matched) + require.Nil(t, evaluation.Replacement) + }) + } +} + +func TestHTTPEvaluatorResponseContract(t *testing.T) { + const text = "my email is john@example.com ok" + for _, tc := range []struct { + name string + response string + expMatched bool + expReplacement string + expErr string + }{ + { + name: "allow ignores findings", + response: `{"action":"allow","findings":[{"type":"PII","start":12,"end":28,"score":0.92}]}`, + }, + { + name: "block without findings", + response: `{"action":"block"}`, + expMatched: true, + }, + { + name: "block with findings provides mask", + response: `{"action":"block","findings":[{"type":"EMAIL","start":12,"end":28,"score":0.92}]}`, + expMatched: true, + expReplacement: "my email is [MASKED] ok", + }, + { + name: "modify with replacement", + response: `{"action":"modify","replacement":"my email is ok"}`, + expMatched: true, + expReplacement: "my email is ok", + }, + { + name: "modify with findings only", + response: `{"action":"modify","findings":[{"type":"EMAIL","start":12,"end":28}]}`, + expMatched: true, + expReplacement: "my email is [MASKED] ok", + }, + { + name: "action is case insensitive", + response: `{"action":"MODIFY","findings":[{"start":0,"end":2}]}`, + expMatched: true, + expReplacement: "[MASKED] email is john@example.com ok", + }, + { + name: "modify without replacement or findings", + response: `{"action":"modify"}`, + expErr: "modify without replacement or findings", + }, + { + name: "unknown action", + response: `{"action":"quarantine"}`, + expErr: `unsupported action "quarantine"`, + }, + { + name: "missing action", + response: `{"findings":[]}`, + expErr: `unsupported action ""`, + }, + { + name: "malformed response", + response: `[1,2,3]`, + expErr: "cannot decode provider response", + }, + } { + t.Run(tc.name, func(t *testing.T) { + server := newHTTPGuardrailTestServer(t, tc.response, nil) + evaluator, err := newHTTPEvaluator(&filterapi.HTTPGuardrailProvider{Endpoint: server.URL}, "[MASKED]", server.Client()) + require.NoError(t, err) + evaluation, err := evaluator.Evaluate(t.Context(), []byte(text), filterapi.GuardrailPhaseRequest) + if tc.expErr != "" { + require.ErrorContains(t, err, tc.expErr) + require.False(t, evaluation.Matched) + return + } + require.NoError(t, err) + require.Equal(t, tc.expMatched, evaluation.Matched) + require.Equal(t, tc.expReplacement, string(evaluation.Replacement)) + }) + } +} + +func TestHTTPEvaluatorProviderError(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "guardrail unavailable", http.StatusBadGateway) + })) + t.Cleanup(server.Close) + + evaluator, err := newHTTPEvaluator(&filterapi.HTTPGuardrailProvider{Endpoint: server.URL}, "", server.Client()) + require.NoError(t, err) + evaluation, err := evaluator.Evaluate(t.Context(), []byte("payload"), filterapi.GuardrailPhaseRequest) + require.False(t, evaluation.Matched) + require.ErrorContains(t, err, "HTTP 502") + require.ErrorContains(t, err, "guardrail unavailable") +} + +func TestNewHTTPEvaluatorValidation(t *testing.T) { + _, err := newHTTPEvaluator(nil, "", http.DefaultClient) + require.ErrorContains(t, err, "endpoint is required") + _, err = newHTTPEvaluator(&filterapi.HTTPGuardrailProvider{Endpoint: "https://guardrail.example.com", Path: "analyze"}, "", http.DefaultClient) + require.ErrorContains(t, err, "path must start with /") +} + +func TestNewEvaluatorHTTPProvider(t *testing.T) { + evaluator, err := NewEvaluator(t.Context(), &filterapi.GuardrailProvider{ + Type: filterapi.GuardrailProviderTypeHTTP, + TimeoutSeconds: 2, + HTTP: &filterapi.HTTPGuardrailProvider{Endpoint: "https://guardrail.example.com/", Path: "/v1/analyze"}, + }) + require.NoError(t, err) + httpEval := evaluator.(*httpEvaluator) + require.Equal(t, "https://guardrail.example.com/v1/analyze", httpEval.url) + require.Equal(t, "[REDACTED]", httpEval.maskReplacement) + require.Equal(t, 2*time.Second, httpEval.client.Timeout) +} + +func TestMaskSpans(t *testing.T) { + for _, tc := range []struct { + name string + body string + spans []textSpan + exp string + }{ + {name: "single", body: "abcdef", spans: []textSpan{{Start: 1, End: 3}}, exp: "a#def"}, + {name: "unsorted", body: "abcdef", spans: []textSpan{{Start: 4, End: 5}, {Start: 0, End: 1}}, exp: "#bcd#f"}, + {name: "overlapping merged", body: "abcdef", spans: []textSpan{{Start: 1, End: 4}, {Start: 2, End: 5}}, exp: "a#f"}, + {name: "adjacent merged", body: "abcdef", spans: []textSpan{{Start: 1, End: 2}, {Start: 2, End: 3}}, exp: "a#def"}, + {name: "unicode offsets", body: "héllo wörld", spans: []textSpan{{Start: 6, End: 11}}, exp: "héllo #"}, + {name: "invalid spans ignored", body: "abcdef", spans: []textSpan{{Start: -1, End: 2}, {Start: 2, End: 99}, {Start: 3, End: 3}, {Start: 0, End: 1}}, exp: "#bcdef"}, + } { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.exp, string(maskSpans([]byte(tc.body), tc.spans, "#"))) + }) + } + require.Nil(t, maskSpans([]byte("abc"), []textSpan{{Start: 5, End: 6}}, "#")) + require.Nil(t, maskSpans([]byte("abc"), nil, "#")) +} diff --git a/internal/guardrails/modelarmor.go b/internal/guardrails/modelarmor.go new file mode 100644 index 0000000000..ae359ada6a --- /dev/null +++ b/internal/guardrails/modelarmor.go @@ -0,0 +1,170 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package guardrails + +import ( + "context" + "fmt" + "net/http" + "net/url" + "strings" + + "golang.org/x/oauth2" + "golang.org/x/oauth2/google" + + "github.com/envoyproxy/ai-gateway/internal/filterapi" + "github.com/envoyproxy/ai-gateway/internal/gcpauth" +) + +const modelArmorScope = "https://www.googleapis.com/auth/cloud-platform" + +// Model Armor sanitization result values. +const ( + modelArmorMatchFound = "MATCH_FOUND" + modelArmorInvocationFailed = "FAILURE" +) + +type modelArmorEvaluator struct { + templateURL string + client *http.Client + tokenSource oauth2.TokenSource +} + +func newModelArmorEvaluator(ctx context.Context, config *filterapi.ModelArmorGuardrailProvider, client *http.Client) (filterapi.GuardrailEvaluator, error) { + if config == nil || config.Project == "" || config.Location == "" || config.Template == "" { + return nil, fmt.Errorf("model Armor project, location, and template are required") + } + tokenSource, err := loadGCPTokenSource(ctx, config, client) + if err != nil { + return nil, err + } + endpoint := config.Endpoint + if endpoint == "" { + endpoint = fmt.Sprintf("https://modelarmor.%s.rep.googleapis.com", config.Location) + } + templateURL := strings.TrimRight(endpoint, "/") + "/v1/projects/" + url.PathEscape(config.Project) + + "/locations/" + url.PathEscape(config.Location) + "/templates/" + url.PathEscape(config.Template) + return &modelArmorEvaluator{templateURL: templateURL, client: client, tokenSource: tokenSource}, nil +} + +// loadGCPTokenSource returns an auto-refreshing token source from the configured service account key, +// or from Application Default Credentials (e.g. GKE Workload Identity) when no key is configured. +func loadGCPTokenSource(ctx context.Context, config *filterapi.ModelArmorGuardrailProvider, client *http.Client) (oauth2.TokenSource, error) { + transport, err := gcpauth.NewTransport() + if err != nil { + return nil, err + } + ctx = context.WithValue(ctx, oauth2.HTTPClient, &http.Client{Transport: transport, Timeout: client.Timeout}) + var credentials *google.Credentials + if config.CredentialsJSON != "" { + credentials, err = google.CredentialsFromJSONWithType(ctx, []byte(config.CredentialsJSON), google.ServiceAccount, modelArmorScope) + } else { + credentials, err = google.FindDefaultCredentials(ctx, modelArmorScope) + } + if err != nil { + return nil, fmt.Errorf("cannot load GCP credentials: %w", err) + } + return credentials.TokenSource, nil +} + +// modelArmorMatch is the common shape of every Model Armor filter result. +type modelArmorMatch struct { + MatchState string `json:"matchState"` +} + +// modelArmorFilterResult holds the result of one filter. At most one field is set. +type modelArmorFilterResult struct { + RaiFilterResult *modelArmorMatch `json:"raiFilterResult,omitempty"` + PiAndJailbreakFilterResult *modelArmorMatch `json:"piAndJailbreakFilterResult,omitempty"` + MaliciousURIFilterResult *modelArmorMatch `json:"maliciousUriFilterResult,omitempty"` + CsamFilterFilterResult *modelArmorMatch `json:"csamFilterFilterResult,omitempty"` + VirusScanFilterResult *modelArmorMatch `json:"virusScanFilterResult,omitempty"` + SdpFilterResult *struct { + InspectResult *modelArmorMatch `json:"inspectResult,omitempty"` + DeidentifyResult *struct { + MatchState string `json:"matchState"` + Data *struct { + Text string `json:"text"` + } `json:"data,omitempty"` + } `json:"deidentifyResult,omitempty"` + } `json:"sdpFilterResult,omitempty"` +} + +type modelArmorResponse struct { + SanitizationResult struct { + FilterMatchState string `json:"filterMatchState"` + InvocationResult string `json:"invocationResult"` + FilterResults map[string]modelArmorFilterResult `json:"filterResults"` + } `json:"sanitizationResult"` +} + +func (e *modelArmorEvaluator) Evaluate(ctx context.Context, body []byte, phase filterapi.GuardrailPhase) (filterapi.GuardrailEvaluationResult, error) { + method := "sanitizeUserPrompt" + var payload any = struct { + UserPromptData struct { + Text string `json:"text"` + } `json:"userPromptData"` + }{UserPromptData: struct { + Text string `json:"text"` + }{Text: string(body)}} + if phase == filterapi.GuardrailPhaseResponse { + method = "sanitizeModelResponse" + payload = struct { + ModelResponseData struct { + Text string `json:"text"` + } `json:"modelResponseData"` + }{ModelResponseData: struct { + Text string `json:"text"` + }{Text: string(body)}} + } + token, err := e.tokenSource.Token() + if err != nil { + return filterapi.GuardrailEvaluationResult{}, fmt.Errorf("cannot retrieve GCP access token: %w", err) + } + + var result modelArmorResponse + if err = doJSON(ctx, e.client, http.MethodPost, e.templateURL+":"+method, payload, func(req *http.Request) { + token.SetAuthHeader(req) + }, &result); err != nil { + return filterapi.GuardrailEvaluationResult{}, fmt.Errorf("model Armor %s request failed: %w", method, err) + } + if result.SanitizationResult.InvocationResult == modelArmorInvocationFailed { + return filterapi.GuardrailEvaluationResult{}, fmt.Errorf("model Armor %s invocation failed", method) + } + if result.SanitizationResult.FilterMatchState != modelArmorMatchFound { + return filterapi.GuardrailEvaluationResult{}, nil + } + return filterapi.GuardrailEvaluationResult{ + Matched: true, + Replacement: modelArmorDeidentifiedText(result.SanitizationResult.FilterResults), + }, nil +} + +// modelArmorDeidentifiedText returns the Sensitive Data Protection de-identified text when it is the +// only filter that matched. It returns nil when any other filter matched, so Mask rules never forward +// content that was flagged for a reason masking cannot fix (e.g. a jailbreak attempt). +func modelArmorDeidentifiedText(results map[string]modelArmorFilterResult) []byte { + var replacement []byte + for _, result := range results { + for _, match := range []*modelArmorMatch{ + result.RaiFilterResult, result.PiAndJailbreakFilterResult, result.MaliciousURIFilterResult, + result.CsamFilterFilterResult, result.VirusScanFilterResult, + } { + if match != nil && match.MatchState == modelArmorMatchFound { + return nil + } + } + if sdp := result.SdpFilterResult; sdp != nil { + if sdp.InspectResult != nil && sdp.InspectResult.MatchState == modelArmorMatchFound { + return nil + } + if d := sdp.DeidentifyResult; d != nil && d.MatchState == modelArmorMatchFound && d.Data != nil { + replacement = []byte(d.Data.Text) + } + } + } + return replacement +} diff --git a/internal/guardrails/presidio.go b/internal/guardrails/presidio.go new file mode 100644 index 0000000000..0830d800c3 --- /dev/null +++ b/internal/guardrails/presidio.go @@ -0,0 +1,63 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package guardrails + +import ( + "context" + "fmt" + "net/http" + "strings" + + "github.com/envoyproxy/ai-gateway/internal/filterapi" +) + +type presidioEvaluator struct { + config *filterapi.PresidioGuardrailProvider + maskReplacement string + client *http.Client +} + +func newPresidioEvaluator(config *filterapi.PresidioGuardrailProvider, maskReplacement string, client *http.Client) (filterapi.GuardrailEvaluator, error) { + if config == nil || config.Endpoint == "" { + return nil, fmt.Errorf("presidio endpoint is required") + } + configCopy := *config + if configCopy.Language == "" { + configCopy.Language = "en" + } + if maskReplacement == "" { + maskReplacement = "[REDACTED]" + } + return &presidioEvaluator{config: &configCopy, maskReplacement: maskReplacement, client: client}, nil +} + +func (e *presidioEvaluator) Evaluate(ctx context.Context, body []byte, _ filterapi.GuardrailPhase) (filterapi.GuardrailEvaluationResult, error) { + payload := struct { + Text string `json:"text"` + Language string `json:"language"` + ScoreThreshold *float64 `json:"score_threshold,omitempty"` + }{Text: string(body), Language: e.config.Language} + if e.config.ScoreThresholdPercent > 0 { + threshold := float64(e.config.ScoreThresholdPercent) / 100 + payload.ScoreThreshold = &threshold + } + + var result []textSpan + if err := doJSON(ctx, e.client, http.MethodPost, strings.TrimRight(e.config.Endpoint, "/")+"/analyze", payload, func(req *http.Request) { + if e.config.APIKey != "" { + req.Header.Set("Authorization", "Bearer "+e.config.APIKey) + } + }, &result); err != nil { + return filterapi.GuardrailEvaluationResult{}, fmt.Errorf("presidio analyze request failed: %w", err) + } + if len(result) == 0 { + return filterapi.GuardrailEvaluationResult{}, nil + } + return filterapi.GuardrailEvaluationResult{ + Matched: true, + Replacement: maskSpans(body, result, e.maskReplacement), + }, nil +} diff --git a/internal/guardrails/presidio_container_test.go b/internal/guardrails/presidio_container_test.go new file mode 100644 index 0000000000..a898349732 --- /dev/null +++ b/internal/guardrails/presidio_container_test.go @@ -0,0 +1,59 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package guardrails + +import ( + "context" + "fmt" + "net" + "net/http" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" + + "github.com/envoyproxy/ai-gateway/internal/filterapi" +) + +const presidioAnalyzerImage = "ghcr.io/data-privacy-stack/presidio-analyzer:2.2.364" + +func TestPresidioEvaluatorContainer(t *testing.T) { + testcontainers.SkipIfProviderIsNotHealthy(t) + + ctx := t.Context() + container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: testcontainers.ContainerRequest{ + Image: presidioAnalyzerImage, + ExposedPorts: []string{"3000/tcp"}, + WaitingFor: wait.ForHTTP("/health"). + WithPort("3000/tcp"). + WithStartupTimeout(2 * time.Minute), + }, + Started: true, + }) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, container.Terminate(context.WithoutCancel(ctx))) + }) + + host, err := container.Host(ctx) + require.NoError(t, err) + port, err := container.MappedPort(ctx, "3000/tcp") + require.NoError(t, err) + + evaluator, err := newPresidioEvaluator(&filterapi.PresidioGuardrailProvider{ + Endpoint: fmt.Sprintf("http://%s", net.JoinHostPort(host, port.Port())), + Language: "en", + ScoreThresholdPercent: 80, + }, "[REDACTED]", &http.Client{Timeout: 10 * time.Second}) + require.NoError(t, err) + + evaluation, err := evaluator.Evaluate(ctx, []byte("Contact me at alice@example.com"), filterapi.GuardrailPhaseRequest) + require.NoError(t, err) + require.True(t, evaluation.Matched) +} diff --git a/internal/metrics/guardrail_metrics.go b/internal/metrics/guardrail_metrics.go new file mode 100644 index 0000000000..b285f63994 --- /dev/null +++ b/internal/metrics/guardrail_metrics.go @@ -0,0 +1,57 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package metrics + +import ( + "context" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" +) + +const ( + guardrailEvaluationCount = "aigateway.guardrail.evaluation.count" + guardrailAttributePhase = "aigateway.guardrail.phase" + guardrailAttributeResult = "aigateway.guardrail.result" +) + +// GuardrailResult is the outcome of evaluating a payload against guardrails. +type GuardrailResult string + +const ( + GuardrailResultAllowed GuardrailResult = "allowed" + GuardrailResultBlocked GuardrailResult = "blocked" + GuardrailResultError GuardrailResult = "error" + GuardrailResultMasked GuardrailResult = "masked" + GuardrailResultMonitored GuardrailResult = "monitored" +) + +// GuardrailMetrics records guardrail evaluation outcomes. +type GuardrailMetrics interface { + RecordEvaluation(ctx context.Context, phase string, result GuardrailResult) +} + +type guardrailMetrics struct { + evaluationCount metric.Float64Counter +} + +// NewGuardrailMetrics creates guardrail metrics backed by the provided meter. +func NewGuardrailMetrics(meter metric.Meter) GuardrailMetrics { + return &guardrailMetrics{ + evaluationCount: mustRegisterCounter( + meter, + guardrailEvaluationCount, + metric.WithDescription("Total number of payloads evaluated by guardrails"), + ), + } +} + +func (m *guardrailMetrics) RecordEvaluation(ctx context.Context, phase string, result GuardrailResult) { + m.evaluationCount.Add(ctx, 1, metric.WithAttributes( + attribute.String(guardrailAttributePhase, phase), + attribute.String(guardrailAttributeResult, string(result)), + )) +} diff --git a/internal/metrics/guardrail_metrics_test.go b/internal/metrics/guardrail_metrics_test.go new file mode 100644 index 0000000000..d78dcac11b --- /dev/null +++ b/internal/metrics/guardrail_metrics_test.go @@ -0,0 +1,31 @@ +// Copyright Envoy AI Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package metrics + +import ( + "testing" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/sdk/metric" + + "github.com/envoyproxy/ai-gateway/internal/testing/testotel" +) + +func TestGuardrailMetricsRecordEvaluation(t *testing.T) { + reader := metric.NewManualReader() + meter := metric.NewMeterProvider(metric.WithReader(reader)).Meter("test") + guardrails := NewGuardrailMetrics(meter) + + guardrails.RecordEvaluation(t.Context(), "Request", GuardrailResultBlocked) + guardrails.RecordEvaluation(t.Context(), "Request", GuardrailResultBlocked) + + value := testotel.GetCounterValue(t, reader, guardrailEvaluationCount, attribute.NewSet( + attribute.String(guardrailAttributePhase, "Request"), + attribute.String(guardrailAttributeResult, string(GuardrailResultBlocked)), + )) + require.Equal(t, float64(2), value) +} diff --git a/internal/tracing/span.go b/internal/tracing/span.go index 4ac15a348f..66fb020b32 100644 --- a/internal/tracing/span.go +++ b/internal/tracing/span.go @@ -6,6 +6,7 @@ package tracing import ( + "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" anthropicschema "github.com/envoyproxy/ai-gateway/internal/apischema/anthropic" @@ -22,6 +23,15 @@ type span[RespT, ChunkT any] struct { chunks []*ChunkT } +// RecordGuardrail records a guardrail evaluation event without payload content. +func (s *span[RespT, ChunkT]) RecordGuardrail(name, phase, result string) { + s.span.AddEvent("guardrail.evaluation", trace.WithAttributes( + attribute.String("aigateway.guardrail.name", name), + attribute.String("aigateway.guardrail.phase", phase), + attribute.String("aigateway.guardrail.result", result), + )) +} + // RecordResponseChunk implements [tracingapi.Span.RecordResponseChunk] func (s *span[RespT, ChunkT]) RecordResponseChunk(resp *ChunkT) { s.chunks = append(s.chunks, resp) diff --git a/internal/tracing/span_test.go b/internal/tracing/span_test.go index 9a19435712..517fae4584 100644 --- a/internal/tracing/span_test.go +++ b/internal/tracing/span_test.go @@ -48,6 +48,22 @@ func TestChatCompletionSpan_RecordResponse(t *testing.T) { }, actualSpan.Attributes) } +func TestChatCompletionSpan_RecordGuardrail(t *testing.T) { + s := &chatCompletionSpan{} + actualSpan := testotel.RecordWithSpan(t, func(span oteltrace.Span) bool { + s.span = span + s.RecordGuardrail("deny-pii", "Request", "blocked") + return false + }) + require.Len(t, actualSpan.Events, 1) + require.Equal(t, "guardrail.evaluation", actualSpan.Events[0].Name) + require.ElementsMatch(t, []attribute.KeyValue{ + attribute.String("aigateway.guardrail.name", "deny-pii"), + attribute.String("aigateway.guardrail.phase", "Request"), + attribute.String("aigateway.guardrail.result", "blocked"), + }, actualSpan.Events[0].Attributes) +} + // TestChatCompletionSpan_RecordBackend pins that the backend only reaches the // span when the convention's recorder asks for it. The backend is resolved for // every request, so a recorder whose convention defines no backend attributes diff --git a/internal/tracing/tracingapi/api.go b/internal/tracing/tracingapi/api.go index d4713af1e8..b34d2e80dc 100644 --- a/internal/tracing/tracingapi/api.go +++ b/internal/tracing/tracingapi/api.go @@ -111,6 +111,10 @@ type ( ) type ( + // GuardrailSpan optionally records guardrail evaluations on a request span. + GuardrailSpan interface { + RecordGuardrail(name, phase, result string) + } // Span standardizes span interfaces, supporting both streaming and non-streaming endpoints. Span[RespT any, RespChunkT any] interface { // RecordResponseChunk records streaming response chunks. Implementations that do not support streaming should provide a no-op implementation. diff --git a/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_guardrailpolicies.yaml b/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_guardrailpolicies.yaml new file mode 100644 index 0000000000..5485cc22cc --- /dev/null +++ b/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_guardrailpolicies.yaml @@ -0,0 +1,1204 @@ +# Copyright Envoy AI Gateway Authors +# SPDX-License-Identifier: Apache-2.0 +# The full text of the Apache license is available in the LICENSE file at +# the root of the repo. + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.0 + labels: + gateway.networking.k8s.io/policy: direct + name: guardrailpolicies.aigateway.envoyproxy.io +spec: + group: aigateway.envoyproxy.io + names: + kind: GuardrailPolicy + listKind: GuardrailPolicyList + plural: guardrailpolicies + singular: guardrailpolicy + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.conditions[-1:].type + name: Status + type: string + deprecated: true + deprecationWarning: aigateway.envoyproxy.io/v1alpha1 is deprecated; use aigateway.envoyproxy.io/v1beta1 + instead + name: v1alpha1 + schema: + openAPIV3Schema: + description: GuardrailPolicy evaluates content safety checks for request and + response payloads. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: GuardrailPolicySpec contains the configured checks attached + to an AIServiceBackend. + properties: + maxRequestBodyBytes: + default: 10485760 + description: MaxRequestBodyBytes is the largest request body evaluated + by this policy. + format: int64 + maximum: 52428800 + minimum: 1024 + type: integer + maxResponseBodyBytes: + default: 10485760 + description: MaxResponseBodyBytes is the largest response body buffered + and evaluated by this policy. + format: int64 + maximum: 52428800 + minimum: 1024 + type: integer + rules: + description: Rules are executed in order and can evaluate request + or response payloads. + items: + description: GuardrailRule defines one content-safety check to apply + to a request or response. + properties: + name: + description: Name is a stable identifier for the rule. + maxLength: 63 + minLength: 1 + type: string + phase: + description: Phase determines whether the rule runs against + the request or the response payload. + enum: + - Request + - Response + type: string + provider: + description: Provider configures how the rule is evaluated. + properties: + action: + default: Block + description: Action is the action taken when the rule is + matched. + enum: + - Block + - Monitor + - Mask + type: string + azureContentSafety: + description: AzureContentSafety configures Azure AI Content + Safety. + properties: + apiKeySecretRef: + description: APIKeySecretRef references a Secret whose + apiKey entry is sent to Azure. + properties: + group: + default: "" + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Secret + description: Kind is kind of the referent. For example + "Secret". + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the referenced object. When unspecified, the local + namespace is inferred. + + Note that when a namespace different than the local namespace is specified, + a ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - name + type: object + apiVersion: + default: "2024-09-01" + type: string + endpoint: + format: uri + type: string + severityThreshold: + format: int32 + maximum: 7 + minimum: 0 + type: integer + required: + - apiKeySecretRef + - endpoint + type: object + bedrock: + description: Bedrock configures AWS Bedrock Guardrails. + properties: + credentialsSecretRef: + description: CredentialsSecretRef optionally references + a Secret whose credentials entry contains an AWS shared + credentials file. + properties: + group: + default: "" + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Secret + description: Kind is kind of the referent. For example + "Secret". + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the referenced object. When unspecified, the local + namespace is inferred. + + Note that when a namespace different than the local namespace is specified, + a ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - name + type: object + endpoint: + description: Endpoint overrides the Bedrock runtime + endpoint, primarily for private endpoints and testing. + format: uri + type: string + guardrailIdentifier: + minLength: 1 + type: string + guardrailVersion: + minLength: 1 + type: string + region: + minLength: 1 + type: string + required: + - guardrailIdentifier + - guardrailVersion + - region + type: object + failureMode: + default: FailClosed + description: FailureMode determines whether provider errors + block or allow the request. + enum: + - FailClosed + - FailOpen + type: string + http: + description: HTTP configures a custom guardrail service + that implements the generic guardrail HTTP contract. + properties: + apiKeySecretRef: + description: APIKeySecretRef optionally references a + Secret whose apiKey entry is sent as a Bearer token. + properties: + group: + default: "" + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Secret + description: Kind is kind of the referent. For example + "Secret". + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the referenced object. When unspecified, the local + namespace is inferred. + + Note that when a namespace different than the local namespace is specified, + a ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - name + type: object + endpoint: + description: Endpoint is the base URL of the guardrail + service. + format: uri + type: string + path: + default: /analyze + description: Path is appended to the endpoint for each + evaluation. + pattern: ^/.*$ + type: string + required: + - endpoint + type: object + maskReplacement: + default: '[REDACTED]' + description: MaskReplacement is used by Regex, Presidio, + and HTTP Mask actions. + type: string + message: + description: Message is returned to the caller when the + rule blocks a request or response. + type: string + modelArmor: + description: ModelArmor configures Google Cloud Model Armor. + properties: + credentialsSecretRef: + description: |- + CredentialsSecretRef optionally references a Secret whose credentials entry contains a Google Cloud + service account key JSON. When omitted, Application Default Credentials are used. + properties: + group: + default: "" + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Secret + description: Kind is kind of the referent. For example + "Secret". + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the referenced object. When unspecified, the local + namespace is inferred. + + Note that when a namespace different than the local namespace is specified, + a ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - name + type: object + endpoint: + description: Endpoint overrides the regional Model Armor + endpoint, primarily for private endpoints and testing. + format: uri + type: string + location: + description: Location is the Google Cloud region of + the Model Armor template, for example us-central1. + minLength: 1 + type: string + project: + description: Project is the Google Cloud project ID + that owns the Model Armor template. + minLength: 1 + type: string + template: + description: Template is the Model Armor template ID. + minLength: 1 + type: string + required: + - location + - project + - template + type: object + pattern: + description: Pattern is used for deterministic regex-based + evaluations. + type: string + presidio: + description: Presidio configures the Presidio analyzer provider. + properties: + apiKeySecretRef: + description: APIKeySecretRef optionally references a + Secret whose apiKey entry is sent as a Bearer token. + properties: + group: + default: "" + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Secret + description: Kind is kind of the referent. For example + "Secret". + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the referenced object. When unspecified, the local + namespace is inferred. + + Note that when a namespace different than the local namespace is specified, + a ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - name + type: object + endpoint: + format: uri + type: string + language: + default: en + type: string + scoreThresholdPercent: + format: int32 + maximum: 100 + minimum: 0 + type: integer + required: + - endpoint + type: object + timeoutSeconds: + default: 10 + description: TimeoutSeconds limits each external provider + evaluation. + format: int32 + maximum: 60 + minimum: 1 + type: integer + type: + description: Type identifies the guardrail implementation. + enum: + - Regex + - Presidio + - Bedrock + - AzureContentSafety + - HTTP + - ModelArmor + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: Regex requires pattern and no external provider configuration + rule: self.type != 'Regex' || (has(self.pattern) && size(self.pattern) + > 0 && !has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety) + && !has(self.http) && !has(self.modelArmor)) + - message: Presidio requires only presidio provider configuration + rule: self.type != 'Presidio' || (!has(self.pattern) && has(self.presidio) + && !has(self.bedrock) && !has(self.azureContentSafety) && + !has(self.http) && !has(self.modelArmor)) + - message: Bedrock requires only bedrock provider configuration + rule: self.type != 'Bedrock' || (!has(self.pattern) && has(self.bedrock) + && !has(self.presidio) && !has(self.azureContentSafety) + && !has(self.http) && !has(self.modelArmor)) + - message: AzureContentSafety requires only azureContentSafety + provider configuration + rule: self.type != 'AzureContentSafety' || (!has(self.pattern) + && has(self.azureContentSafety) && !has(self.presidio) && + !has(self.bedrock) && !has(self.http) && !has(self.modelArmor)) + - message: HTTP requires only http provider configuration + rule: self.type != 'HTTP' || (!has(self.pattern) && has(self.http) + && !has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety) + && !has(self.modelArmor)) + - message: ModelArmor requires only modelArmor provider configuration + rule: self.type != 'ModelArmor' || (!has(self.pattern) && + has(self.modelArmor) && !has(self.presidio) && !has(self.bedrock) + && !has(self.azureContentSafety) && !has(self.http)) + - message: AzureContentSafety does not support Mask + rule: self.action != 'Mask' || self.type != 'AzureContentSafety' + required: + - name + - phase + - provider + type: object + maxItems: 32 + minItems: 1 + type: array + x-kubernetes-validations: + - message: rule name must be unique within the policy + rule: self.all(rule, self.exists_one(other, other.name == rule.name)) + targetRefs: + description: TargetRefs identify the AIServiceBackend resources this + GuardrailPolicy is attached to. + items: + description: |- + LocalPolicyTargetReference identifies an API object to apply a direct or + inherited policy to. This should be used as part of Policy resources + that can target Gateway API resources. For more information on how this + policy attachment model works, and a sample Policy resource, refer to + the policy attachment documentation for Gateway API. + properties: + group: + description: Group is the group of the target resource. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + description: Kind is kind of the target resource. + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the target resource. + maxLength: 253 + minLength: 1 + type: string + required: + - group + - kind + - name + type: object + maxItems: 16 + minItems: 1 + type: array + x-kubernetes-validations: + - message: targetRefs must reference AIServiceBackend resources + rule: self.all(ref, ref.group == 'aigateway.envoyproxy.io' && ref.kind + == 'AIServiceBackend') + type: object + status: + description: Status defines the status details of the GuardrailPolicy. + properties: + conditions: + description: |- + Conditions is the list of conditions by the reconciliation result. + Currently, at most one condition is set. + + Known .status.conditions.type are: "Accepted", "NotAccepted". + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - jsonPath: .status.conditions[-1:].type + name: Status + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: GuardrailPolicy evaluates content safety checks for request and + response payloads. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: GuardrailPolicySpec contains the configured checks attached + to an AIServiceBackend. + properties: + maxRequestBodyBytes: + default: 10485760 + description: MaxRequestBodyBytes is the largest request body evaluated + by this policy. + format: int64 + maximum: 52428800 + minimum: 1024 + type: integer + maxResponseBodyBytes: + default: 10485760 + description: MaxResponseBodyBytes is the largest response body buffered + and evaluated by this policy. + format: int64 + maximum: 52428800 + minimum: 1024 + type: integer + rules: + description: Rules are executed in order and can evaluate request + or response payloads. + items: + description: GuardrailRule defines one content-safety check to apply + to a request or response. + properties: + name: + description: Name is a stable identifier for the rule. + maxLength: 63 + minLength: 1 + type: string + phase: + description: Phase determines whether the rule runs against + the request or the response payload. + enum: + - Request + - Response + type: string + provider: + description: Provider configures how the rule is evaluated. + properties: + action: + default: Block + description: Action is the action taken when the rule is + matched. + enum: + - Block + - Monitor + - Mask + type: string + azureContentSafety: + description: AzureContentSafety configures Azure AI Content + Safety. + properties: + apiKeySecretRef: + description: APIKeySecretRef references a Secret whose + apiKey entry is sent to Azure. + properties: + group: + default: "" + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Secret + description: Kind is kind of the referent. For example + "Secret". + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the referenced object. When unspecified, the local + namespace is inferred. + + Note that when a namespace different than the local namespace is specified, + a ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - name + type: object + apiVersion: + default: "2024-09-01" + type: string + endpoint: + format: uri + type: string + severityThreshold: + format: int32 + maximum: 7 + minimum: 0 + type: integer + required: + - apiKeySecretRef + - endpoint + type: object + bedrock: + description: Bedrock configures AWS Bedrock Guardrails. + properties: + credentialsSecretRef: + description: CredentialsSecretRef optionally references + a Secret whose credentials entry contains an AWS shared + credentials file. + properties: + group: + default: "" + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Secret + description: Kind is kind of the referent. For example + "Secret". + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the referenced object. When unspecified, the local + namespace is inferred. + + Note that when a namespace different than the local namespace is specified, + a ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - name + type: object + endpoint: + description: Endpoint overrides the Bedrock runtime + endpoint, primarily for private endpoints and testing. + format: uri + type: string + guardrailIdentifier: + minLength: 1 + type: string + guardrailVersion: + minLength: 1 + type: string + region: + minLength: 1 + type: string + required: + - guardrailIdentifier + - guardrailVersion + - region + type: object + failureMode: + default: FailClosed + description: FailureMode determines whether provider errors + block or allow the request. + enum: + - FailClosed + - FailOpen + type: string + http: + description: HTTP configures a custom guardrail service + that implements the generic guardrail HTTP contract. + properties: + apiKeySecretRef: + description: APIKeySecretRef optionally references a + Secret whose apiKey entry is sent as a Bearer token. + properties: + group: + default: "" + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Secret + description: Kind is kind of the referent. For example + "Secret". + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the referenced object. When unspecified, the local + namespace is inferred. + + Note that when a namespace different than the local namespace is specified, + a ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - name + type: object + endpoint: + description: Endpoint is the base URL of the guardrail + service. + format: uri + type: string + path: + default: /analyze + description: Path is appended to the endpoint for each + evaluation. + pattern: ^/.*$ + type: string + required: + - endpoint + type: object + maskReplacement: + default: '[REDACTED]' + description: MaskReplacement is used by Regex, Presidio, + and HTTP Mask actions. + type: string + message: + description: Message is returned to the caller when the + rule blocks a request or response. + type: string + modelArmor: + description: ModelArmor configures Google Cloud Model Armor. + properties: + credentialsSecretRef: + description: |- + CredentialsSecretRef optionally references a Secret whose credentials entry contains a Google Cloud + service account key JSON. When omitted, Application Default Credentials are used. + properties: + group: + default: "" + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Secret + description: Kind is kind of the referent. For example + "Secret". + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the referenced object. When unspecified, the local + namespace is inferred. + + Note that when a namespace different than the local namespace is specified, + a ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - name + type: object + endpoint: + description: Endpoint overrides the regional Model Armor + endpoint, primarily for private endpoints and testing. + format: uri + type: string + location: + description: Location is the Google Cloud region of + the Model Armor template, for example us-central1. + minLength: 1 + type: string + project: + description: Project is the Google Cloud project ID + that owns the Model Armor template. + minLength: 1 + type: string + template: + description: Template is the Model Armor template ID. + minLength: 1 + type: string + required: + - location + - project + - template + type: object + pattern: + description: Pattern is used for deterministic regex-based + evaluations. + type: string + presidio: + description: Presidio configures the Presidio analyzer provider. + properties: + apiKeySecretRef: + description: APIKeySecretRef optionally references a + Secret whose apiKey entry is sent as a Bearer token. + properties: + group: + default: "" + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Secret + description: Kind is kind of the referent. For example + "Secret". + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the referenced object. When unspecified, the local + namespace is inferred. + + Note that when a namespace different than the local namespace is specified, + a ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - name + type: object + endpoint: + format: uri + type: string + language: + default: en + type: string + scoreThresholdPercent: + format: int32 + maximum: 100 + minimum: 0 + type: integer + required: + - endpoint + type: object + timeoutSeconds: + default: 10 + description: TimeoutSeconds limits each external provider + evaluation. + format: int32 + maximum: 60 + minimum: 1 + type: integer + type: + description: Type identifies the guardrail implementation. + enum: + - Regex + - Presidio + - Bedrock + - AzureContentSafety + - HTTP + - ModelArmor + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: Regex requires pattern and no external provider configuration + rule: self.type != 'Regex' || (has(self.pattern) && size(self.pattern) + > 0 && !has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety) + && !has(self.http) && !has(self.modelArmor)) + - message: Presidio requires only presidio provider configuration + rule: self.type != 'Presidio' || (!has(self.pattern) && has(self.presidio) + && !has(self.bedrock) && !has(self.azureContentSafety) && + !has(self.http) && !has(self.modelArmor)) + - message: Bedrock requires only bedrock provider configuration + rule: self.type != 'Bedrock' || (!has(self.pattern) && has(self.bedrock) + && !has(self.presidio) && !has(self.azureContentSafety) + && !has(self.http) && !has(self.modelArmor)) + - message: AzureContentSafety requires only azureContentSafety + provider configuration + rule: self.type != 'AzureContentSafety' || (!has(self.pattern) + && has(self.azureContentSafety) && !has(self.presidio) && + !has(self.bedrock) && !has(self.http) && !has(self.modelArmor)) + - message: HTTP requires only http provider configuration + rule: self.type != 'HTTP' || (!has(self.pattern) && has(self.http) + && !has(self.presidio) && !has(self.bedrock) && !has(self.azureContentSafety) + && !has(self.modelArmor)) + - message: ModelArmor requires only modelArmor provider configuration + rule: self.type != 'ModelArmor' || (!has(self.pattern) && + has(self.modelArmor) && !has(self.presidio) && !has(self.bedrock) + && !has(self.azureContentSafety) && !has(self.http)) + - message: AzureContentSafety does not support Mask + rule: self.action != 'Mask' || self.type != 'AzureContentSafety' + required: + - name + - phase + - provider + type: object + maxItems: 32 + minItems: 1 + type: array + x-kubernetes-validations: + - message: rule name must be unique within the policy + rule: self.all(rule, self.exists_one(other, other.name == rule.name)) + targetRefs: + description: TargetRefs identify the AIServiceBackend resources this + GuardrailPolicy is attached to. + items: + description: |- + LocalPolicyTargetReference identifies an API object to apply a direct or + inherited policy to. This should be used as part of Policy resources + that can target Gateway API resources. For more information on how this + policy attachment model works, and a sample Policy resource, refer to + the policy attachment documentation for Gateway API. + properties: + group: + description: Group is the group of the target resource. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + description: Kind is kind of the target resource. + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the target resource. + maxLength: 253 + minLength: 1 + type: string + required: + - group + - kind + - name + type: object + maxItems: 16 + minItems: 1 + type: array + x-kubernetes-validations: + - message: targetRefs must reference AIServiceBackend resources + rule: self.all(ref, ref.group == 'aigateway.envoyproxy.io' && ref.kind + == 'AIServiceBackend') + type: object + status: + description: Status defines the status details of the GuardrailPolicy. + properties: + conditions: + description: |- + Conditions is the list of conditions by the reconciliation result. + Currently, at most one condition is set. + + Known .status.conditions.type are: "Accepted", "NotAccepted". + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/site/docs/api/api.mdx b/site/docs/api/api.mdx index eb0853a31e..ed4fbdcafe 100644 --- a/site/docs/api/api.mdx +++ b/site/docs/api/api.mdx @@ -27,6 +27,8 @@ API group. - [BackendSecurityPolicyList](#github-com-envoyproxy-ai-gateway-api-v1alpha1-backendsecuritypolicylist) - [GatewayConfig](#github-com-envoyproxy-ai-gateway-api-v1alpha1-gatewayconfig) - [GatewayConfigList](#github-com-envoyproxy-ai-gateway-api-v1alpha1-gatewayconfiglist) +- [GuardrailPolicy](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailpolicy) +- [GuardrailPolicyList](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailpolicylist) - [MCPRoute](#github-com-envoyproxy-ai-gateway-api-v1alpha1-mcproute) - [MCPRouteList](#github-com-envoyproxy-ai-gateway-api-v1alpha1-mcproutelist) - [QuotaPolicy](#github-com-envoyproxy-ai-gateway-api-v1alpha1-quotapolicy) @@ -391,6 +393,85 @@ GatewayConfigList contains a list of GatewayConfig. /> +#### GuardrailPolicy + + + +**Appears in:** +- [GuardrailPolicyList](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailpolicylist) + +GuardrailPolicy evaluates content safety checks for request and response payloads. + +##### Fields + + + + + + + + +#### GuardrailPolicyList + + + + +GuardrailPolicyList contains a list of GuardrailPolicy resources. + +##### Fields + + + + + + + + #### MCPRoute @@ -568,6 +649,7 @@ QuotaPolicyList contains a list of QuotaPolicy - [APISchema](#github-com-envoyproxy-ai-gateway-api-v1alpha1-apischema) - [AWSCredentialsFile](#github-com-envoyproxy-ai-gateway-api-v1alpha1-awscredentialsfile) - [AWSOIDCExchangeToken](#github-com-envoyproxy-ai-gateway-api-v1alpha1-awsoidcexchangetoken) +- [AzureContentSafetyGuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1alpha1-azurecontentsafetyguardrailprovider) - [AzureOIDCExchangeToken](#github-com-envoyproxy-ai-gateway-api-v1alpha1-azureoidcexchangetoken) - [BackendSecurityPolicyAPIKey](#github-com-envoyproxy-ai-gateway-api-v1alpha1-backendsecuritypolicyapikey) - [BackendSecurityPolicyAWSCredentials](#github-com-envoyproxy-ai-gateway-api-v1alpha1-backendsecuritypolicyawscredentials) @@ -579,6 +661,7 @@ QuotaPolicyList contains a list of QuotaPolicy - [BackendSecurityPolicySpec](#github-com-envoyproxy-ai-gateway-api-v1alpha1-backendsecuritypolicyspec) - [BackendSecurityPolicyStatus](#github-com-envoyproxy-ai-gateway-api-v1alpha1-backendsecuritypolicystatus) - [BackendSecurityPolicyType](#github-com-envoyproxy-ai-gateway-api-v1alpha1-backendsecuritypolicytype) +- [BedrockGuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1alpha1-bedrockguardrailprovider) - [GCPCredentialsFile](#github-com-envoyproxy-ai-gateway-api-v1alpha1-gcpcredentialsfile) - [GCPOIDCExchangeToken](#github-com-envoyproxy-ai-gateway-api-v1alpha1-gcpoidcexchangetoken) - [GCPServiceAccountImpersonationConfig](#github-com-envoyproxy-ai-gateway-api-v1alpha1-gcpserviceaccountimpersonationconfig) @@ -587,8 +670,17 @@ QuotaPolicyList contains a list of QuotaPolicy - [GatewayConfigForwardProxy](#github-com-envoyproxy-ai-gateway-api-v1alpha1-gatewayconfigforwardproxy) - [GatewayConfigSpec](#github-com-envoyproxy-ai-gateway-api-v1alpha1-gatewayconfigspec) - [GatewayConfigStatus](#github-com-envoyproxy-ai-gateway-api-v1alpha1-gatewayconfigstatus) +- [GuardrailAction](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailaction) +- [GuardrailFailureMode](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailfailuremode) +- [GuardrailPhase](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailphase) +- [GuardrailPolicySpec](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailpolicyspec) +- [GuardrailPolicyStatus](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailpolicystatus) +- [GuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailprovider) +- [GuardrailProviderType](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailprovidertype) +- [GuardrailRule](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailrule) - [HTTPBodyField](#github-com-envoyproxy-ai-gateway-api-v1alpha1-httpbodyfield) - [HTTPBodyMutation](#github-com-envoyproxy-ai-gateway-api-v1alpha1-httpbodymutation) +- [HTTPGuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1alpha1-httpguardrailprovider) - [HTTPHeaderMutation](#github-com-envoyproxy-ai-gateway-api-v1alpha1-httpheadermutation) - [JWKS](#github-com-envoyproxy-ai-gateway-api-v1alpha1-jwks) - [JWTSource](#github-com-envoyproxy-ai-gateway-api-v1alpha1-jwtsource) @@ -610,7 +702,9 @@ QuotaPolicyList contains a list of QuotaPolicy - [MCPRouteSpec](#github-com-envoyproxy-ai-gateway-api-v1alpha1-mcproutespec) - [MCPRouteStatus](#github-com-envoyproxy-ai-gateway-api-v1alpha1-mcproutestatus) - [MCPToolFilter](#github-com-envoyproxy-ai-gateway-api-v1alpha1-mcptoolfilter) +- [ModelArmorGuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1alpha1-modelarmorguardrailprovider) - [PerModelQuota](#github-com-envoyproxy-ai-gateway-api-v1alpha1-permodelquota) +- [PresidioGuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1alpha1-presidioguardrailprovider) - [ProtectedResourceMetadata](#github-com-envoyproxy-ai-gateway-api-v1alpha1-protectedresourcemetadata) - [QuotaBucketMode](#github-com-envoyproxy-ai-gateway-api-v1alpha1-quotabucketmode) - [QuotaDefinition](#github-com-envoyproxy-ai-gateway-api-v1alpha1-quotadefinition) @@ -1009,6 +1103,43 @@ and store them in a temporary credentials file. /> +#### AzureContentSafetyGuardrailProvider + + + +**Appears in:** +- [GuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailprovider) + +AzureContentSafetyGuardrailProvider configures calls to Azure AI Content Safety. + +##### Fields + + + + + + #### AzureOIDCExchangeToken @@ -1379,6 +1510,47 @@ BackendSecurityPolicyType specifies the type of auth mechanism used to access a required="false" description="" /> +#### BedrockGuardrailProvider + + + +**Appears in:** +- [GuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailprovider) + +BedrockGuardrailProvider configures calls to the AWS Bedrock ApplyGuardrail API. + +##### Fields + + + + + + #### GCPCredentialsFile @@ -1587,180 +1759,509 @@ GatewayConfigStatus defines the observed state of GatewayConfig. /> -#### HTTPBodyField - +#### GuardrailAction +**Underlying type:** string **Appears in:** -- [HTTPBodyMutation](#github-com-envoyproxy-ai-gateway-api-v1alpha1-httpbodymutation) +- [GuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailprovider) -HTTPBodyField represents a JSON field name and value for body mutation +GuardrailAction defines the safeguard action. -##### Fields +##### Possible Values +#### GuardrailFailureMode - -#### HTTPBodyMutation - - +**Underlying type:** string **Appears in:** -- [AIGatewayRouteRuleBackendRef](#github-com-envoyproxy-ai-gateway-api-v1alpha1-aigatewayrouterulebackendref) -- [AIServiceBackendSpec](#github-com-envoyproxy-ai-gateway-api-v1alpha1-aiservicebackendspec) +- [GuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailprovider) -HTTPBodyMutation defines the mutation of HTTP request body JSON fields that will be applied to the request +GuardrailFailureMode defines behavior when an external provider cannot evaluate content. -##### Fields +##### Possible Values +#### GuardrailPhase - -#### HTTPHeaderMutation - - +**Underlying type:** string **Appears in:** -- [AIGatewayRouteRuleBackendRef](#github-com-envoyproxy-ai-gateway-api-v1alpha1-aigatewayrouterulebackendref) -- [AIServiceBackendSpec](#github-com-envoyproxy-ai-gateway-api-v1alpha1-aiservicebackendspec) +- [GuardrailRule](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailrule) -HTTPHeaderMutation defines the mutation of HTTP headers that will be applied to the request +GuardrailPhase determines when a guardrail runs. -##### Fields +##### Possible Values - - -#### JWKS +#### GuardrailPolicySpec **Appears in:** -- [MCPRouteOAuth](#github-com-envoyproxy-ai-gateway-api-v1alpha1-mcprouteoauth) +- [GuardrailPolicy](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailpolicy) -JWKS defines how to obtain JSON Web Key Sets (JWKS) either from a remote HTTP/HTTPS endpoint or from a local source. +GuardrailPolicySpec contains the configured checks attached to an AIServiceBackend. ##### Fields -#### JWTSource +#### GuardrailPolicyStatus **Appears in:** -- [MCPAuthorizationSource](#github-com-envoyproxy-ai-gateway-api-v1alpha1-mcpauthorizationsource) +- [GuardrailPolicy](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailpolicy) -JWTSource defines the MCP authorization source for JWT tokens. -At least one of scopes or claims must be provided. -Scopes and claims are AND-ed: when both are specified, both sets must match. +GuardrailPolicyStatus contains the conditions by the reconciliation result. ##### Fields -#### LLMRequestCost +#### GuardrailProvider **Appears in:** -- [AIGatewayRouteSpec](#github-com-envoyproxy-ai-gateway-api-v1alpha1-aigatewayroutespec) -- [GatewayConfigSpec](#github-com-envoyproxy-ai-gateway-api-v1alpha1-gatewayconfigspec) +- [GuardrailRule](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailrule) -LLMRequestCost configures each request cost. +GuardrailProvider describes the implementation used to evaluate a rule. ##### Fields - - -#### LLMRequestCostType - -**Underlying type:** string - -**Appears in:** -- [LLMRequestCost](#github-com-envoyproxy-ai-gateway-api-v1alpha1-llmrequestcost) - -LLMRequestCostType specifies the type of the LLMRequestCost. + description="Pattern is used for deterministic regex-based evaluations." +/> + + +#### GuardrailProviderType + +**Underlying type:** string + +**Appears in:** +- [GuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailprovider) + +GuardrailProviderType is the guardrail implementation. + + + +##### Possible Values + + +#### GuardrailRule + + + +**Appears in:** +- [GuardrailPolicySpec](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailpolicyspec) + +GuardrailRule defines one content-safety check to apply to a request or response. + +##### Fields + + + + + + +#### HTTPBodyField + + + +**Appears in:** +- [HTTPBodyMutation](#github-com-envoyproxy-ai-gateway-api-v1alpha1-httpbodymutation) + +HTTPBodyField represents a JSON field name and value for body mutation + +##### Fields + + + + + + +#### HTTPBodyMutation + + + +**Appears in:** +- [AIGatewayRouteRuleBackendRef](#github-com-envoyproxy-ai-gateway-api-v1alpha1-aigatewayrouterulebackendref) +- [AIServiceBackendSpec](#github-com-envoyproxy-ai-gateway-api-v1alpha1-aiservicebackendspec) + +HTTPBodyMutation defines the mutation of HTTP request body JSON fields that will be applied to the request + +##### Fields + + + + + + +#### HTTPGuardrailProvider + + + +**Appears in:** +- [GuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailprovider) + +HTTPGuardrailProvider configures calls to a custom guardrail service. + +For each extracted text fragment, the gateway sends a JSON POST request with the +text and an evaluation context (stage "input" or "output") to endpoint+path. The +service returns a normalized response with an action ("allow", "block", or "modify"), +optional findings (type, start, end, score), and an optional replacement text. +Finding offsets are Unicode code point positions in text; end is exclusive. + +##### Fields + + + + + + +#### HTTPHeaderMutation + + + +**Appears in:** +- [AIGatewayRouteRuleBackendRef](#github-com-envoyproxy-ai-gateway-api-v1alpha1-aigatewayrouterulebackendref) +- [AIServiceBackendSpec](#github-com-envoyproxy-ai-gateway-api-v1alpha1-aiservicebackendspec) + +HTTPHeaderMutation defines the mutation of HTTP headers that will be applied to the request + +##### Fields + + + + + + +#### JWKS + + + +**Appears in:** +- [MCPRouteOAuth](#github-com-envoyproxy-ai-gateway-api-v1alpha1-mcprouteoauth) + +JWKS defines how to obtain JSON Web Key Sets (JWKS) either from a remote HTTP/HTTPS endpoint or from a local source. + +##### Fields + + + + + + +#### JWTSource + + + +**Appears in:** +- [MCPAuthorizationSource](#github-com-envoyproxy-ai-gateway-api-v1alpha1-mcpauthorizationsource) + +JWTSource defines the MCP authorization source for JWT tokens. +At least one of scopes or claims must be provided. +Scopes and claims are AND-ed: when both are specified, both sets must match. + +##### Fields + + + + + + +#### LLMRequestCost + + + +**Appears in:** +- [AIGatewayRouteSpec](#github-com-envoyproxy-ai-gateway-api-v1alpha1-aigatewayroutespec) +- [GatewayConfigSpec](#github-com-envoyproxy-ai-gateway-api-v1alpha1-gatewayconfigspec) + +LLMRequestCost configures each request cost. + +##### Fields + + + + + + +#### LLMRequestCostType + +**Underlying type:** string + +**Appears in:** +- [LLMRequestCost](#github-com-envoyproxy-ai-gateway-api-v1alpha1-llmrequestcost) + +LLMRequestCostType specifies the type of the LLMRequestCost. @@ -2337,6 +2838,52 @@ a tool must match an include rule AND not match any exclude rule to be allowed. /> +#### ModelArmorGuardrailProvider + + + +**Appears in:** +- [GuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailprovider) + +ModelArmorGuardrailProvider configures calls to the Google Cloud Model Armor sanitize APIs. + +Request rules call sanitizeUserPrompt and Response rules call sanitizeModelResponse on the +configured template. The filters (Responsible AI, prompt injection and jailbreak, malicious URLs, +and Sensitive Data Protection) and their confidence levels are configured in the template. +Mask requires a template with an advanced Sensitive Data Protection de-identify template. + +##### Fields + + + + + + #### PerModelQuota @@ -2363,6 +2910,43 @@ a tool must match an include rule AND not match any exclude rule to be allowed. /> +#### PresidioGuardrailProvider + + + +**Appears in:** +- [GuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1alpha1-guardrailprovider) + +PresidioGuardrailProvider configures calls to a Presidio analyzer service. + +##### Fields + + + + + + #### ProtectedResourceMetadata @@ -2688,6 +3272,8 @@ API group. This is the beta version of the API, preferred over v1alpha1 for new - [BackendSecurityPolicyList](#github-com-envoyproxy-ai-gateway-api-v1beta1-backendsecuritypolicylist) - [GatewayConfig](#github-com-envoyproxy-ai-gateway-api-v1beta1-gatewayconfig) - [GatewayConfigList](#github-com-envoyproxy-ai-gateway-api-v1beta1-gatewayconfiglist) +- [GuardrailPolicy](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailpolicy) +- [GuardrailPolicyList](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailpolicylist) - [MCPRoute](#github-com-envoyproxy-ai-gateway-api-v1beta1-mcproute) - [MCPRouteList](#github-com-envoyproxy-ai-gateway-api-v1beta1-mcproutelist) @@ -3050,6 +3636,85 @@ GatewayConfigList contains a list of GatewayConfig. /> +#### GuardrailPolicy + + + +**Appears in:** +- [GuardrailPolicyList](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailpolicylist) + +GuardrailPolicy evaluates content safety checks for request and response payloads. + +##### Fields + + + + + + + + +#### GuardrailPolicyList + + + + +GuardrailPolicyList contains a list of GuardrailPolicy resources. + +##### Fields + + + + + + + + #### MCPRoute @@ -3145,6 +3810,7 @@ MCPRouteList contains a list of MCPRoute. - [APISchema](#github-com-envoyproxy-ai-gateway-api-v1beta1-apischema) - [AWSCredentialsFile](#github-com-envoyproxy-ai-gateway-api-v1beta1-awscredentialsfile) - [AWSOIDCExchangeToken](#github-com-envoyproxy-ai-gateway-api-v1beta1-awsoidcexchangetoken) +- [AzureContentSafetyGuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1beta1-azurecontentsafetyguardrailprovider) - [AzureOIDCExchangeToken](#github-com-envoyproxy-ai-gateway-api-v1beta1-azureoidcexchangetoken) - [BackendSecurityPolicyAPIKey](#github-com-envoyproxy-ai-gateway-api-v1beta1-backendsecuritypolicyapikey) - [BackendSecurityPolicyAWSCredentials](#github-com-envoyproxy-ai-gateway-api-v1beta1-backendsecuritypolicyawscredentials) @@ -3157,6 +3823,7 @@ MCPRouteList contains a list of MCPRoute. - [BackendSecurityPolicySpec](#github-com-envoyproxy-ai-gateway-api-v1beta1-backendsecuritypolicyspec) - [BackendSecurityPolicyStatus](#github-com-envoyproxy-ai-gateway-api-v1beta1-backendsecuritypolicystatus) - [BackendSecurityPolicyType](#github-com-envoyproxy-ai-gateway-api-v1beta1-backendsecuritypolicytype) +- [BedrockGuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1beta1-bedrockguardrailprovider) - [CredentialOverrideFromDynamicMetadata](#github-com-envoyproxy-ai-gateway-api-v1beta1-credentialoverridefromdynamicmetadata) - [CredentialOverrideFromRequestHeaders](#github-com-envoyproxy-ai-gateway-api-v1beta1-credentialoverridefromrequestheaders) - [GCPCredentialsFile](#github-com-envoyproxy-ai-gateway-api-v1beta1-gcpcredentialsfile) @@ -3167,8 +3834,17 @@ MCPRouteList contains a list of MCPRoute. - [GatewayConfigForwardProxy](#github-com-envoyproxy-ai-gateway-api-v1beta1-gatewayconfigforwardproxy) - [GatewayConfigSpec](#github-com-envoyproxy-ai-gateway-api-v1beta1-gatewayconfigspec) - [GatewayConfigStatus](#github-com-envoyproxy-ai-gateway-api-v1beta1-gatewayconfigstatus) +- [GuardrailAction](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailaction) +- [GuardrailFailureMode](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailfailuremode) +- [GuardrailPhase](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailphase) +- [GuardrailPolicySpec](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailpolicyspec) +- [GuardrailPolicyStatus](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailpolicystatus) +- [GuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailprovider) +- [GuardrailProviderType](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailprovidertype) +- [GuardrailRule](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailrule) - [HTTPBodyField](#github-com-envoyproxy-ai-gateway-api-v1beta1-httpbodyfield) - [HTTPBodyMutation](#github-com-envoyproxy-ai-gateway-api-v1beta1-httpbodymutation) +- [HTTPGuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1beta1-httpguardrailprovider) - [HTTPHeaderMutation](#github-com-envoyproxy-ai-gateway-api-v1beta1-httpheadermutation) - [HTTPHeaderValueFilter](#github-com-envoyproxy-ai-gateway-api-v1beta1-httpheadervaluefilter) - [HTTPHeaderValueFilterMode](#github-com-envoyproxy-ai-gateway-api-v1beta1-httpheadervaluefiltermode) @@ -3195,6 +3871,8 @@ MCPRouteList contains a list of MCPRoute. - [MCPRouteSpec](#github-com-envoyproxy-ai-gateway-api-v1beta1-mcproutespec) - [MCPRouteStatus](#github-com-envoyproxy-ai-gateway-api-v1beta1-mcproutestatus) - [MCPToolFilter](#github-com-envoyproxy-ai-gateway-api-v1beta1-mcptoolfilter) +- [ModelArmorGuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1beta1-modelarmorguardrailprovider) +- [PresidioGuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1beta1-presidioguardrailprovider) - [ProtectedResourceMetadata](#github-com-envoyproxy-ai-gateway-api-v1beta1-protectedresourcemetadata) - [ToolCall](#github-com-envoyproxy-ai-gateway-api-v1beta1-toolcall) - [VersionedAPISchema](#github-com-envoyproxy-ai-gateway-api-v1beta1-versionedapischema) @@ -3579,15 +4257,52 @@ and store them in a temporary credentials file. required="false" description="GrantType is the method application gets access token." /> + + +#### AzureContentSafetyGuardrailProvider + + + +**Appears in:** +- [GuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailprovider) + +AzureContentSafetyGuardrailProvider configures calls to Azure AI Content Safety. + +##### Fields + + + + @@ -3999,6 +4714,47 @@ BackendSecurityPolicyType specifies the type of auth mechanism used to access a required="false" description="" /> +#### BedrockGuardrailProvider + + + +**Appears in:** +- [GuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailprovider) + +BedrockGuardrailProvider configures calls to the AWS Bedrock ApplyGuardrail API. + +##### Fields + + + + + + #### CredentialOverrideFromDynamicMetadata @@ -4079,196 +4835,487 @@ GCPCredentialsFile specifies the service account key json file to authenticate w description="SecretRef is the reference to the credential file.
The secret should contain the GCP service account credentials file keyed on `service_account.json`." /> - -#### GCPOIDCExchangeToken - - + +#### GCPOIDCExchangeToken + + + +**Appears in:** +- [GCPWorkloadIdentityFederationConfig](#github-com-envoyproxy-ai-gateway-api-v1beta1-gcpworkloadidentityfederationconfig) + + + +##### Fields + + + + + + +#### GCPServiceAccountImpersonationConfig + + + +**Appears in:** +- [GCPWorkloadIdentityFederationConfig](#github-com-envoyproxy-ai-gateway-api-v1beta1-gcpworkloadidentityfederationconfig) + + + +##### Fields + + + + + + +#### GCPWorkloadIdentityFederationConfig + + + +**Appears in:** +- [BackendSecurityPolicyGCPCredentials](#github-com-envoyproxy-ai-gateway-api-v1beta1-backendsecuritypolicygcpcredentials) + + + +##### Fields + + + + + + +#### GatewayConfigExtProc + + + +**Appears in:** +- [GatewayConfigSpec](#github-com-envoyproxy-ai-gateway-api-v1beta1-gatewayconfigspec) + +GatewayConfigExtProc holds configuration for the external processor. + +##### Fields + + + + + + +#### GatewayConfigForwardProxy + + + +**Appears in:** +- [GatewayConfigSpec](#github-com-envoyproxy-ai-gateway-api-v1beta1-gatewayconfigspec) + +GatewayConfigForwardProxy configures an HTTP CONNECT forward proxy for upstream egress. + +##### Fields + + + + + + +#### GatewayConfigSpec + + + +**Appears in:** +- [GatewayConfig](#github-com-envoyproxy-ai-gateway-api-v1beta1-gatewayconfig) + +GatewayConfigSpec defines the configuration for the AI Gateway. + +##### Fields + + + + + + +#### GatewayConfigStatus + + + +**Appears in:** +- [GatewayConfig](#github-com-envoyproxy-ai-gateway-api-v1beta1-gatewayconfig) + +GatewayConfigStatus defines the observed state of GatewayConfig. + +##### Fields + + + + + + +#### GuardrailAction + +**Underlying type:** string + +**Appears in:** +- [GuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailprovider) + +GuardrailAction defines the safeguard action. + + + +##### Possible Values + + +#### GuardrailFailureMode + +**Underlying type:** string **Appears in:** -- [GCPWorkloadIdentityFederationConfig](#github-com-envoyproxy-ai-gateway-api-v1beta1-gcpworkloadidentityfederationconfig) +- [GuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailprovider) +GuardrailFailureMode defines behavior when an external provider cannot evaluate content. -##### Fields - +##### Possible Values +#### GuardrailPhase - -#### GCPServiceAccountImpersonationConfig - - +**Underlying type:** string **Appears in:** -- [GCPWorkloadIdentityFederationConfig](#github-com-envoyproxy-ai-gateway-api-v1beta1-gcpworkloadidentityfederationconfig) +- [GuardrailRule](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailrule) +GuardrailPhase determines when a guardrail runs. -##### Fields - +##### Possible Values - - -#### GCPWorkloadIdentityFederationConfig +#### GuardrailPolicySpec **Appears in:** -- [BackendSecurityPolicyGCPCredentials](#github-com-envoyproxy-ai-gateway-api-v1beta1-backendsecuritypolicygcpcredentials) - +- [GuardrailPolicy](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailpolicy) +GuardrailPolicySpec contains the configured checks attached to an AIServiceBackend. ##### Fields -#### GatewayConfigExtProc +#### GuardrailPolicyStatus **Appears in:** -- [GatewayConfigSpec](#github-com-envoyproxy-ai-gateway-api-v1beta1-gatewayconfigspec) +- [GuardrailPolicy](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailpolicy) -GatewayConfigExtProc holds configuration for the external processor. +GuardrailPolicyStatus contains the conditions by the reconciliation result. ##### Fields -#### GatewayConfigForwardProxy +#### GuardrailProvider **Appears in:** -- [GatewayConfigSpec](#github-com-envoyproxy-ai-gateway-api-v1beta1-gatewayconfigspec) +- [GuardrailRule](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailrule) -GatewayConfigForwardProxy configures an HTTP CONNECT forward proxy for upstream egress. +GuardrailProvider describes the implementation used to evaluate a rule. ##### Fields -#### GatewayConfigSpec - +#### GuardrailProviderType +**Underlying type:** string **Appears in:** -- [GatewayConfig](#github-com-envoyproxy-ai-gateway-api-v1beta1-gatewayconfig) +- [GuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailprovider) -GatewayConfigSpec defines the configuration for the AI Gateway. +GuardrailProviderType is the guardrail implementation. -##### Fields +##### Possible Values - - -#### GatewayConfigStatus +#### GuardrailRule **Appears in:** -- [GatewayConfig](#github-com-envoyproxy-ai-gateway-api-v1beta1-gatewayconfig) +- [GuardrailPolicySpec](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailpolicyspec) -GatewayConfigStatus defines the observed state of GatewayConfig. +GuardrailRule defines one content-safety check to apply to a request or response. ##### Fields @@ -4325,6 +5372,44 @@ HTTPBodyMutation defines the mutation of HTTP request body JSON fields that will /> +#### HTTPGuardrailProvider + + + +**Appears in:** +- [GuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailprovider) + +HTTPGuardrailProvider configures calls to a custom guardrail service. + +For each extracted text fragment, the gateway sends a JSON POST request with the +text and an evaluation context (stage "input" or "output") to endpoint+path. The +service returns a normalized response with an action ("allow", "block", or "modify"), +optional findings (type, start, end, score), and an optional replacement text. +Finding offsets are Unicode code point positions in text; end is exclusive. + +##### Fields + + + + + + #### HTTPHeaderMutation @@ -5202,6 +6287,89 @@ a tool must match an include rule AND not match any exclude rule to be allowed. /> +#### ModelArmorGuardrailProvider + + + +**Appears in:** +- [GuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailprovider) + +ModelArmorGuardrailProvider configures calls to the Google Cloud Model Armor sanitize APIs. + +Request rules call sanitizeUserPrompt and Response rules call sanitizeModelResponse on the +configured template. The filters (Responsible AI, prompt injection and jailbreak, malicious URLs, +and Sensitive Data Protection) and their confidence levels are configured in the template. +Mask requires a template with an advanced Sensitive Data Protection de-identify template. + +##### Fields + + + + + + +#### PresidioGuardrailProvider + + + +**Appears in:** +- [GuardrailProvider](#github-com-envoyproxy-ai-gateway-api-v1beta1-guardrailprovider) + +PresidioGuardrailProvider configures calls to a Presidio analyzer service. + +##### Fields + + + + + + #### ProtectedResourceMetadata diff --git a/site/docs/capabilities/security/guardrails.md b/site/docs/capabilities/security/guardrails.md new file mode 100644 index 0000000000..01842f7836 --- /dev/null +++ b/site/docs/capabilities/security/guardrails.md @@ -0,0 +1,269 @@ +--- +id: guardrails +title: Content Guardrails +sidebar_position: 9 +--- + +# Content Guardrails + +`GuardrailPolicy` evaluates request or response payloads for an `AIServiceBackend`. Rules can use local regular expressions, native integrations with external content-safety providers, or any custom guardrail service that implements a small HTTP contract. + +## Apply a regex guardrail + +```yaml +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: GuardrailPolicy +metadata: + name: pii-guardrails +spec: + maxRequestBodyBytes: 10485760 + maxResponseBodyBytes: 10485760 + targetRefs: + - group: aigateway.envoyproxy.io + kind: AIServiceBackend + name: openai + rules: + - name: block-sensitive-input + phase: Request + provider: + type: Regex + pattern: "(?i)password|social security number" + action: Block + message: Request contains sensitive content +``` + +A matched rule returns HTTP `403` with error type `GuardrailViolation`. Rules are scoped to the generated backends that reference the targeted `AIServiceBackend`. + +Rules support three actions: + +- `Block` rejects matching traffic. +- `Monitor` records matching traffic without blocking or changing it. +- `Mask` replaces detected text. Regex and Presidio use `maskReplacement`; Bedrock uses transformed output returned by the provider; HTTP uses the returned `replacement` or masks returned findings with `maskReplacement`; Model Armor uses the text de-identified by its Sensitive Data Protection filter. Azure Text Analysis does not support Mask. + +When multiple policies target one backend, policies are evaluated in namespace/name order and rules retain declaration order. The first Block result stops evaluation. + +## External providers + +### Presidio + +Presidio calls the analyzer `POST /analyze` endpoint. `scoreThresholdPercent` accepts values from 0 through 100. Authentication is optional. + +```yaml +provider: + type: Presidio + timeoutSeconds: 5 + failureMode: FailClosed + presidio: + endpoint: http://presidio-analyzer.presidio.svc.cluster.local:3000 + language: en + scoreThresholdPercent: 70 + apiKeySecretRef: + name: presidio-key +``` + +When configured, the Secret must contain an `apiKey` entry, sent as a bearer token. + +### AWS Bedrock Guardrails + +Bedrock uses the ApplyGuardrail API and SigV4 signing. By default, the ext-proc uses the standard AWS credential chain, including IRSA and EKS Pod Identity. + +```yaml +provider: + type: Bedrock + bedrock: + region: us-east-1 + guardrailIdentifier: my-guardrail + guardrailVersion: "1" +``` + +For static credentials, set `credentialsSecretRef` to a Secret whose `credentials` entry contains an AWS shared credentials file. `endpoint` can override the public Bedrock runtime endpoint for a private endpoint. + +### Azure AI Content Safety + +```yaml +provider: + type: AzureContentSafety + azureContentSafety: + endpoint: https://my-resource.cognitiveservices.azure.com + apiVersion: "2024-09-01" + severityThreshold: 4 + apiKeySecretRef: + name: azure-content-safety-key +``` + +The referenced Secret must contain an `apiKey` entry. + +### Google Cloud Model Armor + +Model Armor screens content with a [Model Armor template](https://docs.cloud.google.com/model-armor/overview). Request rules call `sanitizeUserPrompt` and Response rules call `sanitizeModelResponse` on the regional endpoint `https://modelarmor..rep.googleapis.com`. The filters (Responsible AI, prompt injection and jailbreak, malicious URLs, and Sensitive Data Protection) and their confidence levels are configured in the template, so the rule only references it: + +```yaml +provider: + type: ModelArmor + timeoutSeconds: 10 + failureMode: FailClosed + modelArmor: + project: my-project + location: us-central1 + template: my-template +``` + +A rule matches when Model Armor reports `MATCH_FOUND` for any filter in the template. An invocation result of `FAILURE` is treated as a provider error and follows the rule's failure mode. + +By default, the ext-proc uses Google Application Default Credentials, including GKE Workload Identity. The identity needs the `roles/modelarmor.user` role. For static credentials, set `credentialsSecretRef` to a Secret whose `credentials` entry contains a service account key JSON. `endpoint` can override the regional endpoint for a private endpoint. Token requests honor the `AI_GATEWAY_GCP_AUTH_PROXY_URL` proxy, like GCP backend authentication. + +`Mask` requires a template with advanced Sensitive Data Protection and a de-identify template; the gateway forwards the de-identified text returned by Model Armor. If any other filter also matches (for example, a jailbreak attempt), there is no safe replacement, so the rule fails instead of forwarding the content. + +### Custom HTTP guardrails + +Use the `HTTP` provider to integrate a guardrail service that has no native integration, such as an in-house classifier. The gateway calls the service over HTTP instead of running local executables. + +```yaml +provider: + type: HTTP + action: Mask + timeoutSeconds: 5 + failureMode: FailClosed + http: + endpoint: http://custom-guardrail.guardrails.svc.cluster.local:8080 + path: /analyze + apiKeySecretRef: + name: custom-guardrail-key +``` + +`path` defaults to `/analyze`. When `apiKeySecretRef` is set, the Secret must contain an `apiKey` entry, sent as a bearer token. + +#### Request + +For each extracted text fragment, the gateway sends `POST {endpoint}{path}`: + +```json +{ + "text": "some user input", + "context": { + "stage": "input" + } +} +``` + +`context.stage` is `input` for `Request` rules and `output` for `Response` rules. Services should ignore unknown fields, because more context may be added later. + +#### Response + +The service must return HTTP 2xx with: + +```json +{ + "action": "allow", + "findings": [ + { + "type": "PII", + "start": 10, + "end": 20, + "score": 0.92 + } + ] +} +``` + +| Field | Required | Description | +| ------------- | -------- | ----------------------------------------------------------------------------------------------------------------------------------------- | +| `action` | yes | `allow`, `block`, or `modify`. | +| `findings` | no | Detected spans. `start` and `end` are Unicode code point offsets into `text`, with `end` exclusive. `type` and `score` are informational. | +| `replacement` | no | Replacement text for the whole fragment, used with `modify`. | + +The gateway interprets the response as follows: + +| Service `action` | Result | +| ---------------- | ----------------------------------------------------------------------------------------------------------- | +| `allow` | The rule does not match. | +| `block` | The rule matches. Findings, if present, are masked with `maskReplacement` for `Mask` rules. | +| `modify` | The rule matches. `replacement` is used when present; otherwise findings are masked with `maskReplacement`. | + +The rule's `action` decides what happens on a match: `Block` rejects the request or response, `Monitor` records it, and `Mask` rewrites the fragment. A `Mask` rule fails when the service reports a match without a replacement or valid findings, so unmasked content is never forwarded. + +Non-2xx responses, invalid JSON, and missing or unknown actions are provider errors handled by `failureMode`. + +A minimal service that blocks prompts containing a keyword could look like this: + +```python +from fastapi import FastAPI +from pydantic import BaseModel + +app = FastAPI() + +class Context(BaseModel): + stage: str + +class Request(BaseModel): + text: str + context: Context + +@app.post("/analyze") +def analyze(req: Request): + start = req.text.lower().find("confidential") + if start < 0: + return {"action": "allow"} + return { + "action": "block", + "findings": [{"type": "KEYWORD", "start": start, "end": start + len("confidential"), "score": 1.0}], + } +``` + +Python string indexes are Unicode code points, which matches the offsets expected by the gateway. + +## Failure behavior + +External providers default to `failureMode: FailClosed`. Provider errors, missing Secrets, and revoked credentials prevent unchecked traffic. Use `FailOpen` only when availability is more important than enforcement: + +```yaml +provider: + type: Presidio + failureMode: FailOpen + timeoutSeconds: 3 + presidio: + endpoint: http://presidio-analyzer.presidio.svc.cluster.local:3000 +``` + +Credential Secret changes automatically requeue the policy and regenerate affected gateway configuration. + +## Streaming responses + +When an applicable response guardrail exists, the gateway keeps the upstream response buffered until evaluation completes. This prevents unsafe content from being partially delivered before a blocking or masking decision and gives Monitor rules a complete payload. Routes without response guardrails retain normal streaming behavior. + +Request and response limits default to 10 MiB and can be configured independently with `maxRequestBodyBytes` and `maxResponseBodyBytes`, up to 50 MiB. An oversized payload follows the rule's failure mode. + +## Observability + +The ext-proc emits structured block and provider-failure logs without payload content. It also records `guardrail.evaluation` span events and the counter: + +```text +aigateway.guardrail.evaluation.count +``` + +The counter attributes are `aigateway.guardrail.phase` (`Request` or `Response`) and `aigateway.guardrail.result` (`allowed`, `blocked`, or `error`). With the Prometheus exporter, dots are converted to underscores. + +## More examples + +See [`examples/guardrails`](https://github.com/envoyproxy/ai-gateway/tree/main/examples/guardrails) for local and external-provider manifests. + +## Optional live-provider tests + +Presidio is tested against its official analyzer image with Testcontainers. The test runs automatically when Docker is available and skips otherwise: + +```shell +go test ./internal/guardrails -run '^TestPresidioEvaluatorContainer$' -v +``` + +The Azure, Bedrock, and Model Armor live tests are disabled unless all required environment variables for a provider are set: + +- Presidio managed/external deployment: `TEST_PRESIDIO_ENDPOINT`, `TEST_PRESIDIO_BLOCKED_TEXT`, and optionally `TEST_PRESIDIO_API_KEY`. +- Azure: `TEST_AZURE_CONTENT_SAFETY_ENDPOINT`, `TEST_AZURE_CONTENT_SAFETY_API_KEY`, and `TEST_AZURE_CONTENT_SAFETY_BLOCKED_TEXT`. +- Bedrock: `TEST_AWS_BEDROCK_GUARDRAIL_REGION`, `TEST_AWS_BEDROCK_GUARDRAIL_ID`, `TEST_AWS_BEDROCK_GUARDRAIL_VERSION`, and `TEST_AWS_BEDROCK_GUARDRAIL_BLOCKED_TEXT`. AWS credentials use the standard credential chain. +- Model Armor: `TEST_GCP_MODEL_ARMOR_PROJECT`, `TEST_GCP_MODEL_ARMOR_LOCATION`, `TEST_GCP_MODEL_ARMOR_TEMPLATE`, and `TEST_GCP_MODEL_ARMOR_BLOCKED_TEXT`. Google credentials use Application Default Credentials. + +Run them with: + +```shell +go test ./internal/guardrails -run '^TestLive' -v +``` diff --git a/tests/crdcel/main_test.go b/tests/crdcel/main_test.go index 6a3511a104..8efa73e457 100644 --- a/tests/crdcel/main_test.go +++ b/tests/crdcel/main_test.go @@ -160,6 +160,42 @@ func TestGatewayConfigs(t *testing.T) { } } +func TestGuardrailPolicies(t *testing.T) { + c, _, _ := testsinternal.NewEnvTest(t) + ctx := t.Context() + + for _, tc := range []struct { + name string + expErr string + }{ + {name: "basic.yaml"}, + {name: "regex_missing_pattern.yaml", expErr: "Regex requires pattern and no external provider configuration"}, + {name: "provider_mismatch.yaml", expErr: "Presidio requires only presidio provider configuration"}, + {name: "invalid_action.yaml", expErr: "Unsupported value"}, + {name: "invalid_endpoint.yaml", expErr: "Invalid value"}, + {name: "duplicate_rule_names.yaml", expErr: "rule name must be unique within the policy"}, + {name: "azure_mask.yaml", expErr: "AzureContentSafety does not support Mask"}, + {name: "http_mismatch.yaml", expErr: "HTTP requires only http provider configuration"}, + {name: "http_invalid_path.yaml", expErr: "should match '^/.*$'"}, + {name: "model_armor_mismatch.yaml", expErr: "ModelArmor requires only modelArmor provider configuration"}, + {name: "model_armor_missing_template.yaml", expErr: "should be at least 1 chars long"}, + } { + t.Run(tc.name, func(t *testing.T) { + data, err := testdata.ReadFile(path.Join("testdata/guardrailpolicies", tc.name)) + require.NoError(t, err) + + policy := &aigv1b1.GuardrailPolicy{} + require.NoError(t, yaml.UnmarshalStrict(data, policy)) + if tc.expErr != "" { + require.ErrorContains(t, c.Create(ctx, policy), tc.expErr) + } else { + require.NoError(t, c.Create(ctx, policy)) + require.NoError(t, c.Delete(ctx, policy)) + } + }) + } +} + func TestBackendSecurityPolicies(t *testing.T) { c, _, _ := testsinternal.NewEnvTest(t) ctx := t.Context() diff --git a/tests/crdcel/testdata/guardrailpolicies/azure_mask.yaml b/tests/crdcel/testdata/guardrailpolicies/azure_mask.yaml new file mode 100644 index 0000000000..a2bf275d8d --- /dev/null +++ b/tests/crdcel/testdata/guardrailpolicies/azure_mask.yaml @@ -0,0 +1,25 @@ +# Copyright Envoy AI Gateway Authors +# SPDX-License-Identifier: Apache-2.0 +# The full text of the Apache license is available in the LICENSE file at +# the root of the repo. + +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: GuardrailPolicy +metadata: + name: azure-mask + namespace: default +spec: + targetRefs: + - group: aigateway.envoyproxy.io + kind: AIServiceBackend + name: backend + rules: + - name: invalid-mask + phase: Response + provider: + type: AzureContentSafety + action: Mask + azureContentSafety: + endpoint: https://content-safety.example.com + apiKeySecretRef: + name: azure-key diff --git a/tests/crdcel/testdata/guardrailpolicies/basic.yaml b/tests/crdcel/testdata/guardrailpolicies/basic.yaml new file mode 100644 index 0000000000..9ffbbd8f7e --- /dev/null +++ b/tests/crdcel/testdata/guardrailpolicies/basic.yaml @@ -0,0 +1,84 @@ +# Copyright Envoy AI Gateway Authors +# SPDX-License-Identifier: Apache-2.0 +# The full text of the Apache license is available in the LICENSE file at +# the root of the repo. + +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: GuardrailPolicy +metadata: + name: valid-guardrails + namespace: default +spec: + targetRefs: + - group: aigateway.envoyproxy.io + kind: AIServiceBackend + name: backend + rules: + - name: regex + phase: Request + provider: + type: Regex + pattern: secret + action: Monitor + - name: regex-mask + phase: Response + provider: + type: Regex + pattern: secret + action: Mask + maskReplacement: "[FILTERED]" + - name: presidio + phase: Request + provider: + type: Presidio + timeoutSeconds: 5 + failureMode: FailOpen + presidio: + endpoint: https://presidio.example.com + scoreThresholdPercent: 70 + - name: bedrock + phase: Response + provider: + type: Bedrock + bedrock: + region: us-east-1 + guardrailIdentifier: guardrail-id + guardrailVersion: "1" + - name: azure + phase: Response + provider: + type: AzureContentSafety + azureContentSafety: + endpoint: https://content-safety.example.com + apiKeySecretRef: + name: azure-key + - name: custom-http + phase: Request + provider: + type: HTTP + action: Mask + http: + endpoint: https://guardrail.example.com + path: /v1/analyze + apiKeySecretRef: + name: custom-guardrail-key + - name: model-armor + phase: Request + provider: + type: ModelArmor + modelArmor: + project: my-project + location: us-central1 + template: my-template + - name: model-armor-mask + phase: Response + provider: + type: ModelArmor + action: Mask + modelArmor: + project: my-project + location: us-central1 + template: my-template + endpoint: https://modelarmor.example.com + credentialsSecretRef: + name: model-armor-sa diff --git a/tests/crdcel/testdata/guardrailpolicies/duplicate_rule_names.yaml b/tests/crdcel/testdata/guardrailpolicies/duplicate_rule_names.yaml new file mode 100644 index 0000000000..ff32953938 --- /dev/null +++ b/tests/crdcel/testdata/guardrailpolicies/duplicate_rule_names.yaml @@ -0,0 +1,26 @@ +# Copyright Envoy AI Gateway Authors +# SPDX-License-Identifier: Apache-2.0 +# The full text of the Apache license is available in the LICENSE file at +# the root of the repo. + +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: GuardrailPolicy +metadata: + name: duplicate-rule-names + namespace: default +spec: + targetRefs: + - group: aigateway.envoyproxy.io + kind: AIServiceBackend + name: backend + rules: + - name: duplicate + phase: Request + provider: + type: Regex + pattern: request + - name: duplicate + phase: Response + provider: + type: Regex + pattern: response diff --git a/tests/crdcel/testdata/guardrailpolicies/http_invalid_path.yaml b/tests/crdcel/testdata/guardrailpolicies/http_invalid_path.yaml new file mode 100644 index 0000000000..2d85b17ed9 --- /dev/null +++ b/tests/crdcel/testdata/guardrailpolicies/http_invalid_path.yaml @@ -0,0 +1,22 @@ +# Copyright Envoy AI Gateway Authors +# SPDX-License-Identifier: Apache-2.0 +# The full text of the Apache license is available in the LICENSE file at +# the root of the repo. +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: GuardrailPolicy +metadata: + name: http-invalid-path + namespace: default +spec: + targetRefs: + - group: aigateway.envoyproxy.io + kind: AIServiceBackend + name: backend + rules: + - name: invalid + phase: Request + provider: + type: HTTP + http: + endpoint: https://guardrail.example.com + path: analyze diff --git a/tests/crdcel/testdata/guardrailpolicies/http_mismatch.yaml b/tests/crdcel/testdata/guardrailpolicies/http_mismatch.yaml new file mode 100644 index 0000000000..2bb14f08d3 --- /dev/null +++ b/tests/crdcel/testdata/guardrailpolicies/http_mismatch.yaml @@ -0,0 +1,21 @@ +# Copyright Envoy AI Gateway Authors +# SPDX-License-Identifier: Apache-2.0 +# The full text of the Apache license is available in the LICENSE file at +# the root of the repo. +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: GuardrailPolicy +metadata: + name: http-mismatch + namespace: default +spec: + targetRefs: + - group: aigateway.envoyproxy.io + kind: AIServiceBackend + name: backend + rules: + - name: invalid + phase: Request + provider: + type: HTTP + presidio: + endpoint: https://presidio.example.com diff --git a/tests/crdcel/testdata/guardrailpolicies/invalid_action.yaml b/tests/crdcel/testdata/guardrailpolicies/invalid_action.yaml new file mode 100644 index 0000000000..b201b64007 --- /dev/null +++ b/tests/crdcel/testdata/guardrailpolicies/invalid_action.yaml @@ -0,0 +1,22 @@ +# Copyright Envoy AI Gateway Authors +# SPDX-License-Identifier: Apache-2.0 +# The full text of the Apache license is available in the LICENSE file at +# the root of the repo. + +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: GuardrailPolicy +metadata: + name: invalid-action + namespace: default +spec: + targetRefs: + - group: aigateway.envoyproxy.io + kind: AIServiceBackend + name: backend + rules: + - name: invalid + phase: Request + provider: + type: Regex + pattern: secret + action: Log diff --git a/tests/crdcel/testdata/guardrailpolicies/invalid_endpoint.yaml b/tests/crdcel/testdata/guardrailpolicies/invalid_endpoint.yaml new file mode 100644 index 0000000000..197909a4b2 --- /dev/null +++ b/tests/crdcel/testdata/guardrailpolicies/invalid_endpoint.yaml @@ -0,0 +1,22 @@ +# Copyright Envoy AI Gateway Authors +# SPDX-License-Identifier: Apache-2.0 +# The full text of the Apache license is available in the LICENSE file at +# the root of the repo. + +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: GuardrailPolicy +metadata: + name: invalid-endpoint + namespace: default +spec: + targetRefs: + - group: aigateway.envoyproxy.io + kind: AIServiceBackend + name: backend + rules: + - name: invalid + phase: Request + provider: + type: Presidio + presidio: + endpoint: not-a-uri diff --git a/tests/crdcel/testdata/guardrailpolicies/model_armor_mismatch.yaml b/tests/crdcel/testdata/guardrailpolicies/model_armor_mismatch.yaml new file mode 100644 index 0000000000..7720c9d9b8 --- /dev/null +++ b/tests/crdcel/testdata/guardrailpolicies/model_armor_mismatch.yaml @@ -0,0 +1,26 @@ +# Copyright Envoy AI Gateway Authors +# SPDX-License-Identifier: Apache-2.0 +# The full text of the Apache license is available in the LICENSE file at +# the root of the repo. + +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: GuardrailPolicy +metadata: + name: model-armor-mismatch + namespace: default +spec: + targetRefs: + - group: aigateway.envoyproxy.io + kind: AIServiceBackend + name: backend + rules: + - name: invalid + phase: Request + provider: + type: ModelArmor + modelArmor: + project: my-project + location: us-central1 + template: my-template + http: + endpoint: https://guardrail.example.com diff --git a/tests/crdcel/testdata/guardrailpolicies/model_armor_missing_template.yaml b/tests/crdcel/testdata/guardrailpolicies/model_armor_missing_template.yaml new file mode 100644 index 0000000000..0eed45daf9 --- /dev/null +++ b/tests/crdcel/testdata/guardrailpolicies/model_armor_missing_template.yaml @@ -0,0 +1,24 @@ +# Copyright Envoy AI Gateway Authors +# SPDX-License-Identifier: Apache-2.0 +# The full text of the Apache license is available in the LICENSE file at +# the root of the repo. + +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: GuardrailPolicy +metadata: + name: model-armor-missing-template + namespace: default +spec: + targetRefs: + - group: aigateway.envoyproxy.io + kind: AIServiceBackend + name: backend + rules: + - name: invalid + phase: Request + provider: + type: ModelArmor + modelArmor: + project: my-project + location: us-central1 + template: "" diff --git a/tests/crdcel/testdata/guardrailpolicies/provider_mismatch.yaml b/tests/crdcel/testdata/guardrailpolicies/provider_mismatch.yaml new file mode 100644 index 0000000000..ef323fc426 --- /dev/null +++ b/tests/crdcel/testdata/guardrailpolicies/provider_mismatch.yaml @@ -0,0 +1,24 @@ +# Copyright Envoy AI Gateway Authors +# SPDX-License-Identifier: Apache-2.0 +# The full text of the Apache license is available in the LICENSE file at +# the root of the repo. + +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: GuardrailPolicy +metadata: + name: provider-mismatch + namespace: default +spec: + targetRefs: + - group: aigateway.envoyproxy.io + kind: AIServiceBackend + name: backend + rules: + - name: invalid + phase: Request + provider: + type: Presidio + bedrock: + region: us-east-1 + guardrailIdentifier: guardrail-id + guardrailVersion: "1" diff --git a/tests/crdcel/testdata/guardrailpolicies/regex_missing_pattern.yaml b/tests/crdcel/testdata/guardrailpolicies/regex_missing_pattern.yaml new file mode 100644 index 0000000000..9ec3045cfb --- /dev/null +++ b/tests/crdcel/testdata/guardrailpolicies/regex_missing_pattern.yaml @@ -0,0 +1,20 @@ +# Copyright Envoy AI Gateway Authors +# SPDX-License-Identifier: Apache-2.0 +# The full text of the Apache license is available in the LICENSE file at +# the root of the repo. + +apiVersion: aigateway.envoyproxy.io/v1beta1 +kind: GuardrailPolicy +metadata: + name: regex-missing-pattern + namespace: default +spec: + targetRefs: + - group: aigateway.envoyproxy.io + kind: AIServiceBackend + name: backend + rules: + - name: invalid + phase: Request + provider: + type: Regex diff --git a/tests/data-plane/testupstream_test.go b/tests/data-plane/testupstream_test.go index fd9ecdd8a4..ff46c4d95d 100644 --- a/tests/data-plane/testupstream_test.go +++ b/tests/data-plane/testupstream_test.go @@ -40,6 +40,103 @@ func failIf5xx(t *testing.T, resp *http.Response, was5xx *bool) { } } +func TestGuardrailsWithTestUpstream(t *testing.T) { + configBytes, err := yaml.Marshal(&filterapi.Config{ + Version: version.Parse(), + Backends: []filterapi.Backend{testUpstreamOpenAIBackend}, + Guardrails: []filterapi.Guardrail{ + { + Name: "e2e-request-guardrail", Phase: filterapi.GuardrailPhaseRequest, + Provider: filterapi.GuardrailProvider{ + Type: filterapi.GuardrailProviderTypeRegex, Pattern: "E2E_REQUEST_GUARDRAIL", Message: "request blocked by e2e guardrail", + }, + }, + { + Name: "e2e-response-guardrail", Phase: filterapi.GuardrailPhaseResponse, + Provider: filterapi.GuardrailProvider{ + Type: filterapi.GuardrailProviderTypeRegex, Pattern: "E2E_RESPONSE_GUARDRAIL", Message: "response blocked by e2e guardrail", + }, + }, + { + Name: "e2e-monitor-guardrail", Phase: filterapi.GuardrailPhaseRequest, + Provider: filterapi.GuardrailProvider{ + Type: filterapi.GuardrailProviderTypeRegex, Pattern: "E2E_MONITOR_GUARDRAIL", Action: filterapi.GuardrailActionMonitor, + }, + }, + { + Name: "e2e-request-mask", Phase: filterapi.GuardrailPhaseRequest, + Provider: filterapi.GuardrailProvider{ + Type: filterapi.GuardrailProviderTypeRegex, Pattern: `mask-me@example\.com`, Action: filterapi.GuardrailActionMask, MaskReplacement: "[EMAIL]", + }, + }, + { + Name: "e2e-response-mask", Phase: filterapi.GuardrailPhaseResponse, + Provider: filterapi.GuardrailProvider{ + Type: filterapi.GuardrailProviderTypeRegex, Pattern: `mask-response@example\.com`, Action: filterapi.GuardrailActionMask, MaskReplacement: "[EMAIL]", + }, + }, + }, + }) + require.NoError(t, err) + env := startTestEnvironment(t, string(configBytes), true, false) + endpoint := fmt.Sprintf("http://localhost:%d/v1/chat/completions", env.EnvoyListenerPort()) + + for _, test := range []struct { + name, requestBody, responseBody, wantBody, wantUpstreamBody string + wantStatus int + }{ + { + name: "request", requestBody: `{"model":"something","messages":[{"role":"user","content":"E2E_REQUEST_GUARDRAIL"}]}`, + wantBody: `{"type":"error","error":{"type":"GuardrailViolation","code":"403","message":"request blocked by e2e guardrail"}}`, + wantStatus: http.StatusForbidden, + }, + { + name: "response", requestBody: `{"model":"something","messages":[{"role":"user","content":"hello"}]}`, + responseBody: `{"id":"guardrail-test","object":"chat.completion","created":123,"model":"something","choices":[{"index":0,"message":{"role":"assistant","content":"E2E_RESPONSE_GUARDRAIL"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}`, + wantBody: `{"type":"error","error":{"type":"GuardrailViolation","code":"403","message":"response blocked by e2e guardrail"}}`, + wantStatus: http.StatusForbidden, + }, + { + name: "monitor", requestBody: `{"model":"something","messages":[{"role":"user","content":"E2E_MONITOR_GUARDRAIL"}]}`, + responseBody: `{"id":"monitor","object":"chat.completion","created":123,"model":"something","choices":[{"index":0,"message":{"role":"assistant","content":"safe"},"finish_reason":"stop"}]}`, + wantBody: `{"id":"monitor","object":"chat.completion","created":123,"model":"something","choices":[{"index":0,"message":{"role":"assistant","content":"safe"},"finish_reason":"stop"}]}`, + wantStatus: http.StatusOK, + }, + { + name: "request mask", requestBody: `{"model":"something","messages":[{"role":"user","content":"email mask-me@example.com"}]}`, + wantUpstreamBody: `{"model":"something","messages":[{"role":"user","content":"email [EMAIL]"}]}`, + responseBody: `{"id":"request-mask","object":"chat.completion","created":123,"model":"something","choices":[{"index":0,"message":{"role":"assistant","content":"safe"},"finish_reason":"stop"}]}`, + wantBody: `{"id":"request-mask","object":"chat.completion","created":123,"model":"something","choices":[{"index":0,"message":{"role":"assistant","content":"safe"},"finish_reason":"stop"}]}`, + wantStatus: http.StatusOK, + }, + { + name: "response mask", requestBody: `{"model":"something","messages":[{"role":"user","content":"hello"}]}`, + responseBody: `{"id":"response-mask","object":"chat.completion","created":123,"model":"something","choices":[{"index":0,"message":{"role":"assistant","content":"email mask-response@example.com"},"finish_reason":"stop"}]}`, + wantBody: `{"id":"response-mask","object":"chat.completion","created":123,"model":"something","choices":[{"index":0,"message":{"role":"assistant","content":"email [EMAIL]"},"finish_reason":"stop"}]}`, + wantStatus: http.StatusOK, + }, + } { + t.Run(test.name, func(t *testing.T) { + req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, endpoint, strings.NewReader(test.requestBody)) + require.NoError(t, err) + req.Header.Set("x-test-backend", "openai") + if test.responseBody != "" { + req.Header.Set(testupstreamlib.ResponseBodyHeaderKey, base64.StdEncoding.EncodeToString([]byte(test.responseBody))) + } + if test.wantUpstreamBody != "" { + req.Header.Set(testupstreamlib.ExpectedRequestBodyHeaderKey, base64.StdEncoding.EncodeToString([]byte(test.wantUpstreamBody))) + } + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + defer func() { _ = resp.Body.Close() }() + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.Equal(t, test.wantStatus, resp.StatusCode, "response body: %s", body) + require.JSONEq(t, test.wantBody, string(body)) + }) + } +} + // TestWithTestUpstream tests the end-to-end flow of the external processor with Envoy and the test upstream. // // This does not require any environment variables to be set as it relies on the test upstream.