From e021c5e729dd4cba668fe435d3557083f44f77e7 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 25 Jun 2026 19:48:52 +0000 Subject: [PATCH] Regenerate azure-ai-evaluation API files Co-authored-by: m7md7sien <16615690+m7md7sien@users.noreply.github.com> --- sdk/evaluation/azure-ai-evaluation/api.md | 1717 +++++++++++++++++ .../azure-ai-evaluation/api.metadata.yml | 3 + 2 files changed, 1720 insertions(+) create mode 100644 sdk/evaluation/azure-ai-evaluation/api.md create mode 100644 sdk/evaluation/azure-ai-evaluation/api.metadata.yml diff --git a/sdk/evaluation/azure-ai-evaluation/api.md b/sdk/evaluation/azure-ai-evaluation/api.md new file mode 100644 index 000000000000..6b69ad8edf9d --- /dev/null +++ b/sdk/evaluation/azure-ai-evaluation/api.md @@ -0,0 +1,1717 @@ +```py +namespace azure.ai.evaluation + + def azure.ai.evaluation.evaluate( + *, + azure_ai_project: Optional[Union[str, AzureAIProject]] = ..., + data: Union[str, PathLike], + evaluation_name: Optional[str] = ..., + evaluator_config: Optional[Dict[str, EvaluatorConfig]] = ..., + evaluators: Dict[str, Union[Callable, AzureOpenAIGrader]], + fail_on_evaluator_errors: bool = False, + output_path: Optional[Union[str, PathLike]] = ..., + tags: Optional[Dict[str, str]] = ..., + target: Optional[Callable] = ..., + user_agent: Optional[str] = ..., + **kwargs + ) -> EvaluationResult: ... + + + class azure.ai.evaluation.AzureAIProject(TypedDict): + key "project_name": str + key "resource_group_name": str + key "subscription_id": str + + + @experimental + class azure.ai.evaluation.AzureOpenAIGrader: + id = azureai://built-in/evaluators/azure-openai/custom_grader + + def __init__( + self, + *, + credential: Optional[TokenCredential] = ..., + grader_config: Dict[str, Any], + model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration], + **kwargs: Any + ): ... + + def get_client(self) -> Any: ... + + + @experimental + class azure.ai.evaluation.AzureOpenAILabelGrader(AzureOpenAIGrader): + id = azureai://built-in/evaluators/azure-openai/label_grader + + def __init__( + self, + *, + credential: Optional[TokenCredential] = ..., + input: List[Dict[str, str]], + labels: List[str], + model: str, + model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration], + name: str, + passing_labels: List[str], + **kwargs: Any + ): ... + + def get_client(self) -> Any: ... + + + class azure.ai.evaluation.AzureOpenAIModelConfiguration(TypedDict): + key "azure_deployment": str + key "azure_endpoint": str + api_key: NotRequired[str] + api_version: NotRequired[str] + credential: NotRequired[Any] + type: NotRequired[Literal["azure_openai"]] + + + @experimental + class azure.ai.evaluation.AzureOpenAIPythonGrader(AzureOpenAIGrader): + id = azureai://built-in/evaluators/azure-openai/python_grader + + def __init__( + self, + *, + credential: Optional[TokenCredential] = ..., + image_tag: Optional[str] = ..., + model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration], + name: str, + pass_threshold: float, + source: str, + **kwargs: Any + ): ... + + def get_client(self) -> Any: ... + + + @experimental + class azure.ai.evaluation.AzureOpenAIScoreModelGrader(AzureOpenAIGrader): + id = azureai://built-in/evaluators/azure-openai/score_model_grader + + def __init__( + self, + *, + credential: Optional[TokenCredential] = ..., + input: List[Dict[str, str]], + model: str, + model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration], + name: str, + pass_threshold: Optional[float] = ..., + range: Optional[List[float]] = ..., + sampling_params: Optional[Dict[str, Any]] = ..., + **kwargs: Any + ): ... + + def get_client(self) -> Any: ... + + + @experimental + class azure.ai.evaluation.AzureOpenAIStringCheckGrader(AzureOpenAIGrader): + id = azureai://built-in/evaluators/azure-openai/string_check_grader + + def __init__( + self, + *, + credential: Optional[TokenCredential] = ..., + input: str, + model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration], + name: str, + operation: Literal["eq", "ne", "like", "ilike"], + reference: str, + **kwargs: Any + ): ... + + def get_client(self) -> Any: ... + + + @experimental + class azure.ai.evaluation.AzureOpenAITextSimilarityGrader(AzureOpenAIGrader): + id = azureai://built-in/evaluators/azure-openai/text_similarity_grader + + def __init__( + self, + *, + credential: Optional[TokenCredential] = ..., + evaluation_metric: Literal["fuzzy_match", "bleu", "gleu", "meteor", "rouge_1", "rouge_2", "rouge_3", "rouge_4", "rouge_5", "rouge_l", "cosine"], + input: str, + model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration], + name: str, + pass_threshold: float, + reference: str, + **kwargs: Any + ): ... + + def get_client(self) -> Any: ... + + + class azure.ai.evaluation.BleuScoreEvaluator(EvaluatorBase): + id = azureai://built-in/evaluators/bleu_score + + @overload + def __call__( + self, + *, + ground_truth: str, + response: str + ): ... + + def __init__( + self, + *, + threshold = 0.5 + ): ... + + + @experimental + class azure.ai.evaluation.CodeVulnerabilityEvaluator(RaiServiceEvaluatorBase[Union[str, bool]]): + id = azureai://built-in/evaluators/code_vulnerability + + @overload + def __call__( + self, + *, + query: str, + response: str + ) -> Dict[str, Union[str, float]]: ... + + @override + def __init__( + self, + credential: TokenCredential, + azure_ai_project: Union[str, AzureAIProject], + **kwargs: Any + ): ... + + + class azure.ai.evaluation.CoherenceEvaluator(PromptyEvaluatorBase[Union[str, float]]): + id = azureai://built-in/evaluators/coherence + + @overload + def __call__( + self, + *, + query: str, + response: str + ) -> Dict[str, Union[str, float]]: ... + + @overload + def __call__( + self, + *, + conversation: Conversation + ) -> Dict[str, Union[float, Dict[str, List[Union[str, float]]]]]: ... + + @override + def __init__( + self, + model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration], + *, + credential = ..., + is_reasoning_model: bool = ..., + threshold = 3, + **kwargs + ): ... + + + @experimental + class azure.ai.evaluation.ContentSafetyEvaluator(MultiEvaluatorBase[Union[str, float]]): + id = azureai://built-in/evaluators/content_safety + + @overload + def __call__( + self, + *, + query: str, + response: str + ) -> Dict[str, Union[str, float]]: ... + + @overload + def __call__( + self, + *, + conversation: Conversation + ) -> Dict[str, Union[float, Dict[str, List[Union[str, float]]]]]: ... + + def __init__( + self, + credential: TokenCredential, + azure_ai_project: Union[str, AzureAIProject], + *, + hate_unfairness_threshold: int = 3, + self_harm_threshold: int = 3, + sexual_threshold: int = 3, + violence_threshold: int = 3, + **kwargs: Any + ): ... + + + class azure.ai.evaluation.Conversation(TypedDict): + key "messages": Union[List[Message], List[Dict]] + context: NotRequired[Dict[str, Any]] + + + class azure.ai.evaluation.EvaluationResult(TypedDict): + key "metrics": Dict + key "rows": List[Dict] + oai_eval_run_ids: NotRequired[List[Dict[str, str]]] + studio_url: NotRequired[str] + + + class azure.ai.evaluation.EvaluatorConfig(TypedDict, total=False): + key "column_mapping": Dict[str, str] + + + class azure.ai.evaluation.F1ScoreEvaluator(EvaluatorBase): + id = azureai://built-in/evaluators/f1_score + + @overload + def __call__( + self, + *, + ground_truth: str, + response: str + ) -> Dict[str, float]: ... + + def __init__( + self, + *, + threshold = 0.5 + ): ... + + + class azure.ai.evaluation.FluencyEvaluator(PromptyEvaluatorBase[Union[str, float]]): + id = azureai://built-in/evaluators/fluency + + @overload + def __call__( + self, + *, + response: str + ) -> Dict[str, Union[str, float]]: ... + + @overload + def __call__( + self, + *, + conversation: Conversation + ) -> Dict[str, Union[float, Dict[str, List[Union[str, float]]]]]: ... + + @override + def __init__( + self, + model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration], + *, + credential = ..., + is_reasoning_model: bool = ..., + threshold = 3, + **kwargs + ): ... + + + class azure.ai.evaluation.GleuScoreEvaluator(EvaluatorBase): + id = azureai://built-in/evaluators/gleu_score + + @overload + def __call__( + self, + *, + ground_truth: str, + response: str + ): ... + + @override + def __init__( + self, + *, + threshold = 0.5 + ): ... + + + class azure.ai.evaluation.GroundednessEvaluator(PromptyEvaluatorBase[Union[str, float]]): + id = azureai://built-in/evaluators/groundedness + + @overload + def __call__( + self, + *, + context: str, + query: Optional[str] = ..., + response: str + ) -> Dict[str, Union[str, float]]: ... + + @overload + def __call__( + self, + *, + query: str, + response: List[dict], + tool_definitions: Optional[List[dict]] = ... + ) -> Dict[str, Union[str, float]]: ... + + @overload + def __call__( + self, + *, + conversation: Conversation + ) -> Dict[str, Union[float, Dict[str, List[Union[str, float]]]]]: ... + + @override + def __init__( + self, + model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration], + *, + credential = ..., + is_reasoning_model: bool = ..., + threshold = 3, + **kwargs + ): ... + + + @experimental + class azure.ai.evaluation.GroundednessProEvaluator(RaiServiceEvaluatorBase[Union[str, bool]]): + id = azureai://built-in/evaluators/groundedness_pro + + @overload + def __call__( + self, + *, + context: str, + query: str, + response: str + ) -> Dict[str, Union[str, bool]]: ... + + @overload + def __call__( + self, + *, + conversation: Conversation + ) -> Dict[str, Union[float, Dict[str, List[Union[str, bool]]]]]: ... + + @override + def __init__( + self, + credential: TokenCredential, + azure_ai_project: Union[str, AzureAIProject], + *, + threshold: int = 5, + **kwargs: Any + ): ... + + + @experimental + class azure.ai.evaluation.HateUnfairnessEvaluator(RaiServiceEvaluatorBase[Union[str, float]]): + id = azureai://built-in/evaluators/hate_unfairness + + @overload + def __call__( + self, + *, + query: str, + response: str + ) -> Dict[str, Union[str, float]]: ... + + @overload + def __call__( + self, + *, + conversation: Conversation + ) -> Dict[str, Union[float, Dict[str, List[Union[str, float]]]]]: ... + + @override + def __init__( + self, + credential: TokenCredential, + azure_ai_project: Union[str, AzureAIProject], + *, + threshold: int = 3, + **kwargs + ): ... + + + @experimental + class azure.ai.evaluation.IndirectAttackEvaluator(RaiServiceEvaluatorBase[Union[str, bool]]): + id = azureai://built-in/evaluators/indirect_attack + + @overload + def __call__( + self, + *, + query: str, + response: str + ) -> Dict[str, Union[str, bool]]: ... + + @overload + def __call__( + self, + *, + conversation: Conversation + ) -> Dict[str, Union[float, Dict[str, List[Union[str, bool]]]]]: ... + + @override + def __init__( + self, + credential: TokenCredential, + azure_ai_project: Union[str, AzureAIProject], + **kwargs + ): ... + + + @experimental + class azure.ai.evaluation.IntentResolutionEvaluator(PromptyEvaluatorBase[Union[str, float]]): + id = azureai://built-in/evaluators/intent_resolution + + @overload + def __call__( + self, + *, + query: Union[str, List[dict]], + response: Union[str, List[dict]], + tool_definitions: Optional[Union[dict, List[dict]]] = ... + ) -> Dict[str, Union[str, float]]: ... + + @override + def __init__( + self, + model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration], + *, + credential = ..., + threshold = _DEFAULT_INTENT_RESOLUTION_THRESHOLD, + **kwargs + ): ... + + + class azure.ai.evaluation.Message(TypedDict): + key "content": Union[str, List[Dict]] + key "role": str + context: NotRequired[Dict[str, Any]] + + + class azure.ai.evaluation.MeteorScoreEvaluator(EvaluatorBase): + id = azureai://built-in/evaluators/meteor_score + + @overload + def __call__( + self, + *, + ground_truth: str, + response: str + ) -> Dict[str, float]: ... + + @override + def __init__( + self, + alpha: float = 0.9, + beta: float = 3.0, + gamma: float = 0.5, + *, + threshold: float = 0.5 + ): ... + + + class azure.ai.evaluation.OpenAIModelConfiguration(TypedDict): + key "api_key": str + key "model": str + base_url: NotRequired[str] + extra_headers: NotRequired[Dict[str, str]] + organization: NotRequired[str] + type: NotRequired[Literal["openai"]] + + + @experimental + class azure.ai.evaluation.ProtectedMaterialEvaluator(RaiServiceEvaluatorBase[Union[str, bool]]): + id = azureai://built-in/evaluators/protected_material + + @overload + def __call__( + self, + *, + query: str, + response: str + ) -> Dict[str, Union[str, bool]]: ... + + @overload + def __call__( + self, + *, + conversation: Conversation + ) -> Dict[str, Union[float, Dict[str, List[Union[str, bool]]]]]: ... + + @override + def __init__( + self, + credential: TokenCredential, + azure_ai_project: Union[str, AzureAIProject], + **kwargs + ): ... + + + class azure.ai.evaluation.QAEvaluator(MultiEvaluatorBase[Union[str, float]]): + id = azureai://built-in/evaluators/qa + + @overload + def __call__( + self, + *, + context: str, + ground_truth: str, + query: str, + response: str + ): ... + + def __init__( + self, + model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration], + *, + coherence_threshold: int = 3, + f1_score_threshold: float = 0.5, + fluency_threshold: int = 3, + groundedness_threshold: int = 3, + relevance_threshold: int = 3, + similarity_threshold: int = 3, + **kwargs: Any + ): ... + + + class azure.ai.evaluation.RelevanceEvaluator(PromptyEvaluatorBase): + id = azureai://built-in/evaluators/relevance + + @overload + def __call__( + self, + *, + query: str, + response: str + ) -> Dict[str, Union[str, float]]: ... + + @overload + def __call__( + self, + *, + conversation: Conversation + ) -> Dict[str, Union[float, Dict[str, List[Union[str, float]]]]]: ... + + @override + def __init__( + self, + model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration], + *, + credential = ..., + is_reasoning_model: bool = ..., + threshold = 3, + **kwargs + ): ... + + + @experimental + class azure.ai.evaluation.ResponseCompletenessEvaluator(PromptyEvaluatorBase[Union[str, float]]): + id = azureai://built-in/evaluators/response_completeness + + @overload + def __call__( + self, + *, + ground_truth: str, + response: str + ) -> Dict[str, Union[str, float]]: ... + + @overload + def __call__( + self, + *, + conversation: Conversation + ) -> Dict[str, Union[float, Dict[str, List[Union[str, float]]]]]: ... + + @override + def __init__( + self, + model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration], + *, + credential = ..., + threshold: Optional[float] = _DEFAULT_COMPLETENESS_THRESHOLD, + **kwargs + ): ... + + + class azure.ai.evaluation.RetrievalEvaluator(PromptyEvaluatorBase[Union[str, float]]): + id = azureai://built-in/evaluators/retrieval + + @overload + def __call__( + self, + *, + context: str, + query: str + ) -> Dict[str, Union[str, float]]: ... + + @overload + def __call__( + self, + *, + conversation: Conversation + ) -> Dict[str, Union[float, Dict[str, List[Union[str, float]]]]]: ... + + @override + def __init__( + self, + model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration], + *, + credential = ..., + is_reasoning_model: bool = ..., + threshold: float = 3, + **kwargs + ) -> Callable: ... + + + class azure.ai.evaluation.RougeScoreEvaluator(EvaluatorBase): + id = azureai://built-in/evaluators/rouge_score + + @overload + def __call__( + self, + *, + ground_truth: str, + response: str + ) -> Dict[str, float]: ... + + @override + def __init__( + self, + rouge_type: RougeType, + *, + f1_score_threshold: float = 0.5, + precision_threshold: float = 0.5, + recall_threshold: float = 0.5 + ): ... + + + class azure.ai.evaluation.RougeType(str, Enum): + ROUGE_1 = "rouge1" + ROUGE_2 = "rouge2" + ROUGE_3 = "rouge3" + ROUGE_4 = "rouge4" + ROUGE_5 = "rouge5" + ROUGE_L = "rougeL" + + + @experimental + class azure.ai.evaluation.SelfHarmEvaluator(RaiServiceEvaluatorBase[Union[str, float]]): + id = azureai://built-in/evaluators/self_harm + + @overload + def __call__( + self, + *, + query: str, + response: str + ) -> Dict[str, Union[str, float]]: ... + + @overload + def __call__( + self, + *, + conversation: Conversation + ) -> Dict[str, Union[float, Dict[str, List[Union[str, float]]]]]: ... + + @override + def __init__( + self, + credential: TokenCredential, + azure_ai_project: Union[str, AzureAIProject], + *, + threshold: int = 3, + **kwargs + ): ... + + + @experimental + class azure.ai.evaluation.SexualEvaluator(RaiServiceEvaluatorBase[Union[str, float]]): + id = azureai://built-in/evaluators/sexual + + @overload + def __call__( + self, + *, + query: str, + response: str + ) -> Dict[str, Union[str, float]]: ... + + @overload + def __call__( + self, + *, + conversation: Conversation + ) -> Dict[str, Union[float, Dict[str, List[Union[str, float]]]]]: ... + + @override + def __init__( + self, + credential: TokenCredential, + azure_ai_project: Union[str, AzureAIProject], + *, + threshold: int = 3, + **kwargs + ): ... + + + class azure.ai.evaluation.SimilarityEvaluator(PromptyEvaluatorBase): + id = azureai://built-in/evaluators/similarity + + @overload + def __call__( + self, + *, + ground_truth: str, + query: str, + response: str + ) -> Dict[str, float]: ... + + @override + def __init__( + self, + model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration], + *, + credential = ..., + is_reasoning_model: bool = ..., + threshold = 3, + **kwargs + ): ... + + + @experimental + class azure.ai.evaluation.TaskAdherenceEvaluator(PromptyEvaluatorBase[Union[str, float]]): + id = azureai://built-in/evaluators/task_adherence + + @overload + def __call__( + self, + *, + query: Union[str, List[dict]], + response: Union[str, List[dict]], + tool_definitions: Optional[Union[dict, List[dict]]] = ... + ) -> Dict[str, Union[str, float]]: ... + + @override + def __init__( + self, + model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration], + *, + credential = ..., + threshold = _DEFAULT_TASK_ADHERENCE_SCORE, + **kwargs + ): ... + + + @experimental + class azure.ai.evaluation.ToolCallAccuracyEvaluator(PromptyEvaluatorBase[Union[str, float]]): + id = azureai://built-in/evaluators/tool_call_accuracy + + @overload + def __call__( + self, + *, + query: Union[str, List[dict]], + response: Union[str, List[dict]] = ..., + tool_calls: Union[dict, List[dict]] = ..., + tool_definitions: Union[dict, List[dict]] + ) -> Dict[str, Union[str, float]]: ... + + @override + def __init__( + self, + model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration], + *, + credential = ..., + threshold = _DEFAULT_TOOL_CALL_ACCURACY_SCORE, + **kwargs + ): ... + + + @experimental + class azure.ai.evaluation.UngroundedAttributesEvaluator(RaiServiceEvaluatorBase[Union[str, bool]]): + id = azureai://built-in/evaluators/ungrounded_attributes + + @overload + def __call__( + self, + *, + context: str, + query: str, + response: str + ) -> Dict[str, Union[str, float]]: ... + + @override + def __init__( + self, + credential: TokenCredential, + azure_ai_project: Union[str, AzureAIProject], + **kwargs: Any + ): ... + + + @experimental + class azure.ai.evaluation.ViolenceEvaluator(RaiServiceEvaluatorBase[Union[str, float]]): + id = azureai://built-in/evaluators/violence + + @overload + def __call__( + self, + *, + query: str, + response: str + ) -> Dict[str, Union[str, float]]: ... + + @overload + def __call__( + self, + *, + conversation: Conversation + ) -> Dict[str, Union[float, Dict[str, List[Union[str, float]]]]]: ... + + @override + def __init__( + self, + credential: TokenCredential, + azure_ai_project: Union[str, AzureAIProject], + *, + threshold: int = 3, + **kwargs + ): ... + + +namespace azure.ai.evaluation.autogen.raiclient + + class azure.ai.evaluation.autogen.raiclient.MachineLearningServicesClient: implements ContextManager + rai_svc: RAISvcOperations + + def __init__( + self, + endpoint: str, + subscription_id: str, + resource_group_name: str, + workspace_name: str, + credential: TokenCredential, + *, + api_version: str = ..., + **kwargs: Any + ) -> None: ... + + def close(self) -> None: ... + + def send_request( + self, + request: HttpRequest, + *, + stream: bool = False, + **kwargs: Any + ) -> HttpResponse: ... + + +namespace azure.ai.evaluation.autogen.raiclient.aio + + class azure.ai.evaluation.autogen.raiclient.aio.MachineLearningServicesClient: implements AsyncContextManager + rai_svc: RAISvcOperations + + def __init__( + self, + endpoint: str, + subscription_id: str, + resource_group_name: str, + workspace_name: str, + credential: AsyncTokenCredential, + *, + api_version: str = ..., + **kwargs: Any + ) -> None: ... + + async def close(self) -> None: ... + + def send_request( + self, + request: HttpRequest, + *, + stream: bool = False, + **kwargs: Any + ) -> Awaitable[AsyncHttpResponse]: ... + + +namespace azure.ai.evaluation.autogen.raiclient.aio.operations + + class azure.ai.evaluation.autogen.raiclient.aio.operations.RAISvcOperations: + + def __init__( + self, + *args, + **kwargs + ) -> None: ... + + @distributed_trace_async + async def get_annotation(self, **kwargs: Any) -> List[str]: ... + + @distributed_trace_async + async def get_attack_objectives( + self, + *, + lang: str, + risk_types: List[str], + **kwargs: Any + ) -> str: ... + + @distributed_trace_async + async def get_jail_break_dataset(self, **kwargs: Any) -> str: ... + + @distributed_trace_async + async def get_jail_break_dataset_with_type( + self, + type: str, + **kwargs: Any + ) -> str: ... + + @distributed_trace_async + async def get_operation_result( + self, + operation_id: str, + *, + api_key: Optional[str] = ..., + model_endpoint: Optional[str] = ..., + **kwargs: Any + ) -> str: ... + + @distributed_trace_async + async def get_template_parameters(self, **kwargs: Any) -> str: ... + + @distributed_trace_async + async def get_template_parameters_image( + self, + *, + path: str, + **kwargs: Any + ) -> str: ... + + @distributed_trace_async + async def get_template_parameters_with_type( + self, + type: str, + **kwargs: Any + ) -> str: ... + + @overload + async def submit_annotation( + self, + body: AnnotationDTO, + *, + content_type: str = "application/json", + **kwargs: Any + ) -> LongRunningResponse: ... + + @overload + async def submit_annotation( + self, + body: JSON, + *, + content_type: str = "application/json", + **kwargs: Any + ) -> LongRunningResponse: ... + + @overload + async def submit_annotation( + self, + body: IO[bytes], + *, + content_type: str = "application/json", + **kwargs: Any + ) -> LongRunningResponse: ... + + @overload + async def submit_aoai_evaluation( + self, + body: GradersDTO, + *, + content_type: str = "application/json", + **kwargs: Any + ) -> LongRunningResponse: ... + + @overload + async def submit_aoai_evaluation( + self, + body: JSON, + *, + content_type: str = "application/json", + **kwargs: Any + ) -> LongRunningResponse: ... + + @overload + async def submit_aoai_evaluation( + self, + body: IO[bytes], + *, + content_type: str = "application/json", + **kwargs: Any + ) -> LongRunningResponse: ... + + @overload + async def submit_simulation( + self, + body: SimulationDTO, + *, + content_type: str = "application/json", + **kwargs: Any + ) -> LongRunningResponse: ... + + @overload + async def submit_simulation( + self, + body: JSON, + *, + content_type: str = "application/json", + **kwargs: Any + ) -> LongRunningResponse: ... + + @overload + async def submit_simulation( + self, + body: IO[bytes], + *, + content_type: str = "application/json", + **kwargs: Any + ) -> LongRunningResponse: ... + + +namespace azure.ai.evaluation.autogen.raiclient.models + + class azure.ai.evaluation.autogen.raiclient.models.AnnotationDTO(Model): + annotation_task: str + content_type: str + contents: List[Content] + metric_list: List[str] + prompt_version: str + user_text_list: List[str] + + @overload + def __init__( + self, + *, + annotation_task: str, + content_type: str, + contents: List[Content], + metric_list: List[str], + prompt_version: str, + user_text_list: List[str] + ) -> None: ... + + @overload + def __init__(self, mapping: Mapping[str, Any]) -> None: ... + + + class azure.ai.evaluation.autogen.raiclient.models.Content(Model): + messages: List[Any] + + @overload + def __init__( + self, + *, + messages: List[Any] + ) -> None: ... + + @overload + def __init__(self, mapping: Mapping[str, Any]) -> None: ... + + + class azure.ai.evaluation.autogen.raiclient.models.CustomizationParameters(Model): + application_scenario: Optional[str] + harm_categories: List[str] + + @overload + def __init__( + self, + *, + application_scenario: Optional[str] = ..., + harm_categories: List[str] + ) -> None: ... + + @overload + def __init__(self, mapping: Mapping[str, Any]) -> None: ... + + + class azure.ai.evaluation.autogen.raiclient.models.Data(Model): + asset_id: str + + @overload + def __init__( + self, + *, + asset_id: str + ) -> None: ... + + @overload + def __init__(self, mapping: Mapping[str, Any]) -> None: ... + + + class azure.ai.evaluation.autogen.raiclient.models.Grader(Model): + config: GraderConfigBase + description: str + name: str + + @overload + def __init__( + self, + *, + config: GraderConfigBase, + description: str, + name: str + ) -> None: ... + + @overload + def __init__(self, mapping: Mapping[str, Any]) -> None: ... + + + class azure.ai.evaluation.autogen.raiclient.models.GraderConfigBase(Model): + type: str + + @overload + def __init__( + self, + *, + type: str + ) -> None: ... + + @overload + def __init__(self, mapping: Mapping[str, Any]) -> None: ... + + + class azure.ai.evaluation.autogen.raiclient.models.GradersDTO(Model): + data: Data + graders: List[Grader] + model_config: ModelConfig + sample_generators: List[SampleGenerator] + + @overload + def __init__( + self, + *, + data: Data, + graders: List[Grader], + model_config: ModelConfig, + sample_generators: List[SampleGenerator] + ) -> None: ... + + @overload + def __init__(self, mapping: Mapping[str, Any]) -> None: ... + + + class azure.ai.evaluation.autogen.raiclient.models.LongRunningResponse(Model): + location: str + operation_result: Any + + @overload + def __init__( + self, + *, + location: str, + operation_result: Any + ) -> None: ... + + @overload + def __init__(self, mapping: Mapping[str, Any]) -> None: ... + + + class azure.ai.evaluation.autogen.raiclient.models.ModelConfig(Model): + azure_endpoint: str + + @overload + def __init__( + self, + *, + azure_endpoint: str + ) -> None: ... + + @overload + def __init__(self, mapping: Mapping[str, Any]) -> None: ... + + + class azure.ai.evaluation.autogen.raiclient.models.SampleGenerator(Model): + model_name: str + sampling_params: Any + trajectory_template: Any + type: str + + @overload + def __init__( + self, + *, + model_name: str, + sampling_params: Any, + trajectory_template: Any, + type: str + ) -> None: ... + + @overload + def __init__(self, mapping: Mapping[str, Any]) -> None: ... + + + class azure.ai.evaluation.autogen.raiclient.models.SimulationDTO(Model): + customization_parameters: Optional[CustomizationParameters] + headers: Optional[Dict[str, str]] + is_microsoft_tenant: Optional[bool] + json: Optional[str] + params: Optional[Dict[str, str]] + resource_group_name: Optional[str] + simulation_type: Optional[Union[str, SimulationType]] + subscription_id: Optional[str] + template_key: Optional[str] + template_parameters: Optional[Dict[str, str]] + url: Optional[str] + workspace_name: Optional[str] + + @overload + def __init__( + self, + *, + customization_parameters: Optional[CustomizationParameters] = ..., + headers: Optional[Dict[str, str]] = ..., + is_microsoft_tenant: Optional[bool] = ..., + json: Optional[str] = ..., + params: Optional[Dict[str, str]] = ..., + resource_group_name: Optional[str] = ..., + simulation_type: Optional[Union[str, SimulationType]] = ..., + subscription_id: Optional[str] = ..., + template_key: Optional[str] = ..., + template_parameters: Optional[Dict[str, str]] = ..., + url: Optional[str] = ..., + workspace_name: Optional[str] = ... + ) -> None: ... + + @overload + def __init__(self, mapping: Mapping[str, Any]) -> None: ... + + + class azure.ai.evaluation.autogen.raiclient.models.SimulationType(str, Enum, metaclass=CaseInsensitiveEnumMeta): + CUSTOM_PERSONA = "CustomPersona" + DEFAULT = "Default" + HARM_TURN_GENERATOR = "HarmTurnGenerator" + + +namespace azure.ai.evaluation.autogen.raiclient.operations + + class azure.ai.evaluation.autogen.raiclient.operations.RAISvcOperations: + + def __init__( + self, + *args, + **kwargs + ): ... + + @distributed_trace + def get_annotation(self, **kwargs: Any) -> List[str]: ... + + @distributed_trace + def get_attack_objectives( + self, + *, + lang: str, + risk_category: Optional[str] = ..., + risk_types: List[str], + strategy: Optional[str] = ..., + target_type: Optional[str] = ..., + **kwargs: Any + ) -> str: ... + + @distributed_trace + def get_jail_break_dataset(self, **kwargs: Any) -> str: ... + + @distributed_trace + def get_jail_break_dataset_with_type( + self, + type: str, + **kwargs: Any + ) -> str: ... + + @distributed_trace + def get_operation_result( + self, + operation_id: str, + *, + api_key: Optional[str] = ..., + model_endpoint: Optional[str] = ..., + **kwargs: Any + ) -> str: ... + + @distributed_trace + def get_template_parameters(self, **kwargs: Any) -> str: ... + + @distributed_trace + def get_template_parameters_image( + self, + *, + path: str, + **kwargs: Any + ) -> str: ... + + @distributed_trace + def get_template_parameters_with_type( + self, + type: str, + **kwargs: Any + ) -> str: ... + + @overload + def submit_annotation( + self, + body: AnnotationDTO, + *, + content_type: str = "application/json", + **kwargs: Any + ) -> LongRunningResponse: ... + + @overload + def submit_annotation( + self, + body: JSON, + *, + content_type: str = "application/json", + **kwargs: Any + ) -> LongRunningResponse: ... + + @overload + def submit_annotation( + self, + body: IO[bytes], + *, + content_type: str = "application/json", + **kwargs: Any + ) -> LongRunningResponse: ... + + @overload + def submit_aoai_evaluation( + self, + body: GradersDTO, + *, + content_type: str = "application/json", + **kwargs: Any + ) -> LongRunningResponse: ... + + @overload + def submit_aoai_evaluation( + self, + body: JSON, + *, + content_type: str = "application/json", + **kwargs: Any + ) -> LongRunningResponse: ... + + @overload + def submit_aoai_evaluation( + self, + body: IO[bytes], + *, + content_type: str = "application/json", + **kwargs: Any + ) -> LongRunningResponse: ... + + @overload + def submit_simulation( + self, + body: SimulationDTO, + *, + content_type: str = "application/json", + **kwargs: Any + ) -> LongRunningResponse: ... + + @overload + def submit_simulation( + self, + body: JSON, + *, + content_type: str = "application/json", + **kwargs: Any + ) -> LongRunningResponse: ... + + @overload + def submit_simulation( + self, + body: IO[bytes], + *, + content_type: str = "application/json", + **kwargs: Any + ) -> LongRunningResponse: ... + + +namespace azure.ai.evaluation.red_team + + @experimental + class azure.ai.evaluation.red_team.AttackStrategy(Enum): + AnsiAttack = "ansi_attack" + AsciiArt = "ascii_art" + AsciiSmuggler = "ascii_smuggler" + Atbash = "atbash" + Base64 = "base64" + Baseline = "baseline" + Binary = "binary" + Caesar = "caesar" + CharSwap = "char_swap" + CharacterSpace = "character_space" + Crescendo = "crescendo" + DIFFICULT = "difficult" + Diacritic = "diacritic" + EASY = "easy" + Flip = "flip" + IndirectJailbreak = "indirect_jailbreak" + Jailbreak = "jailbreak" + Leetspeak = "leetspeak" + MODERATE = "moderate" + Morse = "morse" + MultiTurn = "multi_turn" + ROT13 = "rot13" + StringJoin = "string_join" + SuffixAppend = "suffix_append" + Tense = "tense" + UnicodeConfusable = "unicode_confusable" + UnicodeSubstitution = "unicode_substitution" + Url = "url" + + + @experimental + class azure.ai.evaluation.red_team.RedTeam: + + def __init__( + self, + azure_ai_project: Union[dict, str], + credential: TokenCredential, + *, + application_scenario: Optional[str] = ..., + attack_success_thresholds: Optional[Dict[RiskCategory, int]] = ..., + custom_attack_seed_prompts: Optional[str] = ..., + language: SupportedLanguages = SupportedLanguages.English, + num_objectives: int = 10, + output_dir = ".", + risk_categories: Optional[List[RiskCategory]] = ..., + **kwargs + ): ... + + async def scan( + self, + target: Union[Callable, AzureOpenAIModelConfiguration, OpenAIModelConfiguration, PromptChatTarget], + *, + application_scenario: Optional[str] = ..., + attack_strategies: List[Union[AttackStrategy, List[AttackStrategy]]] = [], + max_parallel_tasks: int = 5, + output_path: Optional[Union[str, PathLike]] = ..., + parallel_execution: bool = True, + scan_name: Optional[str] = ..., + skip_evals: bool = False, + skip_upload: bool = False, + timeout: int = 3600, + **kwargs: Any + ) -> RedTeamResult: ... + + + @experimental + class azure.ai.evaluation.red_team.RedTeamResult: + + def __init__( + self, + scan_result: Optional[ScanResult] = None, + attack_details: Optional[List[AttackDetails]] = None + ): ... + + def attack_simulation(self) -> str: ... + + def to_eval_qr_json_lines(self) -> str: ... + + def to_json(self) -> str: ... + + def to_scorecard(self) -> Optional[RedTeamingScorecard]: ... + + + @experimental + class azure.ai.evaluation.red_team.RiskCategory(str, Enum): + CodeVulnerability = "code_vulnerability" + HateUnfairness = "hate_unfairness" + ProhibitedActions = "prohibited_actions" + ProtectedMaterial = "protected_material" + SelfHarm = "self_harm" + SensitiveDataLeakage = "sensitive_data_leakage" + Sexual = "sexual" + TaskAdherence = "task_adherence" + UngroundedAttributes = "ungrounded_attributes" + Violence = "violence" + + + @experimental + class azure.ai.evaluation.red_team.SupportedLanguages(Enum): + English = "en" + French = "fr" + German = "de" + Italian = "it" + Japanese = "ja" + Korean = "ko" + Portuguese = "pt" + SimplifiedChinese = "zh-cn" + Spanish = "es" + + +namespace azure.ai.evaluation.simulator + + @experimental + class azure.ai.evaluation.simulator.AdversarialScenario(Enum): + ADVERSARIAL_CODE_VULNERABILITY = "adv_code_vuln" + ADVERSARIAL_CONTENT_GEN_GROUNDED = "adv_content_gen_grounded" + ADVERSARIAL_CONTENT_GEN_UNGROUNDED = "adv_content_gen_ungrounded" + ADVERSARIAL_CONTENT_PROTECTED_MATERIAL = "adv_content_protected_material" + ADVERSARIAL_CONVERSATION = "adv_conversation" + ADVERSARIAL_QA = "adv_qa" + ADVERSARIAL_QA_DOCUMENTS = "adv_qa_documents" + ADVERSARIAL_REWRITE = "adv_rewrite" + ADVERSARIAL_SEARCH = "adv_search" + ADVERSARIAL_SUMMARIZATION = "adv_summarization" + ADVERSARIAL_UNGROUNDED_ATTRIBUTES = "adv_isa" + + + @experimental + class azure.ai.evaluation.simulator.AdversarialScenarioJailbreak(Enum): + ADVERSARIAL_INDIRECT_JAILBREAK = "adv_xpia" + + + @experimental + class azure.ai.evaluation.simulator.AdversarialSimulator: + + async def __call__( + self, + *, + api_call_delay_sec: int = 0, + api_call_retry_limit: int = 3, + api_call_retry_sleep_sec: int = 1, + concurrent_async_task: int = 3, + language: SupportedLanguages = SupportedLanguages.English, + max_conversation_turns: int = 1, + max_simulation_results: int = 3, + randomization_seed: Optional[int] = ..., + randomize_order: bool = True, + scenario: AdversarialScenario, + target: Callable, + **kwargs + ) -> List[Dict[str, Any]]: ... + + def __init__( + self, + *, + azure_ai_project: Union[str, AzureAIProject], + credential: TokenCredential + ): ... + + def call_sync( + self, + *, + api_call_delay_sec: int, + api_call_retry_limit: int, + api_call_retry_sleep_sec: int, + concurrent_async_task: int, + max_conversation_turns: int, + max_simulation_results: int, + scenario: AdversarialScenario, + target: Callable + ) -> List[Dict[str, Any]]: ... + + + @experimental + class azure.ai.evaluation.simulator.DirectAttackSimulator: + + async def __call__( + self, + *, + api_call_delay_sec: int = 0, + api_call_retry_limit: int = 3, + api_call_retry_sleep_sec: int = 1, + concurrent_async_task: int = 3, + max_conversation_turns: int = 1, + max_simulation_results: int = 3, + randomization_seed: Optional[int] = ..., + scenario: AdversarialScenario, + target: Callable + ) -> Dict[str, [List[Dict[str, Any]]]]: ... + + def __init__( + self, + *, + azure_ai_project: Union[str, AzureAIProject], + credential: TokenCredential + ): ... + + + @experimental + class azure.ai.evaluation.simulator.IndirectAttackSimulator(AdversarialSimulator): + + async def __call__( + self, + *, + api_call_delay_sec: int = 0, + api_call_retry_limit: int = 3, + api_call_retry_sleep_sec: int = 1, + concurrent_async_task: int = 3, + max_simulation_results: int = 3, + randomization_seed: Optional[int] = ..., + target: Callable, + **kwargs + ) -> List[Dict[str, Any]]: ... + + def __init__( + self, + *, + azure_ai_project: Union[str, AzureAIProject], + credential: TokenCredential + ): ... + + def call_sync( + self, + *, + api_call_delay_sec: int, + api_call_retry_limit: int, + api_call_retry_sleep_sec: int, + concurrent_async_task: int, + max_conversation_turns: int, + max_simulation_results: int, + scenario: AdversarialScenario, + target: Callable + ) -> List[Dict[str, Any]]: ... + + + @experimental + class azure.ai.evaluation.simulator.Simulator: + + async def __call__( + self, + *, + api_call_delay_sec: float = 1, + concurrent_async_tasks: int = 5, + conversation_turns: List[List[Union[str, Dict[str, Any]]]] = [], + max_conversation_turns: int = 5, + num_queries: int = 5, + query_response_generating_prompty: Optional[str] = ..., + query_response_generating_prompty_options: Dict[str, Any] = {}, + randomization_seed: Optional[int] = ..., + target: Callable, + tasks: List[str] = [], + text: str = "", + user_simulator_prompty: Optional[str] = ..., + user_simulator_prompty_options: Dict[str, Any] = {}, + **kwargs + ) -> List[JsonLineChatProtocol]: ... + + def __init__(self, model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration]): ... + + + class azure.ai.evaluation.simulator.SupportedLanguages(Enum): + English = "en" + French = "fr" + German = "de" + Italian = "it" + Japanese = "ja" + Korean = "ko" + Portuguese = "pt" + SimplifiedChinese = "zh-cn" + Spanish = "es" + + +``` \ No newline at end of file diff --git a/sdk/evaluation/azure-ai-evaluation/api.metadata.yml b/sdk/evaluation/azure-ai-evaluation/api.metadata.yml new file mode 100644 index 000000000000..58b96aa4ee23 --- /dev/null +++ b/sdk/evaluation/azure-ai-evaluation/api.metadata.yml @@ -0,0 +1,3 @@ +apiMdSha256: 942162ce260c408be1be04a373bc86eef35d0a6e386fd5d530aa3fcb60c691dd +parserVersion: 0.3.28 +pythonVersion: 3.10.20