feat: add OnlineEvaluator API and update Evaluation API

docs/aiplatform_v1beta1/online_evaluator_service.rst

@@ -0,0 +1,10 @@
+OnlineEvaluatorService
+----------------------------------------
+
+.. automodule:: google.cloud.aiplatform_v1beta1.services.online_evaluator_service
+    :members:
+    :inherited-members:
+
+.. automodule:: google.cloud.aiplatform_v1beta1.services.online_evaluator_service.pagers
+    :members:
+    :inherited-members:

docs/aiplatform_v1beta1/services_.rst

@@ -29,6 +29,7 @@ Services for Google Cloud Aiplatform v1beta1 API
     model_monitoring_service
     model_service
     notebook_service
+    online_evaluator_service
     persistent_resource_service
     pipeline_service
     prediction_service

google/cloud/aiplatform_v1/__init__.py

@@ -261,6 +261,9 @@
 from .types.evaluated_annotation import ErrorAnalysisAnnotation
 from .types.evaluated_annotation import EvaluatedAnnotation
 from .types.evaluated_annotation import EvaluatedAnnotationExplanation
+from .types.evaluation_service import AggregationOutput
+from .types.evaluation_service import AggregationResult
+from .types.evaluation_service import AutoraterConfig
 from .types.evaluation_service import BleuInput
 from .types.evaluation_service import BleuInstance
 from .types.evaluation_service import BleuMetricValue
@@ -274,8 +277,14 @@
 from .types.evaluation_service import CometInstance
 from .types.evaluation_service import CometResult
 from .types.evaluation_service import CometSpec
+from .types.evaluation_service import ComputationBasedMetricSpec
+from .types.evaluation_service import ContentMap
+from .types.evaluation_service import CustomOutput
+from .types.evaluation_service import CustomOutputFormatConfig
+from .types.evaluation_service import EvaluateDatasetResponse
 from .types.evaluation_service import EvaluateInstancesRequest
 from .types.evaluation_service import EvaluateInstancesResponse
+from .types.evaluation_service import EvaluationDataset
 from .types.evaluation_service import ExactMatchInput
 from .types.evaluation_service import ExactMatchInstance
 from .types.evaluation_service import ExactMatchMetricValue
@@ -293,10 +302,15 @@
 from .types.evaluation_service import GroundednessInstance
 from .types.evaluation_service import GroundednessResult
 from .types.evaluation_service import GroundednessSpec
+from .types.evaluation_service import LLMBasedMetricSpec
+from .types.evaluation_service import Metric
+from .types.evaluation_service import MetricResult
 from .types.evaluation_service import MetricxInput
 from .types.evaluation_service import MetricxInstance
 from .types.evaluation_service import MetricxResult
 from .types.evaluation_service import MetricxSpec
+from .types.evaluation_service import OutputConfig
+from .types.evaluation_service import OutputInfo
 from .types.evaluation_service import PairwiseMetricInput
 from .types.evaluation_service import PairwiseMetricInstance
 from .types.evaluation_service import PairwiseMetricResult
@@ -313,6 +327,7 @@
 from .types.evaluation_service import PointwiseMetricInstance
 from .types.evaluation_service import PointwiseMetricResult
 from .types.evaluation_service import PointwiseMetricSpec
+from .types.evaluation_service import PredefinedMetricSpec
 from .types.evaluation_service import QuestionAnsweringCorrectnessInput
 from .types.evaluation_service import QuestionAnsweringCorrectnessInstance
 from .types.evaluation_service import QuestionAnsweringCorrectnessResult
@@ -329,6 +344,7 @@
 from .types.evaluation_service import QuestionAnsweringRelevanceInstance
 from .types.evaluation_service import QuestionAnsweringRelevanceResult
 from .types.evaluation_service import QuestionAnsweringRelevanceSpec
+from .types.evaluation_service import RawOutput
 from .types.evaluation_service import RougeInput
 from .types.evaluation_service import RougeInstance
 from .types.evaluation_service import RougeMetricValue
@@ -879,6 +895,11 @@
 from .types.publisher_model import PublisherModel
 from .types.reasoning_engine import ReasoningEngine
 from .types.reasoning_engine import ReasoningEngineSpec
+from .types.reasoning_engine_execution_service import (
+    AsyncQueryReasoningEngineOperationMetadata,
+)
+from .types.reasoning_engine_execution_service import AsyncQueryReasoningEngineRequest
+from .types.reasoning_engine_execution_service import AsyncQueryReasoningEngineResponse
 from .types.reasoning_engine_execution_service import QueryReasoningEngineRequest
 from .types.reasoning_engine_execution_service import QueryReasoningEngineResponse
 from .types.reasoning_engine_execution_service import StreamQueryReasoningEngineRequest
@@ -1023,6 +1044,8 @@
 from .types.training_pipeline import StratifiedSplit
 from .types.training_pipeline import TimestampSplit
 from .types.training_pipeline import TrainingPipeline
+from .types.tuning_job import EvaluateDatasetRun
+from .types.tuning_job import EvaluationConfig
 from .types.tuning_job import PreTunedModel
 from .types.tuning_job import SupervisedHyperParameters
 from .types.tuning_job import SupervisedTuningDatasetDistribution
@@ -1253,6 +1276,8 @@ def _get_version(dependency_name):
     "AddExecutionEventsRequest",
     "AddExecutionEventsResponse",
     "AddTrialMeasurementRequest",
+    "AggregationOutput",
+    "AggregationResult",
     "Annotation",
     "AnnotationSpec",
     "ApiAuth",
@@ -1263,13 +1288,17 @@ def _get_version(dependency_name):
     "AskContextsResponse",
     "AssignNotebookRuntimeOperationMetadata",
     "AssignNotebookRuntimeRequest",
+    "AsyncQueryReasoningEngineOperationMetadata",
+    "AsyncQueryReasoningEngineRequest",
+    "AsyncQueryReasoningEngineResponse",
     "AsyncRetrieveContextsOperationMetadata",
     "AsyncRetrieveContextsRequest",
     "AsyncRetrieveContextsResponse",
     "Attribution",
     "AugmentPromptRequest",
     "AugmentPromptResponse",
     "AutomaticResources",
+    "AutoraterConfig",
     "AutoscalingMetricSpec",
     "AvroSource",
     "BatchCancelPipelineJobsOperationMetadata",
@@ -1338,11 +1367,13 @@ def _get_version(dependency_name):
     "CometSpec",
     "CompleteTrialRequest",
     "CompletionStats",
+    "ComputationBasedMetricSpec",
     "ComputeTokensRequest",
     "ComputeTokensResponse",
     "ContainerRegistryDestination",
     "ContainerSpec",
     "Content",
+    "ContentMap",
     "Context",
     "CopyModelOperationMetadata",
     "CopyModelRequest",
@@ -1419,6 +1450,8 @@ def _get_version(dependency_name):
     "CsvSource",
     "CustomJob",
     "CustomJobSpec",
+    "CustomOutput",
+    "CustomOutputFormatConfig",
     "DataFoundryServiceClient",
     "DataItem",
     "DataItemView",
@@ -1513,10 +1546,14 @@ def _get_version(dependency_name):
     "EntityType",
     "EnvVar",
     "ErrorAnalysisAnnotation",
+    "EvaluateDatasetResponse",
+    "EvaluateDatasetRun",
     "EvaluateInstancesRequest",
     "EvaluateInstancesResponse",
     "EvaluatedAnnotation",
     "EvaluatedAnnotationExplanation",
+    "EvaluationConfig",
+    "EvaluationDataset",
     "EvaluationServiceClient",
     "Event",
     "EventActions",
@@ -1704,6 +1741,7 @@ def _get_version(dependency_name):
     "JiraSource",
     "JobServiceClient",
     "JobState",
+    "LLMBasedMetricSpec",
     "LargeModelReference",
     "LineageSubgraph",
     "ListAnnotationsRequest",
@@ -1828,6 +1866,8 @@ def _get_version(dependency_name):
     "MetadataSchema",
     "MetadataServiceClient",
     "MetadataStore",
+    "Metric",
+    "MetricResult",
     "MetricxInput",
     "MetricxInstance",
     "MetricxResult",
@@ -1885,7 +1925,9 @@ def _get_version(dependency_name):
     "NotebookRuntimeType",
     "NotebookServiceClient",
     "NotebookSoftwareConfig",
+    "OutputConfig",
     "OutputFieldSpec",
+    "OutputInfo",
     "PSCAutomationConfig",
     "PSCAutomationState",
     "PairwiseChoice",
@@ -1924,6 +1966,7 @@ def _get_version(dependency_name):
     "PostStartupScriptConfig",
     "PreTunedModel",
     "PrebuiltVoiceConfig",
+    "PredefinedMetricSpec",
     "PredefinedSplit",
     "PredictRequest",
     "PredictRequestResponseLoggingConfig",
@@ -1984,6 +2027,7 @@ def _get_version(dependency_name):
     "RagQuery",
     "RagRetrievalConfig",
     "RagVectorDbConfig",
+    "RawOutput",
     "RawPredictRequest",
     "RayLogsSpec",
     "RayMetricSpec",

google/cloud/aiplatform_v1/gapic_metadata.json

@@ -4310,6 +4310,11 @@
         "grpc": {
           "libraryClient": "ReasoningEngineExecutionServiceClient",
           "rpcs": {
+            "AsyncQueryReasoningEngine": {
+              "methods": [
+                "async_query_reasoning_engine"
+              ]
+            },
             "QueryReasoningEngine": {
               "methods": [
                 "query_reasoning_engine"
@@ -4325,6 +4330,11 @@
         "grpc-async": {
           "libraryClient": "ReasoningEngineExecutionServiceAsyncClient",
           "rpcs": {
+            "AsyncQueryReasoningEngine": {
+              "methods": [
+                "async_query_reasoning_engine"
+              ]
+            },
             "QueryReasoningEngine": {
               "methods": [
                 "query_reasoning_engine"
@@ -4340,6 +4350,11 @@
         "rest": {
           "libraryClient": "ReasoningEngineExecutionServiceClient",
           "rpcs": {
+            "AsyncQueryReasoningEngine": {
+              "methods": [
+                "async_query_reasoning_engine"
+              ]
+            },
             "QueryReasoningEngine": {
               "methods": [
                 "query_reasoning_engine"

google/cloud/aiplatform_v1/services/reasoning_engine_execution_service/async_client.py

@@ -53,6 +53,8 @@
 from google.iam.v1 import policy_pb2  # type: ignore
 from google.longrunning import operations_pb2  # type: ignore
 import google.api.httpbody_pb2 as httpbody_pb2  # type: ignore
+import google.api_core.operation as operation  # type: ignore
+import google.api_core.operation_async as operation_async  # type: ignore
 import google.protobuf.any_pb2 as any_pb2  # type: ignore
 import google.protobuf.struct_pb2 as struct_pb2  # type: ignore
 from .transports.base import (
@@ -561,6 +563,117 @@ async def sample_stream_query_reasoning_engine():
         # Done; return the response.
         return response
 
+    async def async_query_reasoning_engine(
+        self,
+        request: Optional[
+            Union[
+                reasoning_engine_execution_service.AsyncQueryReasoningEngineRequest,
+                dict,
+            ]
+        ] = None,
+        *,
+        retry: OptionalRetry = gapic_v1.method.DEFAULT,
+        timeout: Union[float, object] = gapic_v1.method.DEFAULT,
+        metadata: Sequence[Tuple[str, Union[str, bytes]]] = (),
+    ) -> operation_async.AsyncOperation:
+        r"""Async query using a reasoning engine.
+
+        .. code-block:: python
+
+            # This snippet has been automatically generated and should be regarded as a
+            # code template only.
+            # It will require modifications to work:
+            # - It may require correct/in-range values for request initialization.
+            # - It may require specifying regional endpoints when creating the service
+            #   client as shown in:
+            #   https://googleapis.dev/python/google-api-core/latest/client_options.html
+            from google.cloud import aiplatform_v1
+
+            async def sample_async_query_reasoning_engine():
+                # Create a client
+                client = aiplatform_v1.ReasoningEngineExecutionServiceAsyncClient()
+
+                # Initialize request argument(s)
+                request = aiplatform_v1.AsyncQueryReasoningEngineRequest(
+                    name="name_value",
+                )
+
+                # Make the request
+                operation = client.async_query_reasoning_engine(request=request)
+
+                print("Waiting for operation to complete...")
+
+                response = (await operation).result()
+
+                # Handle the response
+                print(response)
+
+        Args:
+            request (Optional[Union[google.cloud.aiplatform_v1.types.AsyncQueryReasoningEngineRequest, dict]]):
+                The request object. Request message for
+                [ReasoningEngineExecutionService.AsyncQueryReasoningEngine][google.cloud.aiplatform.v1.ReasoningEngineExecutionService.AsyncQueryReasoningEngine].
+            retry (google.api_core.retry_async.AsyncRetry): Designation of what errors, if any,
+                should be retried.
+            timeout (float): The timeout for this request.
+            metadata (Sequence[Tuple[str, Union[str, bytes]]]): Key/value pairs which should be
+                sent along with the request as metadata. Normally, each value must be of type `str`,
+                but for metadata keys ending with the suffix `-bin`, the corresponding values must
+                be of type `bytes`.
+
+        Returns:
+            google.api_core.operation_async.AsyncOperation:
+                An object representing a long-running operation.
+
+                The result type for the operation will be :class:`google.cloud.aiplatform_v1.types.AsyncQueryReasoningEngineResponse` Response message for
+                   [ReasoningEngineExecutionService.AsyncQueryReasoningEngine][google.cloud.aiplatform.v1.ReasoningEngineExecutionService.AsyncQueryReasoningEngine].
+
+        """
+        # Create or coerce a protobuf request object.
+        # - Use the request object if provided (there's no risk of modifying the input as
+        #   there are no flattened fields), or create one.
+        if not isinstance(
+            request, reasoning_engine_execution_service.AsyncQueryReasoningEngineRequest
+        ):
+            request = (
+                reasoning_engine_execution_service.AsyncQueryReasoningEngineRequest(
+                    request
+                )
+            )
+
+        # Wrap the RPC method; this adds retry and timeout information,
+        # and friendly error handling.
+        rpc = self._client._transport._wrapped_methods[
+            self._client._transport.async_query_reasoning_engine
+        ]
+
+        # Certain fields should be provided within the metadata header;
+        # add these here.
+        metadata = tuple(metadata) + (
+            gapic_v1.routing_header.to_grpc_metadata((("name", request.name),)),
+        )
+
+        # Validate the universe domain.
+        self._client._validate_universe_domain()
+
+        # Send the request.
+        response = await rpc(
+            request,
+            retry=retry,
+            timeout=timeout,
+            metadata=metadata,
+        )
+
+        # Wrap the response in an operation future.
+        response = operation_async.from_gapic(
+            response,
+            self._client._transport.operations_client,
+            reasoning_engine_execution_service.AsyncQueryReasoningEngineResponse,
+            metadata_type=reasoning_engine_execution_service.AsyncQueryReasoningEngineOperationMetadata,
+        )
+
+        # Done; return the response.
+        return response
+
     async def list_operations(
         self,
         request: Optional[operations_pb2.ListOperationsRequest] = None,