TiKV Metadata

.cspell.json

@@ -300,6 +300,7 @@
     "TMPDIR",
     "Tgts",
     "Tolerations",
+    "typedarray",
     "UPSERT",
     "Unmarshal",
     "Upsert",
@@ -311,6 +312,7 @@
     "Vald's",
     "Vals",
     "Vecs",
+    "vercel",
     "Vectorizer",
     "Vqueue",
     "Warnd",
@@ -355,6 +357,7 @@
     "autoremove",
     "autorest",
     "autoscaler",
+    "arraybuffer",
     "azcore",
     "azidentity",
     "backoffmetrics",
@@ -442,6 +445,7 @@
     "daemonset",
     "dataspace",
     "datelier",
+    "dataview",
     "davecgh",
     "dbuild",
     "dbus",
@@ -466,6 +470,7 @@
     "dgryski",
     "dicoverer",
     "difflib",
+    "discordjs",
     "diskv",
     "distroless",
     "dnaeon",
@@ -882,6 +887,7 @@
     "quis",
     "ratelimit",
     "rdonly",
+    "readablestream",
     "readreplica",
     "readyz",
     "rebalance",

.gitfiles

@@ -59,6 +59,7 @@
 .github/helm/values/values-index-management-jobs.yaml
 .github/helm/values/values-lb.yaml
 .github/helm/values/values-max-dim.yaml
+.github/helm/values/values-meta.yaml
 .github/helm/values/values-mirror-01.yaml
 .github/helm/values/values-mirror-02.yaml
 .github/helm/values/values-profile.yaml
@@ -97,6 +98,7 @@
 .github/workflows/dockers-example-client-image.yaml
 .github/workflows/dockers-gateway-filter-image.yaml
 .github/workflows/dockers-gateway-lb-image.yaml
+.github/workflows/dockers-gateway-meta-image.yaml
 .github/workflows/dockers-gateway-mirror-image.yaml
 .github/workflows/dockers-helm-operator-image.yaml
 .github/workflows/dockers-image-scan.yaml
@@ -178,6 +180,7 @@ apis/docs/v1/filter.md
 apis/docs/v1/flush.md
 apis/docs/v1/index.md
 apis/docs/v1/insert.md
+apis/docs/v1/meta.md
 apis/docs/v1/mirror.md
 apis/docs/v1/object.md
 apis/docs/v1/payload.md.tmpl
@@ -211,14 +214,6 @@ apis/grpc/v1/rpc/errdetails/error_details.pb.json.go
 apis/grpc/v1/rpc/errdetails/error_details_vtproto.pb.go
 apis/grpc/v1/rpc/stats/stats.pb.go
 apis/grpc/v1/rpc/stats/stats_vtproto.pb.go
-apis/grpc/v1/tikv/errorpb.pb.go
-apis/grpc/v1/tikv/errorpb.pb.json.go
-apis/grpc/v1/tikv/errorpb_vtproto.pb.go
-apis/grpc/v1/tikv/kvrpcpb.pb.go
-apis/grpc/v1/tikv/kvrpcpb.pb.json.go
-apis/grpc/v1/tikv/kvrpcpb_vtproto.pb.go
-apis/grpc/v1/tikv/tikvpb.pb.go
-apis/grpc/v1/tikv/tikvpb_vtproto.pb.go
 apis/grpc/v1/vald/filter.pb.go
 apis/grpc/v1/vald/filter_vtproto.pb.go
 apis/grpc/v1/vald/flush.pb.go
@@ -227,6 +222,8 @@ apis/grpc/v1/vald/index.pb.go
 apis/grpc/v1/vald/index_vtproto.pb.go
 apis/grpc/v1/vald/insert.pb.go
 apis/grpc/v1/vald/insert_vtproto.pb.go
+apis/grpc/v1/vald/meta.pb.go
+apis/grpc/v1/vald/meta_vtproto.pb.go
 apis/grpc/v1/vald/object.pb.go
 apis/grpc/v1/vald/object_vtproto.pb.go
 apis/grpc/v1/vald/remove.pb.go
@@ -248,15 +245,11 @@ apis/proto/v1/mirror/mirror.proto
 apis/proto/v1/payload/payload.proto
 apis/proto/v1/rpc/errdetails/error_details.proto
 apis/proto/v1/rpc/stats/stats.proto
-apis/proto/v1/tikv/LICENSE
-apis/proto/v1/tikv/README.md
-apis/proto/v1/tikv/errorpb.proto
-apis/proto/v1/tikv/kvrpcpb.proto
-apis/proto/v1/tikv/tikvpb.proto
 apis/proto/v1/vald/filter.proto
 apis/proto/v1/vald/flush.proto
 apis/proto/v1/vald/index.proto
 apis/proto/v1/vald/insert.proto
+apis/proto/v1/vald/meta.proto
 apis/proto/v1/vald/object.proto
 apis/proto/v1/vald/remove.proto
 apis/proto/v1/vald/search.proto
@@ -272,13 +265,11 @@ apis/swagger/v1/mirror/mirror.swagger.json
 apis/swagger/v1/payload/payload.swagger.json
 apis/swagger/v1/rpc/errdetails/error_details.swagger.json
 apis/swagger/v1/rpc/stats/stats.swagger.json
-apis/swagger/v1/tikv/errorpb.swagger.json
-apis/swagger/v1/tikv/kvrpcpb.swagger.json
-apis/swagger/v1/tikv/tikvpb.swagger.json
 apis/swagger/v1/vald/filter.swagger.json
 apis/swagger/v1/vald/flush.swagger.json
 apis/swagger/v1/vald/index.swagger.json
 apis/swagger/v1/vald/insert.swagger.json
+apis/swagger/v1/vald/meta.swagger.json
 apis/swagger/v1/vald/object.swagger.json
 apis/swagger/v1/vald/remove.swagger.json
 apis/swagger/v1/vald/search.swagger.json
@@ -417,6 +408,7 @@ charts/vald/.helmignore
 charts/vald/Chart.yaml
 charts/vald/README.md
 charts/vald/README.md.gotmpl
+charts/vald/crds/tidbcluster.yaml
 charts/vald/crds/valdmirrortarget.yaml
 charts/vald/schemas/mirror-target-values.yaml
 charts/vald/templates/NOTES.txt
@@ -466,6 +458,15 @@ charts/vald/templates/gateway/lb/pdb.yaml
 charts/vald/templates/gateway/lb/priorityclass.yaml
 charts/vald/templates/gateway/lb/serviceaccount.yaml
 charts/vald/templates/gateway/lb/svc.yaml
+charts/vald/templates/gateway/meta/configmap.yaml
+charts/vald/templates/gateway/meta/daemonset.yaml
+charts/vald/templates/gateway/meta/deployment.yaml
+charts/vald/templates/gateway/meta/hpa.yaml
+charts/vald/templates/gateway/meta/networkpolicy.yaml
+charts/vald/templates/gateway/meta/pdb.yaml
+charts/vald/templates/gateway/meta/priorityclass.yaml
+charts/vald/templates/gateway/meta/serviceaccount.yaml
+charts/vald/templates/gateway/meta/svc.yaml
 charts/vald/templates/gateway/mirror/clusterrole.yaml
 charts/vald/templates/gateway/mirror/clusterrolebinding.yaml
 charts/vald/templates/gateway/mirror/configmap.yaml
@@ -504,6 +505,19 @@ charts/vald/templates/index/operator/configmap.yaml
 charts/vald/templates/index/operator/deployment.yaml
 charts/vald/templates/index/operator/priorityclass.yaml
 charts/vald/templates/index/operator/serviceaccount.yaml
+charts/vald/templates/kvs/tikv/pd.yaml
+charts/vald/templates/kvs/tikv/pd_configmap.yaml
+charts/vald/templates/kvs/tikv/tidbcluster.yaml
+charts/vald/templates/kvs/tikv/tikv.yaml
+charts/vald/templates/kvs/tikv/tikv_configmap.yaml
+charts/vald/templates/kvs/tikv/tikv_discovery.yaml
+charts/vald/templates/kvs/tikv/tikv_discovery_role.yaml
+charts/vald/templates/kvs/tikv/tikv_discovery_rolebinding.yaml
+charts/vald/templates/kvs/tikv/tikv_discovery_sa.yaml
+charts/vald/templates/kvs/tikv/tikv_discovery_svc.yaml
+charts/vald/templates/kvs/tikv/tikv_pd_peer_svc.yaml
+charts/vald/templates/kvs/tikv/tikv_pd_svc.yaml
+charts/vald/templates/kvs/tikv/tikv_tikv_peer_svc.yaml
 charts/vald/templates/manager/index/configmap.yaml
 charts/vald/templates/manager/index/daemonset.yaml
 charts/vald/templates/manager/index/deployment.yaml
@@ -558,6 +572,9 @@ cmd/gateway/lb/doc.go
 cmd/gateway/lb/main.go
 cmd/gateway/lb/main_test.go
 cmd/gateway/lb/sample.yaml
+cmd/gateway/meta/doc.go
+cmd/gateway/meta/main.go
+cmd/gateway/meta/sample.yaml
 cmd/gateway/mirror/doc.go
 cmd/gateway/mirror/main.go
 cmd/gateway/mirror/main_test.go
@@ -631,6 +648,7 @@ dockers/gateway/filter/Dockerfile
 dockers/gateway/filter/README.md
 dockers/gateway/lb/Dockerfile
 dockers/gateway/lb/README.md
+dockers/gateway/meta/Dockerfile
 dockers/gateway/mirror/Dockerfile
 dockers/index/job/correction/Dockerfile
 dockers/index/job/creation/Dockerfile
@@ -891,6 +909,9 @@ internal/client/v1/client/filter/ingress/client_test.go
 internal/client/v1/client/filter/ingress/doc.go
 internal/client/v1/client/filter/ingress/option.go
 internal/client/v1/client/filter/ingress/option_test.go
+internal/client/v1/client/meta/client.go
+internal/client/v1/client/meta/doc.go
+internal/client/v1/client/meta/rawkv.go
 internal/client/v1/client/mirror/doc.go
 internal/client/v1/client/mirror/mirror.go
 internal/client/v1/client/mirror/mirror_test.go
@@ -2070,6 +2091,26 @@ pkg/gateway/lb/service/option_test.go
 pkg/gateway/lb/usecase/doc.go
 pkg/gateway/lb/usecase/vald.go
 pkg/gateway/lb/usecase/vald_test.go
+pkg/gateway/meta/config/config.go
+pkg/gateway/meta/config/doc.go
+pkg/gateway/meta/handler/doc.go
+pkg/gateway/meta/handler/grpc/doc.go
+pkg/gateway/meta/handler/grpc/handler.go
+pkg/gateway/meta/handler/grpc/insert_meta.go
+pkg/gateway/meta/handler/grpc/object_meta.go
+pkg/gateway/meta/handler/grpc/option.go
+pkg/gateway/meta/handler/grpc/remove_meta.go
+pkg/gateway/meta/handler/grpc/search_meta.go
+pkg/gateway/meta/handler/grpc/update_meta.go
+pkg/gateway/meta/handler/grpc/upsert_meta.go
+pkg/gateway/meta/handler/rest/doc.go
+pkg/gateway/meta/handler/rest/handler.go
+pkg/gateway/meta/handler/rest/option.go
+pkg/gateway/meta/router/doc.go
+pkg/gateway/meta/router/option.go
+pkg/gateway/meta/router/router.go
+pkg/gateway/meta/usecase/doc.go
+pkg/gateway/meta/usecase/vald.go
 pkg/gateway/mirror/README.md
 pkg/gateway/mirror/config/config.go
 pkg/gateway/mirror/config/config_test.go
@@ -2342,13 +2383,18 @@ rust/libs/proto/src/lib.rs
 rust/libs/proto/src/meta/mod.rs
 rust/libs/proto/src/meta/v1/meta.v1.tonic.rs
 rust/libs/proto/src/meta/v1/mod.rs
+rust/libs/proto/src/metapb/metapb.rs
+rust/libs/proto/src/metapb/metapb.serde.rs
 rust/libs/proto/src/mirror/mod.rs
 rust/libs/proto/src/mirror/v1/mirror.v1.tonic.rs
 rust/libs/proto/src/mirror/v1/mod.rs
 rust/libs/proto/src/payload/mod.rs
 rust/libs/proto/src/payload/v1/mod.rs
 rust/libs/proto/src/payload/v1/payload.v1.rs
 rust/libs/proto/src/payload/v1/payload.v1.serde.rs
+rust/libs/proto/src/pdpb/pdpb.rs
+rust/libs/proto/src/pdpb/pdpb.serde.rs
+rust/libs/proto/src/pdpb/pdpb.tonic.rs
 rust/libs/proto/src/rpc/mod.rs
 rust/libs/proto/src/rpc/v1/mod.rs
 rust/libs/proto/src/rpc/v1/rpc.v1.rs
@@ -2360,6 +2406,7 @@ rust/libs/proto/src/sidecar/v1/sidecar.v1.tonic.rs
 rust/libs/proto/src/tikv/tikv.rs
 rust/libs/proto/src/tikv/tikv.serde.rs
 rust/libs/proto/src/tikv/tikv.tonic.rs
+rust/libs/proto/src/tikvpb/tikvpb.tonic.rs
 rust/libs/proto/src/vald/mod.rs
 rust/libs/proto/src/vald/v1/mod.rs
 rust/libs/proto/src/vald/v1/vald.v1.tonic.rs
@@ -2400,6 +2447,7 @@ tests/v2/e2e/assets/readreplica.yaml
 tests/v2/e2e/assets/rollout.yaml
 tests/v2/e2e/assets/stream_crud.yaml
 tests/v2/e2e/assets/unary_crud.yaml
+tests/v2/e2e/assets/unary_crud_meta.yaml
 tests/v2/e2e/config/config.go
 tests/v2/e2e/config/enums.go
 tests/v2/e2e/crud/agent_test.go

.github/actions/detect-docker-image-tags/action.yaml

@@ -23,7 +23,7 @@ inputs:
   images:
     description: "Image names"
     required: false
-    default: "vdaas/vald-agent-ngt vdaas/vald-agent-faiss vdaas/vald-agent-sidecar vdaas/vald-discoverer-k8s vdaas/vald-lb-gateway vdaas/vald-filter-gateway vdaas/vald-mirror-gateway vdaas/vald-manager-index vdaas/vald-index-operator vdaas/vald-readreplica-rotate"
+    default: "vdaas/vald-agent-ngt vdaas/vald-agent-faiss vdaas/vald-agent-sidecar vdaas/vald-discoverer-k8s vdaas/vald-lb-gateway vdaas/vald-meta-gateway vdaas/vald-filter-gateway vdaas/vald-mirror-gateway vdaas/vald-manager-index vdaas/vald-index-operator vdaas/vald-readreplica-rotate"
 outputs:
   HELM_EXTRA_OPTIONS:
     description: "Helm extra options that specifies image tags"
@@ -44,6 +44,7 @@ runs:
           ["vdaas/vald-agent-sidecar"]="agent.sidecar.image.tag"
           ["vdaas/vald-discoverer-k8s"]="discoverer.image.tag"
           ["vdaas/vald-lb-gateway"]="gateway.lb.image.tag"
+          ["vdaas/vald-meta-gateway"]="gateway.meta.image.tag"
           ["vdaas/vald-filter-gateway"]="gateway.filter.image.tag"
           ["vdaas/vald-mirror-gateway"]="gateway.mirror.image.tag"
           ["vdaas/vald-manager-index"]="manager.index.image.tag"

.github/actions/e2e-deploy-vald/action.yaml

@@ -99,11 +99,13 @@ runs:
         make k8s/vald/deploy HELM_VALUES=${VALUES} HELM_EXTRA_OPTIONS="${HELM_EXTRA_OPTIONS}"
         sleep 3
 
-        kubectl wait --for=condition=Ready pod -l ${WAIT_FOR_SELECTOR} --timeout=${WAIT_FOR_TIMEOUT}
+        kubectl wait --for=condition=Ready pod -l "${WAIT_FOR_SELECTOR}" --timeout=${WAIT_FOR_TIMEOUT}
 
         kubectl get pods
+        kubectl describe pods --selector="${WAIT_FOR_SELECTOR}"
+        kubectl describe cm
 
-        podname=`kubectl get pods --selector=${WAIT_FOR_SELECTOR} | tail -1 | awk '{print $1}'`
+        podname=`kubectl get pods --selector="${WAIT_FOR_SELECTOR}" | tail -1 | awk '{print $1}'`
         echo "POD_NAME=${podname}" >> $GITHUB_OUTPUT
       env:
         VALUES: ${{ inputs.values }}

.github/actions/setup-e2e/action.yaml

@@ -51,7 +51,7 @@ inputs:
   target_images:
     description: "Image names"
     required: false
-    default: "vdaas/vald-agent-ngt vdaas/vald-agent-faiss vdaas/vald-agent-sidecar vdaas/vald-discoverer-k8s vdaas/vald-lb-gateway vdaas/vald-filter-gateway vdaas/vald-mirror-gateway vdaas/vald-manager-index vdaas/vald-index-operator vdaas/vald-readreplica-rotate"
+    default: "vdaas/vald-agent-ngt vdaas/vald-agent-faiss vdaas/vald-agent-sidecar vdaas/vald-discoverer-k8s vdaas/vald-lb-gateway vdaas/vald-meta-gateway vdaas/vald-filter-gateway vdaas/vald-mirror-gateway vdaas/vald-manager-index vdaas/vald-index-operator vdaas/vald-readreplica-rotate"
 outputs:
   HELM_EXTRA_OPTIONS:
     description: "Helm extra options that specifies E2E target image tags"

.github/actions/wait-for-docker-image/action.yaml

@@ -19,7 +19,7 @@ inputs:
   images:
     description: "image names"
     required: false
-    default: "vdaas/vald-agent-ngt vdaas/vald-agent-faiss vdaas/vald-agent-sidecar vdaas/vald-discoverer-k8s vdaas/vald-lb-gateway vdaas/vald-filter-gateway vdaas/vald-mirror-gateway vdaas/vald-manager-index vdaas/vald-index-operator vdaas/vald-readreplica-rotate"
+    default: "vdaas/vald-agent-ngt vdaas/vald-agent-faiss vdaas/vald-agent-sidecar vdaas/vald-discoverer-k8s vdaas/vald-lb-gateway vdaas/vald-meta-gateway vdaas/vald-filter-gateway vdaas/vald-mirror-gateway vdaas/vald-manager-index vdaas/vald-index-operator vdaas/vald-readreplica-rotate"
 outputs: {}
 runs:
   using: "composite"