From fcdb2cb693ae104739e6fadd53ac88587e6633a2 Mon Sep 17 00:00:00 2001 From: Matt Kaniaris Date: Sat, 11 Jul 2026 22:46:39 -0600 Subject: [PATCH 1/3] experimental sync replay functionality --- go.mod | 2 + go.sum | 2 - pkg/connector/connector.go | 29 +- pkg/connector/group_type_scoped.go | 441 +++++++++ pkg/connector/request_log.go | 48 + pkg/connector/sourcecache_fuzz_test.go | 384 ++++++++ pkg/connector/sourcecache_sync_test.go | 880 ++++++++++++++++++ .../baton-sdk/internal/connector/connector.go | 58 +- .../v2/annotation_source_cache.pb.go | 529 +++++++++++ .../v2/annotation_source_cache.pb.validate.go | 352 +++++++ .../annotation_source_cache_protoopaque.pb.go | 475 ++++++++++ .../v2/annotation_type_scoped_grants.pb.go | 234 +++++ ...notation_type_scoped_grants.pb.validate.go | 237 +++++ ...ation_type_scoped_grants_protoopaque.pb.go | 231 +++++ .../connectorapi/baton/v1/source_cache.pb.go | 249 +++++ .../baton/v1/source_cache.pb.validate.go | 271 ++++++ .../baton/v1/source_cache_grpc.pb.go | 145 +++ .../baton/v1/source_cache_protoopaque.pb.go | 235 +++++ .../baton-sdk/pb/c1/storage/v3/records.pb.go | 325 +++++-- .../pb/c1/storage/v3/records.pb.validate.go | 143 +++ .../c1/storage/v3/records_protoopaque.pb.go | 340 +++++-- .../conductorone/baton-sdk/pkg/cli/cli.go | 8 +- .../baton-sdk/pkg/cli/commands.go | 129 ++- .../baton-sdk/pkg/config/config.go | 1 + .../pkg/connectorbuilder/connectorbuilder.go | 26 + .../pkg/connectorbuilder/resource_syncer.go | 73 +- .../pkg/connectorclient/connectorclient.go | 15 + .../baton-sdk/pkg/connectorrunner/runner.go | 25 +- .../pkg/connectorstore/connectorstore.go | 4 +- .../baton-sdk/pkg/dotc1z/c1file.go | 38 +- .../baton-sdk/pkg/dotc1z/c1file_store.go | 56 +- .../baton-sdk/pkg/dotc1z/c1zstore/c1zstore.go | 5 +- .../baton-sdk/pkg/dotc1z/cleanup_policy.go | 2 +- .../baton-sdk/pkg/dotc1z/clone_sync.go | 5 +- .../baton-sdk/pkg/dotc1z/convert_open.go | 3 +- .../pkg/dotc1z/engine/pebble/adapter.go | 20 +- .../engine/pebble/adapter_grants_store.go | 4 + .../pkg/dotc1z/engine/pebble/bulk_import.go | 1 + .../pkg/dotc1z/engine/pebble/cleanup.go | 4 + .../pkg/dotc1z/engine/pebble/engine_stub.go | 2 +- .../pkg/dotc1z/engine/pebble/entitlements.go | 28 + .../pkg/dotc1z/engine/pebble/grants.go | 41 +- .../engine/pebble/grants_synth_encode.go | 8 + .../engine/pebble/id_index_migration.go | 2 +- .../pkg/dotc1z/engine/pebble/if_newer.go | 14 + .../pkg/dotc1z/engine/pebble/keys.go | 154 ++- .../pkg/dotc1z/engine/pebble/lookup.go | 32 +- .../pkg/dotc1z/engine/pebble/raw_records.go | 132 ++- .../pkg/dotc1z/engine/pebble/resources.go | 23 +- .../pkg/dotc1z/engine/pebble/source_cache.go | 835 +++++++++++++++++ .../pkg/dotc1z/engine/pebble/sync_runs.go | 2 +- .../baton-sdk/pkg/dotc1z/engine_registry.go | 51 +- .../baton-sdk/pkg/dotc1z/file_ops.go | 25 + .../baton-sdk/pkg/dotc1z/format.go | 56 ++ .../pkg/dotc1z/format/v3/envelope.go | 96 +- .../baton-sdk/pkg/dotc1z/format/v3/indexed.go | 30 +- .../baton-sdk/pkg/dotc1z/grant_store.go | 19 + .../baton-sdk/pkg/dotc1z/pebble_store.go | 75 +- .../baton-sdk/pkg/dotc1z/source_cache.go | 213 +++++ .../baton-sdk/pkg/dotc1z/sql_helpers.go | 4 +- .../baton-sdk/pkg/dotc1z/store.go | 23 +- .../baton-sdk/pkg/dotc1z/sync_meta.go | 14 + .../baton-sdk/pkg/dotc1z/sync_runs.go | 24 +- .../baton-sdk/pkg/dotc1z/to_pebble.go | 3 +- .../baton-sdk/pkg/field/defaults.go | 5 + .../baton-sdk/pkg/lambda/grpc/client.go | 12 +- .../baton-sdk/pkg/lambda/grpc/server.go | 11 - .../baton-sdk/pkg/lambda/grpc/transport.go | 109 +-- .../baton-sdk/pkg/lambda/grpc/wire.go | 88 -- .../conductorone/baton-sdk/pkg/sdk/version.go | 2 +- .../baton-sdk/pkg/sourcecache/context.go | 20 + .../baton-sdk/pkg/sourcecache/grpc_lookup.go | 54 ++ .../baton-sdk/pkg/sourcecache/grpc_server.go | 71 ++ .../baton-sdk/pkg/sourcecache/sourcecache.go | 135 +++ .../baton-sdk/pkg/sync/expand/expander.go | 4 +- .../baton-sdk/pkg/sync/source_cache.go | 344 +++++++ .../conductorone/baton-sdk/pkg/sync/syncer.go | 215 ++++- .../pkg/synccompactor/attached/attached.go | 3 +- .../baton-sdk/pkg/synccompactor/compactor.go | 29 +- .../pkg/synccompactor/compactor_pebble.go | 5 +- .../pkg/synccompactor/pebble/bucket_plans.go | 28 + .../baton-sdk/pkg/tasks/c1api/full_sync.go | 8 +- .../baton-sdk/pkg/tasks/c1api/manager.go | 6 +- .../baton-sdk/pkg/tasks/local/compactor.go | 6 +- .../baton-sdk/pkg/tasks/local/syncer.go | 14 +- .../baton-sdk/pkg/types/resource/resource.go | 12 +- vendor/modules.txt | 5 +- 87 files changed, 8369 insertions(+), 752 deletions(-) create mode 100644 pkg/connector/group_type_scoped.go create mode 100644 pkg/connector/request_log.go create mode 100644 pkg/connector/sourcecache_fuzz_test.go create mode 100644 pkg/connector/sourcecache_sync_test.go create mode 100644 vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_source_cache.pb.go create mode 100644 vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_source_cache.pb.validate.go create mode 100644 vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_source_cache_protoopaque.pb.go create mode 100644 vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_type_scoped_grants.pb.go create mode 100644 vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_type_scoped_grants.pb.validate.go create mode 100644 vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_type_scoped_grants_protoopaque.pb.go create mode 100644 vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/source_cache.pb.go create mode 100644 vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/source_cache.pb.validate.go create mode 100644 vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/source_cache_grpc.pb.go create mode 100644 vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/source_cache_protoopaque.pb.go create mode 100644 vendor/github.com/conductorone/baton-sdk/pkg/connectorclient/connectorclient.go create mode 100644 vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/source_cache.go create mode 100644 vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/file_ops.go create mode 100644 vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/grant_store.go create mode 100644 vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/source_cache.go create mode 100644 vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sync_meta.go delete mode 100644 vendor/github.com/conductorone/baton-sdk/pkg/lambda/grpc/wire.go create mode 100644 vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/context.go create mode 100644 vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/grpc_lookup.go create mode 100644 vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/grpc_server.go create mode 100644 vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/sourcecache.go create mode 100644 vendor/github.com/conductorone/baton-sdk/pkg/sync/source_cache.go diff --git a/go.mod b/go.mod index e6942672..f3f99788 100644 --- a/go.mod +++ b/go.mod @@ -161,3 +161,5 @@ require ( modernc.org/memory v1.11.0 // indirect modernc.org/sqlite v1.50.0 // indirect ) + +replace github.com/conductorone/baton-sdk => ../baton-sdk-2 diff --git a/go.sum b/go.sum index 63d0a615..1c1cfd68 100644 --- a/go.sum +++ b/go.sum @@ -86,8 +86,6 @@ github.com/cockroachdb/swiss v0.0.0-20251224182025-b0f6560f979b h1:VXvSNzmr8hMj8 github.com/cockroachdb/swiss v0.0.0-20251224182025-b0f6560f979b/go.mod h1:yBRu/cnL4ks9bgy4vAASdjIW+/xMlFwuHKqtmh3GZQg= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= -github.com/conductorone/baton-sdk v0.18.2 h1:2pFlzwSpaFnIv0GbmTmLtIUOqWuNBtlsm28mujt7F5g= -github.com/conductorone/baton-sdk v0.18.2/go.mod h1:xacgmef9cM4dUTdvGN3Qip6fwkRbciqtaZMi5iWnjsY= github.com/conductorone/dpop v0.2.6 h1:fakwai/Xm2b/fcDUwJN41WtcSI/2UhQOyRIVvnnrrNA= github.com/conductorone/dpop v0.2.6/go.mod h1:gyo8TtzB9SCFCsjsICH4IaLZ7y64CcrDXMOPBwfq/3s= github.com/conductorone/dpop/integrations/dpop_grpc v0.2.4 h1:lYxYi9/WTSL9sE96CO0QF2BY3kehs8dTTApI134TGCA= diff --git a/pkg/connector/connector.go b/pkg/connector/connector.go index 9673e30e..a7f910a3 100644 --- a/pkg/connector/connector.go +++ b/pkg/connector/connector.go @@ -54,6 +54,12 @@ func capabilityPermissions(perms ...string) *v2.CapabilityPermissions { return cp } +func groupResourceTypeAnnotations() annotations.Annotations { + annos := v1AnnotationsForResourceType("group", false, capabilityPermissions("okta.groups.read", "okta.groups.manage")) + annos.Update(&v2.TypeScopedGrants{}) + return annos +} + func v1AnnotationsForResourceType(resourceTypeID string, skipEntitlementsAndGrants bool, perms *v2.CapabilityPermissions) annotations.Annotations { annos := annotations.Annotations{} annos.Update(&v2.V1Identifier{ @@ -88,11 +94,15 @@ var ( Traits: []v2.ResourceType_Trait{v2.ResourceType_TRAIT_USER}, Annotations: v1AnnotationsForResourceType(userResourceTypeID, true, capabilityPermissions("okta.users.read", "okta.users.manage")), } + // TypeScopedGrants excludes the group type from the SDK's per-resource + // grants fan-out: full syncs enumerate group grants through the + // planner/cursors in group_type_scoped.go (with source-cache replay for + // clean groups); the per-resource path serves targeted syncs. resourceTypeGroup = &v2.ResourceType{ Id: "group", DisplayName: "Group", Traits: []v2.ResourceType_Trait{v2.ResourceType_TRAIT_GROUP}, - Annotations: v1AnnotationsForResourceType("group", false, capabilityPermissions("okta.groups.read", "okta.groups.manage")), + Annotations: groupResourceTypeAnnotations(), } resourceTypeApp = &v2.ResourceType{ Id: "app", @@ -275,8 +285,14 @@ func (c *Okta) Metadata(ctx context.Context) (*v2.ConnectorMetadata, error) { } func (c *Okta) Validate(ctx context.Context) (annotations.Annotations, error) { + // Source-cache replay opt-in: group member grants are validated by + // lastMembershipUpdated (group_type_scoped.go). + annos := annotations.New(&v2.SourceCacheCapability{ + Mode: v2.SourceCacheCapability_MODE_READ_WRITE, + }) + if c.apiToken == "" { - return nil, nil + return annos, nil } token := newPaginationToken(defaultLimit, "") @@ -296,7 +312,7 @@ func (c *Okta) Validate(ctx context.Context) (annotations.Annotations, error) { return nil, err } - return nil, nil + return annos, nil } func (c *Okta) Asset(ctx context.Context, asset *v2.AssetRef) (string, io.ReadCloser, error) { @@ -320,6 +336,13 @@ func New(ctx context.Context, cc *cfg.Okta, opts *cli.ConnectorOpts) (connectorb return nil, nil, err } + // Demo instrumentation: count/log Okta requests when + // BATON_OKTA_REQUEST_LOG is set (source-cache measurement harness). + client, err = wrapRequestCounting(client) + if err != nil { + return nil, nil, err + } + cacheTTI, err := safeCacheInt32(cc.CacheTti) if err != nil { return nil, nil, err diff --git a/pkg/connector/group_type_scoped.go b/pkg/connector/group_type_scoped.go new file mode 100644 index 00000000..f13e1709 --- /dev/null +++ b/pkg/connector/group_type_scoped.go @@ -0,0 +1,441 @@ +package connector + +// Type-scoped group grants: source-cache replay validated by Okta's +// lastMembershipUpdated timestamp (dirty-scope model — see +// baton-microsoft-entra/docs/okta-replay-brief.md and the probe results in +// docs/replay-probe-results.md). +// +// Okta has no delta endpoints and no useful ETags, but every group row in +// the ordinary GET /api/v1/groups listing carries lastMembershipUpdated, +// which bumps on every membership change (direct add/remove, rule-driven +// evaluation, user deletion — all probed live). The connector compares +// that value ITSELF against the validator stored by the previous sync: +// +// - equal → the group's member grants are REPLAYED (no overlay, no +// tombstones), spending zero API requests on the group; +// - different → the group is dirty and its members are re-enumerated +// cold (there are no membership deltas to apply); +// - missing/empty → cold. Every surprise fails toward cold. +// +// Comparison is per-group EQUALITY against the group's own previous +// value, never "since T" against a clock: Okta's timestamps are Okta's, +// filter results lag writes, and equality is immune to both. +// +// Shape: the group resource type carries TypeScopedGrants, so the SDK +// issues one planning call instead of a per-resource fan-out. Planning +// pages the groups listing (which returns lastMembershipUpdated for free) +// and spawns one cursor per group via SpawnCursors. Each cursor runs two +// legs: +// +// 1. members — replay or cold enumeration as decided above; +// 2. group role assignments — ALWAYS a fresh enumeration. Role +// assign/revoke changes neither group timestamp (probed live), so +// the API offers nothing to validate with; per the brief the leg is +// not forced into the model. This bounds a fully-warm sync at one +// /groups/{id}/roles request per group. +// +// The per-resource Grants path in group.go remains for targeted syncs. + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "sort" + "strings" + "time" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/connectorbuilder" + "github.com/conductorone/baton-sdk/pkg/sourcecache" + sdkResource "github.com/conductorone/baton-sdk/pkg/types/resource" + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "github.com/okta/okta-sdk-golang/v2/okta" + "go.uber.org/zap" +) + +var _ connectorbuilder.TypeScopedGrantsSyncer = (*groupResourceType)(nil) + +const ( + groupCursorPhaseMembers = "m" + groupCursorPhaseRoles = "r" + + replayModeWarm = "warm" + replayModeCold = "cold" +) + +// groupGrantsCursor rides the SDK page token. Planning pages carry Plan + +// PlanPage (the groups listing's after-cursor); per-group cursors carry +// the group id, the validator read from the planning listing, the +// users_count stat (nil when the listing's expand=stats was absent), and +// the current leg's phase + after-cursor. +type groupGrantsCursor struct { + Plan bool `json:"p,omitempty"` + PlanPage string `json:"pp,omitempty"` + + GroupID string `json:"g,omitempty"` + Validator string `json:"v,omitempty"` + UsersCount *int64 `json:"uc,omitempty"` + + Phase string `json:"ph,omitempty"` + Page string `json:"pg,omitempty"` +} + +func (t *groupGrantsCursor) marshal() (string, error) { + b, err := json.Marshal(t) + if err != nil { + return "", fmt.Errorf("okta-connectorv2: failed to marshal group grants cursor: %w", err) + } + return string(b), nil +} + +// scopeSig hashes the request properties that shape a cached row set. +// Any change yields a different scope — a clean lookup miss and a full +// re-enumeration, exactly what an under- or over-filtered cache requires. +func scopeSig(parts ...string) string { + sum := sha256.Sum256([]byte(strings.Join(parts, "\x00"))) + return hex.EncodeToString(sum[:8]) +} + +// memberScope identifies one group's member-grant rows. The signature +// covers the client-side email-domain filter applied during enumeration +// (shouldIncludeUserAndSetCache): a config change must invalidate every +// group's scope. skipAppGroups is deliberately absent — it decides which +// groups get planned at all, not which rows a planned group's scope holds. +func (o *groupResourceType) memberScope(groupID string) string { + parts := []string{"v1"} + if o.connector.userFilters != nil && len(o.connector.userFilters.includedEmailDomains) > 0 { + domains := make([]string, len(o.connector.userFilters.includedEmailDomains)) + copy(domains, o.connector.userFilters.includedEmailDomains) + sort.Strings(domains) + parts = append(parts, domains...) + } + return fmt.Sprintf("groups/%s/users?sig=%s", groupID, scopeSig(parts...)) +} + +// groupMembershipValidator formats a group's lastMembershipUpdated as the +// scope validator. Empty (never observed live; every group gets the value +// at creation) means the scope cannot be validated and stays cold. +func groupMembershipValidator(group *okta.Group) string { + if group.LastMembershipUpdated == nil { + return "" + } + return group.LastMembershipUpdated.UTC().Format(time.RFC3339Nano) +} + +// GrantsForResourceType implements connectorbuilder.TypeScopedGrantsSyncer +// for the group type: the planning walk pages the groups listing and +// spawns one cursor per group; each cursor replays or re-enumerates that +// group's member grants and freshly enumerates its role assignments. +func (o *groupResourceType) GrantsForResourceType( + ctx context.Context, + resourceTypeID string, + attrs sdkResource.SyncOpAttrs, +) ([]*v2.Grant, *sdkResource.SyncOpResults, error) { + if resourceTypeID != resourceTypeGroup.Id { + return nil, nil, fmt.Errorf("okta-connectorv2: type-scoped grants for unexpected resource type %s", resourceTypeID) + } + + tok := &groupGrantsCursor{Plan: true} + if attrs.PageToken.Token != "" { + tok = &groupGrantsCursor{} + if err := json.Unmarshal([]byte(attrs.PageToken.Token), tok); err != nil { + return nil, nil, fmt.Errorf("okta-connectorv2: invalid group grants cursor: %w", err) + } + } + + if tok.Plan { + return o.planGroupCursorsPage(ctx, tok, attrs) + } + if tok.GroupID == "" { + return nil, nil, fmt.Errorf("okta-connectorv2: malformed group grants cursor: missing group id") + } + + switch tok.Phase { + case "": + return o.startGroupCursor(ctx, tok, attrs) + case groupCursorPhaseMembers: + return o.coldMembersPage(ctx, tok, attrs) + case groupCursorPhaseRoles: + return o.groupRolesPage(ctx, tok, attrs) + default: + return nil, nil, fmt.Errorf("okta-connectorv2: malformed group grants cursor: unknown phase %q", tok.Phase) + } +} + +// planGroupCursorsPage processes ONE page of the groups listing and spawns +// a cursor per group carrying (id, lastMembershipUpdated, users_count). +// Planning state never leaves the page token. The listing is the entire +// fixed cost of a warm sync's member legs: expand=stats rides along so the +// users_count==0 shortcut works exactly like the per-resource path. +func (o *groupResourceType) planGroupCursorsPage( + ctx context.Context, + tok *groupGrantsCursor, + attrs sdkResource.SyncOpAttrs, +) ([]*v2.Grant, *sdkResource.SyncOpResults, error) { + l := ctxzap.Extract(ctx) + + token := newPaginationToken(attrs.PageToken.Size, tok.PlanPage) + qp := queryParamsExpand(token.Size, tok.PlanPage, "stats") + groups, respCtx, err := listGroupsHelper(ctx, o.connector.client, token, qp) + if err != nil { + return nil, nil, fmt.Errorf("okta-connectorv2: group grants planning page failed: %w", err) + } + + nextPage, annos, err := parseResp(respCtx.OktaResponse) + if err != nil { + return nil, nil, fmt.Errorf("okta-connectorv2: failed to parse response: %w", err) + } + + tokens := make([]string, 0, len(groups)) + for _, group := range groups { + if o.connector.skipAppGroups && group.Type == appGroupType { + l.Debug("okta-connectorv2: skipping APP_GROUP type group", zap.String("group_id", group.Id)) + continue + } + cursor := &groupGrantsCursor{ + GroupID: group.Id, + Validator: groupMembershipValidator(group), + } + if usersCount, exists := getGroupUserCount(group); exists { + uc := int64(usersCount) + cursor.UsersCount = &uc + } + ct, err := cursor.marshal() + if err != nil { + return nil, nil, err + } + tokens = append(tokens, ct) + } + + nextToken := "" + if nextPage != "" { + nextToken, err = (&groupGrantsCursor{Plan: true, PlanPage: nextPage}).marshal() + if err != nil { + return nil, nil, err + } + } + + l.Debug("okta-connectorv2: planned group grant cursors (page)", + zap.Int("groups", len(groups)), + zap.Int("cursors_spawned", len(tokens)), + zap.Bool("final_page", nextPage == ""), + ) + + if len(tokens) > 0 { + annos.Update(&v2.SpawnCursors{PageTokens: tokens}) + } + return nil, &sdkResource.SyncOpResults{NextPageToken: nextToken, Annotations: annos}, nil +} + +// startGroupCursor runs one group cursor's first call: source-cache +// lookup, then either a replay of the member scope (zero API calls) or +// the first page of a cold member enumeration. The roles leg follows +// either way. +func (o *groupResourceType) startGroupCursor( + ctx context.Context, + tok *groupGrantsCursor, + attrs sdkResource.SyncOpAttrs, +) ([]*v2.Grant, *sdkResource.SyncOpResults, error) { + l := ctxzap.Extract(ctx) + scope := o.memberScope(tok.GroupID) + + lookup := attrs.SourceCache + if lookup == nil { + lookup = sourcecache.NoopLookup{} + } + entry, found, err := lookup.LookupPreviousSourceCache(ctx, sourcecache.RowKindGrants, scope) + if err != nil { + return nil, nil, fmt.Errorf("okta-connectorv2: group member scope lookup failed: %w", err) + } + + if found && tok.Validator != "" && entry.ETag == tok.Validator { + l.Debug("okta-connectorv2: group members scope", + zap.String("mode", replayModeWarm), + zap.String("group_id", tok.GroupID), + zap.String("scope", scope), + ) + var annos annotations.Annotations + annos.Update(&v2.SourceCacheReplay{ + ScopeHash: scope, + Etag: tok.Validator, + }) + next, err := (&groupGrantsCursor{ + GroupID: tok.GroupID, + UsersCount: tok.UsersCount, + Phase: groupCursorPhaseRoles, + }).marshal() + if err != nil { + return nil, nil, err + } + return nil, &sdkResource.SyncOpResults{NextPageToken: next, Annotations: annos}, nil + } + + l.Debug("okta-connectorv2: group members scope", + zap.String("mode", replayModeCold), + zap.String("group_id", tok.GroupID), + zap.String("scope", scope), + zap.Bool("validator_found", found), + ) + tok.Phase = groupCursorPhaseMembers + tok.Page = "" + return o.coldMembersPage(ctx, tok, attrs) +} + +// coldMembersPage serves one page of a group's full member enumeration, +// stamping rows with the member scope. The validator (read from the +// planning listing) is written on the final page; a membership change +// between planning and enumeration stores a validator OLDER than the rows +// it describes, which the next sync sees as dirty — fails toward cold. +func (o *groupResourceType) coldMembersPage( + ctx context.Context, + tok *groupGrantsCursor, + attrs sdkResource.SyncOpAttrs, +) ([]*v2.Grant, *sdkResource.SyncOpResults, error) { + l := ctxzap.Extract(ctx) + scope := o.memberScope(tok.GroupID) + + rolesToken, err := (&groupGrantsCursor{ + GroupID: tok.GroupID, + UsersCount: tok.UsersCount, + Phase: groupCursorPhaseRoles, + }).marshal() + if err != nil { + return nil, nil, err + } + + // users_count==0 shortcut (parity with the per-resource path): skip + // the members call. A zero-row page still persists the scope entry, + // so the group replays next sync if its validator holds. + if tok.Page == "" && tok.UsersCount != nil && *tok.UsersCount == 0 { + l.Debug("okta-connectorv2: skipping list group users (users_count is 0)", + zap.String("group_id", tok.GroupID)) + var annos annotations.Annotations + annos.Update(&v2.SourceCacheScope{ScopeHash: scope, Etag: tok.Validator}) + return nil, &sdkResource.SyncOpResults{NextPageToken: rolesToken, Annotations: annos}, nil + } + + token := newPaginationToken(attrs.PageToken.Size, tok.Page) + qp := queryParams(token.Size, tok.Page) + users, respCtx, err := o.listGroupUsers(ctx, tok.GroupID, token, qp) + if err != nil { + return nil, nil, convertNotFoundError(err, "okta-connectorv2: failed to list group users") + } + + nextPage, annos, err := parseResp(respCtx.OktaResponse) + if err != nil { + return nil, nil, fmt.Errorf("okta-connectorv2: failed to parse response: %w", err) + } + + groupStub := &v2.Resource{Id: fmtResourceId(resourceTypeGroup.Id, tok.GroupID)} + var rv []*v2.Grant + for _, user := range users { + if !o.connector.shouldIncludeUserAndSetCache(ctx, attrs.Session, user) { + continue + } + rv = append(rv, groupGrant(groupStub, user)) + } + + // The validator rides only the final page; interim pages stamp rows + // with an empty etag (the SDK writes the manifest entry when the + // non-empty etag arrives). + etag := tok.Validator + nextToken := rolesToken + if nextPage != "" { + etag = "" + nextToken, err = (&groupGrantsCursor{ + GroupID: tok.GroupID, + Validator: tok.Validator, + UsersCount: tok.UsersCount, + Phase: groupCursorPhaseMembers, + Page: nextPage, + }).marshal() + if err != nil { + return nil, nil, err + } + } + annos.Update(&v2.SourceCacheScope{ScopeHash: scope, Etag: etag}) + + l.Debug("okta-connectorv2: group members page", + zap.String("mode", replayModeCold), + zap.String("group_id", tok.GroupID), + zap.Int("rows", len(rv)), + zap.Bool("final_page", nextPage == ""), + ) + + return rv, &sdkResource.SyncOpResults{NextPageToken: nextToken, Annotations: annos}, nil +} + +// groupRolesPage enumerates the group's role assignments — always fresh, +// never scope-stamped: role assign/revoke does not bump either group +// timestamp (probed live), so there is nothing to validate replay with. +// Mirrors the role leg of the per-resource Grants path, including the +// access-denied skip. The endpoint is effectively unpaginated (the +// per-resource path also issues a single unparameterized call). +func (o *groupResourceType) groupRolesPage( + ctx context.Context, + tok *groupGrantsCursor, + attrs sdkResource.SyncOpAttrs, +) ([]*v2.Grant, *sdkResource.SyncOpResults, error) { + l := ctxzap.Extract(ctx) + + roles, resp, err := listGroupAssignedRoles(ctx, o.connector.client, tok.GroupID, nil) + if err != nil { + if resp == nil { + return nil, nil, fmt.Errorf("okta-connectorv2: failed to list group roles: %w", err) + } + defer resp.Body.Close() + errOkta, err2 := getError(resp) + if err2 != nil { + return nil, nil, err2 + } + if errOkta.ErrorCode == AccessDeniedErrorCode { + l.Debug("okta-connectorv2: skipping group role grants (access denied)", + zap.String("group_id", tok.GroupID)) + return nil, &sdkResource.SyncOpResults{}, nil + } + return nil, nil, convertNotFoundError(&errOkta, "okta-connectorv2: failed to list group roles") + } + + _, annos, err := parseResp(resp) + if err != nil { + return nil, nil, fmt.Errorf("okta-connectorv2: failed to parse response: %w", err) + } + + shouldExpand := tok.UsersCount == nil || *tok.UsersCount > 0 + var rv []*v2.Grant + for _, role := range roles { + if role.Status == roleStatusInactive || role.AssignmentType != "GROUP" { + continue + } + if !o.connector.SyncCustomRoles && role.Type == roleTypeCustom { + continue + } + + var roleResourceVal *v2.Resource + if role.Type == roleTypeCustom { + roleResourceVal, err = roleResource(ctx, &okta.Role{ + Id: role.Role, + Label: role.Label, + }, resourceTypeCustomRole) + } else { + roleResourceVal, err = roleResource(ctx, &okta.Role{ + Id: role.Role, + Label: role.Label, + Type: role.Type, + }, resourceTypeRole) + } + if err != nil { + return nil, nil, err + } + + if !shouldExpand { + l.Debug("okta-connectorv2: skipping expand for role group grant since users_count is 0") + } + rv = append(rv, roleGroupGrant(tok.GroupID, roleResourceVal, shouldExpand)) + } + + return rv, &sdkResource.SyncOpResults{Annotations: annos}, nil +} diff --git a/pkg/connector/request_log.go b/pkg/connector/request_log.go new file mode 100644 index 00000000..03ee326b --- /dev/null +++ b/pkg/connector/request_log.go @@ -0,0 +1,48 @@ +package connector + +import ( + "fmt" + "net/http" + "os" + "sync" +) + +// requestLogEnvVar names a file to append one "METHOD URL" line per Okta +// API request. Demo instrumentation for the source-cache measurement +// harness (warm-vs-cold request counting); unset (the normal case) means +// no wrapping at all. +const requestLogEnvVar = "BATON_OKTA_REQUEST_LOG" + +type countingTransport struct { + base http.RoundTripper + logMu sync.Mutex + logF *os.File +} + +func (t *countingTransport) RoundTrip(req *http.Request) (*http.Response, error) { + t.logMu.Lock() + _, _ = fmt.Fprintf(t.logF, "%s %s\n", req.Method, req.URL.String()) + t.logMu.Unlock() + + base := t.base + if base == nil { + base = http.DefaultTransport + } + return base.RoundTrip(req) +} + +// wrapRequestCounting installs the counting transport when +// BATON_OKTA_REQUEST_LOG is set. Returns the client unchanged otherwise. +func wrapRequestCounting(httpClient *http.Client) (*http.Client, error) { + logPath := os.Getenv(requestLogEnvVar) + if logPath == "" { + return httpClient, nil + } + f, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644) + if err != nil { + return nil, fmt.Errorf("okta-connectorv2: failed to open request log %s: %w", logPath, err) + } + wrapped := *httpClient + wrapped.Transport = &countingTransport{base: httpClient.Transport, logF: f} + return &wrapped, nil +} diff --git a/pkg/connector/sourcecache_fuzz_test.go b/pkg/connector/sourcecache_fuzz_test.go new file mode 100644 index 00000000..00d42d56 --- /dev/null +++ b/pkg/connector/sourcecache_fuzz_test.go @@ -0,0 +1,384 @@ +package connector + +// Randomized churn equivalence fuzzer for the source-cache warm path. +// +// The scripted scenarios in sourcecache_sync_test.go each pin one known +// hazard. This test covers the space BETWEEN them: every round applies a +// random batch of org mutations (user lifecycle, membership, group +// lifecycle, renames, role assignments, validator-only touches, and +// occasional mass invalidation), runs a warm sync chained off the previous +// round's output, runs a fresh uncached control sync of the same org +// state, and requires the two to be byte-identical at the reader surface. +// Any divergence — a stale replay, a missed dirty group, a leaked grant — +// fails with the round's seed and mutation log, which replays +// deterministically. +// +// Runs are deterministic by default (fixed seed) so CI is stable; set +// BATON_FUZZ_SEED to explore a different trajectory, and BATON_FUZZ_ROUNDS +// to run longer soaks. + +import ( + "fmt" + "math/rand" + "os" + "strconv" + "testing" + + "github.com/conductorone/baton-sdk/pkg/logging" + "github.com/stretchr/testify/require" +) + +// fuzzOrgView is a locked copy of the mock org's mutable state, used to +// pick valid mutation targets without racing the handler. +type fuzzOrgView struct { + activeUsers []string + allUsers []string + groups []string + members map[string][]string + roles map[string][]string // group id → role types +} + +func (m *mockOkta) fuzzView() fuzzOrgView { + m.mu.Lock() + defer m.mu.Unlock() + v := fuzzOrgView{ + members: map[string][]string{}, + roles: map[string][]string{}, + } + for _, id := range m.userOrder { + v.allUsers = append(v.allUsers, id) + if m.users[id].Status == "ACTIVE" { + v.activeUsers = append(v.activeUsers, id) + } + } + for _, gid := range m.groupOrder { + g := m.groups[gid] + v.groups = append(v.groups, gid) + v.members[gid] = append([]string{}, g.Members...) + for _, r := range g.Roles { + v.roles[gid] = append(v.roles[gid], r.Type) + } + } + return v +} + +type fuzzOp struct { + name string + // ready reports whether the op has a valid target in this state. + ready func(v fuzzOrgView) bool + apply func(f *fuzzRun, v fuzzOrgView) +} + +type fuzzRun struct { + t *testing.T + m *mockOkta + rng *rand.Rand + nextID int + log []string +} + +func (f *fuzzRun) id(prefix string) string { + f.nextID++ + return fmt.Sprintf("fz-%s-%03d", prefix, f.nextID) +} + +func (f *fuzzRun) pick(items []string) string { + return items[f.rng.Intn(len(items))] +} + +func (f *fuzzRun) note(format string, args ...any) { + f.log = append(f.log, fmt.Sprintf(format, args...)) +} + +// fuzzableRoleTypes are standard org roles the fuzzer assigns to groups +// (drawn from standardRoleTypes so the role resources exist). +var fuzzableRoleTypes = []string{"USER_ADMIN", "HELP_DESK_ADMIN", "APP_ADMIN", "REPORT_ADMIN"} + +func fuzzOps() []fuzzOp { + return []fuzzOp{ + { + name: "add-user", + ready: func(v fuzzOrgView) bool { return true }, + apply: func(f *fuzzRun, v fuzzOrgView) { + id := f.id("user") + f.m.addUser(&mockOktaUser{ + ID: id, FirstName: "Fuzz", LastName: id, Email: id + "@x.test", + }) + f.note("add-user %s", id) + }, + }, + { + name: "deactivate-user", + ready: func(v fuzzOrgView) bool { return len(v.activeUsers) > 1 }, + apply: func(f *fuzzRun, v fuzzOrgView) { + uid := f.pick(v.activeUsers) + f.m.deactivateUser(uid) + f.note("deactivate-user %s", uid) + }, + }, + { + name: "delete-user", + ready: func(v fuzzOrgView) bool { return len(v.allUsers) > 2 }, + apply: func(f *fuzzRun, v fuzzOrgView) { + uid := f.pick(v.allUsers) + f.m.deleteUser(uid) + f.note("delete-user %s", uid) + }, + }, + { + name: "create-group", + ready: func(v fuzzOrgView) bool { return true }, + apply: func(f *fuzzRun, v fuzzOrgView) { + gid := f.id("group") + g := &mockOktaGroup{ID: gid, Name: "Fuzz " + gid} + if len(v.activeUsers) > 0 && f.rng.Intn(2) == 0 { + g.Members = []string{f.pick(v.activeUsers)} + } + f.m.addGroup(g) + f.note("create-group %s (members=%v)", gid, g.Members) + }, + }, + { + name: "delete-group", + ready: func(v fuzzOrgView) bool { return len(v.groups) > 1 }, + apply: func(f *fuzzRun, v fuzzOrgView) { + gid := f.pick(v.groups) + f.m.deleteGroup(gid) + f.note("delete-group %s", gid) + }, + }, + { + name: "add-member", + ready: func(v fuzzOrgView) bool { + return len(v.groups) > 0 && len(v.activeUsers) > 0 + }, + apply: func(f *fuzzRun, v fuzzOrgView) { + gid := f.pick(v.groups) + current := map[string]bool{} + for _, mid := range v.members[gid] { + current[mid] = true + } + var cands []string + for _, uid := range v.activeUsers { + if !current[uid] { + cands = append(cands, uid) + } + } + if len(cands) == 0 { + return + } + uid := f.pick(cands) + f.m.addMember(gid, uid) + f.note("add-member %s -> %s", uid, gid) + }, + }, + { + name: "remove-member", + ready: func(v fuzzOrgView) bool { + for _, gid := range v.groups { + if len(v.members[gid]) > 0 { + return true + } + } + return false + }, + apply: func(f *fuzzRun, v fuzzOrgView) { + var withMembers []string + for _, gid := range v.groups { + if len(v.members[gid]) > 0 { + withMembers = append(withMembers, gid) + } + } + gid := f.pick(withMembers) + uid := f.pick(v.members[gid]) + f.m.removeMember(gid, uid) + f.note("remove-member %s <- %s", uid, gid) + }, + }, + { + name: "rename-group", + ready: func(v fuzzOrgView) bool { return len(v.groups) > 0 }, + apply: func(f *fuzzRun, v fuzzOrgView) { + gid := f.pick(v.groups) + f.m.renameGroup(gid, "Renamed "+f.id("nm")) + f.note("rename-group %s", gid) + }, + }, + { + name: "touch-group", + ready: func(v fuzzOrgView) bool { return len(v.groups) > 0 }, + apply: func(f *fuzzRun, v fuzzOrgView) { + gid := f.pick(v.groups) + f.m.touchGroup(gid) + f.note("touch-group %s", gid) + }, + }, + { + name: "assign-group-role", + ready: func(v fuzzOrgView) bool { + for _, gid := range v.groups { + if len(v.roles[gid]) < len(fuzzableRoleTypes) { + return true + } + } + return false + }, + apply: func(f *fuzzRun, v fuzzOrgView) { + var cands []string + for _, gid := range v.groups { + if len(v.roles[gid]) < len(fuzzableRoleTypes) { + cands = append(cands, gid) + } + } + gid := f.pick(cands) + held := map[string]bool{} + for _, rt := range v.roles[gid] { + held[rt] = true + } + var free []string + for _, rt := range fuzzableRoleTypes { + if !held[rt] { + free = append(free, rt) + } + } + rt := f.pick(free) + f.m.assignGroupRole(gid, mockOktaGroupRole{AssignmentID: f.id("gra"), Type: rt, Label: rt}) + f.note("assign-group-role %s -> %s", rt, gid) + }, + }, + { + name: "revoke-group-role", + ready: func(v fuzzOrgView) bool { + for _, gid := range v.groups { + if len(v.roles[gid]) > 0 { + return true + } + } + return false + }, + apply: func(f *fuzzRun, v fuzzOrgView) { + var withRoles []string + for _, gid := range v.groups { + if len(v.roles[gid]) > 0 { + withRoles = append(withRoles, gid) + } + } + gid := f.pick(withRoles) + rt := f.pick(v.roles[gid]) + f.m.revokeGroupRole(gid, rt) + f.note("revoke-group-role %s <- %s", rt, gid) + }, + }, + } +} + +func fuzzEnvInt(name string, def int) int { + if s := os.Getenv(name); s != "" { + if n, err := strconv.Atoi(s); err == nil { + return n + } + } + return def +} + +func TestSourceCacheChurnFuzz(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + + seed := int64(fuzzEnvInt("BATON_FUZZ_SEED", 20260711)) + rounds := fuzzEnvInt("BATON_FUZZ_ROUNDS", 8) + t.Logf("churn fuzz: seed=%d rounds=%d (override with BATON_FUZZ_SEED / BATON_FUZZ_ROUNDS)", seed, rounds) + + mock := newMockOkta(t) + + // Seed org: shape mirrors the scripted test so every leg has data from + // round zero. + for i := 1; i <= 4; i++ { + mock.addUser(&mockOktaUser{ + ID: fmt.Sprintf("u%d", i), FirstName: "User", LastName: fmt.Sprintf("N%d", i), + Email: fmt.Sprintf("u%d@x.test", i), + }) + } + mock.addGroup(&mockOktaGroup{ID: "g1", Name: "Group One", Members: []string{"u1", "u2"}}) + mock.addGroup(&mockOktaGroup{ID: "g2", Name: "Group Two", Members: []string{"u3"}}) + mock.addGroup(&mockOktaGroup{ID: "g3", Name: "Empty"}) + mock.assignGroupRole("g1", mockOktaGroupRole{AssignmentID: "gra-seed", Type: "USER_ADMIN", Label: "Group Administrator"}) + + h := newSyncHarness(ctx, t, mock) + f := &fuzzRun{t: t, m: mock, rng: rand.New(rand.NewSource(seed))} + ops := fuzzOps() + + prev := h.runSync("fuzz-cold", "") + + for round := 1; round <= rounds; round++ { + nOps := 1 + f.rng.Intn(3) + f.log = f.log[:0] + for i := 0; i < nOps; i++ { + // Re-view after each mutation so ops in the same round compose + // against current state, exactly as real churn would. + v := mock.fuzzView() + var ready []fuzzOp + for _, op := range ops { + if op.ready(v) { + ready = append(ready, op) + } + } + require.NotEmpty(t, ready) + ready[f.rng.Intn(len(ready))].apply(f, v) + } + + // ~1 round in 6 also invalidates every group's validator, so mass + // invalidation is fuzzed IN COMBINATION with churn, not only in + // isolation. + if f.rng.Intn(6) == 0 { + for _, gid := range mock.fuzzView().groups { + mock.touchGroup(gid) + } + f.note("mass-invalidation") + } + + warm := h.runSync(fmt.Sprintf("fuzz-warm-%02d", round), prev) + control := h.runSync(fmt.Sprintf("fuzz-control-%02d", round), "") + + wSnap := h.snapshot(warm) + cSnap := h.snapshot(control) + if !assertSnapshotsEqual(t, cSnap, wSnap) { + t.Fatalf("round %d diverged (seed=%d); mutations this round:\n %s", + round, seed, joinLines(f.log)) + } + prev = warm + } +} + +func assertSnapshotsEqual(t *testing.T, control, warm map[string]string) bool { + t.Helper() + ok := true + for k, cv := range control { + wv, found := warm[k] + if !found { + t.Errorf("warm sync MISSING %s", k) + ok = false + } else if wv != cv { + t.Errorf("warm sync DIFFERS at %s:\n control: %s\n warm: %s", k, cv, wv) + ok = false + } + } + for k := range warm { + if _, found := control[k]; !found { + t.Errorf("warm sync EXTRA %s: %s", k, warm[k]) + ok = false + } + } + return ok +} + +func joinLines(lines []string) string { + out := "" + for i, l := range lines { + if i > 0 { + out += "\n " + } + out += l + } + return out +} diff --git a/pkg/connector/sourcecache_sync_test.go b/pkg/connector/sourcecache_sync_test.go new file mode 100644 index 00000000..52854a9e --- /dev/null +++ b/pkg/connector/sourcecache_sync_test.go @@ -0,0 +1,880 @@ +package connector + +// End-to-end source-cache replay harness for the lastMembershipUpdated +// dirty-scope model (group_type_scoped.go). +// +// Runs the real connector against a strict mock Okta org (exact query +// verification, unknown requests fail the test) through the real SDK sync +// loop on the Pebble engine, chaining each sync's c1z as the next sync's +// replay source. Timestamps follow the semantics probed live +// (docs/replay-probe-results.md): member add/remove and user deletion bump +// lastMembershipUpdated; profile changes bump only lastUpdated; user +// deactivation bumps neither and the user stays in the member listing. +// +// The paramount assertion is equivalence: after every warm sync a control +// sync (no previous c1z) runs against the same org state and the two must +// be identical at the v2 reader surface. Request-count ceilings assert the +// point of the exercise: clean groups spend ZERO member-listing requests. + +import ( + "context" + "encoding/json" + "fmt" + "net" + "net/http" + "net/http/httptest" + "net/url" + "path/filepath" + "sort" + "strings" + "sync" + "testing" + "time" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/connectorbuilder" + "github.com/conductorone/baton-sdk/pkg/connectorclient" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/logging" + "github.com/conductorone/baton-sdk/pkg/sourcecache" + sdkSync "github.com/conductorone/baton-sdk/pkg/sync" + "github.com/conductorone/baton-sdk/pkg/types" + "github.com/okta/okta-sdk-golang/v2/okta" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" +) + +// --- mock Okta org ----------------------------------------------------------- + +// mockPageSize forces pagination everywhere (groups listing, member +// listings) so multi-page scopes and the planner's cross-page SpawnCursors +// path are always exercised. +const mockPageSize = 2 + +var mockTimeBase = time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + +type mockOktaUser struct { + ID string + FirstName string + LastName string + Email string + Status string // ACTIVE | DEPROVISIONED | ... +} + +type mockOktaGroupRole struct { + AssignmentID string + Type string // e.g. USER_ADMIN + Label string +} + +type mockOktaGroup struct { + ID string + Name string + Description string + Type string // OKTA_GROUP | APP_GROUP | BUILT_IN + Members []string + Roles []mockOktaGroupRole + + lastMembershipUpdated int64 // logical seconds since mockTimeBase + lastUpdated int64 +} + +type mockOkta struct { + mu sync.Mutex + t *testing.T + base string // server URL, for Link headers + + clock int64 + + users map[string]*mockOktaUser + userOrder []string + + groups map[string]*mockOktaGroup + groupOrder []string + + counts map[string]int +} + +func newMockOkta(t *testing.T) *mockOkta { + return &mockOkta{ + t: t, + users: map[string]*mockOktaUser{}, + groups: map[string]*mockOktaGroup{}, + counts: map[string]int{}, + } +} + +func (m *mockOkta) tick() int64 { + m.clock++ + return m.clock +} + +func mockTS(n int64) string { + return mockTimeBase.Add(time.Duration(n) * time.Second).Format("2006-01-02T15:04:05.000Z") +} + +func (m *mockOkta) addUser(u *mockOktaUser) { + m.mu.Lock() + defer m.mu.Unlock() + if u.Status == "" { + u.Status = "ACTIVE" + } + m.users[u.ID] = u + m.userOrder = append(m.userOrder, u.ID) +} + +// deactivateUser flips the user's status. Probed live: the DEPROVISIONED +// user REMAINS in group member listings and no group timestamp bumps. +func (m *mockOkta) deactivateUser(id string) { + m.mu.Lock() + defer m.mu.Unlock() + m.users[id].Status = "DEPROVISIONED" +} + +// deleteUser removes the user from the org and from every group's member +// list, bumping those groups' lastMembershipUpdated (probed live). +func (m *mockOkta) deleteUser(id string) { + m.mu.Lock() + defer m.mu.Unlock() + delete(m.users, id) + for i, uid := range m.userOrder { + if uid == id { + m.userOrder = append(m.userOrder[:i], m.userOrder[i+1:]...) + break + } + } + for _, g := range m.groups { + for i, mid := range g.Members { + if mid == id { + g.Members = append(g.Members[:i], g.Members[i+1:]...) + g.lastMembershipUpdated = m.tick() + break + } + } + } +} + +func (m *mockOkta) addGroup(g *mockOktaGroup) { + m.mu.Lock() + defer m.mu.Unlock() + if g.Type == "" { + g.Type = oktaGroupType + } + now := m.tick() + g.lastMembershipUpdated = now + g.lastUpdated = now + m.groups[g.ID] = g + m.groupOrder = append(m.groupOrder, g.ID) +} + +func (m *mockOkta) deleteGroup(id string) { + m.mu.Lock() + defer m.mu.Unlock() + delete(m.groups, id) + for i, gid := range m.groupOrder { + if gid == id { + m.groupOrder = append(m.groupOrder[:i], m.groupOrder[i+1:]...) + break + } + } +} + +func (m *mockOkta) addMember(groupID, userID string) { + m.mu.Lock() + defer m.mu.Unlock() + g := m.groups[groupID] + g.Members = append(g.Members, userID) + g.lastMembershipUpdated = m.tick() +} + +func (m *mockOkta) removeMember(groupID, userID string) { + m.mu.Lock() + defer m.mu.Unlock() + g := m.groups[groupID] + for i, mid := range g.Members { + if mid == userID { + g.Members = append(g.Members[:i], g.Members[i+1:]...) + g.lastMembershipUpdated = m.tick() + return + } + } + m.t.Fatalf("removeMember: %s not in %s", userID, groupID) +} + +// touchGroup bumps lastMembershipUpdated without changing the member set — +// the validator-regression case (e.g. an add+remove that nets to zero, or +// a rule re-evaluation). The connector must re-enumerate and produce +// identical rows. +func (m *mockOkta) touchGroup(groupID string) { + m.mu.Lock() + defer m.mu.Unlock() + m.groups[groupID].lastMembershipUpdated = m.tick() +} + +// renameGroup bumps only lastUpdated (probed live): the member scope's +// validator must NOT rotate. +func (m *mockOkta) renameGroup(groupID, name string) { + m.mu.Lock() + defer m.mu.Unlock() + g := m.groups[groupID] + g.Name = name + g.lastUpdated = m.tick() +} + +// assignGroupRole attaches an admin role to the group. Probed live: role +// assignment changes NEITHER group timestamp. +func (m *mockOkta) assignGroupRole(groupID string, role mockOktaGroupRole) { + m.mu.Lock() + defer m.mu.Unlock() + g := m.groups[groupID] + g.Roles = append(g.Roles, role) +} + +func (m *mockOkta) revokeGroupRole(groupID string, roleType string) { + m.mu.Lock() + defer m.mu.Unlock() + g := m.groups[groupID] + for i, r := range g.Roles { + if r.Type == roleType { + g.Roles = append(g.Roles[:i], g.Roles[i+1:]...) + return + } + } + m.t.Fatalf("revokeGroupRole: %s has no %s", groupID, roleType) +} + +// snapshotCounts returns a copy of the request counters and resets them. +func (m *mockOkta) snapshotCounts() map[string]int { + m.mu.Lock() + defer m.mu.Unlock() + out := map[string]int{} + for k, v := range m.counts { + out[k] = v + } + m.counts = map[string]int{} + return out +} + +func (m *mockOkta) userJSON(u *mockOktaUser) map[string]any { + return map[string]any{ + "id": u.ID, + "status": u.Status, + "created": mockTS(0), + "lastUpdated": mockTS(0), + "profile": map[string]any{ + "firstName": u.FirstName, + "lastName": u.LastName, + "email": u.Email, + "login": u.Email, + }, + } +} + +func (m *mockOkta) groupJSON(g *mockOktaGroup, withStats bool) map[string]any { + obj := map[string]any{ + "id": g.ID, + "type": g.Type, + "created": mockTS(0), + "lastUpdated": mockTS(g.lastUpdated), + "lastMembershipUpdated": mockTS(g.lastMembershipUpdated), + "profile": map[string]any{ + "name": g.Name, + "description": g.Description, + }, + } + if withStats { + obj["_embedded"] = map[string]any{ + "stats": map[string]any{ + "usersCount": float64(len(g.Members)), + "appsCount": float64(0), + "groupPushMappingsCount": float64(0), + }, + } + } + return obj +} + +// pageOf slices order after the given cursor. The cursor is the last id of +// the previous page (Okta's after-cursor is opaque; ids work fine). +func pageOf(order []string, after string, size int) (ids []string, next string) { + start := 0 + if after != "" { + for i, id := range order { + if id == after { + start = i + 1 + break + } + } + } + end := start + size + if end >= len(order) { + return order[start:], "" + } + return order[start:end], order[end-1] +} + +func mockWriteJSON(w http.ResponseWriter, obj any) { + w.Header().Set("Content-Type", "application/json") + data, err := json.Marshal(obj) + if err != nil { + panic(err) + } + _, _ = w.Write(data) +} + +// requireQueryKeys fails the test when the request carries query params +// outside the allowed set — the strict-mock discipline that catches scope +// signature drift and unexpected request shapes. +func (m *mockOkta) requireQueryKeys(r *http.Request, allowed ...string) { + q := r.URL.Query() + for k := range q { + ok := false + for _, a := range allowed { + if k == a { + ok = true + break + } + } + if !ok { + m.t.Errorf("mock okta: unexpected query param %q on %s (allowed: %v)", k, r.URL.String(), allowed) + } + } +} + +func (m *mockOkta) linkNext(w http.ResponseWriter, path string, next string) { + if next == "" { + return + } + w.Header().Set("Link", fmt.Sprintf("<%s%s?after=%s&limit=%d>; rel=\"next\"", m.base, path, url.QueryEscape(next), mockPageSize)) +} + +func (m *mockOkta) handler() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + m.mu.Lock() + defer m.mu.Unlock() + + if r.Method != http.MethodGet { + m.t.Errorf("mock okta: unexpected method %s %s", r.Method, r.URL.String()) + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + + path := r.URL.Path + q := r.URL.Query() + parts := strings.Split(strings.TrimPrefix(path, "/"), "/") + + switch { + case path == "/api/v1/org": + m.counts["org"]++ + mockWriteJSON(w, map[string]any{"id": "org1", "companyName": "Mock Org", "subdomain": "mock"}) + + case path == "/api/v1/users": + m.counts["users-list"]++ + m.requireQueryKeys(r, "limit", "after", "search") + if q.Get("search") != "status pr" { + m.t.Errorf("mock okta: users listing missing search=\"status pr\": %s", r.URL.String()) + } + ids, next := pageOf(m.userOrder, q.Get("after"), mockPageSize) + out := make([]map[string]any, 0, len(ids)) + for _, id := range ids { + out = append(out, m.userJSON(m.users[id])) + } + m.linkNext(w, "/api/v1/users", next) + mockWriteJSON(w, out) + + case path == "/api/v1/groups": + m.counts["groups-list"]++ + m.requireQueryKeys(r, "limit", "after", "expand") + if q.Get("expand") != "stats" { + m.t.Errorf("mock okta: groups listing missing expand=stats: %s", r.URL.String()) + } + ids, next := pageOf(m.groupOrder, q.Get("after"), mockPageSize) + out := make([]map[string]any, 0, len(ids)) + for _, id := range ids { + out = append(out, m.groupJSON(m.groups[id], true)) + } + m.linkNext(w, "/api/v1/groups", next) + mockWriteJSON(w, out) + + case len(parts) == 5 && parts[2] == "groups" && parts[4] == "users": + gid := parts[3] + m.counts["group-users:"+gid]++ + m.requireQueryKeys(r, "limit", "after") + g, ok := m.groups[gid] + if !ok { + w.WriteHeader(http.StatusNotFound) + mockWriteJSON(w, map[string]any{"errorCode": "E0000007", "errorSummary": "Not found: " + gid}) + return + } + ids, next := pageOf(g.Members, q.Get("after"), mockPageSize) + out := make([]map[string]any, 0, len(ids)) + for _, id := range ids { + out = append(out, m.userJSON(m.users[id])) + } + m.linkNext(w, path, next) + mockWriteJSON(w, out) + + case len(parts) == 5 && parts[2] == "groups" && parts[4] == "roles": + gid := parts[3] + m.counts["group-roles:"+gid]++ + m.requireQueryKeys(r) + g, ok := m.groups[gid] + if !ok { + w.WriteHeader(http.StatusNotFound) + mockWriteJSON(w, map[string]any{"errorCode": "E0000007", "errorSummary": "Not found: " + gid}) + return + } + out := make([]map[string]any, 0, len(g.Roles)) + for _, role := range g.Roles { + out = append(out, map[string]any{ + "id": role.AssignmentID, + "type": role.Type, + "label": role.Label, + "status": "ACTIVE", + "assignmentType": "GROUP", + }) + } + mockWriteJSON(w, out) + + case path == "/api/v1/iam/assignees/users": + m.counts["role-assignees"]++ + mockWriteJSON(w, map[string]any{"value": []any{}}) + + default: + m.t.Errorf("mock okta: unexpected request %s %s", r.Method, r.URL.String()) + w.WriteHeader(http.StatusNotFound) + mockWriteJSON(w, map[string]any{"errorCode": "E0000007", "errorSummary": "unhandled: " + path}) + } + } +} + +// --- harness ----------------------------------------------------------------- + +var harnessSyncResourceTypes = []string{"user", "group", "role"} + +type syncHarness struct { + t *testing.T + ctx context.Context + mock *mockOkta + cc types.ConnectorClient + tmpDir string + syncN int +} + +func newSyncHarness(ctx context.Context, t *testing.T, mock *mockOkta) *syncHarness { + t.Helper() + + server := httptest.NewServer(mock.handler()) + t.Cleanup(server.Close) + mock.base = server.URL + serverURL, err := url.Parse(server.URL) + require.NoError(t, err) + + _, oktaClient, err := okta.NewClient(ctx, + okta.WithOrgUrl(server.URL), + okta.WithToken("test-token"), + okta.WithTestingDisableHttpsCheck(true), + okta.WithHttpClientPtr(server.Client()), + okta.WithCache(false), + okta.WithRateLimitMaxRetries(0), + ) + require.NoError(t, err) + + c := &Okta{ + client: oktaClient, + domain: serverURL.Host, + apiToken: "test-token", + userFilters: &userFilterConfig{}, + } + + srv, err := connectorbuilder.NewConnector(ctx, c) + require.NoError(t, err) + + // Serve the connector over local gRPC and talk to it through the real + // connector client, mirroring how the CLI runs syncs. + gs := grpc.NewServer() + v2.RegisterConnectorServiceServer(gs, srv) + v2.RegisterGrantsServiceServer(gs, srv) + v2.RegisterEntitlementsServiceServer(gs, srv) + v2.RegisterResourcesServiceServer(gs, srv) + v2.RegisterResourceTypesServiceServer(gs, srv) + v2.RegisterAssetServiceServer(gs, srv) + v2.RegisterEventServiceServer(gs, srv) + v2.RegisterResourceGetterServiceServer(gs, srv) + v2.RegisterTicketsServiceServer(gs, srv) + v2.RegisterActionServiceServer(gs, srv) + v2.RegisterGrantManagerServiceServer(gs, srv) + v2.RegisterResourceManagerServiceServer(gs, srv) + v2.RegisterResourceDeleterServiceServer(gs, srv) + v2.RegisterAccountManagerServiceServer(gs, srv) + v2.RegisterCredentialManagerServiceServer(gs, srv) + + lis, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + go func() { _ = gs.Serve(lis) }() + t.Cleanup(gs.Stop) + + conn, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(t, err) + t.Cleanup(func() { _ = conn.Close() }) + + cc := connectorclient.NewConnectorClient(ctx, conn) + + // In-process lookup delivery: the syncer installs its per-sync lookup on + // the client, which forwards it to the builder (the CLI wrapper does + // this same wiring in internal/connector). + setter, ok := cc.(interface { + SetSourceCacheSetter(sourcecache.SetLookup) + }) + require.True(t, ok, "connector client must accept a source-cache setter") + lookupSink, ok := srv.(sourcecache.SetLookup) + require.True(t, ok, "connectorbuilder server must implement sourcecache.SetLookup") + setter.SetSourceCacheSetter(lookupSink) + + return &syncHarness{t: t, ctx: ctx, mock: mock, cc: cc, tmpDir: t.TempDir()} +} + +// runSync executes one full sync into a fresh Pebble c1z, optionally +// replaying from prevPath. Returns the new file's path. +func (h *syncHarness) runSync(name string, prevPath string) string { + h.t.Helper() + h.syncN++ + path := filepath.Join(h.tmpDir, fmt.Sprintf("%02d-%s.c1z", h.syncN, name)) + + store, err := dotc1z.NewStore(h.ctx, path, + dotc1z.WithEngine(dotc1z.EnginePebble), + dotc1z.WithTmpDir(h.tmpDir), + ) + require.NoError(h.t, err) + + opts := []sdkSync.SyncOpt{ + sdkSync.WithConnectorStore(store), + sdkSync.WithTmpDir(h.tmpDir), + sdkSync.WithSyncResourceTypes(harnessSyncResourceTypes), + } + if prevPath != "" { + opts = append(opts, sdkSync.WithPreviousSyncC1ZPath(prevPath)) + } + + syncer, err := sdkSync.NewSyncer(h.ctx, h.cc, opts...) + require.NoError(h.t, err) + require.NoError(h.t, syncer.Sync(h.ctx)) + require.NoError(h.t, syncer.Close(h.ctx)) + return path +} + +// runControlSync runs an uncached control sync and discards its request +// counts, so the next scenario's counters see only its own warm sync. +func (h *syncHarness) runControlSync(name string) string { + h.t.Helper() + path := h.runSync(name, "") + h.mock.snapshotCounts() + return path +} + +// snapshot reads a finished c1z at the v2 reader surface and returns +// id → canonical JSON for resources, entitlements, and grants. +func (h *syncHarness) snapshot(path string) map[string]string { + h.t.Helper() + store, err := dotc1z.NewStore(h.ctx, path, + dotc1z.WithEngine(dotc1z.EnginePebble), + dotc1z.WithReadOnly(true), + dotc1z.WithTmpDir(h.tmpDir), + ) + require.NoError(h.t, err) + defer func() { _ = store.Close(h.ctx) }() + + latest, err := store.SyncMeta().LatestFullSync(h.ctx) + require.NoError(h.t, err) + require.NotNil(h.t, latest) + require.NoError(h.t, store.SetCurrentSync(h.ctx, latest.ID)) + + out := map[string]string{} + put := func(prefix, id string, msg proto.Message) { + jb, err := protojson.Marshal(msg) + require.NoError(h.t, err) + // protojson output spacing is deliberately unstable; re-marshal + // through encoding/json for canonical (sorted-key) bytes. + var v any + require.NoError(h.t, json.Unmarshal(jb, &v)) + cb, err := json.Marshal(v) + require.NoError(h.t, err) + key := prefix + ":" + id + require.NotContains(h.t, out, key, "duplicate id at reader surface") + out[key] = string(cb) + } + + for _, rt := range harnessSyncResourceTypes { + pageToken := "" + for { + resp, err := store.ListResources(h.ctx, v2.ResourcesServiceListResourcesRequest_builder{ + ResourceTypeId: rt, + PageToken: pageToken, + }.Build()) + require.NoError(h.t, err) + for _, r := range resp.GetList() { + put("resource", r.GetId().GetResourceType()+"/"+r.GetId().GetResource(), r) + } + pageToken = resp.GetNextPageToken() + if pageToken == "" { + break + } + } + } + + pageToken := "" + for { + resp, err := store.ListEntitlements(h.ctx, v2.EntitlementsServiceListEntitlementsRequest_builder{ + PageToken: pageToken, + }.Build()) + require.NoError(h.t, err) + for _, e := range resp.GetList() { + put("entitlement", e.GetId(), e) + } + pageToken = resp.GetNextPageToken() + if pageToken == "" { + break + } + } + + pageToken = "" + for { + resp, err := store.ListGrants(h.ctx, v2.GrantsServiceListGrantsRequest_builder{ + PageToken: pageToken, + }.Build()) + require.NoError(h.t, err) + for _, g := range resp.GetList() { + put("grant", g.GetId(), g) + } + pageToken = resp.GetNextPageToken() + if pageToken == "" { + break + } + } + + return out +} + +// requireEquivalent is the release-blocker check: a warm (replayed) sync +// must be byte-identical to an uncached control sync at the reader surface. +func (h *syncHarness) requireEquivalent(warmPath, controlPath string, scenario string) { + h.t.Helper() + warm := h.snapshot(warmPath) + control := h.snapshot(controlPath) + require.Equal(h.t, control, warm, + "%s: warm sync diverged from uncached control sync — replay equivalence violated", scenario) +} + +// memberListingCalls sums group-users request counts, keyed per group. +func memberListingCalls(counts map[string]int) map[string]int { + out := map[string]int{} + for k, v := range counts { + if gid, ok := strings.CutPrefix(k, "group-users:"); ok { + out[gid] = v + } + } + return out +} + +func sortedKeys(m map[string]int) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} + +func memberGrantID(groupID, userID string) string { + return fmt.Sprintf("group:%s:member:user:%s", groupID, userID) +} + +func roleGroupGrantID(roleType, groupID string) string { + return fmt.Sprintf("role:%s:assigned:group:%s", roleType, groupID) +} + +// --- the scenarios ----------------------------------------------------------- + +func TestSourceCacheReplayEndToEnd(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + + mock := newMockOkta(t) + + // Org: five users; g1 (3 members, paginates at mockPageSize=2, carries + // an admin role for the always-fresh roles leg + grant expansion), + // g2 (1 member), g3 (empty — exercises the users_count==0 skip). + for i := 1; i <= 5; i++ { + mock.addUser(&mockOktaUser{ + ID: fmt.Sprintf("u%d", i), + FirstName: "User", + LastName: fmt.Sprintf("Number%d", i), + Email: fmt.Sprintf("u%d@x.test", i), + }) + } + mock.addGroup(&mockOktaGroup{ID: "g1", Name: "Engineering", Members: []string{"u1", "u2", "u3"}}) + mock.addGroup(&mockOktaGroup{ID: "g2", Name: "Sales", Members: []string{"u4"}}) + mock.addGroup(&mockOktaGroup{ID: "g3", Name: "Empty"}) + mock.assignGroupRole("g1", mockOktaGroupRole{AssignmentID: "gra1", Type: "USER_ADMIN", Label: "Group Administrator"}) + + h := newSyncHarness(ctx, t, mock) + + // --- Sync 1: cold --------------------------------------------------------- + sync1 := h.runSync("cold", "") + c1 := mock.snapshotCounts() + mc1 := memberListingCalls(c1) + require.Equal(t, 2, mc1["g1"], "3 members at page size 2 = 2 requests") + require.Equal(t, 1, mc1["g2"]) + require.Zero(t, mc1["g3"], "users_count==0 skips the member listing even cold") + require.Equal(t, 1, c1["group-roles:g1"], "roles leg runs once per group") + require.Equal(t, 1, c1["group-roles:g2"]) + require.Equal(t, 1, c1["group-roles:g3"]) + + snap1 := h.snapshot(sync1) + require.Contains(t, snap1, "resource:user/u1") + require.Contains(t, snap1, "resource:group/g1") + require.Contains(t, snap1, "resource:group/g3") + require.Contains(t, snap1, "grant:"+memberGrantID("g1", "u1")) + require.Contains(t, snap1, "grant:"+memberGrantID("g1", "u3")) + require.Contains(t, snap1, "grant:"+memberGrantID("g2", "u4")) + require.Contains(t, snap1, "grant:"+roleGroupGrantID("USER_ADMIN", "g1"), "group role grant from the fresh roles leg") + // Grant expansion: g1's members must hold derived USER_ADMIN grants. + foundDerived := false + for k := range snap1 { + if strings.HasPrefix(k, "grant:role:USER_ADMIN:assigned:user:u1") { + foundDerived = true + } + } + require.True(t, foundDerived, "expansion must derive u1's USER_ADMIN grant via g1") + + // --- Scenario 1: no-op round ---------------------------------------------- + sync2 := h.runSync("noop", sync1) + c2 := mock.snapshotCounts() + mc2 := memberListingCalls(c2) + require.Empty(t, sortedKeys(mc2), "no-op warm sync must spend ZERO member-listing requests, got %v", mc2) + require.Equal(t, 1, c2["group-roles:g1"], "roles leg stays fresh on warm rounds") + control2 := h.runControlSync("noop-control") + h.requireEquivalent(sync2, control2, "no-op round") + + // --- Scenario 2: member ADD ----------------------------------------------- + mock.addMember("g2", "u5") + sync3 := h.runSync("add", sync2) + mc3 := memberListingCalls(mock.snapshotCounts()) + require.Equal(t, []string{"g2"}, sortedKeys(mc3), "only the dirty group re-enumerates, got %v", mc3) + require.Contains(t, h.snapshot(sync3), "grant:"+memberGrantID("g2", "u5")) + control3 := h.runControlSync("add-control") + h.requireEquivalent(sync3, control3, "member add") + + // --- Scenario 3: member REMOVE (the model-critical direction) -------------- + mock.removeMember("g1", "u2") + sync4 := h.runSync("remove", sync3) + mc4 := memberListingCalls(mock.snapshotCounts()) + require.Equal(t, []string{"g1"}, sortedKeys(mc4), "only the dirty group re-enumerates, got %v", mc4) + snap4 := h.snapshot(sync4) + require.NotContains(t, snap4, "grant:"+memberGrantID("g1", "u2"), "revoked membership must disappear from the warm sync") + require.Contains(t, snap4, "grant:"+memberGrantID("g1", "u1")) + control4 := h.runControlSync("remove-control") + h.requireEquivalent(sync4, control4, "member remove") + + // --- Scenario 4: empty group gains its first member ------------------------ + mock.addMember("g3", "u2") + sync5 := h.runSync("empty-fill", sync4) + mc5 := memberListingCalls(mock.snapshotCounts()) + require.Equal(t, []string{"g3"}, sortedKeys(mc5), "formerly-empty dirty group must enumerate, got %v", mc5) + require.Contains(t, h.snapshot(sync5), "grant:"+memberGrantID("g3", "u2")) + control5 := h.runControlSync("empty-fill-control") + h.requireEquivalent(sync5, control5, "empty group fill") + + // --- Scenario 5: group create + delete ------------------------------------- + mock.addGroup(&mockOktaGroup{ID: "g4", Name: "Newcomers", Members: []string{"u5"}}) + sync6 := h.runSync("create", sync5) + mc6 := memberListingCalls(mock.snapshotCounts()) + require.Equal(t, []string{"g4"}, sortedKeys(mc6), "only the new group enumerates, got %v", mc6) + require.Contains(t, h.snapshot(sync6), "grant:"+memberGrantID("g4", "u5")) + control6 := h.runControlSync("create-control") + h.requireEquivalent(sync6, control6, "group create") + + mock.deleteGroup("g4") + sync7 := h.runSync("delete", sync6) + mc7 := memberListingCalls(mock.snapshotCounts()) + require.Empty(t, sortedKeys(mc7), "deleting a group must not dirty the others, got %v", mc7) + snap7 := h.snapshot(sync7) + require.NotContains(t, snap7, "resource:group/g4") + require.NotContains(t, snap7, "grant:"+memberGrantID("g4", "u5")) + control7 := h.runControlSync("delete-control") + h.requireEquivalent(sync7, control7, "group delete") + + // --- Scenario 6: validator regression (bump without member change) --------- + mock.touchGroup("g2") + sync8 := h.runSync("touch", sync7) + mc8 := memberListingCalls(mock.snapshotCounts()) + require.Equal(t, []string{"g2"}, sortedKeys(mc8), "touched group re-enumerates (fails toward cold), got %v", mc8) + control8 := h.runControlSync("touch-control") + h.requireEquivalent(sync8, control8, "validator regression") + + // --- Scenario 7: profile rename must NOT rotate the member validator ------- + mock.renameGroup("g1", "Engineering Platform") + sync9 := h.runSync("rename", sync8) + mc9 := memberListingCalls(mock.snapshotCounts()) + require.Empty(t, sortedKeys(mc9), "profile change must not dirty the member scope, got %v", mc9) + snap9 := h.snapshot(sync9) + require.Contains(t, snap9["resource:group/g1"], "Engineering Platform", "rename lands via the resources phase") + control9 := h.runControlSync("rename-control") + h.requireEquivalent(sync9, control9, "group rename") + + // --- Scenario 8: user deactivation (member stays; probed live) ------------- + mock.deactivateUser("u4") + sync10 := h.runSync("deactivate", sync9) + mc10 := memberListingCalls(mock.snapshotCounts()) + require.Empty(t, sortedKeys(mc10), "deactivation bumps no group timestamp; replay must hold, got %v", mc10) + snap10 := h.snapshot(sync10) + require.Contains(t, snap10, "grant:"+memberGrantID("g2", "u4"), "deprovisioned member remains in the listing (probed) — grant stays") + control10 := h.runControlSync("deactivate-control") + h.requireEquivalent(sync10, control10, "user deactivation") + + // --- Scenario 9: user DELETION (removes membership + bumps; probed live) --- + mock.deleteUser("u4") + sync11 := h.runSync("user-delete", sync10) + mc11 := memberListingCalls(mock.snapshotCounts()) + require.Equal(t, []string{"g2"}, sortedKeys(mc11), "user deletion dirties exactly the groups they belonged to, got %v", mc11) + snap11 := h.snapshot(sync11) + require.NotContains(t, snap11, "resource:user/u4") + require.NotContains(t, snap11, "grant:"+memberGrantID("g2", "u4")) + control11 := h.runControlSync("user-delete-control") + h.requireEquivalent(sync11, control11, "user deletion") + + // --- Scenario 10: role assignment changes ride the fresh leg --------------- + mock.assignGroupRole("g2", mockOktaGroupRole{AssignmentID: "gra2", Type: "HELP_DESK_ADMIN", Label: "Help Desk Administrator"}) + sync12 := h.runSync("role-assign", sync11) + mc12 := memberListingCalls(mock.snapshotCounts()) + require.Empty(t, sortedKeys(mc12), "role assignment must not dirty the member scope, got %v", mc12) + require.Contains(t, h.snapshot(sync12), "grant:"+roleGroupGrantID("HELP_DESK_ADMIN", "g2"), "new role grant arrives on a fully-warm round via the fresh leg") + control12 := h.runControlSync("role-assign-control") + h.requireEquivalent(sync12, control12, "group role assignment") + + mock.revokeGroupRole("g2", "HELP_DESK_ADMIN") + sync13 := h.runSync("role-revoke", sync12) + require.NotContains(t, h.snapshot(sync13), "grant:"+roleGroupGrantID("HELP_DESK_ADMIN", "g2")) + control13 := h.runControlSync("role-revoke-control") + h.requireEquivalent(sync13, control13, "group role revocation") + + // --- Scenario 11: mass invalidation + recovery ------------------------------ + for _, gid := range []string{"g1", "g2", "g3"} { + mock.touchGroup(gid) + } + sync14 := h.runSync("mass-invalidation", sync13) + mc14 := memberListingCalls(mock.snapshotCounts()) + // g3 gained u2 in scenario 4, so all three groups are non-empty here. + require.Equal(t, []string{"g1", "g2", "g3"}, sortedKeys(mc14), "every non-empty group re-enumerates, got %v", mc14) + control14 := h.runControlSync("mass-invalidation-control") + h.requireEquivalent(sync14, control14, "mass invalidation") + + sync15 := h.runSync("recovery", sync14) + mc15 := memberListingCalls(mock.snapshotCounts()) + require.Empty(t, sortedKeys(mc15), "the round after mass invalidation must be fully warm again, got %v", mc15) + control15 := h.runControlSync("recovery-control") + h.requireEquivalent(sync15, control15, "post-invalidation recovery") +} diff --git a/vendor/github.com/conductorone/baton-sdk/internal/connector/connector.go b/vendor/github.com/conductorone/baton-sdk/internal/connector/connector.go index 04170efd..066fcff7 100644 --- a/vendor/github.com/conductorone/baton-sdk/internal/connector/connector.go +++ b/vendor/github.com/conductorone/baton-sdk/internal/connector/connector.go @@ -23,11 +23,13 @@ import ( connectorV2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" connectorwrapperV1 "github.com/conductorone/baton-sdk/pb/c1/connector_wrapper/v1" + batonV1 "github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1" ratelimitV1 "github.com/conductorone/baton-sdk/pb/c1/ratelimit/v1" tlsV1 "github.com/conductorone/baton-sdk/pb/c1/utls/v1" "github.com/conductorone/baton-sdk/pkg/bid" ratelimit2 "github.com/conductorone/baton-sdk/pkg/ratelimit" "github.com/conductorone/baton-sdk/pkg/session" + "github.com/conductorone/baton-sdk/pkg/sourcecache" "github.com/conductorone/baton-sdk/pkg/types" "github.com/conductorone/baton-sdk/pkg/types/sessions" "github.com/conductorone/baton-sdk/pkg/ugrpc" @@ -55,19 +57,42 @@ type connectorClient struct { connectorV2.ActionServiceClient sessionStoreSetter sessions.SetSessionStore // this is the session store server + sourceCacheSetter sourcecache.SetLookup // this is the source-cache lookup server } var _ sessions.SetSessionStore = (*connectorClient)(nil) +var _ sourcecache.SetLookup = (*connectorClient)(nil) var _ SetSessionStoreSetter = (*connectorClient)(nil) +var _ SetSourceCacheSetter = (*connectorClient)(nil) type SetSessionStoreSetter interface { SetSessionStoreSetter(setsessionStoreSetter sessions.SetSessionStore) } +type SetSourceCacheSetter interface { + SetSourceCacheSetter(sourceCacheSetter sourcecache.SetLookup) +} + func (c *connectorClient) SetSessionStoreSetter(sessionStoreSetter sessions.SetSessionStore) { c.sessionStoreSetter = sessionStoreSetter } +func (c *connectorClient) SetSourceCacheSetter(sourceCacheSetter sourcecache.SetLookup) { + c.sourceCacheSetter = sourceCacheSetter +} + +// SetSourceCache forwards the syncer's per-sync lookup to whatever +// receives it: the subprocess-mode BatonSourceCacheService server, or the +// in-process builder. A nil setter means the connector never opted in; +// the syncer calls this unconditionally, so stay quiet at debug level. +func (c *connectorClient) SetSourceCache(ctx context.Context, lookup sourcecache.Lookup) { + if c.sourceCacheSetter == nil { + ctxzap.Extract(ctx).Debug("connectorClient's source cache setter is nil — connector did not opt into source caching") + return + } + c.sourceCacheSetter.SetSourceCache(ctx, lookup) +} + func (c *connectorClient) SetSessionStore(ctx context.Context, store sessions.SessionStore) { if c.sessionStoreSetter == nil { // Demoted from Warn to Debug: this path is the normal case for @@ -109,7 +134,8 @@ type wrapper struct { now func() time.Time - SessionServer sessions.SetSessionStore + SessionServer sessions.SetSessionStore + SourceCacheServer sourcecache.SetLookup } type Option func(ctx context.Context, w *wrapper) error @@ -197,6 +223,12 @@ func NewWrapper(ctx context.Context, server interface{}, opts ...Option) (*wrapp server: connectorServer, now: time.Now, } + // In-process delivery: a connectorbuilder-based server implements + // sourcecache.SetLookup itself, so the syncer's per-sync lookup can be + // installed without the subprocess gRPC hop. + if sourceCacheServer, ok := connectorServer.(sourcecache.SetLookup); ok { + w.SourceCacheServer = sourceCacheServer + } for _, o := range opts { err := o(ctx, w) @@ -276,18 +308,29 @@ func (cw *wrapper) runServer(ctx context.Context, serverCred *tlsV1.Credential) return 0, fmt.Errorf("failed to create session listener config: %w", err) } - // TODO(kans): block until we send a request or something/error handling in general. + // One listener serves BatonSessionService (connector session data) + // and BatonSourceCacheService (source-cache scope lookups). Keeping + // them as separate RPCs keeps validator lookups out of the + // connector's local MemorySessionCache and its TTL/eviction rules. l.Info("starting session store server") - server := session.NewGRPCSessionServer() - cw.SessionServer = server + sessionServer := session.NewGRPCSessionServer() + sourceCacheServer := sourcecache.NewGRPCServer() + cw.SessionServer = sessionServer + cw.SourceCacheServer = sourceCacheServer go func() { defer sessionListener.Close() - serverErr := session.StartGRPCSessionServerWithOptions(ctx, sessionListener, server, + grpcServer := grpc.NewServer( grpc.Creds(credentials.NewTLS(tlsConfig)), grpc.ChainUnaryInterceptor(ugrpc.UnaryServerInterceptor(ctx)...), ) - if serverErr != nil { - l.Error("failed to create session store server", zap.Error(serverErr)) + batonV1.RegisterBatonSessionServiceServer(grpcServer, sessionServer) + batonV1.RegisterBatonSourceCacheServiceServer(grpcServer, sourceCacheServer) + go func() { + <-ctx.Done() + grpcServer.GracefulStop() + }() + if serveErr := grpcServer.Serve(sessionListener); serveErr != nil { + l.Error("session/source-cache server stopped", zap.Error(serveErr)) return } }() @@ -440,6 +483,7 @@ func (cw *wrapper) C(ctx context.Context) (types.ConnectorClient, error) { cw.conn = conn client := NewConnectorClient(ctx, cw.conn) client.SetSessionStoreSetter(cw.SessionServer) + client.SetSourceCacheSetter(cw.SourceCacheServer) cw.client = client return client, nil diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_source_cache.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_source_cache.pb.go new file mode 100644 index 00000000..8796df98 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_source_cache.pb.go @@ -0,0 +1,529 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.10 +// protoc (unknown) +// source: c1/connector/v2/annotation_source_cache.proto + +//go:build !protoopaque + +package v2 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type SourceCacheCapability_Mode int32 + +const ( + SourceCacheCapability_MODE_UNSPECIFIED SourceCacheCapability_Mode = 0 + SourceCacheCapability_MODE_DISABLED SourceCacheCapability_Mode = 1 + SourceCacheCapability_MODE_READ_WRITE SourceCacheCapability_Mode = 2 +) + +// Enum value maps for SourceCacheCapability_Mode. +var ( + SourceCacheCapability_Mode_name = map[int32]string{ + 0: "MODE_UNSPECIFIED", + 1: "MODE_DISABLED", + 2: "MODE_READ_WRITE", + } + SourceCacheCapability_Mode_value = map[string]int32{ + "MODE_UNSPECIFIED": 0, + "MODE_DISABLED": 1, + "MODE_READ_WRITE": 2, + } +) + +func (x SourceCacheCapability_Mode) Enum() *SourceCacheCapability_Mode { + p := new(SourceCacheCapability_Mode) + *p = x + return p +} + +func (x SourceCacheCapability_Mode) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (SourceCacheCapability_Mode) Descriptor() protoreflect.EnumDescriptor { + return file_c1_connector_v2_annotation_source_cache_proto_enumTypes[0].Descriptor() +} + +func (SourceCacheCapability_Mode) Type() protoreflect.EnumType { + return &file_c1_connector_v2_annotation_source_cache_proto_enumTypes[0] +} + +func (x SourceCacheCapability_Mode) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// SourceCacheCapability is attached to ConnectorServiceValidateResponse +// annotations to opt in to source-cache replay. Absent or any mode other +// than MODE_READ_WRITE means all source-cache annotations are ignored. +type SourceCacheCapability struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + Mode SourceCacheCapability_Mode `protobuf:"varint,1,opt,name=mode,proto3,enum=c1.connector.v2.SourceCacheCapability_Mode" json:"mode,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheCapability) Reset() { + *x = SourceCacheCapability{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheCapability) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheCapability) ProtoMessage() {} + +func (x *SourceCacheCapability) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheCapability) GetMode() SourceCacheCapability_Mode { + if x != nil { + return x.Mode + } + return SourceCacheCapability_MODE_UNSPECIFIED +} + +func (x *SourceCacheCapability) SetMode(v SourceCacheCapability_Mode) { + x.Mode = v +} + +type SourceCacheCapability_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + Mode SourceCacheCapability_Mode +} + +func (b0 SourceCacheCapability_builder) Build() *SourceCacheCapability { + m0 := &SourceCacheCapability{} + b, x := &b0, m0 + _, _ = b, x + x.Mode = b.Mode + return m0 +} + +// SourceCacheScope is attached to a list-response page whose rows were +// freshly fetched from upstream. The SDK stamps the page's rows with +// scope_hash so a future sync can replay them as a unit. +type SourceCacheScope struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + // Connector-computed stable identifier for the canonical scope. Must be + // byte-stable across syncs for the same logical scope. Prefer an ORDERED + // natural identifier (e.g. the request URL, or "groups/{id}/members") + // over a random hash: scope-index writes lead with this value, so + // identifiers that correlate with fetch order keep index writes nearly + // append-ordered at scale. sourcecache.HashScope is the fallback when no + // compact natural form exists. + ScopeHash string `protobuf:"bytes,1,opt,name=scope_hash,json=scopeHash,proto3" json:"scope_hash,omitempty"` + // Opaque validator to persist for this scope. May be empty on interim + // pages of a multi-page scope (e.g. Graph @odata.nextLink pages); the + // SDK writes the scope's manifest entry when a non-empty etag arrives. + // A 200 response with zero rows still persists the entry. + Etag string `protobuf:"bytes,2,opt,name=etag,proto3" json:"etag,omitempty"` + // Tombstones applied after this page's rows commit. Lets every page of + // a multi-page delta round carry its own deletions as the provider + // delivers them, instead of buffering a whole round onto the first + // (replay-annotated) page. Same formats as SourceCacheReplay. + // + // PRECONDITION for tombstones anywhere in a round: the provider's delta + // must be coalesced — at most one add-or-tombstone per object per round + // (Microsoft Graph guarantees this by returning final object state). + // With interleaved add/remove events for one object, per-page ordering + // is deterministic (a page's rows upsert before its deletions apply) + // but cross-page re-adds after a tombstone are the connector's + // responsibility to order. + DeletedIds []string `protobuf:"bytes,3,rep,name=deleted_ids,json=deletedIds,proto3" json:"deleted_ids,omitempty"` + // Principal-scoped grant tombstones: for RowKindGrants pages, each + // entry deletes EVERY grant row stamped with this scope whose principal + // id equals the entry — no principal resource type and no canonical + // grant-id reconstruction required (delta tombstones usually carry only + // a bare object id, and the object may no longer exist to look up). + // + // PRECONDITION: the scope must be partitioned so that "principal + // removed from scope" means every grant they have in the scope is gone + // — one scope per navigation with independent removal semantics (e.g. + // members and owners of a group are separate scopes, or membership + // removal would take the owner grant with it). + // + // For RowKindResources pages, each entry deletes the resource row(s) + // stamped with this scope whose resource id equals the entry, any + // resource type. + DeletedPrincipalIds []string `protobuf:"bytes,4,rep,name=deleted_principal_ids,json=deletedPrincipalIds,proto3" json:"deleted_principal_ids,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheScope) Reset() { + *x = SourceCacheScope{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheScope) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheScope) ProtoMessage() {} + +func (x *SourceCacheScope) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheScope) GetScopeHash() string { + if x != nil { + return x.ScopeHash + } + return "" +} + +func (x *SourceCacheScope) GetEtag() string { + if x != nil { + return x.Etag + } + return "" +} + +func (x *SourceCacheScope) GetDeletedIds() []string { + if x != nil { + return x.DeletedIds + } + return nil +} + +func (x *SourceCacheScope) GetDeletedPrincipalIds() []string { + if x != nil { + return x.DeletedPrincipalIds + } + return nil +} + +func (x *SourceCacheScope) SetScopeHash(v string) { + x.ScopeHash = v +} + +func (x *SourceCacheScope) SetEtag(v string) { + x.Etag = v +} + +func (x *SourceCacheScope) SetDeletedIds(v []string) { + x.DeletedIds = v +} + +func (x *SourceCacheScope) SetDeletedPrincipalIds(v []string) { + x.DeletedPrincipalIds = v +} + +type SourceCacheScope_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Connector-computed stable identifier for the canonical scope. Must be + // byte-stable across syncs for the same logical scope. Prefer an ORDERED + // natural identifier (e.g. the request URL, or "groups/{id}/members") + // over a random hash: scope-index writes lead with this value, so + // identifiers that correlate with fetch order keep index writes nearly + // append-ordered at scale. sourcecache.HashScope is the fallback when no + // compact natural form exists. + ScopeHash string + // Opaque validator to persist for this scope. May be empty on interim + // pages of a multi-page scope (e.g. Graph @odata.nextLink pages); the + // SDK writes the scope's manifest entry when a non-empty etag arrives. + // A 200 response with zero rows still persists the entry. + Etag string + // Tombstones applied after this page's rows commit. Lets every page of + // a multi-page delta round carry its own deletions as the provider + // delivers them, instead of buffering a whole round onto the first + // (replay-annotated) page. Same formats as SourceCacheReplay. + // + // PRECONDITION for tombstones anywhere in a round: the provider's delta + // must be coalesced — at most one add-or-tombstone per object per round + // (Microsoft Graph guarantees this by returning final object state). + // With interleaved add/remove events for one object, per-page ordering + // is deterministic (a page's rows upsert before its deletions apply) + // but cross-page re-adds after a tombstone are the connector's + // responsibility to order. + DeletedIds []string + // Principal-scoped grant tombstones: for RowKindGrants pages, each + // entry deletes EVERY grant row stamped with this scope whose principal + // id equals the entry — no principal resource type and no canonical + // grant-id reconstruction required (delta tombstones usually carry only + // a bare object id, and the object may no longer exist to look up). + // + // PRECONDITION: the scope must be partitioned so that "principal + // removed from scope" means every grant they have in the scope is gone + // — one scope per navigation with independent removal semantics (e.g. + // members and owners of a group are separate scopes, or membership + // removal would take the owner grant with it). + // + // For RowKindResources pages, each entry deletes the resource row(s) + // stamped with this scope whose resource id equals the entry, any + // resource type. + DeletedPrincipalIds []string +} + +func (b0 SourceCacheScope_builder) Build() *SourceCacheScope { + m0 := &SourceCacheScope{} + b, x := &b0, m0 + _, _ = b, x + x.ScopeHash = b.ScopeHash + x.Etag = b.Etag + x.DeletedIds = b.DeletedIds + x.DeletedPrincipalIds = b.DeletedPrincipalIds + return m0 +} + +// SourceCacheReplay is attached to a list-response page to tell the SDK to +// copy the previous sync's rows for scope_hash into the current sync. +// +// The row kind (resources, entitlements, grants) is determined by which +// RPC the annotation arrived on, never by the annotation itself. +// +// The connector must only emit this for a scope whose etag it received +// from the SDK's source-cache lookup during this same sync. A replay for +// an unknown scope fails the sync: the connector has already skipped row +// generation, so there is nothing to fall back to. +type SourceCacheReplay struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + ScopeHash string `protobuf:"bytes,1,opt,name=scope_hash,json=scopeHash,proto3" json:"scope_hash,omitempty"` + // Validator to persist for this scope in the current sync. For an HTTP + // 304 this is the unchanged etag. For a delta query this is the NEW + // token; it may instead be supplied by the final overlay page's + // SourceCacheScope.etag, in which case this field may be left empty. + Etag string `protobuf:"bytes,2,opt,name=etag,proto3" json:"etag,omitempty"` + // When true, the response (and subsequent pages carrying + // SourceCacheScope with the same scope_hash) contains changed rows to + // upsert on top of the replayed base. When false the response must + // contain no rows for this scope. + Overlay bool `protobuf:"varint,3,opt,name=overlay,proto3" json:"overlay,omitempty"` + // Public canonical IDs (grant/entitlement IDs, or resource BIDs for + // RowKindResources) to delete from the current sync after the replay + // copy and this page's upserts. Used for delta-query tombstones (e.g. + // Microsoft Graph @removed entries). Subsequent pages of the round + // carry their tombstones on SourceCacheScope.deleted_ids. + DeletedIds []string `protobuf:"bytes,4,rep,name=deleted_ids,json=deletedIds,proto3" json:"deleted_ids,omitempty"` + // Principal-scoped tombstones for this page; see + // SourceCacheScope.deleted_principal_ids for semantics and + // preconditions. + DeletedPrincipalIds []string `protobuf:"bytes,5,rep,name=deleted_principal_ids,json=deletedPrincipalIds,proto3" json:"deleted_principal_ids,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheReplay) Reset() { + *x = SourceCacheReplay{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheReplay) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheReplay) ProtoMessage() {} + +func (x *SourceCacheReplay) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheReplay) GetScopeHash() string { + if x != nil { + return x.ScopeHash + } + return "" +} + +func (x *SourceCacheReplay) GetEtag() string { + if x != nil { + return x.Etag + } + return "" +} + +func (x *SourceCacheReplay) GetOverlay() bool { + if x != nil { + return x.Overlay + } + return false +} + +func (x *SourceCacheReplay) GetDeletedIds() []string { + if x != nil { + return x.DeletedIds + } + return nil +} + +func (x *SourceCacheReplay) GetDeletedPrincipalIds() []string { + if x != nil { + return x.DeletedPrincipalIds + } + return nil +} + +func (x *SourceCacheReplay) SetScopeHash(v string) { + x.ScopeHash = v +} + +func (x *SourceCacheReplay) SetEtag(v string) { + x.Etag = v +} + +func (x *SourceCacheReplay) SetOverlay(v bool) { + x.Overlay = v +} + +func (x *SourceCacheReplay) SetDeletedIds(v []string) { + x.DeletedIds = v +} + +func (x *SourceCacheReplay) SetDeletedPrincipalIds(v []string) { + x.DeletedPrincipalIds = v +} + +type SourceCacheReplay_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + ScopeHash string + // Validator to persist for this scope in the current sync. For an HTTP + // 304 this is the unchanged etag. For a delta query this is the NEW + // token; it may instead be supplied by the final overlay page's + // SourceCacheScope.etag, in which case this field may be left empty. + Etag string + // When true, the response (and subsequent pages carrying + // SourceCacheScope with the same scope_hash) contains changed rows to + // upsert on top of the replayed base. When false the response must + // contain no rows for this scope. + Overlay bool + // Public canonical IDs (grant/entitlement IDs, or resource BIDs for + // RowKindResources) to delete from the current sync after the replay + // copy and this page's upserts. Used for delta-query tombstones (e.g. + // Microsoft Graph @removed entries). Subsequent pages of the round + // carry their tombstones on SourceCacheScope.deleted_ids. + DeletedIds []string + // Principal-scoped tombstones for this page; see + // SourceCacheScope.deleted_principal_ids for semantics and + // preconditions. + DeletedPrincipalIds []string +} + +func (b0 SourceCacheReplay_builder) Build() *SourceCacheReplay { + m0 := &SourceCacheReplay{} + b, x := &b0, m0 + _, _ = b, x + x.ScopeHash = b.ScopeHash + x.Etag = b.Etag + x.Overlay = b.Overlay + x.DeletedIds = b.DeletedIds + x.DeletedPrincipalIds = b.DeletedPrincipalIds + return m0 +} + +var File_c1_connector_v2_annotation_source_cache_proto protoreflect.FileDescriptor + +const file_c1_connector_v2_annotation_source_cache_proto_rawDesc = "" + + "\n" + + "-c1/connector/v2/annotation_source_cache.proto\x12\x0fc1.connector.v2\"\x9e\x01\n" + + "\x15SourceCacheCapability\x12?\n" + + "\x04mode\x18\x01 \x01(\x0e2+.c1.connector.v2.SourceCacheCapability.ModeR\x04mode\"D\n" + + "\x04Mode\x12\x14\n" + + "\x10MODE_UNSPECIFIED\x10\x00\x12\x11\n" + + "\rMODE_DISABLED\x10\x01\x12\x13\n" + + "\x0fMODE_READ_WRITE\x10\x02\"\x9a\x01\n" + + "\x10SourceCacheScope\x12\x1d\n" + + "\n" + + "scope_hash\x18\x01 \x01(\tR\tscopeHash\x12\x12\n" + + "\x04etag\x18\x02 \x01(\tR\x04etag\x12\x1f\n" + + "\vdeleted_ids\x18\x03 \x03(\tR\n" + + "deletedIds\x122\n" + + "\x15deleted_principal_ids\x18\x04 \x03(\tR\x13deletedPrincipalIds\"\xb5\x01\n" + + "\x11SourceCacheReplay\x12\x1d\n" + + "\n" + + "scope_hash\x18\x01 \x01(\tR\tscopeHash\x12\x12\n" + + "\x04etag\x18\x02 \x01(\tR\x04etag\x12\x18\n" + + "\aoverlay\x18\x03 \x01(\bR\aoverlay\x12\x1f\n" + + "\vdeleted_ids\x18\x04 \x03(\tR\n" + + "deletedIds\x122\n" + + "\x15deleted_principal_ids\x18\x05 \x03(\tR\x13deletedPrincipalIdsB6Z4github.com/conductorone/baton-sdk/pb/c1/connector/v2b\x06proto3" + +var file_c1_connector_v2_annotation_source_cache_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_c1_connector_v2_annotation_source_cache_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_c1_connector_v2_annotation_source_cache_proto_goTypes = []any{ + (SourceCacheCapability_Mode)(0), // 0: c1.connector.v2.SourceCacheCapability.Mode + (*SourceCacheCapability)(nil), // 1: c1.connector.v2.SourceCacheCapability + (*SourceCacheScope)(nil), // 2: c1.connector.v2.SourceCacheScope + (*SourceCacheReplay)(nil), // 3: c1.connector.v2.SourceCacheReplay +} +var file_c1_connector_v2_annotation_source_cache_proto_depIdxs = []int32{ + 0, // 0: c1.connector.v2.SourceCacheCapability.mode:type_name -> c1.connector.v2.SourceCacheCapability.Mode + 1, // [1:1] is the sub-list for method output_type + 1, // [1:1] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name +} + +func init() { file_c1_connector_v2_annotation_source_cache_proto_init() } +func file_c1_connector_v2_annotation_source_cache_proto_init() { + if File_c1_connector_v2_annotation_source_cache_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_connector_v2_annotation_source_cache_proto_rawDesc), len(file_c1_connector_v2_annotation_source_cache_proto_rawDesc)), + NumEnums: 1, + NumMessages: 3, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_c1_connector_v2_annotation_source_cache_proto_goTypes, + DependencyIndexes: file_c1_connector_v2_annotation_source_cache_proto_depIdxs, + EnumInfos: file_c1_connector_v2_annotation_source_cache_proto_enumTypes, + MessageInfos: file_c1_connector_v2_annotation_source_cache_proto_msgTypes, + }.Build() + File_c1_connector_v2_annotation_source_cache_proto = out.File + file_c1_connector_v2_annotation_source_cache_proto_goTypes = nil + file_c1_connector_v2_annotation_source_cache_proto_depIdxs = nil +} diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_source_cache.pb.validate.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_source_cache.pb.validate.go new file mode 100644 index 00000000..e35ec1ac --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_source_cache.pb.validate.go @@ -0,0 +1,352 @@ +// Code generated by protoc-gen-validate. DO NOT EDIT. +// source: c1/connector/v2/annotation_source_cache.proto + +package v2 + +import ( + "bytes" + "errors" + "fmt" + "net" + "net/mail" + "net/url" + "regexp" + "sort" + "strings" + "time" + "unicode/utf8" + + "google.golang.org/protobuf/types/known/anypb" +) + +// ensure the imports are used +var ( + _ = bytes.MinRead + _ = errors.New("") + _ = fmt.Print + _ = utf8.UTFMax + _ = (*regexp.Regexp)(nil) + _ = (*strings.Reader)(nil) + _ = net.IPv4len + _ = time.Duration(0) + _ = (*url.URL)(nil) + _ = (*mail.Address)(nil) + _ = anypb.Any{} + _ = sort.Sort +) + +// Validate checks the field values on SourceCacheCapability with the rules +// defined in the proto definition for this message. If any rules are +// violated, the first error encountered is returned, or nil if there are no violations. +func (m *SourceCacheCapability) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SourceCacheCapability with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// SourceCacheCapabilityMultiError, or nil if none found. +func (m *SourceCacheCapability) ValidateAll() error { + return m.validate(true) +} + +func (m *SourceCacheCapability) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + // no validation rules for Mode + + if len(errors) > 0 { + return SourceCacheCapabilityMultiError(errors) + } + + return nil +} + +// SourceCacheCapabilityMultiError is an error wrapping multiple validation +// errors returned by SourceCacheCapability.ValidateAll() if the designated +// constraints aren't met. +type SourceCacheCapabilityMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SourceCacheCapabilityMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SourceCacheCapabilityMultiError) AllErrors() []error { return m } + +// SourceCacheCapabilityValidationError is the validation error returned by +// SourceCacheCapability.Validate if the designated constraints aren't met. +type SourceCacheCapabilityValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SourceCacheCapabilityValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SourceCacheCapabilityValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SourceCacheCapabilityValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SourceCacheCapabilityValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SourceCacheCapabilityValidationError) ErrorName() string { + return "SourceCacheCapabilityValidationError" +} + +// Error satisfies the builtin error interface +func (e SourceCacheCapabilityValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSourceCacheCapability.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SourceCacheCapabilityValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SourceCacheCapabilityValidationError{} + +// Validate checks the field values on SourceCacheScope with the rules defined +// in the proto definition for this message. If any rules are violated, the +// first error encountered is returned, or nil if there are no violations. +func (m *SourceCacheScope) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SourceCacheScope with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// SourceCacheScopeMultiError, or nil if none found. +func (m *SourceCacheScope) ValidateAll() error { + return m.validate(true) +} + +func (m *SourceCacheScope) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + // no validation rules for ScopeHash + + // no validation rules for Etag + + if len(errors) > 0 { + return SourceCacheScopeMultiError(errors) + } + + return nil +} + +// SourceCacheScopeMultiError is an error wrapping multiple validation errors +// returned by SourceCacheScope.ValidateAll() if the designated constraints +// aren't met. +type SourceCacheScopeMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SourceCacheScopeMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SourceCacheScopeMultiError) AllErrors() []error { return m } + +// SourceCacheScopeValidationError is the validation error returned by +// SourceCacheScope.Validate if the designated constraints aren't met. +type SourceCacheScopeValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SourceCacheScopeValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SourceCacheScopeValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SourceCacheScopeValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SourceCacheScopeValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SourceCacheScopeValidationError) ErrorName() string { return "SourceCacheScopeValidationError" } + +// Error satisfies the builtin error interface +func (e SourceCacheScopeValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSourceCacheScope.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SourceCacheScopeValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SourceCacheScopeValidationError{} + +// Validate checks the field values on SourceCacheReplay with the rules defined +// in the proto definition for this message. If any rules are violated, the +// first error encountered is returned, or nil if there are no violations. +func (m *SourceCacheReplay) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SourceCacheReplay with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// SourceCacheReplayMultiError, or nil if none found. +func (m *SourceCacheReplay) ValidateAll() error { + return m.validate(true) +} + +func (m *SourceCacheReplay) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + // no validation rules for ScopeHash + + // no validation rules for Etag + + // no validation rules for Overlay + + if len(errors) > 0 { + return SourceCacheReplayMultiError(errors) + } + + return nil +} + +// SourceCacheReplayMultiError is an error wrapping multiple validation errors +// returned by SourceCacheReplay.ValidateAll() if the designated constraints +// aren't met. +type SourceCacheReplayMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SourceCacheReplayMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SourceCacheReplayMultiError) AllErrors() []error { return m } + +// SourceCacheReplayValidationError is the validation error returned by +// SourceCacheReplay.Validate if the designated constraints aren't met. +type SourceCacheReplayValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SourceCacheReplayValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SourceCacheReplayValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SourceCacheReplayValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SourceCacheReplayValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SourceCacheReplayValidationError) ErrorName() string { + return "SourceCacheReplayValidationError" +} + +// Error satisfies the builtin error interface +func (e SourceCacheReplayValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSourceCacheReplay.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SourceCacheReplayValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SourceCacheReplayValidationError{} diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_source_cache_protoopaque.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_source_cache_protoopaque.pb.go new file mode 100644 index 00000000..08ab696f --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_source_cache_protoopaque.pb.go @@ -0,0 +1,475 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.10 +// protoc (unknown) +// source: c1/connector/v2/annotation_source_cache.proto + +//go:build protoopaque + +package v2 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type SourceCacheCapability_Mode int32 + +const ( + SourceCacheCapability_MODE_UNSPECIFIED SourceCacheCapability_Mode = 0 + SourceCacheCapability_MODE_DISABLED SourceCacheCapability_Mode = 1 + SourceCacheCapability_MODE_READ_WRITE SourceCacheCapability_Mode = 2 +) + +// Enum value maps for SourceCacheCapability_Mode. +var ( + SourceCacheCapability_Mode_name = map[int32]string{ + 0: "MODE_UNSPECIFIED", + 1: "MODE_DISABLED", + 2: "MODE_READ_WRITE", + } + SourceCacheCapability_Mode_value = map[string]int32{ + "MODE_UNSPECIFIED": 0, + "MODE_DISABLED": 1, + "MODE_READ_WRITE": 2, + } +) + +func (x SourceCacheCapability_Mode) Enum() *SourceCacheCapability_Mode { + p := new(SourceCacheCapability_Mode) + *p = x + return p +} + +func (x SourceCacheCapability_Mode) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (SourceCacheCapability_Mode) Descriptor() protoreflect.EnumDescriptor { + return file_c1_connector_v2_annotation_source_cache_proto_enumTypes[0].Descriptor() +} + +func (SourceCacheCapability_Mode) Type() protoreflect.EnumType { + return &file_c1_connector_v2_annotation_source_cache_proto_enumTypes[0] +} + +func (x SourceCacheCapability_Mode) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// SourceCacheCapability is attached to ConnectorServiceValidateResponse +// annotations to opt in to source-cache replay. Absent or any mode other +// than MODE_READ_WRITE means all source-cache annotations are ignored. +type SourceCacheCapability struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Mode SourceCacheCapability_Mode `protobuf:"varint,1,opt,name=mode,proto3,enum=c1.connector.v2.SourceCacheCapability_Mode"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheCapability) Reset() { + *x = SourceCacheCapability{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheCapability) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheCapability) ProtoMessage() {} + +func (x *SourceCacheCapability) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheCapability) GetMode() SourceCacheCapability_Mode { + if x != nil { + return x.xxx_hidden_Mode + } + return SourceCacheCapability_MODE_UNSPECIFIED +} + +func (x *SourceCacheCapability) SetMode(v SourceCacheCapability_Mode) { + x.xxx_hidden_Mode = v +} + +type SourceCacheCapability_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + Mode SourceCacheCapability_Mode +} + +func (b0 SourceCacheCapability_builder) Build() *SourceCacheCapability { + m0 := &SourceCacheCapability{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_Mode = b.Mode + return m0 +} + +// SourceCacheScope is attached to a list-response page whose rows were +// freshly fetched from upstream. The SDK stamps the page's rows with +// scope_hash so a future sync can replay them as a unit. +type SourceCacheScope struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_ScopeHash string `protobuf:"bytes,1,opt,name=scope_hash,json=scopeHash,proto3"` + xxx_hidden_Etag string `protobuf:"bytes,2,opt,name=etag,proto3"` + xxx_hidden_DeletedIds []string `protobuf:"bytes,3,rep,name=deleted_ids,json=deletedIds,proto3"` + xxx_hidden_DeletedPrincipalIds []string `protobuf:"bytes,4,rep,name=deleted_principal_ids,json=deletedPrincipalIds,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheScope) Reset() { + *x = SourceCacheScope{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheScope) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheScope) ProtoMessage() {} + +func (x *SourceCacheScope) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheScope) GetScopeHash() string { + if x != nil { + return x.xxx_hidden_ScopeHash + } + return "" +} + +func (x *SourceCacheScope) GetEtag() string { + if x != nil { + return x.xxx_hidden_Etag + } + return "" +} + +func (x *SourceCacheScope) GetDeletedIds() []string { + if x != nil { + return x.xxx_hidden_DeletedIds + } + return nil +} + +func (x *SourceCacheScope) GetDeletedPrincipalIds() []string { + if x != nil { + return x.xxx_hidden_DeletedPrincipalIds + } + return nil +} + +func (x *SourceCacheScope) SetScopeHash(v string) { + x.xxx_hidden_ScopeHash = v +} + +func (x *SourceCacheScope) SetEtag(v string) { + x.xxx_hidden_Etag = v +} + +func (x *SourceCacheScope) SetDeletedIds(v []string) { + x.xxx_hidden_DeletedIds = v +} + +func (x *SourceCacheScope) SetDeletedPrincipalIds(v []string) { + x.xxx_hidden_DeletedPrincipalIds = v +} + +type SourceCacheScope_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Connector-computed stable identifier for the canonical scope. Must be + // byte-stable across syncs for the same logical scope. Prefer an ORDERED + // natural identifier (e.g. the request URL, or "groups/{id}/members") + // over a random hash: scope-index writes lead with this value, so + // identifiers that correlate with fetch order keep index writes nearly + // append-ordered at scale. sourcecache.HashScope is the fallback when no + // compact natural form exists. + ScopeHash string + // Opaque validator to persist for this scope. May be empty on interim + // pages of a multi-page scope (e.g. Graph @odata.nextLink pages); the + // SDK writes the scope's manifest entry when a non-empty etag arrives. + // A 200 response with zero rows still persists the entry. + Etag string + // Tombstones applied after this page's rows commit. Lets every page of + // a multi-page delta round carry its own deletions as the provider + // delivers them, instead of buffering a whole round onto the first + // (replay-annotated) page. Same formats as SourceCacheReplay. + // + // PRECONDITION for tombstones anywhere in a round: the provider's delta + // must be coalesced — at most one add-or-tombstone per object per round + // (Microsoft Graph guarantees this by returning final object state). + // With interleaved add/remove events for one object, per-page ordering + // is deterministic (a page's rows upsert before its deletions apply) + // but cross-page re-adds after a tombstone are the connector's + // responsibility to order. + DeletedIds []string + // Principal-scoped grant tombstones: for RowKindGrants pages, each + // entry deletes EVERY grant row stamped with this scope whose principal + // id equals the entry — no principal resource type and no canonical + // grant-id reconstruction required (delta tombstones usually carry only + // a bare object id, and the object may no longer exist to look up). + // + // PRECONDITION: the scope must be partitioned so that "principal + // removed from scope" means every grant they have in the scope is gone + // — one scope per navigation with independent removal semantics (e.g. + // members and owners of a group are separate scopes, or membership + // removal would take the owner grant with it). + // + // For RowKindResources pages, each entry deletes the resource row(s) + // stamped with this scope whose resource id equals the entry, any + // resource type. + DeletedPrincipalIds []string +} + +func (b0 SourceCacheScope_builder) Build() *SourceCacheScope { + m0 := &SourceCacheScope{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_ScopeHash = b.ScopeHash + x.xxx_hidden_Etag = b.Etag + x.xxx_hidden_DeletedIds = b.DeletedIds + x.xxx_hidden_DeletedPrincipalIds = b.DeletedPrincipalIds + return m0 +} + +// SourceCacheReplay is attached to a list-response page to tell the SDK to +// copy the previous sync's rows for scope_hash into the current sync. +// +// The row kind (resources, entitlements, grants) is determined by which +// RPC the annotation arrived on, never by the annotation itself. +// +// The connector must only emit this for a scope whose etag it received +// from the SDK's source-cache lookup during this same sync. A replay for +// an unknown scope fails the sync: the connector has already skipped row +// generation, so there is nothing to fall back to. +type SourceCacheReplay struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_ScopeHash string `protobuf:"bytes,1,opt,name=scope_hash,json=scopeHash,proto3"` + xxx_hidden_Etag string `protobuf:"bytes,2,opt,name=etag,proto3"` + xxx_hidden_Overlay bool `protobuf:"varint,3,opt,name=overlay,proto3"` + xxx_hidden_DeletedIds []string `protobuf:"bytes,4,rep,name=deleted_ids,json=deletedIds,proto3"` + xxx_hidden_DeletedPrincipalIds []string `protobuf:"bytes,5,rep,name=deleted_principal_ids,json=deletedPrincipalIds,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheReplay) Reset() { + *x = SourceCacheReplay{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheReplay) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheReplay) ProtoMessage() {} + +func (x *SourceCacheReplay) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheReplay) GetScopeHash() string { + if x != nil { + return x.xxx_hidden_ScopeHash + } + return "" +} + +func (x *SourceCacheReplay) GetEtag() string { + if x != nil { + return x.xxx_hidden_Etag + } + return "" +} + +func (x *SourceCacheReplay) GetOverlay() bool { + if x != nil { + return x.xxx_hidden_Overlay + } + return false +} + +func (x *SourceCacheReplay) GetDeletedIds() []string { + if x != nil { + return x.xxx_hidden_DeletedIds + } + return nil +} + +func (x *SourceCacheReplay) GetDeletedPrincipalIds() []string { + if x != nil { + return x.xxx_hidden_DeletedPrincipalIds + } + return nil +} + +func (x *SourceCacheReplay) SetScopeHash(v string) { + x.xxx_hidden_ScopeHash = v +} + +func (x *SourceCacheReplay) SetEtag(v string) { + x.xxx_hidden_Etag = v +} + +func (x *SourceCacheReplay) SetOverlay(v bool) { + x.xxx_hidden_Overlay = v +} + +func (x *SourceCacheReplay) SetDeletedIds(v []string) { + x.xxx_hidden_DeletedIds = v +} + +func (x *SourceCacheReplay) SetDeletedPrincipalIds(v []string) { + x.xxx_hidden_DeletedPrincipalIds = v +} + +type SourceCacheReplay_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + ScopeHash string + // Validator to persist for this scope in the current sync. For an HTTP + // 304 this is the unchanged etag. For a delta query this is the NEW + // token; it may instead be supplied by the final overlay page's + // SourceCacheScope.etag, in which case this field may be left empty. + Etag string + // When true, the response (and subsequent pages carrying + // SourceCacheScope with the same scope_hash) contains changed rows to + // upsert on top of the replayed base. When false the response must + // contain no rows for this scope. + Overlay bool + // Public canonical IDs (grant/entitlement IDs, or resource BIDs for + // RowKindResources) to delete from the current sync after the replay + // copy and this page's upserts. Used for delta-query tombstones (e.g. + // Microsoft Graph @removed entries). Subsequent pages of the round + // carry their tombstones on SourceCacheScope.deleted_ids. + DeletedIds []string + // Principal-scoped tombstones for this page; see + // SourceCacheScope.deleted_principal_ids for semantics and + // preconditions. + DeletedPrincipalIds []string +} + +func (b0 SourceCacheReplay_builder) Build() *SourceCacheReplay { + m0 := &SourceCacheReplay{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_ScopeHash = b.ScopeHash + x.xxx_hidden_Etag = b.Etag + x.xxx_hidden_Overlay = b.Overlay + x.xxx_hidden_DeletedIds = b.DeletedIds + x.xxx_hidden_DeletedPrincipalIds = b.DeletedPrincipalIds + return m0 +} + +var File_c1_connector_v2_annotation_source_cache_proto protoreflect.FileDescriptor + +const file_c1_connector_v2_annotation_source_cache_proto_rawDesc = "" + + "\n" + + "-c1/connector/v2/annotation_source_cache.proto\x12\x0fc1.connector.v2\"\x9e\x01\n" + + "\x15SourceCacheCapability\x12?\n" + + "\x04mode\x18\x01 \x01(\x0e2+.c1.connector.v2.SourceCacheCapability.ModeR\x04mode\"D\n" + + "\x04Mode\x12\x14\n" + + "\x10MODE_UNSPECIFIED\x10\x00\x12\x11\n" + + "\rMODE_DISABLED\x10\x01\x12\x13\n" + + "\x0fMODE_READ_WRITE\x10\x02\"\x9a\x01\n" + + "\x10SourceCacheScope\x12\x1d\n" + + "\n" + + "scope_hash\x18\x01 \x01(\tR\tscopeHash\x12\x12\n" + + "\x04etag\x18\x02 \x01(\tR\x04etag\x12\x1f\n" + + "\vdeleted_ids\x18\x03 \x03(\tR\n" + + "deletedIds\x122\n" + + "\x15deleted_principal_ids\x18\x04 \x03(\tR\x13deletedPrincipalIds\"\xb5\x01\n" + + "\x11SourceCacheReplay\x12\x1d\n" + + "\n" + + "scope_hash\x18\x01 \x01(\tR\tscopeHash\x12\x12\n" + + "\x04etag\x18\x02 \x01(\tR\x04etag\x12\x18\n" + + "\aoverlay\x18\x03 \x01(\bR\aoverlay\x12\x1f\n" + + "\vdeleted_ids\x18\x04 \x03(\tR\n" + + "deletedIds\x122\n" + + "\x15deleted_principal_ids\x18\x05 \x03(\tR\x13deletedPrincipalIdsB6Z4github.com/conductorone/baton-sdk/pb/c1/connector/v2b\x06proto3" + +var file_c1_connector_v2_annotation_source_cache_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_c1_connector_v2_annotation_source_cache_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_c1_connector_v2_annotation_source_cache_proto_goTypes = []any{ + (SourceCacheCapability_Mode)(0), // 0: c1.connector.v2.SourceCacheCapability.Mode + (*SourceCacheCapability)(nil), // 1: c1.connector.v2.SourceCacheCapability + (*SourceCacheScope)(nil), // 2: c1.connector.v2.SourceCacheScope + (*SourceCacheReplay)(nil), // 3: c1.connector.v2.SourceCacheReplay +} +var file_c1_connector_v2_annotation_source_cache_proto_depIdxs = []int32{ + 0, // 0: c1.connector.v2.SourceCacheCapability.mode:type_name -> c1.connector.v2.SourceCacheCapability.Mode + 1, // [1:1] is the sub-list for method output_type + 1, // [1:1] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name +} + +func init() { file_c1_connector_v2_annotation_source_cache_proto_init() } +func file_c1_connector_v2_annotation_source_cache_proto_init() { + if File_c1_connector_v2_annotation_source_cache_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_connector_v2_annotation_source_cache_proto_rawDesc), len(file_c1_connector_v2_annotation_source_cache_proto_rawDesc)), + NumEnums: 1, + NumMessages: 3, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_c1_connector_v2_annotation_source_cache_proto_goTypes, + DependencyIndexes: file_c1_connector_v2_annotation_source_cache_proto_depIdxs, + EnumInfos: file_c1_connector_v2_annotation_source_cache_proto_enumTypes, + MessageInfos: file_c1_connector_v2_annotation_source_cache_proto_msgTypes, + }.Build() + File_c1_connector_v2_annotation_source_cache_proto = out.File + file_c1_connector_v2_annotation_source_cache_proto_goTypes = nil + file_c1_connector_v2_annotation_source_cache_proto_depIdxs = nil +} diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_type_scoped_grants.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_type_scoped_grants.pb.go new file mode 100644 index 00000000..189ef309 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_type_scoped_grants.pb.go @@ -0,0 +1,234 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.10 +// protoc (unknown) +// source: c1/connector/v2/annotation_type_scoped_grants.proto + +//go:build !protoopaque + +package v2 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Type-scoped grant ingestion. +// +// Some providers expose change/enumeration APIs whose natural unit is a +// whole collection (or a connector-defined shard of one), not a single +// resource — e.g. Microsoft Graph delta queries, where one stream yields +// membership changes for up to 50 groups. Forcing that shape through the +// per-resource ListGrants fan-out costs one request per resource even when +// nothing changed. +// +// A resource type annotated with TypeScopedGrants is EXCLUDED from the +// per-resource grants fan-out. Instead the syncer issues ListGrants calls +// whose resource carries ONLY the resource type (empty resource id); the +// connector answers with grants for the whole type, across as many +// paginated cursors as it chooses to spawn (see SpawnCursors). +// +// This is the grants-phase analogue of StaticEntitlements: the connector +// takes over enumeration for the type, and every downstream behavior +// (storage, source-cache scopes/replay/tombstones, grant expansion, rate +// limiting, page-token checkpointing) is unchanged because grants are +// self-describing rows. +// +// COMPLETENESS CONTRACT: a full sync of an annotated type must emit (or +// replay, via source-cache annotations) EVERY grant of that type. The +// syncer no longer visits each resource, so completeness rests entirely on +// the connector's enumeration. +type TypeScopedGrants struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TypeScopedGrants) Reset() { + *x = TypeScopedGrants{} + mi := &file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TypeScopedGrants) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TypeScopedGrants) ProtoMessage() {} + +func (x *TypeScopedGrants) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +type TypeScopedGrants_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + +} + +func (b0 TypeScopedGrants_builder) Build() *TypeScopedGrants { + m0 := &TypeScopedGrants{} + b, x := &b0, m0 + _, _ = b, x + return m0 +} + +// SpawnCursors is attached to a type-scoped ListGrants response to enqueue +// additional independent cursors for the same resource type. Each token is +// delivered back to the connector as the page token of its own action — +// scheduled by the syncer's worker pool, rate-limited, and checkpointed +// like any other pagination. +// +// Typical use: the first (planning) call computes shard assignments — e.g. +// one 50-id delta filter chunk per cursor — returns no rows, and spawns +// one cursor per shard. Cold enumeration then parallelizes across shards +// instead of serializing through one stream. +// +// Tokens are opaque to the SDK. They must be self-contained: a cursor may +// execute after a suspend/resume, on a different worker, so anything the +// connector needs to serve the cursor must be inside the token (or +// re-derivable from it). +// +// Honored only on responses to type-scoped ListGrants calls; ignored (with +// a warning) elsewhere. +type SpawnCursors struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + // Page tokens for the sibling cursors to enqueue, one action each. + PageTokens []string `protobuf:"bytes,1,rep,name=page_tokens,json=pageTokens,proto3" json:"page_tokens,omitempty"` + // Optional connector estimate of total grants across all cursors of this + // type, for progress reporting. Zero means unknown. + EstimatedTotal int64 `protobuf:"varint,2,opt,name=estimated_total,json=estimatedTotal,proto3" json:"estimated_total,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SpawnCursors) Reset() { + *x = SpawnCursors{} + mi := &file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SpawnCursors) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SpawnCursors) ProtoMessage() {} + +func (x *SpawnCursors) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SpawnCursors) GetPageTokens() []string { + if x != nil { + return x.PageTokens + } + return nil +} + +func (x *SpawnCursors) GetEstimatedTotal() int64 { + if x != nil { + return x.EstimatedTotal + } + return 0 +} + +func (x *SpawnCursors) SetPageTokens(v []string) { + x.PageTokens = v +} + +func (x *SpawnCursors) SetEstimatedTotal(v int64) { + x.EstimatedTotal = v +} + +type SpawnCursors_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Page tokens for the sibling cursors to enqueue, one action each. + PageTokens []string + // Optional connector estimate of total grants across all cursors of this + // type, for progress reporting. Zero means unknown. + EstimatedTotal int64 +} + +func (b0 SpawnCursors_builder) Build() *SpawnCursors { + m0 := &SpawnCursors{} + b, x := &b0, m0 + _, _ = b, x + x.PageTokens = b.PageTokens + x.EstimatedTotal = b.EstimatedTotal + return m0 +} + +var File_c1_connector_v2_annotation_type_scoped_grants_proto protoreflect.FileDescriptor + +const file_c1_connector_v2_annotation_type_scoped_grants_proto_rawDesc = "" + + "\n" + + "3c1/connector/v2/annotation_type_scoped_grants.proto\x12\x0fc1.connector.v2\"\x12\n" + + "\x10TypeScopedGrants\"X\n" + + "\fSpawnCursors\x12\x1f\n" + + "\vpage_tokens\x18\x01 \x03(\tR\n" + + "pageTokens\x12'\n" + + "\x0festimated_total\x18\x02 \x01(\x03R\x0eestimatedTotalB6Z4github.com/conductorone/baton-sdk/pb/c1/connector/v2b\x06proto3" + +var file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_c1_connector_v2_annotation_type_scoped_grants_proto_goTypes = []any{ + (*TypeScopedGrants)(nil), // 0: c1.connector.v2.TypeScopedGrants + (*SpawnCursors)(nil), // 1: c1.connector.v2.SpawnCursors +} +var file_c1_connector_v2_annotation_type_scoped_grants_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_c1_connector_v2_annotation_type_scoped_grants_proto_init() } +func file_c1_connector_v2_annotation_type_scoped_grants_proto_init() { + if File_c1_connector_v2_annotation_type_scoped_grants_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_connector_v2_annotation_type_scoped_grants_proto_rawDesc), len(file_c1_connector_v2_annotation_type_scoped_grants_proto_rawDesc)), + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_c1_connector_v2_annotation_type_scoped_grants_proto_goTypes, + DependencyIndexes: file_c1_connector_v2_annotation_type_scoped_grants_proto_depIdxs, + MessageInfos: file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes, + }.Build() + File_c1_connector_v2_annotation_type_scoped_grants_proto = out.File + file_c1_connector_v2_annotation_type_scoped_grants_proto_goTypes = nil + file_c1_connector_v2_annotation_type_scoped_grants_proto_depIdxs = nil +} diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_type_scoped_grants.pb.validate.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_type_scoped_grants.pb.validate.go new file mode 100644 index 00000000..5679e6e9 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_type_scoped_grants.pb.validate.go @@ -0,0 +1,237 @@ +// Code generated by protoc-gen-validate. DO NOT EDIT. +// source: c1/connector/v2/annotation_type_scoped_grants.proto + +package v2 + +import ( + "bytes" + "errors" + "fmt" + "net" + "net/mail" + "net/url" + "regexp" + "sort" + "strings" + "time" + "unicode/utf8" + + "google.golang.org/protobuf/types/known/anypb" +) + +// ensure the imports are used +var ( + _ = bytes.MinRead + _ = errors.New("") + _ = fmt.Print + _ = utf8.UTFMax + _ = (*regexp.Regexp)(nil) + _ = (*strings.Reader)(nil) + _ = net.IPv4len + _ = time.Duration(0) + _ = (*url.URL)(nil) + _ = (*mail.Address)(nil) + _ = anypb.Any{} + _ = sort.Sort +) + +// Validate checks the field values on TypeScopedGrants with the rules defined +// in the proto definition for this message. If any rules are violated, the +// first error encountered is returned, or nil if there are no violations. +func (m *TypeScopedGrants) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on TypeScopedGrants with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// TypeScopedGrantsMultiError, or nil if none found. +func (m *TypeScopedGrants) ValidateAll() error { + return m.validate(true) +} + +func (m *TypeScopedGrants) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + if len(errors) > 0 { + return TypeScopedGrantsMultiError(errors) + } + + return nil +} + +// TypeScopedGrantsMultiError is an error wrapping multiple validation errors +// returned by TypeScopedGrants.ValidateAll() if the designated constraints +// aren't met. +type TypeScopedGrantsMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m TypeScopedGrantsMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m TypeScopedGrantsMultiError) AllErrors() []error { return m } + +// TypeScopedGrantsValidationError is the validation error returned by +// TypeScopedGrants.Validate if the designated constraints aren't met. +type TypeScopedGrantsValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e TypeScopedGrantsValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e TypeScopedGrantsValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e TypeScopedGrantsValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e TypeScopedGrantsValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e TypeScopedGrantsValidationError) ErrorName() string { return "TypeScopedGrantsValidationError" } + +// Error satisfies the builtin error interface +func (e TypeScopedGrantsValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sTypeScopedGrants.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = TypeScopedGrantsValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = TypeScopedGrantsValidationError{} + +// Validate checks the field values on SpawnCursors with the rules defined in +// the proto definition for this message. If any rules are violated, the first +// error encountered is returned, or nil if there are no violations. +func (m *SpawnCursors) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SpawnCursors with the rules defined +// in the proto definition for this message. If any rules are violated, the +// result is a list of violation errors wrapped in SpawnCursorsMultiError, or +// nil if none found. +func (m *SpawnCursors) ValidateAll() error { + return m.validate(true) +} + +func (m *SpawnCursors) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + // no validation rules for EstimatedTotal + + if len(errors) > 0 { + return SpawnCursorsMultiError(errors) + } + + return nil +} + +// SpawnCursorsMultiError is an error wrapping multiple validation errors +// returned by SpawnCursors.ValidateAll() if the designated constraints aren't met. +type SpawnCursorsMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SpawnCursorsMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SpawnCursorsMultiError) AllErrors() []error { return m } + +// SpawnCursorsValidationError is the validation error returned by +// SpawnCursors.Validate if the designated constraints aren't met. +type SpawnCursorsValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SpawnCursorsValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SpawnCursorsValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SpawnCursorsValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SpawnCursorsValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SpawnCursorsValidationError) ErrorName() string { return "SpawnCursorsValidationError" } + +// Error satisfies the builtin error interface +func (e SpawnCursorsValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSpawnCursors.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SpawnCursorsValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SpawnCursorsValidationError{} diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_type_scoped_grants_protoopaque.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_type_scoped_grants_protoopaque.pb.go new file mode 100644 index 00000000..d49107b0 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_type_scoped_grants_protoopaque.pb.go @@ -0,0 +1,231 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.10 +// protoc (unknown) +// source: c1/connector/v2/annotation_type_scoped_grants.proto + +//go:build protoopaque + +package v2 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Type-scoped grant ingestion. +// +// Some providers expose change/enumeration APIs whose natural unit is a +// whole collection (or a connector-defined shard of one), not a single +// resource — e.g. Microsoft Graph delta queries, where one stream yields +// membership changes for up to 50 groups. Forcing that shape through the +// per-resource ListGrants fan-out costs one request per resource even when +// nothing changed. +// +// A resource type annotated with TypeScopedGrants is EXCLUDED from the +// per-resource grants fan-out. Instead the syncer issues ListGrants calls +// whose resource carries ONLY the resource type (empty resource id); the +// connector answers with grants for the whole type, across as many +// paginated cursors as it chooses to spawn (see SpawnCursors). +// +// This is the grants-phase analogue of StaticEntitlements: the connector +// takes over enumeration for the type, and every downstream behavior +// (storage, source-cache scopes/replay/tombstones, grant expansion, rate +// limiting, page-token checkpointing) is unchanged because grants are +// self-describing rows. +// +// COMPLETENESS CONTRACT: a full sync of an annotated type must emit (or +// replay, via source-cache annotations) EVERY grant of that type. The +// syncer no longer visits each resource, so completeness rests entirely on +// the connector's enumeration. +type TypeScopedGrants struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TypeScopedGrants) Reset() { + *x = TypeScopedGrants{} + mi := &file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TypeScopedGrants) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TypeScopedGrants) ProtoMessage() {} + +func (x *TypeScopedGrants) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +type TypeScopedGrants_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + +} + +func (b0 TypeScopedGrants_builder) Build() *TypeScopedGrants { + m0 := &TypeScopedGrants{} + b, x := &b0, m0 + _, _ = b, x + return m0 +} + +// SpawnCursors is attached to a type-scoped ListGrants response to enqueue +// additional independent cursors for the same resource type. Each token is +// delivered back to the connector as the page token of its own action — +// scheduled by the syncer's worker pool, rate-limited, and checkpointed +// like any other pagination. +// +// Typical use: the first (planning) call computes shard assignments — e.g. +// one 50-id delta filter chunk per cursor — returns no rows, and spawns +// one cursor per shard. Cold enumeration then parallelizes across shards +// instead of serializing through one stream. +// +// Tokens are opaque to the SDK. They must be self-contained: a cursor may +// execute after a suspend/resume, on a different worker, so anything the +// connector needs to serve the cursor must be inside the token (or +// re-derivable from it). +// +// Honored only on responses to type-scoped ListGrants calls; ignored (with +// a warning) elsewhere. +type SpawnCursors struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_PageTokens []string `protobuf:"bytes,1,rep,name=page_tokens,json=pageTokens,proto3"` + xxx_hidden_EstimatedTotal int64 `protobuf:"varint,2,opt,name=estimated_total,json=estimatedTotal,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SpawnCursors) Reset() { + *x = SpawnCursors{} + mi := &file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SpawnCursors) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SpawnCursors) ProtoMessage() {} + +func (x *SpawnCursors) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SpawnCursors) GetPageTokens() []string { + if x != nil { + return x.xxx_hidden_PageTokens + } + return nil +} + +func (x *SpawnCursors) GetEstimatedTotal() int64 { + if x != nil { + return x.xxx_hidden_EstimatedTotal + } + return 0 +} + +func (x *SpawnCursors) SetPageTokens(v []string) { + x.xxx_hidden_PageTokens = v +} + +func (x *SpawnCursors) SetEstimatedTotal(v int64) { + x.xxx_hidden_EstimatedTotal = v +} + +type SpawnCursors_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Page tokens for the sibling cursors to enqueue, one action each. + PageTokens []string + // Optional connector estimate of total grants across all cursors of this + // type, for progress reporting. Zero means unknown. + EstimatedTotal int64 +} + +func (b0 SpawnCursors_builder) Build() *SpawnCursors { + m0 := &SpawnCursors{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_PageTokens = b.PageTokens + x.xxx_hidden_EstimatedTotal = b.EstimatedTotal + return m0 +} + +var File_c1_connector_v2_annotation_type_scoped_grants_proto protoreflect.FileDescriptor + +const file_c1_connector_v2_annotation_type_scoped_grants_proto_rawDesc = "" + + "\n" + + "3c1/connector/v2/annotation_type_scoped_grants.proto\x12\x0fc1.connector.v2\"\x12\n" + + "\x10TypeScopedGrants\"X\n" + + "\fSpawnCursors\x12\x1f\n" + + "\vpage_tokens\x18\x01 \x03(\tR\n" + + "pageTokens\x12'\n" + + "\x0festimated_total\x18\x02 \x01(\x03R\x0eestimatedTotalB6Z4github.com/conductorone/baton-sdk/pb/c1/connector/v2b\x06proto3" + +var file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_c1_connector_v2_annotation_type_scoped_grants_proto_goTypes = []any{ + (*TypeScopedGrants)(nil), // 0: c1.connector.v2.TypeScopedGrants + (*SpawnCursors)(nil), // 1: c1.connector.v2.SpawnCursors +} +var file_c1_connector_v2_annotation_type_scoped_grants_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_c1_connector_v2_annotation_type_scoped_grants_proto_init() } +func file_c1_connector_v2_annotation_type_scoped_grants_proto_init() { + if File_c1_connector_v2_annotation_type_scoped_grants_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_connector_v2_annotation_type_scoped_grants_proto_rawDesc), len(file_c1_connector_v2_annotation_type_scoped_grants_proto_rawDesc)), + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_c1_connector_v2_annotation_type_scoped_grants_proto_goTypes, + DependencyIndexes: file_c1_connector_v2_annotation_type_scoped_grants_proto_depIdxs, + MessageInfos: file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes, + }.Build() + File_c1_connector_v2_annotation_type_scoped_grants_proto = out.File + file_c1_connector_v2_annotation_type_scoped_grants_proto_goTypes = nil + file_c1_connector_v2_annotation_type_scoped_grants_proto_depIdxs = nil +} diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/source_cache.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/source_cache.pb.go new file mode 100644 index 00000000..5d0c2090 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/source_cache.pb.go @@ -0,0 +1,249 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.10 +// protoc (unknown) +// source: c1/connectorapi/baton/v1/source_cache.proto + +//go:build !protoopaque + +package v1 + +import ( + _ "github.com/envoyproxy/protoc-gen-validate/validate" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type LookupRequest struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + // Row kind: resources / entitlements / grants + // (pkg/sourcecache.RowKind values). Entries are partitioned by row + // kind, so one scope hash can carry a different validator per kind. + RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3" json:"row_kind,omitempty"` + // Connector-defined stable scope identifier (conventionally a hex hash + // of the canonical scope; see pkg/sourcecache.HashScope). Opaque to the + // parent; matched verbatim against the previous sync's source-cache + // entries. + ScopeHash string `protobuf:"bytes,2,opt,name=scope_hash,json=scopeHash,proto3" json:"scope_hash,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LookupRequest) Reset() { + *x = LookupRequest{} + mi := &file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LookupRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LookupRequest) ProtoMessage() {} + +func (x *LookupRequest) ProtoReflect() protoreflect.Message { + mi := &file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *LookupRequest) GetRowKind() string { + if x != nil { + return x.RowKind + } + return "" +} + +func (x *LookupRequest) GetScopeHash() string { + if x != nil { + return x.ScopeHash + } + return "" +} + +func (x *LookupRequest) SetRowKind(v string) { + x.RowKind = v +} + +func (x *LookupRequest) SetScopeHash(v string) { + x.ScopeHash = v +} + +type LookupRequest_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Row kind: resources / entitlements / grants + // (pkg/sourcecache.RowKind values). Entries are partitioned by row + // kind, so one scope hash can carry a different validator per kind. + RowKind string + // Connector-defined stable scope identifier (conventionally a hex hash + // of the canonical scope; see pkg/sourcecache.HashScope). Opaque to the + // parent; matched verbatim against the previous sync's source-cache + // entries. + ScopeHash string +} + +func (b0 LookupRequest_builder) Build() *LookupRequest { + m0 := &LookupRequest{} + b, x := &b0, m0 + _, _ = b, x + x.RowKind = b.RowKind + x.ScopeHash = b.ScopeHash + return m0 +} + +type LookupResponse struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + // False means no prior entry exists for (row_kind, scope_hash): the + // connector must fetch fresh and must not emit SourceCacheReplay for + // this scope. + Found bool `protobuf:"varint,1,opt,name=found,proto3" json:"found,omitempty"` + // The opaque validator the previous sync recorded for this scope (HTTP + // ETag, delta token, ...). Empty when found is false. The cap is a + // sanity bound sized for Microsoft Graph delta tokens, which are known + // to run to thousands of characters; storage imposes no limit. + Etag string `protobuf:"bytes,2,opt,name=etag,proto3" json:"etag,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LookupResponse) Reset() { + *x = LookupResponse{} + mi := &file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LookupResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LookupResponse) ProtoMessage() {} + +func (x *LookupResponse) ProtoReflect() protoreflect.Message { + mi := &file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *LookupResponse) GetFound() bool { + if x != nil { + return x.Found + } + return false +} + +func (x *LookupResponse) GetEtag() string { + if x != nil { + return x.Etag + } + return "" +} + +func (x *LookupResponse) SetFound(v bool) { + x.Found = v +} + +func (x *LookupResponse) SetEtag(v string) { + x.Etag = v +} + +type LookupResponse_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // False means no prior entry exists for (row_kind, scope_hash): the + // connector must fetch fresh and must not emit SourceCacheReplay for + // this scope. + Found bool + // The opaque validator the previous sync recorded for this scope (HTTP + // ETag, delta token, ...). Empty when found is false. The cap is a + // sanity bound sized for Microsoft Graph delta tokens, which are known + // to run to thousands of characters; storage imposes no limit. + Etag string +} + +func (b0 LookupResponse_builder) Build() *LookupResponse { + m0 := &LookupResponse{} + b, x := &b0, m0 + _, _ = b, x + x.Found = b.Found + x.Etag = b.Etag + return m0 +} + +var File_c1_connectorapi_baton_v1_source_cache_proto protoreflect.FileDescriptor + +const file_c1_connectorapi_baton_v1_source_cache_proto_rawDesc = "" + + "\n" + + "+c1/connectorapi/baton/v1/source_cache.proto\x12\x18c1.connectorapi.baton.v1\x1a\x17validate/validate.proto\"`\n" + + "\rLookupRequest\x12$\n" + + "\brow_kind\x18\x01 \x01(\tB\t\xfaB\x06r\x04\x10\x01\x18@R\arowKind\x12)\n" + + "\n" + + "scope_hash\x18\x02 \x01(\tB\n" + + "\xfaB\ar\x05\x10\x01\x18\x80\x02R\tscopeHash\"E\n" + + "\x0eLookupResponse\x12\x14\n" + + "\x05found\x18\x01 \x01(\bR\x05found\x12\x1d\n" + + "\x04etag\x18\x02 \x01(\tB\t\xfaB\x06r\x04\x18\x80\x80\x04R\x04etag2x\n" + + "\x17BatonSourceCacheService\x12]\n" + + "\x06Lookup\x12'.c1.connectorapi.baton.v1.LookupRequest\x1a(.c1.connectorapi.baton.v1.LookupResponse\"\x00B7Z5gitlab.com/ductone/c1/pkg/pb/c1/connectorapi/baton/v1b\x06proto3" + +var file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_c1_connectorapi_baton_v1_source_cache_proto_goTypes = []any{ + (*LookupRequest)(nil), // 0: c1.connectorapi.baton.v1.LookupRequest + (*LookupResponse)(nil), // 1: c1.connectorapi.baton.v1.LookupResponse +} +var file_c1_connectorapi_baton_v1_source_cache_proto_depIdxs = []int32{ + 0, // 0: c1.connectorapi.baton.v1.BatonSourceCacheService.Lookup:input_type -> c1.connectorapi.baton.v1.LookupRequest + 1, // 1: c1.connectorapi.baton.v1.BatonSourceCacheService.Lookup:output_type -> c1.connectorapi.baton.v1.LookupResponse + 1, // [1:2] is the sub-list for method output_type + 0, // [0:1] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_c1_connectorapi_baton_v1_source_cache_proto_init() } +func file_c1_connectorapi_baton_v1_source_cache_proto_init() { + if File_c1_connectorapi_baton_v1_source_cache_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_connectorapi_baton_v1_source_cache_proto_rawDesc), len(file_c1_connectorapi_baton_v1_source_cache_proto_rawDesc)), + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_c1_connectorapi_baton_v1_source_cache_proto_goTypes, + DependencyIndexes: file_c1_connectorapi_baton_v1_source_cache_proto_depIdxs, + MessageInfos: file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes, + }.Build() + File_c1_connectorapi_baton_v1_source_cache_proto = out.File + file_c1_connectorapi_baton_v1_source_cache_proto_goTypes = nil + file_c1_connectorapi_baton_v1_source_cache_proto_depIdxs = nil +} diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/source_cache.pb.validate.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/source_cache.pb.validate.go new file mode 100644 index 00000000..dfc7cca7 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/source_cache.pb.validate.go @@ -0,0 +1,271 @@ +// Code generated by protoc-gen-validate. DO NOT EDIT. +// source: c1/connectorapi/baton/v1/source_cache.proto + +package v1 + +import ( + "bytes" + "errors" + "fmt" + "net" + "net/mail" + "net/url" + "regexp" + "sort" + "strings" + "time" + "unicode/utf8" + + "google.golang.org/protobuf/types/known/anypb" +) + +// ensure the imports are used +var ( + _ = bytes.MinRead + _ = errors.New("") + _ = fmt.Print + _ = utf8.UTFMax + _ = (*regexp.Regexp)(nil) + _ = (*strings.Reader)(nil) + _ = net.IPv4len + _ = time.Duration(0) + _ = (*url.URL)(nil) + _ = (*mail.Address)(nil) + _ = anypb.Any{} + _ = sort.Sort +) + +// Validate checks the field values on LookupRequest with the rules defined in +// the proto definition for this message. If any rules are violated, the first +// error encountered is returned, or nil if there are no violations. +func (m *LookupRequest) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on LookupRequest with the rules defined +// in the proto definition for this message. If any rules are violated, the +// result is a list of violation errors wrapped in LookupRequestMultiError, or +// nil if none found. +func (m *LookupRequest) ValidateAll() error { + return m.validate(true) +} + +func (m *LookupRequest) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + if l := utf8.RuneCountInString(m.GetRowKind()); l < 1 || l > 64 { + err := LookupRequestValidationError{ + field: "RowKind", + reason: "value length must be between 1 and 64 runes, inclusive", + } + if !all { + return err + } + errors = append(errors, err) + } + + if l := utf8.RuneCountInString(m.GetScopeHash()); l < 1 || l > 256 { + err := LookupRequestValidationError{ + field: "ScopeHash", + reason: "value length must be between 1 and 256 runes, inclusive", + } + if !all { + return err + } + errors = append(errors, err) + } + + if len(errors) > 0 { + return LookupRequestMultiError(errors) + } + + return nil +} + +// LookupRequestMultiError is an error wrapping multiple validation errors +// returned by LookupRequest.ValidateAll() if the designated constraints +// aren't met. +type LookupRequestMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m LookupRequestMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m LookupRequestMultiError) AllErrors() []error { return m } + +// LookupRequestValidationError is the validation error returned by +// LookupRequest.Validate if the designated constraints aren't met. +type LookupRequestValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e LookupRequestValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e LookupRequestValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e LookupRequestValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e LookupRequestValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e LookupRequestValidationError) ErrorName() string { return "LookupRequestValidationError" } + +// Error satisfies the builtin error interface +func (e LookupRequestValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sLookupRequest.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = LookupRequestValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = LookupRequestValidationError{} + +// Validate checks the field values on LookupResponse with the rules defined in +// the proto definition for this message. If any rules are violated, the first +// error encountered is returned, or nil if there are no violations. +func (m *LookupResponse) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on LookupResponse with the rules defined +// in the proto definition for this message. If any rules are violated, the +// result is a list of violation errors wrapped in LookupResponseMultiError, +// or nil if none found. +func (m *LookupResponse) ValidateAll() error { + return m.validate(true) +} + +func (m *LookupResponse) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + // no validation rules for Found + + if utf8.RuneCountInString(m.GetEtag()) > 65536 { + err := LookupResponseValidationError{ + field: "Etag", + reason: "value length must be at most 65536 runes", + } + if !all { + return err + } + errors = append(errors, err) + } + + if len(errors) > 0 { + return LookupResponseMultiError(errors) + } + + return nil +} + +// LookupResponseMultiError is an error wrapping multiple validation errors +// returned by LookupResponse.ValidateAll() if the designated constraints +// aren't met. +type LookupResponseMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m LookupResponseMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m LookupResponseMultiError) AllErrors() []error { return m } + +// LookupResponseValidationError is the validation error returned by +// LookupResponse.Validate if the designated constraints aren't met. +type LookupResponseValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e LookupResponseValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e LookupResponseValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e LookupResponseValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e LookupResponseValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e LookupResponseValidationError) ErrorName() string { return "LookupResponseValidationError" } + +// Error satisfies the builtin error interface +func (e LookupResponseValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sLookupResponse.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = LookupResponseValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = LookupResponseValidationError{} diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/source_cache_grpc.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/source_cache_grpc.pb.go new file mode 100644 index 00000000..8db25cd3 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/source_cache_grpc.pb.go @@ -0,0 +1,145 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.5.1 +// - protoc (unknown) +// source: c1/connectorapi/baton/v1/source_cache.proto + +package v1 + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + BatonSourceCacheService_Lookup_FullMethodName = "/c1.connectorapi.baton.v1.BatonSourceCacheService/Lookup" +) + +// BatonSourceCacheServiceClient is the client API for BatonSourceCacheService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// BatonSourceCacheService is the dedicated parent-side RPC the connector +// subprocess uses to ask "do I have a previous validator (etag / delta +// token) for this scope?" before revalidating upstream. +// +// This is intentionally NOT routed through the session-store gRPC service: +// session data goes through the connector's local MemorySessionCache +// (otter), which would subject sync-scoped validator state to generic +// TTL/eviction policies and burn bounded cache weight on it. A dedicated +// service keeps the message shape explicit and the path uncached. +// +// The parent has exactly one active lookup registered at a time (set +// per-sync, cleared at sync end), so no sync_id travels on the wire. +type BatonSourceCacheServiceClient interface { + Lookup(ctx context.Context, in *LookupRequest, opts ...grpc.CallOption) (*LookupResponse, error) +} + +type batonSourceCacheServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewBatonSourceCacheServiceClient(cc grpc.ClientConnInterface) BatonSourceCacheServiceClient { + return &batonSourceCacheServiceClient{cc} +} + +func (c *batonSourceCacheServiceClient) Lookup(ctx context.Context, in *LookupRequest, opts ...grpc.CallOption) (*LookupResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(LookupResponse) + err := c.cc.Invoke(ctx, BatonSourceCacheService_Lookup_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// BatonSourceCacheServiceServer is the server API for BatonSourceCacheService service. +// All implementations should embed UnimplementedBatonSourceCacheServiceServer +// for forward compatibility. +// +// BatonSourceCacheService is the dedicated parent-side RPC the connector +// subprocess uses to ask "do I have a previous validator (etag / delta +// token) for this scope?" before revalidating upstream. +// +// This is intentionally NOT routed through the session-store gRPC service: +// session data goes through the connector's local MemorySessionCache +// (otter), which would subject sync-scoped validator state to generic +// TTL/eviction policies and burn bounded cache weight on it. A dedicated +// service keeps the message shape explicit and the path uncached. +// +// The parent has exactly one active lookup registered at a time (set +// per-sync, cleared at sync end), so no sync_id travels on the wire. +type BatonSourceCacheServiceServer interface { + Lookup(context.Context, *LookupRequest) (*LookupResponse, error) +} + +// UnimplementedBatonSourceCacheServiceServer should be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedBatonSourceCacheServiceServer struct{} + +func (UnimplementedBatonSourceCacheServiceServer) Lookup(context.Context, *LookupRequest) (*LookupResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method Lookup not implemented") +} +func (UnimplementedBatonSourceCacheServiceServer) testEmbeddedByValue() {} + +// UnsafeBatonSourceCacheServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to BatonSourceCacheServiceServer will +// result in compilation errors. +type UnsafeBatonSourceCacheServiceServer interface { + mustEmbedUnimplementedBatonSourceCacheServiceServer() +} + +func RegisterBatonSourceCacheServiceServer(s grpc.ServiceRegistrar, srv BatonSourceCacheServiceServer) { + // If the following call pancis, it indicates UnimplementedBatonSourceCacheServiceServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&BatonSourceCacheService_ServiceDesc, srv) +} + +func _BatonSourceCacheService_Lookup_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(LookupRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(BatonSourceCacheServiceServer).Lookup(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: BatonSourceCacheService_Lookup_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(BatonSourceCacheServiceServer).Lookup(ctx, req.(*LookupRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// BatonSourceCacheService_ServiceDesc is the grpc.ServiceDesc for BatonSourceCacheService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var BatonSourceCacheService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "c1.connectorapi.baton.v1.BatonSourceCacheService", + HandlerType: (*BatonSourceCacheServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Lookup", + Handler: _BatonSourceCacheService_Lookup_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "c1/connectorapi/baton/v1/source_cache.proto", +} diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/source_cache_protoopaque.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/source_cache_protoopaque.pb.go new file mode 100644 index 00000000..9bfc8953 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/source_cache_protoopaque.pb.go @@ -0,0 +1,235 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.10 +// protoc (unknown) +// source: c1/connectorapi/baton/v1/source_cache.proto + +//go:build protoopaque + +package v1 + +import ( + _ "github.com/envoyproxy/protoc-gen-validate/validate" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type LookupRequest struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3"` + xxx_hidden_ScopeHash string `protobuf:"bytes,2,opt,name=scope_hash,json=scopeHash,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LookupRequest) Reset() { + *x = LookupRequest{} + mi := &file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LookupRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LookupRequest) ProtoMessage() {} + +func (x *LookupRequest) ProtoReflect() protoreflect.Message { + mi := &file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *LookupRequest) GetRowKind() string { + if x != nil { + return x.xxx_hidden_RowKind + } + return "" +} + +func (x *LookupRequest) GetScopeHash() string { + if x != nil { + return x.xxx_hidden_ScopeHash + } + return "" +} + +func (x *LookupRequest) SetRowKind(v string) { + x.xxx_hidden_RowKind = v +} + +func (x *LookupRequest) SetScopeHash(v string) { + x.xxx_hidden_ScopeHash = v +} + +type LookupRequest_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Row kind: resources / entitlements / grants + // (pkg/sourcecache.RowKind values). Entries are partitioned by row + // kind, so one scope hash can carry a different validator per kind. + RowKind string + // Connector-defined stable scope identifier (conventionally a hex hash + // of the canonical scope; see pkg/sourcecache.HashScope). Opaque to the + // parent; matched verbatim against the previous sync's source-cache + // entries. + ScopeHash string +} + +func (b0 LookupRequest_builder) Build() *LookupRequest { + m0 := &LookupRequest{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_RowKind = b.RowKind + x.xxx_hidden_ScopeHash = b.ScopeHash + return m0 +} + +type LookupResponse struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Found bool `protobuf:"varint,1,opt,name=found,proto3"` + xxx_hidden_Etag string `protobuf:"bytes,2,opt,name=etag,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LookupResponse) Reset() { + *x = LookupResponse{} + mi := &file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LookupResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LookupResponse) ProtoMessage() {} + +func (x *LookupResponse) ProtoReflect() protoreflect.Message { + mi := &file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *LookupResponse) GetFound() bool { + if x != nil { + return x.xxx_hidden_Found + } + return false +} + +func (x *LookupResponse) GetEtag() string { + if x != nil { + return x.xxx_hidden_Etag + } + return "" +} + +func (x *LookupResponse) SetFound(v bool) { + x.xxx_hidden_Found = v +} + +func (x *LookupResponse) SetEtag(v string) { + x.xxx_hidden_Etag = v +} + +type LookupResponse_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // False means no prior entry exists for (row_kind, scope_hash): the + // connector must fetch fresh and must not emit SourceCacheReplay for + // this scope. + Found bool + // The opaque validator the previous sync recorded for this scope (HTTP + // ETag, delta token, ...). Empty when found is false. The cap is a + // sanity bound sized for Microsoft Graph delta tokens, which are known + // to run to thousands of characters; storage imposes no limit. + Etag string +} + +func (b0 LookupResponse_builder) Build() *LookupResponse { + m0 := &LookupResponse{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_Found = b.Found + x.xxx_hidden_Etag = b.Etag + return m0 +} + +var File_c1_connectorapi_baton_v1_source_cache_proto protoreflect.FileDescriptor + +const file_c1_connectorapi_baton_v1_source_cache_proto_rawDesc = "" + + "\n" + + "+c1/connectorapi/baton/v1/source_cache.proto\x12\x18c1.connectorapi.baton.v1\x1a\x17validate/validate.proto\"`\n" + + "\rLookupRequest\x12$\n" + + "\brow_kind\x18\x01 \x01(\tB\t\xfaB\x06r\x04\x10\x01\x18@R\arowKind\x12)\n" + + "\n" + + "scope_hash\x18\x02 \x01(\tB\n" + + "\xfaB\ar\x05\x10\x01\x18\x80\x02R\tscopeHash\"E\n" + + "\x0eLookupResponse\x12\x14\n" + + "\x05found\x18\x01 \x01(\bR\x05found\x12\x1d\n" + + "\x04etag\x18\x02 \x01(\tB\t\xfaB\x06r\x04\x18\x80\x80\x04R\x04etag2x\n" + + "\x17BatonSourceCacheService\x12]\n" + + "\x06Lookup\x12'.c1.connectorapi.baton.v1.LookupRequest\x1a(.c1.connectorapi.baton.v1.LookupResponse\"\x00B7Z5gitlab.com/ductone/c1/pkg/pb/c1/connectorapi/baton/v1b\x06proto3" + +var file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_c1_connectorapi_baton_v1_source_cache_proto_goTypes = []any{ + (*LookupRequest)(nil), // 0: c1.connectorapi.baton.v1.LookupRequest + (*LookupResponse)(nil), // 1: c1.connectorapi.baton.v1.LookupResponse +} +var file_c1_connectorapi_baton_v1_source_cache_proto_depIdxs = []int32{ + 0, // 0: c1.connectorapi.baton.v1.BatonSourceCacheService.Lookup:input_type -> c1.connectorapi.baton.v1.LookupRequest + 1, // 1: c1.connectorapi.baton.v1.BatonSourceCacheService.Lookup:output_type -> c1.connectorapi.baton.v1.LookupResponse + 1, // [1:2] is the sub-list for method output_type + 0, // [0:1] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_c1_connectorapi_baton_v1_source_cache_proto_init() } +func file_c1_connectorapi_baton_v1_source_cache_proto_init() { + if File_c1_connectorapi_baton_v1_source_cache_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_connectorapi_baton_v1_source_cache_proto_rawDesc), len(file_c1_connectorapi_baton_v1_source_cache_proto_rawDesc)), + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_c1_connectorapi_baton_v1_source_cache_proto_goTypes, + DependencyIndexes: file_c1_connectorapi_baton_v1_source_cache_proto_depIdxs, + MessageInfos: file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes, + }.Build() + File_c1_connectorapi_baton_v1_source_cache_proto = out.File + file_c1_connectorapi_baton_v1_source_cache_proto_goTypes = nil + file_c1_connectorapi_baton_v1_source_cache_proto_depIdxs = nil +} diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records.pb.go index 312f32a0..29552e74 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records.pb.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records.pb.go @@ -455,8 +455,11 @@ type ResourceRecord struct { Parent *ResourceRef `protobuf:"bytes,6,opt,name=parent,proto3" json:"parent,omitempty"` Annotations []*anypb.Any `protobuf:"bytes,7,rep,name=annotations,proto3" json:"annotations,omitempty"` DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,8,opt,name=discovered_at,json=discoveredAt,proto3" json:"discovered_at,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope. + SourceScopeHash string `protobuf:"bytes,9,opt,name=source_scope_hash,json=sourceScopeHash,proto3" json:"source_scope_hash,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ResourceRecord) Reset() { @@ -533,6 +536,13 @@ func (x *ResourceRecord) GetDiscoveredAt() *timestamppb.Timestamp { return nil } +func (x *ResourceRecord) GetSourceScopeHash() string { + if x != nil { + return x.SourceScopeHash + } + return "" +} + func (x *ResourceRecord) SetResourceTypeId(v string) { x.ResourceTypeId = v } @@ -561,6 +571,10 @@ func (x *ResourceRecord) SetDiscoveredAt(v *timestamppb.Timestamp) { x.DiscoveredAt = v } +func (x *ResourceRecord) SetSourceScopeHash(v string) { + x.SourceScopeHash = v +} + func (x *ResourceRecord) HasParent() bool { if x == nil { return false @@ -593,6 +607,9 @@ type ResourceRecord_builder struct { Parent *ResourceRef Annotations []*anypb.Any DiscoveredAt *timestamppb.Timestamp + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope. + SourceScopeHash string } func (b0 ResourceRecord_builder) Build() *ResourceRecord { @@ -606,6 +623,7 @@ func (b0 ResourceRecord_builder) Build() *ResourceRecord { x.Parent = b.Parent x.Annotations = b.Annotations x.DiscoveredAt = b.DiscoveredAt + x.SourceScopeHash = b.SourceScopeHash return m0 } @@ -632,8 +650,11 @@ type EntitlementRecord struct { // resource_types table when needed. Consumed by // pkg/sync/syncer.go's principal-type narrowing. GrantableToResourceTypeIds []string `protobuf:"bytes,10,rep,name=grantable_to_resource_type_ids,json=grantableToResourceTypeIds,proto3" json:"grantable_to_resource_type_ids,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope. + SourceScopeHash string `protobuf:"bytes,11,opt,name=source_scope_hash,json=sourceScopeHash,proto3" json:"source_scope_hash,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *EntitlementRecord) Reset() { @@ -724,6 +745,13 @@ func (x *EntitlementRecord) GetGrantableToResourceTypeIds() []string { return nil } +func (x *EntitlementRecord) GetSourceScopeHash() string { + if x != nil { + return x.SourceScopeHash + } + return "" +} + func (x *EntitlementRecord) SetExternalId(v string) { x.ExternalId = v } @@ -760,6 +788,10 @@ func (x *EntitlementRecord) SetGrantableToResourceTypeIds(v []string) { x.GrantableToResourceTypeIds = v } +func (x *EntitlementRecord) SetSourceScopeHash(v string) { + x.SourceScopeHash = v +} + func (x *EntitlementRecord) HasResource() bool { if x == nil { return false @@ -806,6 +838,9 @@ type EntitlementRecord_builder struct { // resource_types table when needed. Consumed by // pkg/sync/syncer.go's principal-type narrowing. GrantableToResourceTypeIds []string + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope. + SourceScopeHash string } func (b0 EntitlementRecord_builder) Build() *EntitlementRecord { @@ -821,6 +856,7 @@ func (b0 EntitlementRecord_builder) Build() *EntitlementRecord { x.DiscoveredAt = b.DiscoveredAt x.Slug = b.Slug x.GrantableToResourceTypeIds = b.GrantableToResourceTypeIds + x.SourceScopeHash = b.SourceScopeHash return m0 } @@ -846,9 +882,15 @@ type GrantRecord struct { Annotations []*anypb.Any `protobuf:"bytes,8,rep,name=annotations,proto3" json:"annotations,omitempty"` // map — same wire shape as // c1.connector.v2.GrantSources.sources. - Sources map[string]*GrantSourceRecord `protobuf:"bytes,9,rep,name=sources,proto3" json:"sources,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + Sources map[string]*GrantSourceRecord `protobuf:"bytes,9,rep,name=sources,proto3" json:"sources,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope — notably + // expander-derived grants, which are recreated by expansion each sync + // and never replayed. StoreExpandedGrants preserves this field on + // rewrites of existing rows, exactly like expansion/needs_expansion. + SourceScopeHash string `protobuf:"bytes,10,opt,name=source_scope_hash,json=sourceScopeHash,proto3" json:"source_scope_hash,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *GrantRecord) Reset() { @@ -932,6 +974,13 @@ func (x *GrantRecord) GetSources() map[string]*GrantSourceRecord { return nil } +func (x *GrantRecord) GetSourceScopeHash() string { + if x != nil { + return x.SourceScopeHash + } + return "" +} + func (x *GrantRecord) SetExternalId(v string) { x.ExternalId = v } @@ -964,6 +1013,10 @@ func (x *GrantRecord) SetSources(v map[string]*GrantSourceRecord) { x.Sources = v } +func (x *GrantRecord) SetSourceScopeHash(v string) { + x.SourceScopeHash = v +} + func (x *GrantRecord) HasEntitlement() bool { if x == nil { return false @@ -1032,6 +1085,12 @@ type GrantRecord_builder struct { // map — same wire shape as // c1.connector.v2.GrantSources.sources. Sources map[string]*GrantSourceRecord + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope — notably + // expander-derived grants, which are recreated by expansion each sync + // and never replayed. StoreExpandedGrants preserves this field on + // rewrites of existing rows, exactly like expansion/needs_expansion. + SourceScopeHash string } func (b0 GrantRecord_builder) Build() *GrantRecord { @@ -1046,6 +1105,7 @@ func (b0 GrantRecord_builder) Build() *GrantRecord { x.NeedsExpansion = b.NeedsExpansion x.Annotations = b.Annotations x.Sources = b.Sources + x.SourceScopeHash = b.SourceScopeHash return m0 } @@ -1653,6 +1713,130 @@ func (b0 SessionRecord_builder) Build() *SessionRecord { return m0 } +// SourceCacheEntryRecord is the per-scope manifest entry for source-cache +// replay (see c1/connector/v2/annotation_source_cache.proto). One entry +// per (row_kind, scope_hash), written for every freshly fetched scope — +// including zero-row responses — and rewritten on replay with the scope's +// current validator. The previous sync's entries (read from the previous +// c1z) are the lookup surface connectors revalidate against. +type SourceCacheEntryRecord struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + // Row kind partition: "resources", "entitlements", or "grants" + // (pkg/sourcecache.RowKind values). + RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3" json:"row_kind,omitempty"` + // Connector-computed scope hash (lowercase hex). + ScopeHash string `protobuf:"bytes,2,opt,name=scope_hash,json=scopeHash,proto3" json:"scope_hash,omitempty"` + // Opaque upstream validator: HTTP ETag, delta token, etc. + Etag string `protobuf:"bytes,3,opt,name=etag,proto3" json:"etag,omitempty"` + DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=discovered_at,json=discoveredAt,proto3" json:"discovered_at,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheEntryRecord) Reset() { + *x = SourceCacheEntryRecord{} + mi := &file_c1_storage_v3_records_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheEntryRecord) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheEntryRecord) ProtoMessage() {} + +func (x *SourceCacheEntryRecord) ProtoReflect() protoreflect.Message { + mi := &file_c1_storage_v3_records_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheEntryRecord) GetRowKind() string { + if x != nil { + return x.RowKind + } + return "" +} + +func (x *SourceCacheEntryRecord) GetScopeHash() string { + if x != nil { + return x.ScopeHash + } + return "" +} + +func (x *SourceCacheEntryRecord) GetEtag() string { + if x != nil { + return x.Etag + } + return "" +} + +func (x *SourceCacheEntryRecord) GetDiscoveredAt() *timestamppb.Timestamp { + if x != nil { + return x.DiscoveredAt + } + return nil +} + +func (x *SourceCacheEntryRecord) SetRowKind(v string) { + x.RowKind = v +} + +func (x *SourceCacheEntryRecord) SetScopeHash(v string) { + x.ScopeHash = v +} + +func (x *SourceCacheEntryRecord) SetEtag(v string) { + x.Etag = v +} + +func (x *SourceCacheEntryRecord) SetDiscoveredAt(v *timestamppb.Timestamp) { + x.DiscoveredAt = v +} + +func (x *SourceCacheEntryRecord) HasDiscoveredAt() bool { + if x == nil { + return false + } + return x.DiscoveredAt != nil +} + +func (x *SourceCacheEntryRecord) ClearDiscoveredAt() { + x.DiscoveredAt = nil +} + +type SourceCacheEntryRecord_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Row kind partition: "resources", "entitlements", or "grants" + // (pkg/sourcecache.RowKind values). + RowKind string + // Connector-computed scope hash (lowercase hex). + ScopeHash string + // Opaque upstream validator: HTTP ETag, delta token, etc. + Etag string + DiscoveredAt *timestamppb.Timestamp +} + +func (b0 SourceCacheEntryRecord_builder) Build() *SourceCacheEntryRecord { + m0 := &SourceCacheEntryRecord{} + b, x := &b0, m0 + _, _ = b, x + x.RowKind = b.RowKind + x.ScopeHash = b.ScopeHash + x.Etag = b.Etag + x.DiscoveredAt = b.DiscoveredAt + return m0 +} + var File_c1_storage_v3_records_proto protoreflect.FileDescriptor const file_c1_storage_v3_records_proto_rawDesc = "" + @@ -1677,7 +1861,7 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\rdiscovered_at\x18\x06 \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt\x12 \n" + "\vdescription\x18\a \x01(\tR\vdescription\x12-\n" + "\x12sourced_externally\x18\b \x01(\bR\x11sourcedExternally:!\x82\xf9+\x1d\n" + - "\x0eresource_types\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\xbc\x03\n" + + "\x0eresource_types\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\x98\x04\n" + "\x0eResourceRecord\x12(\n" + "\x10resource_type_id\x18\x02 \x01(\tR\x0eresourceTypeId\x12\x1f\n" + "\vresource_id\x18\x03 \x01(\tR\n" + @@ -1687,8 +1871,10 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\x06parent\x18\x06 \x01(\v2\x1a.c1.storage.v3.ResourceRefB.\x8a\xf9+*\n" + "\tby_parent\x1a\x10resource_type_id\x1a\vresource_idR\x06parent\x126\n" + "\vannotations\x18\a \x03(\v2\x14.google.protobuf.AnyR\vannotations\x12?\n" + - "\rdiscovered_at\x18\b \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt:.\x82\xf9+*\n" + - "\tresources\x12\x10resource_type_id\x12\vresource_idJ\x04\b\x01\x10\x02R\async_id\"\xfe\x03\n" + + "\rdiscovered_at\x18\b \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt\x12Z\n" + + "\x11source_scope_hash\x18\t \x01(\tB.\x8a\xf9+*\n" + + "\x0fby_source_scope\"\x17source_scope_hash != ''R\x0fsourceScopeHash:.\x82\xf9+*\n" + + "\tresources\x12\x10resource_type_id\x12\vresource_idJ\x04\b\x01\x10\x02R\async_id\"\xda\x04\n" + "\x11EntitlementRecord\x12\x1f\n" + "\vexternal_id\x18\x02 \x01(\tR\n" + "externalId\x12h\n" + @@ -1701,8 +1887,10 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\rdiscovered_at\x18\b \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt\x12\x12\n" + "\x04slug\x18\t \x01(\tR\x04slug\x12B\n" + "\x1egrantable_to_resource_type_ids\x18\n" + - " \x03(\tR\x1agrantableToResourceTypeIds:\x1f\x82\xf9+\x1b\n" + - "\fentitlements\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\x9a\x06\n" + + " \x03(\tR\x1agrantableToResourceTypeIds\x12Z\n" + + "\x11source_scope_hash\x18\v \x01(\tB.\x8a\xf9+*\n" + + "\x0fby_source_scope\"\x17source_scope_hash != ''R\x0fsourceScopeHash:\x1f\x82\xf9+\x1b\n" + + "\fentitlements\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\xf6\x06\n" + "\vGrantRecord\x12\x1f\n" + "\vexternal_id\x18\x02 \x01(\tR\n" + "externalId\x12\x98\x01\n" + @@ -1715,7 +1903,10 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\x0fneeds_expansion\x18\a \x01(\bB0\x8a\xf9+,\n" + "\x12by_needs_expansion\"\x16needs_expansion = trueR\x0eneedsExpansion\x126\n" + "\vannotations\x18\b \x03(\v2\x14.google.protobuf.AnyR\vannotations\x12A\n" + - "\asources\x18\t \x03(\v2'.c1.storage.v3.GrantRecord.SourcesEntryR\asources\x1a\\\n" + + "\asources\x18\t \x03(\v2'.c1.storage.v3.GrantRecord.SourcesEntryR\asources\x12Z\n" + + "\x11source_scope_hash\x18\n" + + " \x01(\tB.\x8a\xf9+*\n" + + "\x0fby_source_scope\"\x17source_scope_hash != ''R\x0fsourceScopeHash\x1a\\\n" + "\fSourcesEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x126\n" + "\x05value\x18\x02 \x01(\v2 .c1.storage.v3.GrantSourceRecordR\x05value:\x028\x01:\x19\x82\xf9+\x15\n" + @@ -1765,7 +1956,15 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\async_id\x18\x01 \x01(\tR\x06syncId\x12\x10\n" + "\x03key\x18\x02 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x03 \x01(\fR\x05value:\x1c\x82\xf9+\x18\n" + - "\bsessions\x12\async_id\x12\x03key*\xae\x01\n" + + "\bsessions\x12\async_id\x12\x03key\"\xd9\x01\n" + + "\x16SourceCacheEntryRecord\x12\x19\n" + + "\brow_kind\x18\x01 \x01(\tR\arowKind\x12\x1d\n" + + "\n" + + "scope_hash\x18\x02 \x01(\tR\tscopeHash\x12\x12\n" + + "\x04etag\x18\x03 \x01(\tR\x04etag\x12?\n" + + "\rdiscovered_at\x18\x04 \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt:0\x82\xf9+,\n" + + "\x14source_cache_entries\x12\brow_kind\x12\n" + + "scope_hash*\xae\x01\n" + "\bSyncType\x12\x19\n" + "\x15SYNC_TYPE_UNSPECIFIED\x10\x00\x12\x12\n" + "\x0eSYNC_TYPE_FULL\x10\x01\x12\x15\n" + @@ -1775,58 +1974,60 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\x1bSYNC_TYPE_PARTIAL_DELETIONS\x10\x05B4Z2github.com/conductorone/baton-sdk/pb/c1/storage/v3b\x06proto3" var file_c1_storage_v3_records_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_c1_storage_v3_records_proto_msgTypes = make([]protoimpl.MessageInfo, 14) +var file_c1_storage_v3_records_proto_msgTypes = make([]protoimpl.MessageInfo, 15) var file_c1_storage_v3_records_proto_goTypes = []any{ - (SyncType)(0), // 0: c1.storage.v3.SyncType - (*GrantExpandableRecord)(nil), // 1: c1.storage.v3.GrantExpandableRecord - (*GrantSourceRecord)(nil), // 2: c1.storage.v3.GrantSourceRecord - (*ResourceTypeRecord)(nil), // 3: c1.storage.v3.ResourceTypeRecord - (*ResourceRecord)(nil), // 4: c1.storage.v3.ResourceRecord - (*EntitlementRecord)(nil), // 5: c1.storage.v3.EntitlementRecord - (*GrantRecord)(nil), // 6: c1.storage.v3.GrantRecord - (*AssetRecord)(nil), // 7: c1.storage.v3.AssetRecord - (*SyncRunRecord)(nil), // 8: c1.storage.v3.SyncRunRecord - (*SyncStatsRecord)(nil), // 9: c1.storage.v3.SyncStatsRecord - (*SessionRecord)(nil), // 10: c1.storage.v3.SessionRecord - nil, // 11: c1.storage.v3.GrantRecord.SourcesEntry - nil, // 12: c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry - nil, // 13: c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry - nil, // 14: c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry - (*anypb.Any)(nil), // 15: google.protobuf.Any - (*timestamppb.Timestamp)(nil), // 16: google.protobuf.Timestamp - (*ResourceRef)(nil), // 17: c1.storage.v3.ResourceRef - (*EntitlementRef)(nil), // 18: c1.storage.v3.EntitlementRef - (*PrincipalRef)(nil), // 19: c1.storage.v3.PrincipalRef + (SyncType)(0), // 0: c1.storage.v3.SyncType + (*GrantExpandableRecord)(nil), // 1: c1.storage.v3.GrantExpandableRecord + (*GrantSourceRecord)(nil), // 2: c1.storage.v3.GrantSourceRecord + (*ResourceTypeRecord)(nil), // 3: c1.storage.v3.ResourceTypeRecord + (*ResourceRecord)(nil), // 4: c1.storage.v3.ResourceRecord + (*EntitlementRecord)(nil), // 5: c1.storage.v3.EntitlementRecord + (*GrantRecord)(nil), // 6: c1.storage.v3.GrantRecord + (*AssetRecord)(nil), // 7: c1.storage.v3.AssetRecord + (*SyncRunRecord)(nil), // 8: c1.storage.v3.SyncRunRecord + (*SyncStatsRecord)(nil), // 9: c1.storage.v3.SyncStatsRecord + (*SessionRecord)(nil), // 10: c1.storage.v3.SessionRecord + (*SourceCacheEntryRecord)(nil), // 11: c1.storage.v3.SourceCacheEntryRecord + nil, // 12: c1.storage.v3.GrantRecord.SourcesEntry + nil, // 13: c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry + nil, // 14: c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry + nil, // 15: c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry + (*anypb.Any)(nil), // 16: google.protobuf.Any + (*timestamppb.Timestamp)(nil), // 17: google.protobuf.Timestamp + (*ResourceRef)(nil), // 18: c1.storage.v3.ResourceRef + (*EntitlementRef)(nil), // 19: c1.storage.v3.EntitlementRef + (*PrincipalRef)(nil), // 20: c1.storage.v3.PrincipalRef } var file_c1_storage_v3_records_proto_depIdxs = []int32{ - 15, // 0: c1.storage.v3.ResourceTypeRecord.annotations:type_name -> google.protobuf.Any - 16, // 1: c1.storage.v3.ResourceTypeRecord.discovered_at:type_name -> google.protobuf.Timestamp - 17, // 2: c1.storage.v3.ResourceRecord.parent:type_name -> c1.storage.v3.ResourceRef - 15, // 3: c1.storage.v3.ResourceRecord.annotations:type_name -> google.protobuf.Any - 16, // 4: c1.storage.v3.ResourceRecord.discovered_at:type_name -> google.protobuf.Timestamp - 17, // 5: c1.storage.v3.EntitlementRecord.resource:type_name -> c1.storage.v3.ResourceRef - 15, // 6: c1.storage.v3.EntitlementRecord.annotations:type_name -> google.protobuf.Any - 16, // 7: c1.storage.v3.EntitlementRecord.discovered_at:type_name -> google.protobuf.Timestamp - 18, // 8: c1.storage.v3.GrantRecord.entitlement:type_name -> c1.storage.v3.EntitlementRef - 19, // 9: c1.storage.v3.GrantRecord.principal:type_name -> c1.storage.v3.PrincipalRef - 16, // 10: c1.storage.v3.GrantRecord.discovered_at:type_name -> google.protobuf.Timestamp + 16, // 0: c1.storage.v3.ResourceTypeRecord.annotations:type_name -> google.protobuf.Any + 17, // 1: c1.storage.v3.ResourceTypeRecord.discovered_at:type_name -> google.protobuf.Timestamp + 18, // 2: c1.storage.v3.ResourceRecord.parent:type_name -> c1.storage.v3.ResourceRef + 16, // 3: c1.storage.v3.ResourceRecord.annotations:type_name -> google.protobuf.Any + 17, // 4: c1.storage.v3.ResourceRecord.discovered_at:type_name -> google.protobuf.Timestamp + 18, // 5: c1.storage.v3.EntitlementRecord.resource:type_name -> c1.storage.v3.ResourceRef + 16, // 6: c1.storage.v3.EntitlementRecord.annotations:type_name -> google.protobuf.Any + 17, // 7: c1.storage.v3.EntitlementRecord.discovered_at:type_name -> google.protobuf.Timestamp + 19, // 8: c1.storage.v3.GrantRecord.entitlement:type_name -> c1.storage.v3.EntitlementRef + 20, // 9: c1.storage.v3.GrantRecord.principal:type_name -> c1.storage.v3.PrincipalRef + 17, // 10: c1.storage.v3.GrantRecord.discovered_at:type_name -> google.protobuf.Timestamp 1, // 11: c1.storage.v3.GrantRecord.expansion:type_name -> c1.storage.v3.GrantExpandableRecord - 15, // 12: c1.storage.v3.GrantRecord.annotations:type_name -> google.protobuf.Any - 11, // 13: c1.storage.v3.GrantRecord.sources:type_name -> c1.storage.v3.GrantRecord.SourcesEntry - 16, // 14: c1.storage.v3.AssetRecord.discovered_at:type_name -> google.protobuf.Timestamp + 16, // 12: c1.storage.v3.GrantRecord.annotations:type_name -> google.protobuf.Any + 12, // 13: c1.storage.v3.GrantRecord.sources:type_name -> c1.storage.v3.GrantRecord.SourcesEntry + 17, // 14: c1.storage.v3.AssetRecord.discovered_at:type_name -> google.protobuf.Timestamp 0, // 15: c1.storage.v3.SyncRunRecord.type:type_name -> c1.storage.v3.SyncType - 16, // 16: c1.storage.v3.SyncRunRecord.started_at:type_name -> google.protobuf.Timestamp - 16, // 17: c1.storage.v3.SyncRunRecord.ended_at:type_name -> google.protobuf.Timestamp - 12, // 18: c1.storage.v3.SyncStatsRecord.resources_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry - 13, // 19: c1.storage.v3.SyncStatsRecord.grants_by_entitlement_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry - 14, // 20: c1.storage.v3.SyncStatsRecord.entitlements_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry - 16, // 21: c1.storage.v3.SyncStatsRecord.written_at:type_name -> google.protobuf.Timestamp - 2, // 22: c1.storage.v3.GrantRecord.SourcesEntry.value:type_name -> c1.storage.v3.GrantSourceRecord - 23, // [23:23] is the sub-list for method output_type - 23, // [23:23] is the sub-list for method input_type - 23, // [23:23] is the sub-list for extension type_name - 23, // [23:23] is the sub-list for extension extendee - 0, // [0:23] is the sub-list for field type_name + 17, // 16: c1.storage.v3.SyncRunRecord.started_at:type_name -> google.protobuf.Timestamp + 17, // 17: c1.storage.v3.SyncRunRecord.ended_at:type_name -> google.protobuf.Timestamp + 13, // 18: c1.storage.v3.SyncStatsRecord.resources_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry + 14, // 19: c1.storage.v3.SyncStatsRecord.grants_by_entitlement_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry + 15, // 20: c1.storage.v3.SyncStatsRecord.entitlements_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry + 17, // 21: c1.storage.v3.SyncStatsRecord.written_at:type_name -> google.protobuf.Timestamp + 17, // 22: c1.storage.v3.SourceCacheEntryRecord.discovered_at:type_name -> google.protobuf.Timestamp + 2, // 23: c1.storage.v3.GrantRecord.SourcesEntry.value:type_name -> c1.storage.v3.GrantSourceRecord + 24, // [24:24] is the sub-list for method output_type + 24, // [24:24] is the sub-list for method input_type + 24, // [24:24] is the sub-list for extension type_name + 24, // [24:24] is the sub-list for extension extendee + 0, // [0:24] is the sub-list for field type_name } func init() { file_c1_storage_v3_records_proto_init() } @@ -1842,7 +2043,7 @@ func file_c1_storage_v3_records_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_storage_v3_records_proto_rawDesc), len(file_c1_storage_v3_records_proto_rawDesc)), NumEnums: 1, - NumMessages: 14, + NumMessages: 15, NumExtensions: 0, NumServices: 0, }, diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records.pb.validate.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records.pb.validate.go index 29287724..0c0b299e 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records.pb.validate.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records.pb.validate.go @@ -544,6 +544,8 @@ func (m *ResourceRecord) validate(all bool) error { } } + // no validation rules for SourceScopeHash + if len(errors) > 0 { return ResourceRecordMultiError(errors) } @@ -746,6 +748,8 @@ func (m *EntitlementRecord) validate(all bool) error { // no validation rules for Slug + // no validation rules for SourceScopeHash + if len(errors) > 0 { return EntitlementRecordMultiError(errors) } @@ -1048,6 +1052,8 @@ func (m *GrantRecord) validate(all bool) error { } } + // no validation rules for SourceScopeHash + if len(errors) > 0 { return GrantRecordMultiError(errors) } @@ -1683,3 +1689,140 @@ var _ interface { Cause() error ErrorName() string } = SessionRecordValidationError{} + +// Validate checks the field values on SourceCacheEntryRecord with the rules +// defined in the proto definition for this message. If any rules are +// violated, the first error encountered is returned, or nil if there are no violations. +func (m *SourceCacheEntryRecord) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SourceCacheEntryRecord with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// SourceCacheEntryRecordMultiError, or nil if none found. +func (m *SourceCacheEntryRecord) ValidateAll() error { + return m.validate(true) +} + +func (m *SourceCacheEntryRecord) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + // no validation rules for RowKind + + // no validation rules for ScopeHash + + // no validation rules for Etag + + if all { + switch v := interface{}(m.GetDiscoveredAt()).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, SourceCacheEntryRecordValidationError{ + field: "DiscoveredAt", + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, SourceCacheEntryRecordValidationError{ + field: "DiscoveredAt", + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(m.GetDiscoveredAt()).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return SourceCacheEntryRecordValidationError{ + field: "DiscoveredAt", + reason: "embedded message failed validation", + cause: err, + } + } + } + + if len(errors) > 0 { + return SourceCacheEntryRecordMultiError(errors) + } + + return nil +} + +// SourceCacheEntryRecordMultiError is an error wrapping multiple validation +// errors returned by SourceCacheEntryRecord.ValidateAll() if the designated +// constraints aren't met. +type SourceCacheEntryRecordMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SourceCacheEntryRecordMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SourceCacheEntryRecordMultiError) AllErrors() []error { return m } + +// SourceCacheEntryRecordValidationError is the validation error returned by +// SourceCacheEntryRecord.Validate if the designated constraints aren't met. +type SourceCacheEntryRecordValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SourceCacheEntryRecordValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SourceCacheEntryRecordValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SourceCacheEntryRecordValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SourceCacheEntryRecordValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SourceCacheEntryRecordValidationError) ErrorName() string { + return "SourceCacheEntryRecordValidationError" +} + +// Error satisfies the builtin error interface +func (e SourceCacheEntryRecordValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSourceCacheEntryRecord.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SourceCacheEntryRecordValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SourceCacheEntryRecordValidationError{} diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records_protoopaque.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records_protoopaque.pb.go index bf2acc8c..797ad824 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records_protoopaque.pb.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records_protoopaque.pb.go @@ -441,16 +441,17 @@ func (b0 ResourceTypeRecord_builder) Build() *ResourceTypeRecord { } type ResourceRecord struct { - state protoimpl.MessageState `protogen:"opaque.v1"` - xxx_hidden_ResourceTypeId string `protobuf:"bytes,2,opt,name=resource_type_id,json=resourceTypeId,proto3"` - xxx_hidden_ResourceId string `protobuf:"bytes,3,opt,name=resource_id,json=resourceId,proto3"` - xxx_hidden_DisplayName string `protobuf:"bytes,4,opt,name=display_name,json=displayName,proto3"` - xxx_hidden_Description string `protobuf:"bytes,5,opt,name=description,proto3"` - xxx_hidden_Parent *ResourceRef `protobuf:"bytes,6,opt,name=parent,proto3"` - xxx_hidden_Annotations *[]*anypb.Any `protobuf:"bytes,7,rep,name=annotations,proto3"` - xxx_hidden_DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,8,opt,name=discovered_at,json=discoveredAt,proto3"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_ResourceTypeId string `protobuf:"bytes,2,opt,name=resource_type_id,json=resourceTypeId,proto3"` + xxx_hidden_ResourceId string `protobuf:"bytes,3,opt,name=resource_id,json=resourceId,proto3"` + xxx_hidden_DisplayName string `protobuf:"bytes,4,opt,name=display_name,json=displayName,proto3"` + xxx_hidden_Description string `protobuf:"bytes,5,opt,name=description,proto3"` + xxx_hidden_Parent *ResourceRef `protobuf:"bytes,6,opt,name=parent,proto3"` + xxx_hidden_Annotations *[]*anypb.Any `protobuf:"bytes,7,rep,name=annotations,proto3"` + xxx_hidden_DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,8,opt,name=discovered_at,json=discoveredAt,proto3"` + xxx_hidden_SourceScopeHash string `protobuf:"bytes,9,opt,name=source_scope_hash,json=sourceScopeHash,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ResourceRecord) Reset() { @@ -529,6 +530,13 @@ func (x *ResourceRecord) GetDiscoveredAt() *timestamppb.Timestamp { return nil } +func (x *ResourceRecord) GetSourceScopeHash() string { + if x != nil { + return x.xxx_hidden_SourceScopeHash + } + return "" +} + func (x *ResourceRecord) SetResourceTypeId(v string) { x.xxx_hidden_ResourceTypeId = v } @@ -557,6 +565,10 @@ func (x *ResourceRecord) SetDiscoveredAt(v *timestamppb.Timestamp) { x.xxx_hidden_DiscoveredAt = v } +func (x *ResourceRecord) SetSourceScopeHash(v string) { + x.xxx_hidden_SourceScopeHash = v +} + func (x *ResourceRecord) HasParent() bool { if x == nil { return false @@ -589,6 +601,9 @@ type ResourceRecord_builder struct { Parent *ResourceRef Annotations []*anypb.Any DiscoveredAt *timestamppb.Timestamp + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope. + SourceScopeHash string } func (b0 ResourceRecord_builder) Build() *ResourceRecord { @@ -602,6 +617,7 @@ func (b0 ResourceRecord_builder) Build() *ResourceRecord { x.xxx_hidden_Parent = b.Parent x.xxx_hidden_Annotations = &b.Annotations x.xxx_hidden_DiscoveredAt = b.DiscoveredAt + x.xxx_hidden_SourceScopeHash = b.SourceScopeHash return m0 } @@ -616,6 +632,7 @@ type EntitlementRecord struct { xxx_hidden_DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,8,opt,name=discovered_at,json=discoveredAt,proto3"` xxx_hidden_Slug string `protobuf:"bytes,9,opt,name=slug,proto3"` xxx_hidden_GrantableToResourceTypeIds []string `protobuf:"bytes,10,rep,name=grantable_to_resource_type_ids,json=grantableToResourceTypeIds,proto3"` + xxx_hidden_SourceScopeHash string `protobuf:"bytes,11,opt,name=source_scope_hash,json=sourceScopeHash,proto3"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -710,6 +727,13 @@ func (x *EntitlementRecord) GetGrantableToResourceTypeIds() []string { return nil } +func (x *EntitlementRecord) GetSourceScopeHash() string { + if x != nil { + return x.xxx_hidden_SourceScopeHash + } + return "" +} + func (x *EntitlementRecord) SetExternalId(v string) { x.xxx_hidden_ExternalId = v } @@ -746,6 +770,10 @@ func (x *EntitlementRecord) SetGrantableToResourceTypeIds(v []string) { x.xxx_hidden_GrantableToResourceTypeIds = v } +func (x *EntitlementRecord) SetSourceScopeHash(v string) { + x.xxx_hidden_SourceScopeHash = v +} + func (x *EntitlementRecord) HasResource() bool { if x == nil { return false @@ -792,6 +820,9 @@ type EntitlementRecord_builder struct { // resource_types table when needed. Consumed by // pkg/sync/syncer.go's principal-type narrowing. GrantableToResourceTypeIds []string + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope. + SourceScopeHash string } func (b0 EntitlementRecord_builder) Build() *EntitlementRecord { @@ -807,21 +838,23 @@ func (b0 EntitlementRecord_builder) Build() *EntitlementRecord { x.xxx_hidden_DiscoveredAt = b.DiscoveredAt x.xxx_hidden_Slug = b.Slug x.xxx_hidden_GrantableToResourceTypeIds = b.GrantableToResourceTypeIds + x.xxx_hidden_SourceScopeHash = b.SourceScopeHash return m0 } type GrantRecord struct { - state protoimpl.MessageState `protogen:"opaque.v1"` - xxx_hidden_ExternalId string `protobuf:"bytes,2,opt,name=external_id,json=externalId,proto3"` - xxx_hidden_Entitlement *EntitlementRef `protobuf:"bytes,3,opt,name=entitlement,proto3"` - xxx_hidden_Principal *PrincipalRef `protobuf:"bytes,4,opt,name=principal,proto3"` - xxx_hidden_DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=discovered_at,json=discoveredAt,proto3"` - xxx_hidden_Expansion *GrantExpandableRecord `protobuf:"bytes,6,opt,name=expansion,proto3"` - xxx_hidden_NeedsExpansion bool `protobuf:"varint,7,opt,name=needs_expansion,json=needsExpansion,proto3"` - xxx_hidden_Annotations *[]*anypb.Any `protobuf:"bytes,8,rep,name=annotations,proto3"` - xxx_hidden_Sources map[string]*GrantSourceRecord `protobuf:"bytes,9,rep,name=sources,proto3" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_ExternalId string `protobuf:"bytes,2,opt,name=external_id,json=externalId,proto3"` + xxx_hidden_Entitlement *EntitlementRef `protobuf:"bytes,3,opt,name=entitlement,proto3"` + xxx_hidden_Principal *PrincipalRef `protobuf:"bytes,4,opt,name=principal,proto3"` + xxx_hidden_DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=discovered_at,json=discoveredAt,proto3"` + xxx_hidden_Expansion *GrantExpandableRecord `protobuf:"bytes,6,opt,name=expansion,proto3"` + xxx_hidden_NeedsExpansion bool `protobuf:"varint,7,opt,name=needs_expansion,json=needsExpansion,proto3"` + xxx_hidden_Annotations *[]*anypb.Any `protobuf:"bytes,8,rep,name=annotations,proto3"` + xxx_hidden_Sources map[string]*GrantSourceRecord `protobuf:"bytes,9,rep,name=sources,proto3" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + xxx_hidden_SourceScopeHash string `protobuf:"bytes,10,opt,name=source_scope_hash,json=sourceScopeHash,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *GrantRecord) Reset() { @@ -907,6 +940,13 @@ func (x *GrantRecord) GetSources() map[string]*GrantSourceRecord { return nil } +func (x *GrantRecord) GetSourceScopeHash() string { + if x != nil { + return x.xxx_hidden_SourceScopeHash + } + return "" +} + func (x *GrantRecord) SetExternalId(v string) { x.xxx_hidden_ExternalId = v } @@ -939,6 +979,10 @@ func (x *GrantRecord) SetSources(v map[string]*GrantSourceRecord) { x.xxx_hidden_Sources = v } +func (x *GrantRecord) SetSourceScopeHash(v string) { + x.xxx_hidden_SourceScopeHash = v +} + func (x *GrantRecord) HasEntitlement() bool { if x == nil { return false @@ -1007,6 +1051,12 @@ type GrantRecord_builder struct { // map — same wire shape as // c1.connector.v2.GrantSources.sources. Sources map[string]*GrantSourceRecord + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope — notably + // expander-derived grants, which are recreated by expansion each sync + // and never replayed. StoreExpandedGrants preserves this field on + // rewrites of existing rows, exactly like expansion/needs_expansion. + SourceScopeHash string } func (b0 GrantRecord_builder) Build() *GrantRecord { @@ -1021,6 +1071,7 @@ func (b0 GrantRecord_builder) Build() *GrantRecord { x.xxx_hidden_NeedsExpansion = b.NeedsExpansion x.xxx_hidden_Annotations = &b.Annotations x.xxx_hidden_Sources = b.Sources + x.xxx_hidden_SourceScopeHash = b.SourceScopeHash return m0 } @@ -1622,6 +1673,126 @@ func (b0 SessionRecord_builder) Build() *SessionRecord { return m0 } +// SourceCacheEntryRecord is the per-scope manifest entry for source-cache +// replay (see c1/connector/v2/annotation_source_cache.proto). One entry +// per (row_kind, scope_hash), written for every freshly fetched scope — +// including zero-row responses — and rewritten on replay with the scope's +// current validator. The previous sync's entries (read from the previous +// c1z) are the lookup surface connectors revalidate against. +type SourceCacheEntryRecord struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3"` + xxx_hidden_ScopeHash string `protobuf:"bytes,2,opt,name=scope_hash,json=scopeHash,proto3"` + xxx_hidden_Etag string `protobuf:"bytes,3,opt,name=etag,proto3"` + xxx_hidden_DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=discovered_at,json=discoveredAt,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheEntryRecord) Reset() { + *x = SourceCacheEntryRecord{} + mi := &file_c1_storage_v3_records_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheEntryRecord) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheEntryRecord) ProtoMessage() {} + +func (x *SourceCacheEntryRecord) ProtoReflect() protoreflect.Message { + mi := &file_c1_storage_v3_records_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheEntryRecord) GetRowKind() string { + if x != nil { + return x.xxx_hidden_RowKind + } + return "" +} + +func (x *SourceCacheEntryRecord) GetScopeHash() string { + if x != nil { + return x.xxx_hidden_ScopeHash + } + return "" +} + +func (x *SourceCacheEntryRecord) GetEtag() string { + if x != nil { + return x.xxx_hidden_Etag + } + return "" +} + +func (x *SourceCacheEntryRecord) GetDiscoveredAt() *timestamppb.Timestamp { + if x != nil { + return x.xxx_hidden_DiscoveredAt + } + return nil +} + +func (x *SourceCacheEntryRecord) SetRowKind(v string) { + x.xxx_hidden_RowKind = v +} + +func (x *SourceCacheEntryRecord) SetScopeHash(v string) { + x.xxx_hidden_ScopeHash = v +} + +func (x *SourceCacheEntryRecord) SetEtag(v string) { + x.xxx_hidden_Etag = v +} + +func (x *SourceCacheEntryRecord) SetDiscoveredAt(v *timestamppb.Timestamp) { + x.xxx_hidden_DiscoveredAt = v +} + +func (x *SourceCacheEntryRecord) HasDiscoveredAt() bool { + if x == nil { + return false + } + return x.xxx_hidden_DiscoveredAt != nil +} + +func (x *SourceCacheEntryRecord) ClearDiscoveredAt() { + x.xxx_hidden_DiscoveredAt = nil +} + +type SourceCacheEntryRecord_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Row kind partition: "resources", "entitlements", or "grants" + // (pkg/sourcecache.RowKind values). + RowKind string + // Connector-computed scope hash (lowercase hex). + ScopeHash string + // Opaque upstream validator: HTTP ETag, delta token, etc. + Etag string + DiscoveredAt *timestamppb.Timestamp +} + +func (b0 SourceCacheEntryRecord_builder) Build() *SourceCacheEntryRecord { + m0 := &SourceCacheEntryRecord{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_RowKind = b.RowKind + x.xxx_hidden_ScopeHash = b.ScopeHash + x.xxx_hidden_Etag = b.Etag + x.xxx_hidden_DiscoveredAt = b.DiscoveredAt + return m0 +} + var File_c1_storage_v3_records_proto protoreflect.FileDescriptor const file_c1_storage_v3_records_proto_rawDesc = "" + @@ -1646,7 +1817,7 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\rdiscovered_at\x18\x06 \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt\x12 \n" + "\vdescription\x18\a \x01(\tR\vdescription\x12-\n" + "\x12sourced_externally\x18\b \x01(\bR\x11sourcedExternally:!\x82\xf9+\x1d\n" + - "\x0eresource_types\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\xbc\x03\n" + + "\x0eresource_types\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\x98\x04\n" + "\x0eResourceRecord\x12(\n" + "\x10resource_type_id\x18\x02 \x01(\tR\x0eresourceTypeId\x12\x1f\n" + "\vresource_id\x18\x03 \x01(\tR\n" + @@ -1656,8 +1827,10 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\x06parent\x18\x06 \x01(\v2\x1a.c1.storage.v3.ResourceRefB.\x8a\xf9+*\n" + "\tby_parent\x1a\x10resource_type_id\x1a\vresource_idR\x06parent\x126\n" + "\vannotations\x18\a \x03(\v2\x14.google.protobuf.AnyR\vannotations\x12?\n" + - "\rdiscovered_at\x18\b \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt:.\x82\xf9+*\n" + - "\tresources\x12\x10resource_type_id\x12\vresource_idJ\x04\b\x01\x10\x02R\async_id\"\xfe\x03\n" + + "\rdiscovered_at\x18\b \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt\x12Z\n" + + "\x11source_scope_hash\x18\t \x01(\tB.\x8a\xf9+*\n" + + "\x0fby_source_scope\"\x17source_scope_hash != ''R\x0fsourceScopeHash:.\x82\xf9+*\n" + + "\tresources\x12\x10resource_type_id\x12\vresource_idJ\x04\b\x01\x10\x02R\async_id\"\xda\x04\n" + "\x11EntitlementRecord\x12\x1f\n" + "\vexternal_id\x18\x02 \x01(\tR\n" + "externalId\x12h\n" + @@ -1670,8 +1843,10 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\rdiscovered_at\x18\b \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt\x12\x12\n" + "\x04slug\x18\t \x01(\tR\x04slug\x12B\n" + "\x1egrantable_to_resource_type_ids\x18\n" + - " \x03(\tR\x1agrantableToResourceTypeIds:\x1f\x82\xf9+\x1b\n" + - "\fentitlements\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\x9a\x06\n" + + " \x03(\tR\x1agrantableToResourceTypeIds\x12Z\n" + + "\x11source_scope_hash\x18\v \x01(\tB.\x8a\xf9+*\n" + + "\x0fby_source_scope\"\x17source_scope_hash != ''R\x0fsourceScopeHash:\x1f\x82\xf9+\x1b\n" + + "\fentitlements\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\xf6\x06\n" + "\vGrantRecord\x12\x1f\n" + "\vexternal_id\x18\x02 \x01(\tR\n" + "externalId\x12\x98\x01\n" + @@ -1684,7 +1859,10 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\x0fneeds_expansion\x18\a \x01(\bB0\x8a\xf9+,\n" + "\x12by_needs_expansion\"\x16needs_expansion = trueR\x0eneedsExpansion\x126\n" + "\vannotations\x18\b \x03(\v2\x14.google.protobuf.AnyR\vannotations\x12A\n" + - "\asources\x18\t \x03(\v2'.c1.storage.v3.GrantRecord.SourcesEntryR\asources\x1a\\\n" + + "\asources\x18\t \x03(\v2'.c1.storage.v3.GrantRecord.SourcesEntryR\asources\x12Z\n" + + "\x11source_scope_hash\x18\n" + + " \x01(\tB.\x8a\xf9+*\n" + + "\x0fby_source_scope\"\x17source_scope_hash != ''R\x0fsourceScopeHash\x1a\\\n" + "\fSourcesEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x126\n" + "\x05value\x18\x02 \x01(\v2 .c1.storage.v3.GrantSourceRecordR\x05value:\x028\x01:\x19\x82\xf9+\x15\n" + @@ -1734,7 +1912,15 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\async_id\x18\x01 \x01(\tR\x06syncId\x12\x10\n" + "\x03key\x18\x02 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x03 \x01(\fR\x05value:\x1c\x82\xf9+\x18\n" + - "\bsessions\x12\async_id\x12\x03key*\xae\x01\n" + + "\bsessions\x12\async_id\x12\x03key\"\xd9\x01\n" + + "\x16SourceCacheEntryRecord\x12\x19\n" + + "\brow_kind\x18\x01 \x01(\tR\arowKind\x12\x1d\n" + + "\n" + + "scope_hash\x18\x02 \x01(\tR\tscopeHash\x12\x12\n" + + "\x04etag\x18\x03 \x01(\tR\x04etag\x12?\n" + + "\rdiscovered_at\x18\x04 \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt:0\x82\xf9+,\n" + + "\x14source_cache_entries\x12\brow_kind\x12\n" + + "scope_hash*\xae\x01\n" + "\bSyncType\x12\x19\n" + "\x15SYNC_TYPE_UNSPECIFIED\x10\x00\x12\x12\n" + "\x0eSYNC_TYPE_FULL\x10\x01\x12\x15\n" + @@ -1744,58 +1930,60 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\x1bSYNC_TYPE_PARTIAL_DELETIONS\x10\x05B4Z2github.com/conductorone/baton-sdk/pb/c1/storage/v3b\x06proto3" var file_c1_storage_v3_records_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_c1_storage_v3_records_proto_msgTypes = make([]protoimpl.MessageInfo, 14) +var file_c1_storage_v3_records_proto_msgTypes = make([]protoimpl.MessageInfo, 15) var file_c1_storage_v3_records_proto_goTypes = []any{ - (SyncType)(0), // 0: c1.storage.v3.SyncType - (*GrantExpandableRecord)(nil), // 1: c1.storage.v3.GrantExpandableRecord - (*GrantSourceRecord)(nil), // 2: c1.storage.v3.GrantSourceRecord - (*ResourceTypeRecord)(nil), // 3: c1.storage.v3.ResourceTypeRecord - (*ResourceRecord)(nil), // 4: c1.storage.v3.ResourceRecord - (*EntitlementRecord)(nil), // 5: c1.storage.v3.EntitlementRecord - (*GrantRecord)(nil), // 6: c1.storage.v3.GrantRecord - (*AssetRecord)(nil), // 7: c1.storage.v3.AssetRecord - (*SyncRunRecord)(nil), // 8: c1.storage.v3.SyncRunRecord - (*SyncStatsRecord)(nil), // 9: c1.storage.v3.SyncStatsRecord - (*SessionRecord)(nil), // 10: c1.storage.v3.SessionRecord - nil, // 11: c1.storage.v3.GrantRecord.SourcesEntry - nil, // 12: c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry - nil, // 13: c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry - nil, // 14: c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry - (*anypb.Any)(nil), // 15: google.protobuf.Any - (*timestamppb.Timestamp)(nil), // 16: google.protobuf.Timestamp - (*ResourceRef)(nil), // 17: c1.storage.v3.ResourceRef - (*EntitlementRef)(nil), // 18: c1.storage.v3.EntitlementRef - (*PrincipalRef)(nil), // 19: c1.storage.v3.PrincipalRef + (SyncType)(0), // 0: c1.storage.v3.SyncType + (*GrantExpandableRecord)(nil), // 1: c1.storage.v3.GrantExpandableRecord + (*GrantSourceRecord)(nil), // 2: c1.storage.v3.GrantSourceRecord + (*ResourceTypeRecord)(nil), // 3: c1.storage.v3.ResourceTypeRecord + (*ResourceRecord)(nil), // 4: c1.storage.v3.ResourceRecord + (*EntitlementRecord)(nil), // 5: c1.storage.v3.EntitlementRecord + (*GrantRecord)(nil), // 6: c1.storage.v3.GrantRecord + (*AssetRecord)(nil), // 7: c1.storage.v3.AssetRecord + (*SyncRunRecord)(nil), // 8: c1.storage.v3.SyncRunRecord + (*SyncStatsRecord)(nil), // 9: c1.storage.v3.SyncStatsRecord + (*SessionRecord)(nil), // 10: c1.storage.v3.SessionRecord + (*SourceCacheEntryRecord)(nil), // 11: c1.storage.v3.SourceCacheEntryRecord + nil, // 12: c1.storage.v3.GrantRecord.SourcesEntry + nil, // 13: c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry + nil, // 14: c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry + nil, // 15: c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry + (*anypb.Any)(nil), // 16: google.protobuf.Any + (*timestamppb.Timestamp)(nil), // 17: google.protobuf.Timestamp + (*ResourceRef)(nil), // 18: c1.storage.v3.ResourceRef + (*EntitlementRef)(nil), // 19: c1.storage.v3.EntitlementRef + (*PrincipalRef)(nil), // 20: c1.storage.v3.PrincipalRef } var file_c1_storage_v3_records_proto_depIdxs = []int32{ - 15, // 0: c1.storage.v3.ResourceTypeRecord.annotations:type_name -> google.protobuf.Any - 16, // 1: c1.storage.v3.ResourceTypeRecord.discovered_at:type_name -> google.protobuf.Timestamp - 17, // 2: c1.storage.v3.ResourceRecord.parent:type_name -> c1.storage.v3.ResourceRef - 15, // 3: c1.storage.v3.ResourceRecord.annotations:type_name -> google.protobuf.Any - 16, // 4: c1.storage.v3.ResourceRecord.discovered_at:type_name -> google.protobuf.Timestamp - 17, // 5: c1.storage.v3.EntitlementRecord.resource:type_name -> c1.storage.v3.ResourceRef - 15, // 6: c1.storage.v3.EntitlementRecord.annotations:type_name -> google.protobuf.Any - 16, // 7: c1.storage.v3.EntitlementRecord.discovered_at:type_name -> google.protobuf.Timestamp - 18, // 8: c1.storage.v3.GrantRecord.entitlement:type_name -> c1.storage.v3.EntitlementRef - 19, // 9: c1.storage.v3.GrantRecord.principal:type_name -> c1.storage.v3.PrincipalRef - 16, // 10: c1.storage.v3.GrantRecord.discovered_at:type_name -> google.protobuf.Timestamp + 16, // 0: c1.storage.v3.ResourceTypeRecord.annotations:type_name -> google.protobuf.Any + 17, // 1: c1.storage.v3.ResourceTypeRecord.discovered_at:type_name -> google.protobuf.Timestamp + 18, // 2: c1.storage.v3.ResourceRecord.parent:type_name -> c1.storage.v3.ResourceRef + 16, // 3: c1.storage.v3.ResourceRecord.annotations:type_name -> google.protobuf.Any + 17, // 4: c1.storage.v3.ResourceRecord.discovered_at:type_name -> google.protobuf.Timestamp + 18, // 5: c1.storage.v3.EntitlementRecord.resource:type_name -> c1.storage.v3.ResourceRef + 16, // 6: c1.storage.v3.EntitlementRecord.annotations:type_name -> google.protobuf.Any + 17, // 7: c1.storage.v3.EntitlementRecord.discovered_at:type_name -> google.protobuf.Timestamp + 19, // 8: c1.storage.v3.GrantRecord.entitlement:type_name -> c1.storage.v3.EntitlementRef + 20, // 9: c1.storage.v3.GrantRecord.principal:type_name -> c1.storage.v3.PrincipalRef + 17, // 10: c1.storage.v3.GrantRecord.discovered_at:type_name -> google.protobuf.Timestamp 1, // 11: c1.storage.v3.GrantRecord.expansion:type_name -> c1.storage.v3.GrantExpandableRecord - 15, // 12: c1.storage.v3.GrantRecord.annotations:type_name -> google.protobuf.Any - 11, // 13: c1.storage.v3.GrantRecord.sources:type_name -> c1.storage.v3.GrantRecord.SourcesEntry - 16, // 14: c1.storage.v3.AssetRecord.discovered_at:type_name -> google.protobuf.Timestamp + 16, // 12: c1.storage.v3.GrantRecord.annotations:type_name -> google.protobuf.Any + 12, // 13: c1.storage.v3.GrantRecord.sources:type_name -> c1.storage.v3.GrantRecord.SourcesEntry + 17, // 14: c1.storage.v3.AssetRecord.discovered_at:type_name -> google.protobuf.Timestamp 0, // 15: c1.storage.v3.SyncRunRecord.type:type_name -> c1.storage.v3.SyncType - 16, // 16: c1.storage.v3.SyncRunRecord.started_at:type_name -> google.protobuf.Timestamp - 16, // 17: c1.storage.v3.SyncRunRecord.ended_at:type_name -> google.protobuf.Timestamp - 12, // 18: c1.storage.v3.SyncStatsRecord.resources_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry - 13, // 19: c1.storage.v3.SyncStatsRecord.grants_by_entitlement_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry - 14, // 20: c1.storage.v3.SyncStatsRecord.entitlements_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry - 16, // 21: c1.storage.v3.SyncStatsRecord.written_at:type_name -> google.protobuf.Timestamp - 2, // 22: c1.storage.v3.GrantRecord.SourcesEntry.value:type_name -> c1.storage.v3.GrantSourceRecord - 23, // [23:23] is the sub-list for method output_type - 23, // [23:23] is the sub-list for method input_type - 23, // [23:23] is the sub-list for extension type_name - 23, // [23:23] is the sub-list for extension extendee - 0, // [0:23] is the sub-list for field type_name + 17, // 16: c1.storage.v3.SyncRunRecord.started_at:type_name -> google.protobuf.Timestamp + 17, // 17: c1.storage.v3.SyncRunRecord.ended_at:type_name -> google.protobuf.Timestamp + 13, // 18: c1.storage.v3.SyncStatsRecord.resources_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry + 14, // 19: c1.storage.v3.SyncStatsRecord.grants_by_entitlement_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry + 15, // 20: c1.storage.v3.SyncStatsRecord.entitlements_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry + 17, // 21: c1.storage.v3.SyncStatsRecord.written_at:type_name -> google.protobuf.Timestamp + 17, // 22: c1.storage.v3.SourceCacheEntryRecord.discovered_at:type_name -> google.protobuf.Timestamp + 2, // 23: c1.storage.v3.GrantRecord.SourcesEntry.value:type_name -> c1.storage.v3.GrantSourceRecord + 24, // [24:24] is the sub-list for method output_type + 24, // [24:24] is the sub-list for method input_type + 24, // [24:24] is the sub-list for extension type_name + 24, // [24:24] is the sub-list for extension extendee + 0, // [0:24] is the sub-list for field type_name } func init() { file_c1_storage_v3_records_proto_init() } @@ -1811,7 +1999,7 @@ func file_c1_storage_v3_records_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_storage_v3_records_proto_rawDesc), len(file_c1_storage_v3_records_proto_rawDesc)), NumEnums: 1, - NumMessages: 14, + NumMessages: 15, NumExtensions: 0, NumServices: 0, }, diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/cli/cli.go b/vendor/github.com/conductorone/baton-sdk/pkg/cli/cli.go index 036e9ee9..59d87248 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/cli/cli.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/cli/cli.go @@ -11,6 +11,7 @@ import ( "github.com/conductorone/baton-sdk/pkg/connectorbuilder" "github.com/conductorone/baton-sdk/pkg/field" + "github.com/conductorone/baton-sdk/pkg/sourcecache" "github.com/conductorone/baton-sdk/pkg/types" "github.com/conductorone/baton-sdk/pkg/types/sessions" "github.com/spf13/cobra" @@ -20,7 +21,12 @@ import ( ) type RunTimeOpts struct { - SessionStore sessions.SessionStore + SessionStore sessions.SessionStore + // SourceCacheLookup resolves a scope's previous-sync validator for + // source-cache replay (see pkg/sourcecache). In subprocess mode this + // is a gRPC client to the parent SDK's BatonSourceCacheService; when + // unset the framework falls back to NoopLookup. + SourceCacheLookup sourcecache.Lookup TokenSource oauth2.TokenSource SelectedAuthMethod string SyncResourceTypeIDs []string diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/cli/commands.go b/vendor/github.com/conductorone/baton-sdk/pkg/cli/commands.go index 19dd598f..a221be47 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/cli/commands.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/cli/commands.go @@ -9,10 +9,10 @@ import ( "fmt" "io" "os" + "sync" "time" "github.com/conductorone/baton-sdk/pkg/connectorbuilder" - "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" "github.com/conductorone/baton-sdk/pkg/types" "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" "github.com/maypok86/otter/v2" @@ -32,9 +32,11 @@ import ( baton_v1 "github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1" "github.com/conductorone/baton-sdk/pkg/connectorrunner" "github.com/conductorone/baton-sdk/pkg/crypto" + "github.com/conductorone/baton-sdk/pkg/dotc1z" "github.com/conductorone/baton-sdk/pkg/field" "github.com/conductorone/baton-sdk/pkg/logging" "github.com/conductorone/baton-sdk/pkg/session" + "github.com/conductorone/baton-sdk/pkg/sourcecache" "github.com/conductorone/baton-sdk/pkg/tempdir" "github.com/conductorone/baton-sdk/pkg/types/sessions" "github.com/conductorone/baton-sdk/pkg/uhttp" @@ -53,29 +55,55 @@ type eventLogEnabledKey struct{} type ContrainstSetter func(*cobra.Command, field.Configuration) error -// In one shot & service mode, the child process uses this client to connect to the session store server... +// parentControlPlaneDialer opens (lazily, at most once) a single gRPC +// connection from the connector subprocess back to the parent SDK's +// control-plane listener. The same connection multiplexes +// BatonSessionService (connector session data) and BatonSourceCacheService +// (source-cache scope lookups) — the two services share a listener on the +// parent (internal/connector.runServer), so they share a client conn here. // -// which uses the C1Z for storage. Unfortunately the C1Z is instantiated well after we fork the child process, -// so there is quite a bit of pass through. -func getGRPCSessionStoreClient(ctx context.Context, serverCfg *v1.ServerConfig) func(ctx context.Context, opt ...sessions.SessionStoreConstructorOption) (sessions.SessionStore, error) { - return func(_ context.Context, opt ...sessions.SessionStoreConstructorOption) (sessions.SessionStore, error) { - l := ctxzap.Extract(ctx) - clientTLSConfig, err := utls2.ClientConfig(ctx, serverCfg.GetCredential()) - if err != nil { - return nil, err +// dial returns (nil, nil) when the parent did not start a control-plane +// listener (session store disabled); callers substitute no-op +// implementations in that case. +type parentControlPlaneDialer struct { + once sync.Once + conn *grpc.ClientConn + dialErr error + ctx context.Context + cfg *v1.ServerConfig +} + +func newParentControlPlaneDialer(ctx context.Context, serverCfg *v1.ServerConfig) *parentControlPlaneDialer { + return &parentControlPlaneDialer{ctx: ctx, cfg: serverCfg} +} + +// Close releases the underlying gRPC connection if dial ever succeeded. +// Safe when dial never ran or failed, and safe to call multiple times. +func (d *parentControlPlaneDialer) Close() { + if d.conn == nil { + return + } + _ = d.conn.Close() +} + +func (d *parentControlPlaneDialer) dial() (*grpc.ClientConn, error) { + d.once.Do(func() { + if d.cfg.GetSessionStoreListenPort() == 0 { + return } - if serverCfg.GetSessionStoreListenPort() == 0 { - return &session.NoOpSessionStore{}, nil + clientTLSConfig, err := utls2.ClientConfig(d.ctx, d.cfg.GetCredential()) + if err != nil { + d.dialErr = err + return } - // connected, grpc will handle retries for us. - dialCtx, canc := context.WithTimeout(ctx, 5*time.Second) + dialCtx, canc := context.WithTimeout(d.ctx, 5*time.Second) defer canc() var dialErr error var conn *grpc.ClientConn for { conn, err = grpc.DialContext( //nolint:staticcheck // grpc.DialContext is deprecated but we are using it still. - ctx, - fmt.Sprintf("127.0.0.1:%d", serverCfg.GetSessionStoreListenPort()), + d.ctx, + fmt.Sprintf("127.0.0.1:%d", d.cfg.GetSessionStoreListenPort()), grpc.WithTransportCredentials(credentials.NewTLS(clientTLSConfig)), grpc.WithBlock(), //nolint:staticcheck // grpc.WithBlock is deprecated but we are using it still. ) @@ -84,26 +112,59 @@ func getGRPCSessionStoreClient(ctx context.Context, serverCfg *v1.ServerConfig) select { case <-time.After(time.Millisecond * 500): case <-dialCtx.Done(): - return nil, dialErr + d.dialErr = dialErr + return } continue } break } + d.conn = conn + }) + return d.conn, d.dialErr +} +// In one shot & service mode, the child process uses this client to connect to the session store server... +// +// which uses the C1Z for storage. Unfortunately the C1Z is instantiated well after we fork the child process, +// so there is quite a bit of pass through. +func getGRPCSessionStoreClient(ctx context.Context, dialer *parentControlPlaneDialer) func(ctx context.Context, opt ...sessions.SessionStoreConstructorOption) (sessions.SessionStore, error) { + return func(_ context.Context, _ ...sessions.SessionStoreConstructorOption) (sessions.SessionStore, error) { + l := ctxzap.Extract(ctx) + conn, err := dialer.dial() + if err != nil { + return nil, err + } + if conn == nil { + return &session.NoOpSessionStore{}, nil + } client := baton_v1.NewBatonSessionServiceClient(conn) - ss, err := session.NewGRPCSessionStore(ctx, client, opt...) + ss, err := session.NewGRPCSessionStore(ctx, client) if err != nil { - err2 := conn.Close() - if err2 != nil { - l.Error("error closing connection", zap.Error(err2)) - } + l.Error("error creating session store client", zap.Error(err)) return nil, err } return ss, nil } } +// buildGRPCSourceCacheLookup returns the connector-side source-cache Lookup +// that talks to the parent's BatonSourceCacheService over the same +// loopback-TLS connection the session client uses. Returns NoopLookup when +// no parent control-plane listener is configured (matches the "no previous +// sync" behavior). +func buildGRPCSourceCacheLookup(dialer *parentControlPlaneDialer) (sourcecache.Lookup, error) { + conn, err := dialer.dial() + if err != nil { + return nil, err + } + if conn == nil { + return sourcecache.NoopLookup{}, nil + } + client := baton_v1.NewBatonSourceCacheServiceClient(conn) + return sourcecache.NewGRPCLookup(client), nil +} + func MakeMainCommand[T field.Configurable]( ctx context.Context, name string, @@ -403,6 +464,15 @@ func MakeMainCommand[T field.Configurable]( opts = append(opts, connectorrunner.WithExternalResourceC1Z(externalResourceC1ZPath)) } + if v.GetString(field.PreviousSyncC1ZField.GetName()) != "" { + previousSyncC1ZPath := v.GetString(field.PreviousSyncC1ZField.GetName()) + _, err := os.Open(previousSyncC1ZPath) + if err != nil { + return fmt.Errorf("the specified previous sync c1z file does not exist: %s", previousSyncC1ZPath) + } + opts = append(opts, connectorrunner.WithPreviousSyncC1Z(previousSyncC1ZPath)) + } + if v.GetString("external-resource-entitlement-id-filter") != "" { externalResourceEntitlementIdFilter := v.GetString("external-resource-entitlement-id-filter") opts = append(opts, connectorrunner.WithExternalResourceEntitlementFilter(externalResourceEntitlementIdFilter)) @@ -439,7 +509,7 @@ func MakeMainCommand[T field.Configurable]( return err } if storageEngine != "" { - opts = append(opts, connectorrunner.WithStorageEngine(c1zstore.Engine(storageEngine))) + opts = append(opts, connectorrunner.WithStorageEngine(dotc1z.Engine(storageEngine))) } taskConcurrency := v.GetInt(field.TaskConcurrencyField.GetName()) @@ -626,7 +696,17 @@ func MakeGRPCServerCommand[T field.Configurable]( runCtx = context.WithValue(runCtx, uhttp.ContextHTTPTimeoutKey, time.Duration(httpTimeout)*time.Second) sessionStoreMaximumSize := v.GetInt(field.ServerSessionStoreMaximumSizeField.GetName()) - sessionConstructor := getGRPCSessionStoreClient(runCtx, serverCfg) + // One dialer per subprocess; the session store client and the + // source-cache lookup client share the underlying gRPC connection + // (both services are registered on the same parent listener in + // internal/connector.runServer). + controlPlaneDialer := newParentControlPlaneDialer(runCtx, serverCfg) + defer controlPlaneDialer.Close() + sessionConstructor := getGRPCSessionStoreClient(runCtx, controlPlaneDialer) + sourceCacheLookup, err := buildGRPCSourceCacheLookup(controlPlaneDialer) + if err != nil { + return fmt.Errorf("failed to build source cache lookup: %w", err) + } c, err := getconnector(runCtx, t, RunTimeOpts{ SessionStore: NewLazyCachingSessionStore(sessionConstructor, func(otterOptions *otter.Options[string, []byte]) { if sessionStoreMaximumSize <= 0 { @@ -635,6 +715,7 @@ func MakeGRPCServerCommand[T field.Configurable]( otterOptions.MaximumWeight = uint64(sessionStoreMaximumSize) } }), + SourceCacheLookup: sourceCacheLookup, SelectedAuthMethod: v.GetString("auth-method"), SyncResourceTypeIDs: v.GetStringSlice("sync-resource-types"), }) diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/config/config.go b/vendor/github.com/conductorone/baton-sdk/pkg/config/config.go index 3586605f..d937dd0b 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/config/config.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/config/config.go @@ -40,6 +40,7 @@ func RunConnector[T field.Configurable]( } builderOpts = append(builderOpts, connectorbuilder.WithSessionStore(runTimeOpts.SessionStore)) + builderOpts = append(builderOpts, connectorbuilder.WithSourceCache(runTimeOpts.SourceCacheLookup)) c, err := connectorbuilder.NewConnector(ctx, connector, builderOpts...) if err != nil { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/connectorbuilder.go b/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/connectorbuilder.go index bfc1a39e..68fb27d8 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/connectorbuilder.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/connectorbuilder.go @@ -22,6 +22,7 @@ import ( "github.com/conductorone/baton-sdk/pkg/metrics" "github.com/conductorone/baton-sdk/pkg/retry" "github.com/conductorone/baton-sdk/pkg/sdk" + "github.com/conductorone/baton-sdk/pkg/sourcecache" "github.com/conductorone/baton-sdk/pkg/types" "github.com/conductorone/baton-sdk/pkg/types/sessions" "github.com/conductorone/baton-sdk/pkg/types/tasks" @@ -76,6 +77,7 @@ type builder struct { nowFunc func() time.Time clientSecret *jose.JSONWebKey sessionStore sessions.SessionStore + sourceCache sourcecache.Lookup metadataProvider MetadataProvider validateProvider ValidateProvider ticketManager TicketManagerLimited @@ -241,6 +243,30 @@ func WithSessionStore(ss sessions.SessionStore) Opt { } } +// WithSourceCache supplies the connector-facing source-cache Lookup exposed +// on SyncOpAttrs.SourceCache (see pkg/sourcecache). A nil lookup is +// normalized to NoopLookup so connectors never nil-check. +func WithSourceCache(lookup sourcecache.Lookup) Opt { + return func(b *builder) error { + if lookup == nil { + lookup = sourcecache.NoopLookup{} + } + b.sourceCache = lookup + return nil + } +} + +// SetSourceCache implements sourcecache.SetLookup so an in-process runner +// (the syncer, via its connector client) can install/clear the active +// lookup per sync. Subprocess mode instead routes lookups over +// BatonSourceCacheService and never calls this. +func (b *builder) SetSourceCache(_ context.Context, lookup sourcecache.Lookup) { + if lookup == nil { + lookup = sourcecache.NoopLookup{} + } + b.sourceCache = lookup +} + func (b *builder) options(opts ...Opt) error { for _, opt := range opts { if err := opt(b); err != nil { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/resource_syncer.go b/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/resource_syncer.go index a7d99394..37d7764a 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/resource_syncer.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/resource_syncer.go @@ -7,6 +7,8 @@ import ( v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" "github.com/conductorone/baton-sdk/pkg/annotations" "github.com/conductorone/baton-sdk/pkg/pagination" + "github.com/conductorone/baton-sdk/pkg/session" + "github.com/conductorone/baton-sdk/pkg/sourcecache" "github.com/conductorone/baton-sdk/pkg/types/resource" "github.com/conductorone/baton-sdk/pkg/types/tasks" "github.com/conductorone/baton-sdk/pkg/uotel" @@ -64,6 +66,46 @@ type StaticEntitlementSyncerV2 interface { StaticEntitlements(ctx context.Context, opts resource.SyncOpAttrs) ([]*v2.Entitlement, *resource.SyncOpResults, error) } +// TypeScopedGrantsSyncer is the grants-phase analogue of +// StaticEntitlementSyncerV2: the connector enumerates grants for a WHOLE +// resource type instead of being called once per resource. Implement it on +// a resource syncer whose ResourceType carries the v2.TypeScopedGrants +// annotation; the syncer then issues ListGrants calls with an empty +// resource id, which the builder routes here. +// +// The first call of a sync arrives with an empty page token (the planning +// call); the connector may answer with rows directly and/or spawn +// additional independent cursors by attaching a v2.SpawnCursors annotation +// whose page tokens are delivered back through opts.PageToken, one action +// each. Source-cache scope/replay/tombstone annotations work exactly as on +// per-resource grants pages. +type TypeScopedGrantsSyncer interface { + GrantsForResourceType(ctx context.Context, resourceTypeID string, opts resource.SyncOpAttrs) ([]*v2.Grant, *resource.SyncOpResults, error) +} + +// syncOpAttrs assembles the per-call SyncOpAttrs handed to V2 resource +// syncers. SourceCache is never nil: when the runner supplied no lookup +// (source cache disabled/degraded) connectors see NoopLookup and every +// lookup misses. Session is likewise never a nil-backed wrapper: without a +// configured store, connectors see the erroring no-op store instead of a +// panic on first use. +func (b *builder) syncOpAttrs(activeSyncID string, token pagination.Token) resource.SyncOpAttrs { + sc := b.sourceCache + if sc == nil { + sc = sourcecache.NoopLookup{} + } + ss := b.sessionStore + if ss == nil { + ss = &session.NoOpSessionStore{} + } + return resource.SyncOpAttrs{ + SyncID: activeSyncID, + PageToken: token, + Session: WithSyncId(ss, activeSyncID), + SourceCache: sc, + } +} + // ResourceTargetedSyncer extends ResourceSyncer to add capabilities for directly syncing an individual resource // // Implementing this interface indicates the connector supports calling "get" on a resource @@ -129,7 +171,7 @@ func (b *builder) ListResources(ctx context.Context, request *v2.ResourcesServic Size: int(request.GetPageSize()), Token: request.GetPageToken(), } - opts := resource.SyncOpAttrs{SyncID: request.GetActiveSyncId(), PageToken: token, Session: WithSyncId(b.sessionStore, request.GetActiveSyncId())} + opts := b.syncOpAttrs(request.GetActiveSyncId(), token) out, retOptions, err := rb.List(ctx, request.GetParentResourceId(), opts) if retOptions == nil { retOptions = &resource.SyncOpResults{} @@ -217,7 +259,7 @@ func (b *builder) ListStaticEntitlements(ctx context.Context, request *v2.Entitl Size: int(request.GetPageSize()), Token: request.GetPageToken(), } - opts := resource.SyncOpAttrs{SyncID: request.GetActiveSyncId(), PageToken: token, Session: WithSyncId(b.sessionStore, request.GetActiveSyncId())} + opts := b.syncOpAttrs(request.GetActiveSyncId(), token) out, retOptions, err := rbse.StaticEntitlements(ctx, opts) if retOptions == nil { retOptions = &resource.SyncOpResults{} @@ -265,7 +307,7 @@ func (b *builder) ListEntitlements(ctx context.Context, request *v2.Entitlements Size: int(request.GetPageSize()), Token: request.GetPageToken(), } - opts := resource.SyncOpAttrs{SyncID: request.GetActiveSyncId(), PageToken: token, Session: WithSyncId(b.sessionStore, request.GetActiveSyncId())} + opts := b.syncOpAttrs(request.GetActiveSyncId(), token) out, retOptions, err := rb.Entitlements(ctx, request.GetResource(), opts) if retOptions == nil { retOptions = &resource.SyncOpResults{} @@ -322,8 +364,29 @@ func (b *builder) ListGrants(ctx context.Context, request *v2.GrantsServiceListG Size: int(request.GetPageSize()), Token: request.GetPageToken(), } - opts := resource.SyncOpAttrs{SyncID: request.GetActiveSyncId(), PageToken: token, Session: WithSyncId(b.sessionStore, request.GetActiveSyncId())} - out, retOptions, err := rb.Grants(ctx, request.GetResource(), opts) + opts := b.syncOpAttrs(request.GetActiveSyncId(), token) + + reqAnnos := annotations.Annotations(request.GetAnnotations()) + typeScoped := reqAnnos.Contains(&v2.TypeScopedGrants{}) + + var out []*v2.Grant + var retOptions *resource.SyncOpResults + if typeScoped { + // Type-scoped grants call (see v2.TypeScopedGrants): the request + // annotation is the routing marker (the resource is a + // self-referential {type, type} stub to satisfy wire validation). + // Only legal against a syncer that opted in. + tsgs, tsOk := rb.(TypeScopedGrantsSyncer) + if !tsOk { + err = status.Errorf(codes.InvalidArgument, + "error: type-scoped list grants for resource type %s, but its syncer does not implement TypeScopedGrantsSyncer", rid.GetResourceType()) + b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start), err) + return nil, err + } + out, retOptions, err = tsgs.GrantsForResourceType(ctx, rid.GetResourceType(), opts) + } else { + out, retOptions, err = rb.Grants(ctx, request.GetResource(), opts) + } if retOptions == nil { retOptions = &resource.SyncOpResults{} } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/connectorclient/connectorclient.go b/vendor/github.com/conductorone/baton-sdk/pkg/connectorclient/connectorclient.go new file mode 100644 index 00000000..bd085ac0 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/connectorclient/connectorclient.go @@ -0,0 +1,15 @@ +package connectorclient + +import ( + "context" + + "github.com/conductorone/baton-sdk/internal/connector" + "github.com/conductorone/baton-sdk/pkg/types" + "google.golang.org/grpc" +) + +// NewConnectorClient takes a grpc.ClientConnInterface and returns an implementation of the ConnectorClient interface. +// Note: lambda functions directly instantiate the connector client, so this function is not used in this package. +func NewConnectorClient(ctx context.Context, cc grpc.ClientConnInterface) types.ConnectorClient { + return connector.NewConnectorClient(ctx, cc) +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/connectorrunner/runner.go b/vendor/github.com/conductorone/baton-sdk/pkg/connectorrunner/runner.go index c404401b..dd6ec4ee 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/connectorrunner/runner.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/connectorrunner/runner.go @@ -13,7 +13,7 @@ import ( "github.com/conductorone/baton-sdk/pkg/bid" "github.com/conductorone/baton-sdk/pkg/connectorbuilder" - "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z" "github.com/conductorone/baton-sdk/pkg/field" "github.com/conductorone/baton-sdk/pkg/healthcheck" "github.com/conductorone/baton-sdk/pkg/synccompactor" @@ -422,10 +422,11 @@ type runnerConfig struct { syncDifferConfig *syncDifferConfig syncCompactorConfig *syncCompactorConfig skipFullSync bool - storageEngine c1zstore.Engine + storageEngine dotc1z.Engine workerCount int targetedSyncResourceIDs []string externalResourceC1Z string + previousSyncC1Z string externalResourceEntitlementIdFilter string keepPreviousSyncC1ZCapable bool keepPreviousSyncC1ZEnabled bool @@ -669,7 +670,7 @@ func WithWorkerCount(workerCount int) Option { } } -func WithStorageEngine(engine c1zstore.Engine) Option { +func WithStorageEngine(engine dotc1z.Engine) Option { return func(ctx context.Context, cfg *runnerConfig) error { cfg.storageEngine = engine return nil @@ -760,6 +761,13 @@ func WithExternalResourceC1Z(externalResourceC1Z string) Option { } } +func WithPreviousSyncC1Z(previousSyncC1Z string) Option { + return func(ctx context.Context, cfg *runnerConfig) error { + cfg.previousSyncC1Z = previousSyncC1Z + return nil + } +} + func WithExternalResourceEntitlementFilter(entitlementId string) Option { return func(ctx context.Context, cfg *runnerConfig) error { cfg.externalResourceEntitlementIdFilter = entitlementId @@ -973,7 +981,15 @@ func NewConnectorRunner(ctx context.Context, c types.ConnectorServer, opts ...Op wrapperOpts = append(wrapperOpts, connector.WithTargetedSyncResources(cfg.targetedSyncResourceIDs)) } - if cfg.sessionStoreEnabled { + // The parent control-plane listener serves BOTH BatonSessionService and + // BatonSourceCacheService (internal/connector.runServer). Source-cache + // replay therefore needs the listener even when the connector never uses + // sessions: without it the subprocess connector's lookup degrades to + // NoopLookup, every scope misses, and every sync runs a full cold + // enumeration despite a valid --previous-sync-c1z. Start it whenever + // replay could be in play: the author declared the capability + // (WithKeepPreviousSyncC1Z) or a previous-sync c1z was configured. + if cfg.sessionStoreEnabled || cfg.keepPreviousSyncC1ZCapable || cfg.previousSyncC1Z != "" { wrapperOpts = append(wrapperOpts, connector.WithSessionStoreEnabled()) } @@ -1081,6 +1097,7 @@ func NewConnectorRunner(ctx context.Context, c types.ConnectorServer, opts ...Op tm, err = local.NewSyncer(ctx, cfg.c1zPath, local.WithTmpDir(cfg.tempDir), local.WithExternalResourceC1Z(cfg.externalResourceC1Z), + local.WithPreviousSyncC1Z(cfg.previousSyncC1Z), local.WithExternalResourceEntitlementIdFilter(cfg.externalResourceEntitlementIdFilter), local.WithTargetedSyncResources(resources), local.WithSkipEntitlementsAndGrants(cfg.skipEntitlementsAndGrants), diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/connectorstore/connectorstore.go b/vendor/github.com/conductorone/baton-sdk/pkg/connectorstore/connectorstore.go index fa1f03fe..17045488 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/connectorstore/connectorstore.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/connectorstore/connectorstore.go @@ -39,7 +39,7 @@ var AllSyncTypes = []SyncType{ // unless the value is recognized. type StoreMetadata struct { // Engine identifies the storage backend. Values match - // c1zstore.Engine string values; using string here keeps + // dotc1z.Engine string values; using string here keeps // connectorstore from depending on dotc1z (avoids an import // cycle). // "sqlite" — original .c1z, v1 magic + zstd-compressed SQLite @@ -56,7 +56,7 @@ type StoreMetadata struct { // PayloadEncoding identifies the v3 envelope payload framing. // Empty for v1 / SQLite. Values match - // c1zstore.PayloadEncoding.String(): + // dotc1z.PayloadEncoding.String(): // "tar_zstd" — Pebble checkpoint as zstd-compressed tar // "tar" — Pebble checkpoint as uncompressed tar // "" — N/A or unset diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file.go index 2845ae7f..d6d3a8f1 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file.go @@ -31,7 +31,6 @@ import ( v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" reader_v2 "github.com/conductorone/baton-sdk/pb/c1/reader/v2" "github.com/conductorone/baton-sdk/pkg/connectorstore" - "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" "github.com/conductorone/baton-sdk/pkg/uotel" ) @@ -74,7 +73,7 @@ type C1File struct { deferredIndexTables []tableDescriptor // Cached sync run for listConnectorObjects (avoids N+1 queries) - cachedViewSyncRun *c1zstore.SyncRun + cachedViewSyncRun *SyncRun cachedViewSyncMu sync.Mutex cachedViewSyncErr error @@ -95,22 +94,22 @@ type C1File struct { // engine is the storage engine to use for newly created files. // Reads dispatch on magic byte regardless of this value. Default // is EngineSQLite (v1 .c1z format). - engine c1zstore.Engine + engine Engine // payloadEncoding selects the v3 envelope payload framing for // Pebble-written files. Zero value = PayloadEncodingTarZstd // (default). Ignored by the SQLite engine. - payloadEncoding c1zstore.PayloadEncoding + payloadEncoding PayloadEncoding } // *C1File satisfies connectorstore.Writer (the connector-facing contract), // connectorstore.LatestFinishedSyncIDFetcher (narrow optional capability -// added in PR #774), and c1zstore.Store (the internal sync-pipeline +// added in PR #774), and dotc1z.C1ZStore (the internal sync-pipeline // contract asserted in c1file_store.go alongside the sub-store assertions). var ( _ connectorstore.Writer = (*C1File)(nil) _ connectorstore.LatestFinishedSyncIDFetcher = (*C1File)(nil) - _ c1zstore.Store = (*C1File)(nil) + _ C1ZStore = (*C1File)(nil) ) type C1FOption func(*C1File) @@ -211,7 +210,7 @@ func WithC1FSyncCountLimit(limit int) C1FOption { // Engine selection only affects newly created files. Existing files // dispatch on their magic byte; readers handle both v1 and v3 // regardless of this option. -func WithC1FEngine(engine c1zstore.Engine) C1FOption { +func WithC1FEngine(engine Engine) C1FOption { return func(o *C1File) { o.engine = engine } @@ -219,7 +218,7 @@ func WithC1FEngine(engine c1zstore.Engine) C1FOption { // WithC1FPayloadEncoding selects the v3 envelope payload encoding // (TAR_ZSTD default, TAR uncompressed). No-op for SQLite engines. -func WithC1FPayloadEncoding(enc c1zstore.PayloadEncoding) C1FOption { +func WithC1FPayloadEncoding(enc PayloadEncoding) C1FOption { return func(o *C1File) { o.payloadEncoding = enc } @@ -294,7 +293,7 @@ func NewC1File(ctx context.Context, dbFilePath string, opts ...C1FOption) (*C1Fi // engine manages its own storage and does not use those indexes, so bulk // load does not apply there. Make the combination an explicit, logged // no-op rather than leaving it silently unspecified. - if c1File.bulkLoad && c1File.engine == c1zstore.EnginePebble { + if c1File.bulkLoad && c1File.engine == EnginePebble { l.Info("new-c1-file: bulk load ignored for the pebble engine; the deferred-index optimization applies only to the sqlite engine") c1File.bulkLoad = false } @@ -312,7 +311,7 @@ func NewC1File(ctx context.Context, dbFilePath string, opts ...C1FOption) (*C1Fi // Normalize the engine zero value so downstream switch/if-eq // checks treat an unset engine as EngineSQLite. if c1File.engine == "" { - c1File.engine = c1zstore.EngineSQLite + c1File.engine = EngineSQLite } err = c1File.validateDb(ctx) @@ -342,13 +341,13 @@ type c1zOptions struct { // engine is the storage engine to use for newly created files. // Reads dispatch on magic byte regardless. Default EngineSQLite. - engine c1zstore.Engine + engine Engine // payloadEncoding controls the v3 envelope payload framing. Only // honored when engine == EnginePebble (the v3 path). Allowed // values: PayloadEncodingTarZstd (default), PayloadEncodingTar. // Zero value means PayloadEncodingTarZstd. - payloadEncoding c1zstore.PayloadEncoding + payloadEncoding PayloadEncoding // decoderPool optionally scopes v3 payload-decoder reuse to the // caller's operation. See WithDecoderPool. @@ -431,7 +430,7 @@ func WithSyncLimit(limit int) C1ZOption { // // Reading existing files dispatches on the file's magic byte and is // independent of this option. -func WithEngine(engine c1zstore.Engine) C1ZOption { +func WithEngine(engine Engine) C1ZOption { return func(o *c1zOptions) { o.engine = engine } @@ -466,7 +465,7 @@ func WithBulkLoad(enabled bool) C1ZOption { // // No-op for SQLite engines; the encoding selector applies only to // the v3 envelope written by Pebble. -func WithPayloadEncoding(enc c1zstore.PayloadEncoding) C1ZOption { +func WithPayloadEncoding(enc PayloadEncoding) C1ZOption { return func(o *c1zOptions) { o.payloadEncoding = enc } @@ -492,11 +491,8 @@ func NewC1ZFile(ctx context.Context, outputFilePath string, opts ...C1ZOption) ( return nil, err } - if options.engine == c1zstore.EnginePebble && !options.readOnly { - err = fmt.Errorf( - "new-c1z-file: %s is a v1/sqlite c1z and engine %q was requested; "+ - "NewC1ZFile cannot return a *C1File for it — open with NewStore to convert", - outputFilePath, c1zstore.EnginePebble) + if options.engine == EnginePebble && !options.readOnly { + err = fmt.Errorf("new-c1z-file: %s is a v1/sqlite c1z and engine %q was requested; NewC1ZFile cannot return a *C1File for it — open with NewStore to convert", outputFilePath, EnginePebble) return nil, err } @@ -539,7 +535,7 @@ func NewC1ZFile(ctx context.Context, outputFilePath string, opts ...C1ZOption) ( if options.engine != "" { c1fopts = append(c1fopts, WithC1FEngine(options.engine)) } - if options.payloadEncoding != c1zstore.PayloadEncodingUnspecified { + if options.payloadEncoding != PayloadEncodingUnspecified { c1fopts = append(c1fopts, WithC1FPayloadEncoding(options.payloadEncoding)) } @@ -1424,7 +1420,7 @@ func (c *C1File) OutputFilepath() (string, error) { func (c *C1File) Metadata() connectorstore.StoreMetadata { engine := c.engine if engine == "" { - engine = c1zstore.EngineSQLite + engine = EngineSQLite } return connectorstore.StoreMetadata{ Engine: string(engine), diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file_store.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file_store.go index dbc4cb35..226a1783 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file_store.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file_store.go @@ -17,21 +17,21 @@ import ( // wrapper structs satisfy each sub-interface. These assertions catch // signature drift at build time rather than at the first runtime call. var ( - _ c1zstore.Store = (*C1File)(nil) - _ c1zstore.GrantStore = c1FileGrantStore{} - _ c1zstore.SyncMeta = c1FileSyncMeta{} - _ c1zstore.FileOps = c1FileFileOps{} - _ SessionStore = c1FileSessionStore{} + _ C1ZStore = (*C1File)(nil) + _ GrantStore = c1FileGrantStore{} + _ SyncMeta = c1FileSyncMeta{} + _ FileOps = c1FileFileOps{} + _ SessionStore = c1FileSessionStore{} ) // Grants returns the grant-store slice of this c1z. -func (c *C1File) Grants() c1zstore.GrantStore { return c1FileGrantStore{c} } +func (c *C1File) Grants() GrantStore { return c1FileGrantStore{c} } // SyncMeta returns the sync-metadata slice of this c1z. -func (c *C1File) SyncMeta() c1zstore.SyncMeta { return c1FileSyncMeta{c} } +func (c *C1File) SyncMeta() SyncMeta { return c1FileSyncMeta{c} } // FileOps returns the file-operations slice of this c1z. -func (c *C1File) FileOps() c1zstore.FileOps { return c1FileFileOps{c} } +func (c *C1File) FileOps() FileOps { return c1FileFileOps{c} } // SessionStore returns the session-store slice of this c1z. func (c *C1File) SessionStore() sessions.SessionStore { return c1FileSessionStore{c} } @@ -97,7 +97,7 @@ func (c *C1File) StoreExpandedGrants(ctx context.Context, grants ...*v2.Grant) e // PendingExpansionPage implements GrantStore. Thin wrapper over // listExpandableGrantsInternal(Mode: ExpansionNeedsOnly) that reshapes // the internal row struct into the exported PendingExpansion shape. -func (g c1FileGrantStore) PendingExpansionPage(ctx context.Context, pageToken string) ([]c1zstore.PendingExpansion, string, error) { +func (g c1FileGrantStore) PendingExpansionPage(ctx context.Context, pageToken string) ([]PendingExpansion, string, error) { defs, nextPageToken, err := g.c.listExpandableGrantsInternal(ctx, grantListOptions{ Mode: grantListModeExpansionNeedsOnly, PageToken: pageToken, @@ -105,12 +105,12 @@ func (g c1FileGrantStore) PendingExpansionPage(ctx context.Context, pageToken st if err != nil { return nil, "", err } - out := make([]c1zstore.PendingExpansion, 0, len(defs)) + out := make([]PendingExpansion, 0, len(defs)) for _, def := range defs { if def == nil { continue } - out = append(out, c1zstore.PendingExpansion{ + out = append(out, PendingExpansion{ GrantExternalID: def.GrantExternalID, TargetEntitlementID: def.TargetEntitlementID, PrincipalResourceTypeID: def.PrincipalResourceTypeID, @@ -136,17 +136,17 @@ func (g c1FileGrantStore) PendingExpansionPage(ctx context.Context, pageToken st // walks terminate promptly when the caller's deadline/cancel fires; // rows within a single page are still delivered (they are already in // memory), so cancellation responsiveness is page-grained, not row-grained. -func (g c1FileGrantStore) PendingExpansion(ctx context.Context) iter.Seq2[c1zstore.PendingExpansion, error] { - return func(yield func(c1zstore.PendingExpansion, error) bool) { +func (g c1FileGrantStore) PendingExpansion(ctx context.Context) iter.Seq2[PendingExpansion, error] { + return func(yield func(PendingExpansion, error) bool) { pageToken := "" for { if err := ctx.Err(); err != nil { - _ = yield(c1zstore.PendingExpansion{}, err) + _ = yield(PendingExpansion{}, err) return } page, nextPageToken, err := g.PendingExpansionPage(ctx, pageToken) if err != nil { - _ = yield(c1zstore.PendingExpansion{}, err) + _ = yield(PendingExpansion{}, err) return } for _, pe := range page { @@ -172,7 +172,7 @@ func (g c1FileGrantStore) ListWithAnnotationsForResourcePage( syncID string, pageToken string, pageSize uint32, -) ([]c1zstore.GrantAnnotation, string, error) { +) ([]GrantAnnotation, string, error) { resp, err := g.c.listGrantsWithExpansionInternal(ctx, grantListOptions{ Mode: grantListModePayloadWithExpansion, Resource: resource, @@ -189,13 +189,13 @@ func (g c1FileGrantStore) ListWithAnnotationsForResourcePage( // grantAnnotationRowsFromInternal converts the internal row shape into // the exported GrantAnnotation shape, unifying the code path between // ListWithAnnotationsPage and ListWithAnnotationsForResourcePage. -func grantAnnotationRowsFromInternal(rows []*internalGrantRow) []c1zstore.GrantAnnotation { - out := make([]c1zstore.GrantAnnotation, 0, len(rows)) +func grantAnnotationRowsFromInternal(rows []*internalGrantRow) []GrantAnnotation { + out := make([]GrantAnnotation, 0, len(rows)) for _, row := range rows { if row == nil { continue } - ga := c1zstore.GrantAnnotation{ + ga := GrantAnnotation{ Grant: row.Grant, GrantExternalID: row.Grant.GetId(), TargetEntitlementID: row.Grant.GetEntitlement().GetId(), @@ -223,7 +223,7 @@ func grantAnnotationRowsFromInternal(rows []*internalGrantRow) []c1zstore.GrantA // from the underlying grant proto, regardless of whether the grant has // an expansion annotation, so callers don't need to branch on // Annotation-nil to get identity. -func (g c1FileGrantStore) ListWithAnnotationsPage(ctx context.Context, pageToken string) ([]c1zstore.GrantAnnotation, string, error) { +func (g c1FileGrantStore) ListWithAnnotationsPage(ctx context.Context, pageToken string) ([]GrantAnnotation, string, error) { resp, err := g.c.listGrantsWithExpansionInternal(ctx, grantListOptions{ Mode: grantListModePayloadWithExpansion, PageToken: pageToken, @@ -237,17 +237,17 @@ func (g c1FileGrantStore) ListWithAnnotationsPage(ctx context.Context, pageToken // ListWithAnnotations implements GrantStore. Convenience iterator that // walks every page via ListWithAnnotationsPage. Cancellation behavior is // identical to PendingExpansion (page-grained). -func (g c1FileGrantStore) ListWithAnnotations(ctx context.Context) iter.Seq2[c1zstore.GrantAnnotation, error] { - return func(yield func(c1zstore.GrantAnnotation, error) bool) { +func (g c1FileGrantStore) ListWithAnnotations(ctx context.Context) iter.Seq2[GrantAnnotation, error] { + return func(yield func(GrantAnnotation, error) bool) { pageToken := "" for { if err := ctx.Err(); err != nil { - _ = yield(c1zstore.GrantAnnotation{}, err) + _ = yield(GrantAnnotation{}, err) return } page, nextPageToken, err := g.ListWithAnnotationsPage(ctx, pageToken) if err != nil { - _ = yield(c1zstore.GrantAnnotation{}, err) + _ = yield(GrantAnnotation{}, err) return } for _, ga := range page { @@ -276,7 +276,7 @@ func (s c1FileSyncMeta) MarkSyncSupportsDiff(ctx context.Context, syncID string) // LatestFullSync implements SyncMeta. Returns the most-recent finished // SyncTypeFull run, or nil if none. -func (s c1FileSyncMeta) LatestFullSync(ctx context.Context) (*c1zstore.SyncRun, error) { +func (s c1FileSyncMeta) LatestFullSync(ctx context.Context) (*SyncRun, error) { run, err := s.c.getFinishedSync(ctx, 0, connectorstore.SyncTypeFull) if err != nil { return nil, err @@ -286,7 +286,7 @@ func (s c1FileSyncMeta) LatestFullSync(ctx context.Context) (*c1zstore.SyncRun, // LatestFinishedSyncOfAnyType implements SyncMeta. Returns the most-recent // finished sync of any type (including diff types), or nil if none. -func (s c1FileSyncMeta) LatestFinishedSyncOfAnyType(ctx context.Context) (*c1zstore.SyncRun, error) { +func (s c1FileSyncMeta) LatestFinishedSyncOfAnyType(ctx context.Context) (*SyncRun, error) { run, err := s.c.getFinishedSync(ctx, 0, connectorstore.SyncTypeAny) if err != nil { return nil, err @@ -314,7 +314,7 @@ type c1FileFileOps struct{ c *C1File } // CloneSync implements FileOps. Translates the engine-neutral // CloneSyncOptions into the SQLite-specific C1FOptions applied to the // destination file. -func (f c1FileFileOps) CloneSync(ctx context.Context, outPath string, syncID string, opts ...c1zstore.CloneSyncOption) error { +func (f c1FileFileOps) CloneSync(ctx context.Context, outPath string, syncID string, opts ...CloneSyncOption) error { cloneOpts := c1zstore.NewCloneSyncOptions(opts...) var c1fOpts []C1FOption if cloneOpts.TmpDir != "" { @@ -326,7 +326,7 @@ func (f c1FileFileOps) CloneSync(ctx context.Context, outPath string, syncID str // CopyIsolateSync implements FileOps. Translates the engine-neutral // CloneSyncOptions into the SQLite-specific C1FOptions applied to the // destination file. -func (f c1FileFileOps) CopyIsolateSync(ctx context.Context, outPath string, syncID string, opts ...c1zstore.CloneSyncOption) error { +func (f c1FileFileOps) CopyIsolateSync(ctx context.Context, outPath string, syncID string, opts ...CloneSyncOption) error { cloneOpts := c1zstore.NewCloneSyncOptions(opts...) var c1fOpts []C1FOption if cloneOpts.TmpDir != "" { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore/c1zstore.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore/c1zstore.go index 8aa50210..5ab9d50f 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore/c1zstore.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore/c1zstore.go @@ -7,8 +7,9 @@ // pkg/connectorstore. Storage engines (pkg/dotc1z's SQLite C1File, // pkg/dotc1z/engine/pebble's Adapter) implement these interfaces without // importing pkg/dotc1z, which lets dotc1z import the engines and register -// them statically. Callers reference these types directly through this -// package (c1zstore.Store, c1zstore.Engine, etc.). +// them statically. pkg/dotc1z re-exports every type here under its +// historical name (dotc1z.C1ZStore = c1zstore.Store, etc.), so callers +// outside the engine packages can keep using the dotc1z names. package c1zstore import ( diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/cleanup_policy.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/cleanup_policy.go index 8f1b07e8..e9aa16b6 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/cleanup_policy.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/cleanup_policy.go @@ -9,7 +9,7 @@ import "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" // SelectSyncsToDelete applies the SDK retention policy to a snapshot of sync // runs and returns the IDs whose data should be deleted. See // c1zstore.SelectSyncsToDelete for the policy details. -func SelectSyncsToDelete(candidates []c1zstore.SyncRun, currentSyncID string, syncLimit int) []string { +func SelectSyncsToDelete(candidates []SyncRun, currentSyncID string, syncLimit int) []string { return c1zstore.SelectSyncsToDelete(candidates, currentSyncID, syncLimit) } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/clone_sync.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/clone_sync.go index 9c34ed97..429df6b7 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/clone_sync.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/clone_sync.go @@ -12,7 +12,6 @@ import ( "strings" "github.com/conductorone/baton-sdk/pkg/connectorstore" - "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" "github.com/conductorone/baton-sdk/pkg/uotel" "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" "go.uber.org/zap" @@ -196,8 +195,8 @@ func (c *C1File) SnapshotTo(ctx context.Context, outPath string, opts ...C1FOpti return err } - if c.engine == c1zstore.EnginePebble { - err = fmt.Errorf("snapshot-to: unsupported for the %q engine; it manages its own storage", c1zstore.EnginePebble) + if c.engine == EnginePebble { + err = fmt.Errorf("snapshot-to: unsupported for the %q engine; it manages its own storage", EnginePebble) return err } if c.readOnly { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/convert_open.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/convert_open.go index c6d5086b..b69b0f81 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/convert_open.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/convert_open.go @@ -9,7 +9,6 @@ import ( "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" "go.uber.org/zap" - "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" "github.com/conductorone/baton-sdk/pkg/uotel" ) @@ -23,7 +22,7 @@ type pebbleOpenOptions struct { skipCleanup bool skipVacuum bool v2GrantsWriter bool - payloadEncoding c1zstore.PayloadEncoding + payloadEncoding PayloadEncoding } func pebbleOpenOptionsFromC1Z(options *c1zOptions) pebbleOpenOptions { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter.go index c18ef1b5..28b37a91 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter.go @@ -21,6 +21,7 @@ import ( v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" "github.com/conductorone/baton-sdk/pkg/connectorstore" "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/sourcecache" ) // Adapter wraps an *Engine and implements connectorstore.Writer @@ -400,12 +401,27 @@ func (a *Adapter) PutGrants(ctx context.Context, grants ...*v2.Grant) error { return ErrNoCurrentSync } records := translateGrants(syncID, grants) + stampSourceScope(ctx, records, func(r *v3.GrantRecord, s string) { r.SetSourceScopeHash(s) }) if err := a.engine.PutGrantRecords(ctx, records...); err != nil { return fmt.Errorf("PutGrants: %w", err) } return nil } +// stampSourceScope stamps the source-cache scope hash carried by ctx +// (sourcecache.WithScope) onto freshly translated records. The syncer +// sets the scope around a page's store writes when the page carried a +// SourceCacheScope annotation; everything else writes unstamped rows. +func stampSourceScope[T any](ctx context.Context, records []T, set func(T, string)) { + scope := sourcecache.ScopeFromContext(ctx) + if scope == "" { + return + } + for _, r := range records { + set(r, scope) + } +} + // UnsafePutUniqueGrants writes grants on the trusted-import path: records // are encoded in parallel and written unconditionally, with no read-before-write // and no dedup pass. Do not use it for live connector output. The destination @@ -563,6 +579,7 @@ func (a *Adapter) PutResources(ctx context.Context, resources ...*v2.Resource) e } records = append(records, rec) } + stampSourceScope(ctx, records, func(r *v3.ResourceRecord, s string) { r.SetSourceScopeHash(s) }) if err := a.engine.PutResourceRecords(ctx, records...); err != nil { return fmt.Errorf("PutResources: %w", err) } @@ -590,6 +607,7 @@ func (a *Adapter) PutEntitlements(ctx context.Context, entitlements ...*v2.Entit } records = append(records, rec) } + stampSourceScope(ctx, records, func(r *v3.EntitlementRecord, s string) { r.SetSourceScopeHash(s) }) if err := a.engine.PutEntitlementRecords(ctx, records...); err != nil { return fmt.Errorf("PutEntitlements: %w", err) } @@ -967,7 +985,7 @@ func (a *Adapter) CurrentDBSizeBytes() (int64, error) { // // Strings are inlined rather than referencing dotc1z constants // because this subpackage is imported by dotc1z, so the reverse -// import would cycle. The values match c1zstore.EnginePebble.String() +// import would cycle. The values match dotc1z.EnginePebble.String() // and dotc1z.C1ZFormatV3.String() — see connectorstore.StoreMetadata // docs for the canonical value list. func (a *Adapter) Metadata() connectorstore.StoreMetadata { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter_grants_store.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter_grants_store.go index 30736c59..fe19c1f2 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter_grants_store.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter_grants_store.go @@ -220,6 +220,10 @@ func (g pebbleGrantStore) translateExpanded(syncID string, grants []*v2.Grant) [ // because the caller left a residual GrantExpandable annotation. newRec.SetExpansion(nil) newRec.SetNeedsExpansion(false) + // Same shape for the source-cache scope stamp: existing records get + // their prior stamp restored in PutExpandedGrantRecords; brand-new + // expander-derived rows are never part of a source scope. + newRec.SetSourceScopeHash("") merged = append(merged, newRec) } return merged diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/bulk_import.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/bulk_import.go index 621779a3..fb27441f 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/bulk_import.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/bulk_import.go @@ -54,6 +54,7 @@ const bulkSpillBufferSize = 1 << 20 var grantIndexFamilies = []byte{ idxGrantByPrincipal, idxGrantByNeedsExpansion, + idxGrantBySourceScope, } // bulkSSTWriter builds one SST file for a single disjoint key bucket. diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/cleanup.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/cleanup.go index a132572d..f412b874 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/cleanup.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/cleanup.go @@ -35,6 +35,10 @@ func scopedRanges() [][2][]byte { {GrantByPrincipalLowerBound(), GrantByPrincipalUpperBound()}, {GrantByPrincipalResourceTypeLowerBound(), GrantByPrincipalResourceTypeUpperBound()}, {GrantByNeedsExpansionLowerBound(), GrantByNeedsExpansionUpperBound()}, + {GrantBySourceScopeLowerBound(), GrantBySourceScopeUpperBound()}, + {EntitlementBySourceScopeLowerBound(), EntitlementBySourceScopeUpperBound()}, + {ResourceBySourceScopeLowerBound(), ResourceBySourceScopeUpperBound()}, + {SourceCacheEntryLowerBound(), SourceCacheEntryUpperBound()}, {encodeAssetPrefix(), upperBoundOf(encodeAssetPrefix())}, // Stats sidecar — single key; the half-open range shape // contains exactly that one key. diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/engine_stub.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/engine_stub.go index fd15e349..2ac92fbc 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/engine_stub.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/engine_stub.go @@ -1,5 +1,5 @@ // Package pebble is the v3 storage engine for baton-sdk. It is the -// implementation behind c1zstore.EnginePebble and the v3 envelope. +// implementation behind dotc1z.EnginePebble and the v3 envelope. package pebble import ( diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/entitlements.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/entitlements.go index 2182c3a2..7037b4db 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/entitlements.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/entitlements.go @@ -75,6 +75,18 @@ func (e *Engine) PutEntitlementRecords(ctx context.Context, records ...*v3.Entit if err := priBatch.Set(key, val, nil); err != nil { return err } + // by_source_scope is the only entitlement secondary index. + // No read-before-write cleanup here (matching this method's + // no-Get philosophy): a same-identity rewrite under a + // different scope within one sync can leave a stale index + // entry, which replay tolerates — the copied record carries + // its true scope and the file holds a single sync, so the + // staleness cannot outlive it. + if sh := r.GetSourceScopeHash(); sh != "" { + if err := priBatch.Set(encodeEntitlementBySourceScopeIndexKey(sh, id), nil, nil); err != nil { + return err + } + } } opts := writeOpts(e.opts.durability) if fresh { @@ -122,6 +134,22 @@ func (e *Engine) DeleteEntitlementRecord(ctx context.Context, externalID string) key := encodeEntitlementIdentityKey(id) batch := e.db.NewBatch() defer batch.Close() + // Clean up the source-scope index entry (the only entitlement + // secondary index) so a replayed scope can't resurrect the row. + if oldVal, closer, getErr := e.db.Get(key); getErr == nil { + oldScope, scanErr := scanEntitlementSourceScopeRaw(oldVal) + closer.Close() + if scanErr != nil { + return scanErr + } + if oldScope != "" { + if err := batch.Delete(encodeEntitlementBySourceScopeIndexKey(oldScope, id), nil); err != nil { + return err + } + } + } else if !errors.Is(getErr, pebble.ErrNotFound) { + return getErr + } if err := batch.Delete(key, nil); err != nil { return err } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grants.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grants.go index e004584f..a3758116 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grants.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grants.go @@ -263,6 +263,12 @@ func (e *Engine) PutExpandedGrantRecords(ctx context.Context, records []*v3.Gran r.SetExpansion(prior.GetExpansion()) r.SetNeedsExpansion(prior.GetNeedsExpansion()) r.SetDiscoveredAt(prior.GetDiscoveredAt()) + // Preserve the source-cache scope stamp exactly like the + // expansion side-state: the expander rewrites existing + // direct grants to bake in Sources, and clobbering the + // stamp here would silently drop every expander-touched + // grant from the next sync's replay of its scope. + r.SetSourceScopeHash(prior.GetSourceScopeHash()) var err error idxScratch, err = e.deleteGrantIndexesScratch(idxBatch, ext, oldVal, idxScratch) if err != nil { @@ -746,14 +752,17 @@ func (e *Engine) putSynthesizedGrantContributionsBatch(ctx context.Context, reco if err != nil { return err } - // sources (field 9) is the record's highest field, so appending - // it after the base marshal matches the deterministic byte order. + // sources (field 9) is the highest field this record carries, so + // appending it after the base marshal matches the deterministic + // byte order. (source_scope_hash is field 10, but synthesized + // grants never carry a scope stamp — fillSynthGrantRecord leaves + // it unset — so field 9 stays last on the wire.) val, srcScratch = appendGrantSourcesWire(val, srcScratch, rec.sources) valScratch = val if err := priBatch.Set(keyScratch, val, nil); err != nil { return err } - idxScratch, err = e.writeGrantIndexesForIdentityScratch(idxBatch, rec.id, false, idxScratch) + idxScratch, err = e.writeGrantIndexesForIdentityScratch(idxBatch, rec.id, false, "", idxScratch) if err != nil { return err } @@ -986,11 +995,14 @@ func grantIndexKeys(r *v3.GrantRecord) [][]byte { if err != nil { return nil } - keys := make([][]byte, 0, 2) + keys := make([][]byte, 0, 3) keys = append(keys, encodeGrantByPrincipalIdentityIndexKey(id)) if r.GetNeedsExpansion() { keys = append(keys, encodeGrantByNeedsExpansionIdentityIndexKey(id)) } + if sh := r.GetSourceScopeHash(); sh != "" { + keys = append(keys, encodeGrantBySourceScopeIndexKey(sh, id)) + } return keys } @@ -1015,7 +1027,7 @@ func (e *Engine) writeGrantIndexesScratch(batch *pebble.Batch, r *v3.GrantRecord if err != nil { return scratch, err } - return e.writeGrantIndexesForIdentityScratch(batch, id, r.GetNeedsExpansion(), scratch) + return e.writeGrantIndexesForIdentityScratch(batch, id, r.GetNeedsExpansion(), r.GetSourceScopeHash(), scratch) } // markDeferredIdxPending arms the deferred by_principal rebuild, durably. @@ -1058,7 +1070,10 @@ func (e *Engine) clearDeferredIdxPending() error { // for one grant. by_principal is never written inline: it is scattered // relative to the entitlement-first write order, so it is always rebuilt as // one sorted SST at EndSync (deferredIdxPending → BuildDeferredGrantIndexes). -func (e *Engine) writeGrantIndexesForIdentityScratch(batch *pebble.Batch, id grantIdentity, needsExpansion bool, scratch []byte) ([]byte, error) { +// by_needs_expansion and by_source_scope are written inline: both share the +// entitlement-first ordering of the primary keyspace, so their writes stay +// sorted. +func (e *Engine) writeGrantIndexesForIdentityScratch(batch *pebble.Batch, id grantIdentity, needsExpansion bool, sourceScopeHash string, scratch []byte) ([]byte, error) { if err := e.markDeferredIdxPending(); err != nil { return scratch, err } @@ -1068,6 +1083,12 @@ func (e *Engine) writeGrantIndexesForIdentityScratch(batch *pebble.Batch, id gra return scratch, err } } + if sourceScopeHash != "" { + scratch = appendGrantBySourceScopeIndexKey(scratch[:0], sourceScopeHash, id) + if err := batch.Set(scratch, nil, nil); err != nil { + return scratch, err + } + } return scratch, nil } @@ -1080,7 +1101,7 @@ func (e *Engine) writeGrantIndexesForIdentityScratch(batch *pebble.Batch, id gra // which also clears any stale entries an overwrite would have left. Returns // the (possibly grown) scratch buffer. func (e *Engine) deleteGrantIndexesScratch(batch *pebble.Batch, externalID string, value, scratch []byte) ([]byte, error) { - entRT, entRID, entID, principalRT, principalID, _, err := scanGrantIndexFieldsRaw(value) + entRT, entRID, entID, principalRT, principalID, _, sourceScopeHash, err := scanGrantIndexFieldsRaw(value) if err != nil { return scratch, err } @@ -1096,6 +1117,12 @@ func (e *Engine) deleteGrantIndexesScratch(batch *pebble.Batch, externalID strin if err := batch.Delete(scratch, nil); err != nil { return scratch, err } + if sourceScopeHash != "" { + scratch = appendGrantBySourceScopeIndexKey(scratch[:0], sourceScopeHash, id) + if err := batch.Delete(scratch, nil); err != nil { + return scratch, err + } + } return scratch, nil } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grants_synth_encode.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grants_synth_encode.go index 8fa0a780..e63e72a3 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grants_synth_encode.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grants_synth_encode.go @@ -30,11 +30,19 @@ var expandedGrantImmutableAnnotations = []*anypb.Any{expandedGrantImmutableAnnot // appendSynthGrantExternalIDWire) and sources (field 9, appended by // appendGrantSourcesWire). Expansion/NeedsExpansion stay zero for // synthesized grants. +// +// SourceScopeHash is cleared unconditionally, for two reasons: synthesized +// (expander-derived) grants are never part of a source-cache scope, and — +// load-bearing for appendGrantSourcesWire — an empty proto3 scalar emits no +// bytes, which keeps sources (field 9) the highest field on the wire so the +// hand encoder's append-after-base-marshal stays canonical. See +// TestGrantSourcesWireSchemaPin. func fillSynthGrantRecord(r *v3.GrantRecord, rec *synthesizedGrantRecord, now *timestamppb.Timestamp) { r.SetEntitlement(rec.entitlement) r.SetPrincipal(rec.principal) r.SetAnnotations(expandedGrantImmutableAnnotations) r.SetDiscoveredAt(now) + r.SetSourceScopeHash("") } // grantExternalIDFieldTag is the wire tag for GrantRecord.external_id (2). diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/id_index_migration.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/id_index_migration.go index 019aa715..c592f89f 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/id_index_migration.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/id_index_migration.go @@ -213,7 +213,7 @@ func (e *Engine) emitStructuredGrantMigration(ctx context.Context, primary *spil lastLog = now } } - entRT, entRID, entID, principalRT, principalID, _, err := scanGrantIndexFieldsRaw(iter.Value()) + entRT, entRID, entID, principalRT, principalID, _, _, err := scanGrantIndexFieldsRaw(iter.Value()) if err != nil { return rows, fmt.Errorf("id-index migration: scan grant: %w", err) } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/if_newer.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/if_newer.go index 347091a0..e166f8dc 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/if_newer.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/if_newer.go @@ -182,7 +182,16 @@ func (e *Engine) PutEntitlementRecordsIfNewer(ctx context.Context, records ...*v closer.Close() continue } + oldScope, scanErr := scanEntitlementSourceScopeRaw(oldVal) closer.Close() + if scanErr != nil { + return scanErr + } + if oldScope != "" && oldScope != r.GetSourceScopeHash() { + if err := batch.Delete(encodeEntitlementBySourceScopeIndexKey(oldScope, id), nil); err != nil { + return err + } + } case errors.Is(getErr, pebble.ErrNotFound): default: return fmt.Errorf("PutEntitlementRecordsIfNewer: get: %w", getErr) @@ -194,6 +203,11 @@ func (e *Engine) PutEntitlementRecordsIfNewer(ctx context.Context, records ...*v if err := batch.Set(key, val, nil); err != nil { return err } + if sh := r.GetSourceScopeHash(); sh != "" { + if err := batch.Set(encodeEntitlementBySourceScopeIndexKey(sh, id), nil, nil); err != nil { + return err + } + } written++ } if written == 0 { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/keys.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/keys.go index f1484d08..a9ca13c6 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/keys.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/keys.go @@ -63,7 +63,16 @@ const ( typeIndex byte = 0x07 typeCounter byte = 0x08 typeSession byte = 0x09 - typeEngineMeta byte = 0xFF + // typeSourceCache holds one SourceCacheEntryRecord per + // (row_kind, scope_hash) — the source-cache replay manifest (see + // proto/c1/connector/v2/annotation_source_cache.proto). Placed + // after typeSession so the clone path's counter/session excise + // span ([typeCounter, upperBoundOf(typeSession))) leaves it in the + // clone: a cloned sync artifact stays usable as a replay source. + // ResetForNewSync's [typeResourceType, typeEngineMeta) span wipes + // it with the rest of the sync-scoped data. + typeSourceCache byte = 0x0A + typeEngineMeta byte = 0xFF ) // Index-discriminator bytes (second byte after typeIndex). One byte @@ -78,6 +87,13 @@ const ( idxGrantByNeedsExpansion byte = 0x05 idxGrantByPrincipalResourceType byte = 0x06 // retired: served by idxGrantByPrincipal prefix scans. idxGrantByEntitlementResource byte = 0x07 // retired: served by grant primary entitlement-resource prefix scans. + // by_source_scope families: partial indexes over records whose + // source_scope_hash is non-empty (source-cache replay). Tails are + // identity tuples per the keys.go convention, so replay derives + // each primary key from the index key without touching the record. + idxGrantBySourceScope byte = 0x08 + idxEntitlementBySourceScope byte = 0x09 + idxResourceBySourceScope byte = 0x0A ) // --- Grant --- @@ -270,6 +286,120 @@ func encodeGrantByPrincipalResourceTypeIdentityPrefix(principalRT string) []byte return codec.AppendTupleSeparator(buf) } +// --- Source-cache (by_source_scope indexes + entry keyspace) --- + +// encodeGrantBySourceScopeIndexKey is the partial by_source_scope index +// over grants whose SourceScopeHash is non-empty: +// +// v3 | typeIndex | idxGrantBySourceScope | 0x00 | +// scope_hash | 0x00 | +// ent_rt | 0x00 | ent_rid | 0x00 | ent_flag | 0x00 | ent_tail | 0x00 | +// principal_rt | 0x00 | principal_id +// +// The tail after scope_hash is the grant identity tuple — byte-identical +// to the grant primary key's tail — so replay derives the primary key from +// the index key alone. Paired with encodeGrantBySourceScopePrefix +// (by-value prefix, with trailing sep). +func encodeGrantBySourceScopeIndexKey(scopeHash string, id grantIdentity) []byte { + return appendGrantBySourceScopeIndexKey(make([]byte, 0, 128), scopeHash, id) +} + +func appendGrantBySourceScopeIndexKey(dst []byte, scopeHash string, id grantIdentity) []byte { + dst = append(dst, versionV3, typeIndex, idxGrantBySourceScope) + dst = codec.AppendTupleSeparator(dst) + return codec.AppendTupleStrings( + dst, + scopeHash, + id.entitlement.resourceTypeID, + id.entitlement.resourceID, + id.entitlement.flagComponent(), + id.entitlement.tail, + id.principalTypeID, + id.principalID, + ) +} + +// encodeGrantBySourceScopePrefix is the by-value prefix for "all grants +// stamped with this scope hash". Trailing sep is load-bearing — see the +// keys.go convention doc. +func encodeGrantBySourceScopePrefix(scopeHash string) []byte { + buf := make([]byte, 0, 32+len(scopeHash)) + buf = append(buf, versionV3, typeIndex, idxGrantBySourceScope) + buf = codec.AppendTupleSeparator(buf) + buf = codec.AppendTupleStrings(buf, scopeHash) + return codec.AppendTupleSeparator(buf) +} + +// encodeEntitlementBySourceScopeIndexKey: +// +// v3 | typeIndex | idxEntitlementBySourceScope | 0x00 | +// scope_hash | 0x00 | rt | 0x00 | rid | 0x00 | flag | 0x00 | tail +// +// Tail is the entitlement identity tuple, byte-identical to the +// entitlement primary key's tail. Paired with +// encodeEntitlementBySourceScopePrefix. +func encodeEntitlementBySourceScopeIndexKey(scopeHash string, id entitlementIdentity) []byte { + return appendEntitlementBySourceScopeIndexKey(make([]byte, 0, 128), scopeHash, id) +} + +func appendEntitlementBySourceScopeIndexKey(dst []byte, scopeHash string, id entitlementIdentity) []byte { + dst = append(dst, versionV3, typeIndex, idxEntitlementBySourceScope) + dst = codec.AppendTupleSeparator(dst) + return codec.AppendTupleStrings(dst, scopeHash, id.resourceTypeID, id.resourceID, id.flagComponent(), id.tail) +} + +func encodeEntitlementBySourceScopePrefix(scopeHash string) []byte { + buf := make([]byte, 0, 32+len(scopeHash)) + buf = append(buf, versionV3, typeIndex, idxEntitlementBySourceScope) + buf = codec.AppendTupleSeparator(buf) + buf = codec.AppendTupleStrings(buf, scopeHash) + return codec.AppendTupleSeparator(buf) +} + +// encodeResourceBySourceScopeIndexKey: +// +// v3 | typeIndex | idxResourceBySourceScope | 0x00 | +// scope_hash | 0x00 | resource_type_id | 0x00 | resource_id +// +// Tail is the resource primary tuple. Paired with +// encodeResourceBySourceScopePrefix. +func encodeResourceBySourceScopeIndexKey(scopeHash, resourceTypeID, resourceID string) []byte { + return appendResourceBySourceScopeIndexKey(make([]byte, 0, 96), scopeHash, resourceTypeID, resourceID) +} + +func appendResourceBySourceScopeIndexKey(dst []byte, scopeHash, resourceTypeID, resourceID string) []byte { + dst = append(dst, versionV3, typeIndex, idxResourceBySourceScope) + dst = codec.AppendTupleSeparator(dst) + return codec.AppendTupleStrings(dst, scopeHash, resourceTypeID, resourceID) +} + +func encodeResourceBySourceScopePrefix(scopeHash string) []byte { + buf := make([]byte, 0, 32+len(scopeHash)) + buf = append(buf, versionV3, typeIndex, idxResourceBySourceScope) + buf = codec.AppendTupleSeparator(buf) + buf = codec.AppendTupleStrings(buf, scopeHash) + return codec.AppendTupleSeparator(buf) +} + +// encodeSourceCacheEntryKey is the primary key for one source-cache +// manifest entry: +// +// v3 | typeSourceCache | 0x00 | row_kind | 0x00 | scope_hash +// +// Paired with encodeSourceCachePrefix (by-type prefix). +func encodeSourceCacheEntryKey(rowKind, scopeHash string) []byte { + buf := make([]byte, 0, 32+len(rowKind)+len(scopeHash)) + buf = append(buf, versionV3, typeSourceCache) + buf = codec.AppendTupleSeparator(buf) + return codec.AppendTupleStrings(buf, rowKind, scopeHash) +} + +// encodeSourceCachePrefix is the by-type prefix for all source-cache +// entries. +func encodeSourceCachePrefix() []byte { + return []byte{versionV3, typeSourceCache} +} + // --- ResourceType --- // encodeResourceTypeKey returns the primary key for a resource_type: @@ -498,6 +628,28 @@ func GrantByEntitlementResourceUpperBound() []byte { return upperBoundOf(GrantByEntitlementResourceLowerBound()) } +func GrantBySourceScopeLowerBound() []byte { + return []byte{versionV3, typeIndex, idxGrantBySourceScope} +} +func GrantBySourceScopeUpperBound() []byte { return upperBoundOf(GrantBySourceScopeLowerBound()) } + +func EntitlementBySourceScopeLowerBound() []byte { + return []byte{versionV3, typeIndex, idxEntitlementBySourceScope} +} +func EntitlementBySourceScopeUpperBound() []byte { + return upperBoundOf(EntitlementBySourceScopeLowerBound()) +} + +func ResourceBySourceScopeLowerBound() []byte { + return []byte{versionV3, typeIndex, idxResourceBySourceScope} +} +func ResourceBySourceScopeUpperBound() []byte { + return upperBoundOf(ResourceBySourceScopeLowerBound()) +} + +func SourceCacheEntryLowerBound() []byte { return encodeSourceCachePrefix() } +func SourceCacheEntryUpperBound() []byte { return upperBoundOf(encodeSourceCachePrefix()) } + func AssetLowerBound() []byte { return encodeAssetPrefix() } func AssetUpperBound() []byte { return upperBoundOf(encodeAssetPrefix()) } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/lookup.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/lookup.go index 2b70e59f..282c3a9c 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/lookup.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/lookup.go @@ -265,6 +265,28 @@ func (e *Engine) grantPrimaryPrefixNonEmpty(prefix []byte) (bool, error) { // the row instead of the concat). Exactly one hit wins; zero is // pebble.ErrNotFound; several is ErrAmbiguousExternalID. func (e *Engine) resolveGrantIdentityByExternalID(ctx context.Context, grantID string) (grantIdentity, error) { + id, err := e.resolveGrantIdentityByCandidates(ctx, grantID) + if err == nil || !errors.Is(err, errNoGrantCandidateHits) { + return id, err + } + // Candidate probing proved nothing either way: the id may still be a + // connector-custom STORED external id (with or without colons), which + // only the O(all grants) scan can find. Interactive edges pay it; + // bounded callers (DeleteGrantRecordBounded) stop before this line. + return e.scanGrantIdentityByStoredExternalID(ctx, grantID) +} + +// errNoGrantCandidateHits reports that combinatorial candidate probing +// completed without a hit — distinct from pebble.ErrNotFound, which the +// full resolution reserves for "provably absent after the scan of last +// resort". Internal to the resolution layer. +var errNoGrantCandidateHits = errors.New("pebble: no grant candidate parse hit") + +// resolveGrantIdentityByCandidates is the bounded stage of grant-id +// resolution: combinatorial concat splits probed with point Gets, no +// keyspace scan. Returns errNoGrantCandidateHits when the shape yields no +// candidates (fewer than two colons) or every candidate missed. +func (e *Engine) resolveGrantIdentityByCandidates(ctx context.Context, grantID string) (grantIdentity, error) { var colons []int for i := 0; i < len(grantID); i++ { if grantID[i] == ':' { @@ -272,10 +294,8 @@ func (e *Engine) resolveGrantIdentityByExternalID(ctx context.Context, grantID s } } if len(colons) < 2 { - // No concat shape to split: connector-custom ids (SQLite keyed rows - // by these, and provisioner revokes address grants with them) are - // findable only by their STORED external id. - return e.scanGrantIdentityByStoredExternalID(ctx, grantID) + // No concat shape to split: findable only by STORED external id. + return grantIdentity{}, errNoGrantCandidateHits } if len(colons) > maxBareIDColons { return grantIdentity{}, fmt.Errorf("%w: grant id has %d colons; too complex to resolve safely by string", ErrAmbiguousExternalID, len(colons)) @@ -378,9 +398,7 @@ func (e *Engine) resolveGrantIdentityByExternalID(ctx context.Context, grantID s } switch len(hits) { case 0: - // Every concat split missed: the id may still be a connector-custom - // STORED external id that merely contains colons. - return e.scanGrantIdentityByStoredExternalID(ctx, grantID) + return grantIdentity{}, errNoGrantCandidateHits case 1: return hits[0], nil default: diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/raw_records.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/raw_records.go index 89ddfd37..40640954 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/raw_records.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/raw_records.go @@ -110,18 +110,25 @@ func rawTimestampNanos(value []byte) (int64, error) { } func (e *Engine) deleteResourceIndexesRaw(batch *pebble.Batch, resourceTypeID string, resourceID string, value []byte) error { - parentRT, parentID, err := scanResourceParentRaw(value) + parentRT, parentID, sourceScopeHash, err := scanResourceIndexFieldsRaw(value) if err != nil { return err } - if parentID == "" { - return nil + if parentID != "" { + if err := batch.Delete(encodeResourceByParentIndexKey(parentRT, parentID, resourceTypeID, resourceID), nil); err != nil { + return err + } + } + if sourceScopeHash != "" { + if err := batch.Delete(encodeResourceBySourceScopeIndexKey(sourceScopeHash, resourceTypeID, resourceID), nil); err != nil { + return err + } } - return batch.Delete(encodeResourceByParentIndexKey(parentRT, parentID, resourceTypeID, resourceID), nil) + return nil } func (e *Engine) deleteGrantIndexesRaw(batch *pebble.Batch, externalID string, value []byte) error { - entRT, entRID, entID, principalRT, principalID, _, err := scanGrantIndexFieldsRaw(value) + entRT, entRID, entID, principalRT, principalID, _, sourceScopeHash, err := scanGrantIndexFieldsRaw(value) if err != nil { return err } @@ -136,6 +143,11 @@ func (e *Engine) deleteGrantIndexesRaw(batch *pebble.Batch, externalID string, v if err := batch.Delete(encodeGrantByPrincipalIdentityIndexKey(id), nil); err != nil { return err } + if sourceScopeHash != "" { + if err := batch.Delete(encodeGrantBySourceScopeIndexKey(sourceScopeHash, id), nil); err != nil { + return err + } + } return batch.Delete(encodeGrantByNeedsExpansionIdentityIndexKey(id), nil) } @@ -257,37 +269,83 @@ func scanEntitlementResourceTypeRaw(value []byte) ([]byte, error) { // occurrence of the target field, matching scanGrantIndexFieldsRaw and // approximating proto merge semantics. Values written by this SDK carry // at most one occurrence, so this only matters for foreign writers. -func scanResourceParentRaw(value []byte) (string, string, error) { - var rt, id string +// scanResourceIndexFieldsRaw extracts the parent ref (field 6) and +// source_scope_hash (field 9) from a marshaled ResourceRecord — the +// fields that key resource secondary indexes. +func scanResourceIndexFieldsRaw(value []byte) (string, string, string, error) { + var rt, id, sourceScopeHash string for len(value) > 0 { num, typ, n := protowire.ConsumeTag(value) if n < 0 { - return "", "", protowire.ParseError(n) + return "", "", "", protowire.ParseError(n) } value = value[n:] - if num != 6 { + switch num { + case 6: + if typ != protowire.BytesType { + return "", "", "", fmt.Errorf("raw record: resource parent has wire type %v", typ) + } + msg, n := protowire.ConsumeBytes(value) + if n < 0 { + return "", "", "", protowire.ParseError(n) + } + var err error + rt, id, err = scanResourceRefRaw(msg) + if err != nil { + return "", "", "", err + } + value = value[n:] + case 9: + if typ != protowire.BytesType { + return "", "", "", fmt.Errorf("raw record: resource source_scope_hash has wire type %v", typ) + } + s, n := protowire.ConsumeBytes(value) + if n < 0 { + return "", "", "", protowire.ParseError(n) + } + sourceScopeHash = string(s) + value = value[n:] + default: n = protowire.ConsumeFieldValue(num, typ, value) if n < 0 { - return "", "", protowire.ParseError(n) + return "", "", "", protowire.ParseError(n) + } + value = value[n:] + } + } + return rt, id, sourceScopeHash, nil +} + +// scanEntitlementSourceScopeRaw extracts source_scope_hash (field 11) +// from a marshaled EntitlementRecord. Keys the entitlement +// by_source_scope index — the only entitlement secondary index. +func scanEntitlementSourceScopeRaw(value []byte) (string, error) { + var sourceScopeHash string + for len(value) > 0 { + num, typ, n := protowire.ConsumeTag(value) + if n < 0 { + return "", protowire.ParseError(n) + } + value = value[n:] + if num != 11 { + n = protowire.ConsumeFieldValue(num, typ, value) + if n < 0 { + return "", protowire.ParseError(n) } value = value[n:] continue } if typ != protowire.BytesType { - return "", "", fmt.Errorf("raw record: resource parent has wire type %v", typ) + return "", fmt.Errorf("raw record: entitlement source_scope_hash has wire type %v", typ) } - msg, n := protowire.ConsumeBytes(value) + s, n := protowire.ConsumeBytes(value) if n < 0 { - return "", "", protowire.ParseError(n) - } - var err error - rt, id, err = scanResourceRefRaw(msg) - if err != nil { - return "", "", err + return "", protowire.ParseError(n) } + sourceScopeHash = string(s) value = value[n:] } - return rt, id, nil + return sourceScopeHash, nil } func scanEntitlementResourceRaw(value []byte) (string, string, error) { @@ -367,63 +425,73 @@ func scanEntitlementIdentityFieldsRaw(value []byte) (string, string, string, err return rt, id, externalID, nil } -func scanGrantIndexFieldsRaw(value []byte) (string, string, string, string, string, bool, error) { - var entRT, entRID, entID, principalRT, principalID string +func scanGrantIndexFieldsRaw(value []byte) (string, string, string, string, string, bool, string, error) { + var entRT, entRID, entID, principalRT, principalID, sourceScopeHash string var needsExpansion bool for len(value) > 0 { num, typ, n := protowire.ConsumeTag(value) if n < 0 { - return "", "", "", "", "", false, protowire.ParseError(n) + return "", "", "", "", "", false, "", protowire.ParseError(n) } value = value[n:] switch num { case 3: if typ != protowire.BytesType { - return "", "", "", "", "", false, fmt.Errorf("raw record: grant entitlement has wire type %v", typ) + return "", "", "", "", "", false, "", fmt.Errorf("raw record: grant entitlement has wire type %v", typ) } msg, n := protowire.ConsumeBytes(value) if n < 0 { - return "", "", "", "", "", false, protowire.ParseError(n) + return "", "", "", "", "", false, "", protowire.ParseError(n) } var err error entRT, entRID, entID, err = scanEntitlementRefRaw(msg) if err != nil { - return "", "", "", "", "", false, err + return "", "", "", "", "", false, "", err } value = value[n:] case 4: if typ != protowire.BytesType { - return "", "", "", "", "", false, fmt.Errorf("raw record: grant principal has wire type %v", typ) + return "", "", "", "", "", false, "", fmt.Errorf("raw record: grant principal has wire type %v", typ) } msg, n := protowire.ConsumeBytes(value) if n < 0 { - return "", "", "", "", "", false, protowire.ParseError(n) + return "", "", "", "", "", false, "", protowire.ParseError(n) } var err error principalRT, principalID, err = scanPrincipalRefRaw(msg) if err != nil { - return "", "", "", "", "", false, err + return "", "", "", "", "", false, "", err } value = value[n:] case 7: if typ != protowire.VarintType { - return "", "", "", "", "", false, fmt.Errorf("raw record: grant needs_expansion has wire type %v", typ) + return "", "", "", "", "", false, "", fmt.Errorf("raw record: grant needs_expansion has wire type %v", typ) } v, n := protowire.ConsumeVarint(value) if n < 0 { - return "", "", "", "", "", false, protowire.ParseError(n) + return "", "", "", "", "", false, "", protowire.ParseError(n) } needsExpansion = v != 0 value = value[n:] + case 10: + if typ != protowire.BytesType { + return "", "", "", "", "", false, "", fmt.Errorf("raw record: grant source_scope_hash has wire type %v", typ) + } + s, n := protowire.ConsumeBytes(value) + if n < 0 { + return "", "", "", "", "", false, "", protowire.ParseError(n) + } + sourceScopeHash = string(s) + value = value[n:] default: n = protowire.ConsumeFieldValue(num, typ, value) if n < 0 { - return "", "", "", "", "", false, protowire.ParseError(n) + return "", "", "", "", "", false, "", protowire.ParseError(n) } value = value[n:] } } - return entRT, entRID, entID, principalRT, principalID, needsExpansion, nil + return entRT, entRID, entID, principalRT, principalID, needsExpansion, sourceScopeHash, nil } // scanGrantNeedsExpansionRaw extracts only the needs_expansion flag diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/resources.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/resources.go index 564a0430..0f0d820c 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/resources.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/resources.go @@ -151,15 +151,22 @@ func (e *Engine) DeleteResourceRecord(ctx context.Context, resourceTypeID, resou } func (e *Engine) writeResourceIndexes(batch *pebble.Batch, r *v3.ResourceRecord) error { - parent := r.GetParent() - if parent == nil || parent.GetResourceId() == "" { - return nil + if parent := r.GetParent(); parent != nil && parent.GetResourceId() != "" { + k := encodeResourceByParentIndexKey( + parent.GetResourceTypeId(), parent.GetResourceId(), + r.GetResourceTypeId(), r.GetResourceId(), + ) + if err := batch.Set(k, nil, nil); err != nil { + return err + } + } + if sh := r.GetSourceScopeHash(); sh != "" { + k := encodeResourceBySourceScopeIndexKey(sh, r.GetResourceTypeId(), r.GetResourceId()) + if err := batch.Set(k, nil, nil); err != nil { + return err + } } - k := encodeResourceByParentIndexKey( - parent.GetResourceTypeId(), parent.GetResourceId(), - r.GetResourceTypeId(), r.GetResourceId(), - ) - return batch.Set(k, nil, nil) + return nil } func (e *Engine) IterateResources(ctx context.Context, yield func(*v3.ResourceRecord) bool) error { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/source_cache.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/source_cache.go new file mode 100644 index 00000000..dfb238c4 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/source_cache.go @@ -0,0 +1,835 @@ +package pebble + +import ( + "context" + "errors" + "fmt" + + "github.com/cockroachdb/pebble/v2" + "google.golang.org/protobuf/encoding/protowire" + "google.golang.org/protobuf/types/known/timestamppb" + + v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" + "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/codec" +) + +// Source-cache replay, engine side. +// +// The typeSourceCache keyspace holds one SourceCacheEntryRecord per +// (row_kind, scope_hash): the opaque upstream validator (etag / delta +// token) the sync recorded for that scope. Rows produced under a scope +// are stamped with source_scope_hash and indexed under the +// by_source_scope families, whose tails are identity tuples — so a +// replay derives every primary key from the index key alone and copies +// raw values across files without a proto unmarshal. +// +// The previous sync lives in a separate read-only engine (a Pebble c1z +// holds exactly one sync); replay copies from prev into the receiver. + +// replayBatchRows bounds how many rows accumulate in one pebble.Batch +// before an intermediate commit. Replay of a delta-query collection can +// be the whole previous row set, so the batch must not grow unbounded. +const replayBatchRows = 10_000 + +// SourceCacheReplayResult reports what one scope's replay copied. +type SourceCacheReplayResult struct { + Rows int64 + // NeedsExpansion is true when at least one copied grant row carried + // needs_expansion. The syncer must arm grant expansion in this case: + // replayed pages never pass GrantExpandable-annotated rows through + // the syncer's connector-response path, which is otherwise the only + // thing that enables the expansion phase. + NeedsExpansion bool +} + +// PutSourceCacheEntry writes the manifest entry for (rowKind, scopeHash). +// Zero-row scopes still get entries — the validator must survive to the +// next sync even when the scope produced no rows. +func (e *Engine) PutSourceCacheEntry(ctx context.Context, rowKind, scopeHash, etag string) error { + return e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + rec := &v3.SourceCacheEntryRecord{} + rec.SetRowKind(rowKind) + rec.SetScopeHash(scopeHash) + rec.SetEtag(etag) + rec.SetDiscoveredAt(timestamppb.Now()) + val, err := marshalRecord(rec) + if err != nil { + return err + } + opts := writeOpts(e.opts.durability) + if e.IsFreshSync() { + opts = pebble.NoSync + } + return e.db.Set(encodeSourceCacheEntryKey(rowKind, scopeHash), val, opts) + }) +} + +// GetSourceCacheEntry returns the manifest entry for (rowKind, scopeHash), +// or pebble.ErrNotFound. +func (e *Engine) GetSourceCacheEntry(ctx context.Context, rowKind, scopeHash string) (*v3.SourceCacheEntryRecord, error) { + val, closer, err := e.db.Get(encodeSourceCacheEntryKey(rowKind, scopeHash)) + if err != nil { + return nil, err + } + defer closer.Close() + rec := &v3.SourceCacheEntryRecord{} + if err := unmarshalRecord(val, rec); err != nil { + return nil, fmt.Errorf("GetSourceCacheEntry: unmarshal: %w", err) + } + return rec, nil +} + +// DeleteGrantRecordBounded deletes a grant by canonical public id WITHOUT +// the O(all grants) stored-external-id scan fallback that the interactive +// DeleteGrantRecord path is allowed to take. Used by the source-cache +// tombstone path, where a mass-removal round would otherwise pay a full +// keyspace scan PER already-absent id. +// +// Consequence, by design: a grant stored under a connector-CUSTOM id (one +// that isn't the SDK concat shape) is unreachable here and the delete +// no-ops. Connectors with custom grant ids must use principal-scoped +// tombstones (SourceCacheScope.deleted_principal_ids) instead — documented +// in the annotation proto. +func (e *Engine) DeleteGrantRecordBounded(ctx context.Context, externalID string) error { + return e.withWrite(func() error { + id, err := e.resolveGrantIdentityByCandidates(ctx, externalID) + if err != nil { + if errors.Is(err, errNoGrantCandidateHits) || errors.Is(err, pebble.ErrNotFound) { + return nil // absent (or custom-id) — tombstone no-op + } + return err + } + return e.deleteGrantByIdentityLocked(id) + }) +} + +// DeleteGrantsByPrincipalsInScope deletes every grant row in the CURRENT +// store stamped with scopeHash whose principal id is in principalIDs — +// the engine side of principal-scoped delta tombstones +// (SourceCacheScope.deleted_principal_ids). +// +// One prefix scan of the scope's by_source_scope index resolves +// everything: the index tail IS the grant identity, so the primary key +// and every secondary index key for a match are constructible from the +// index key alone — no value reads, no string resolution, no guessing. +// A principal with no rows in the scope is a no-op (providers tombstone +// objects the client never synced). Deleting a missing secondary index +// entry is a pebble no-op, which covers the mixed inline/deferred +// by_principal state mid-sync. +// +// Complexity: O(scope size) tuple-walks per call regardless of tombstone +// count — callers batch a page's tombstones into one call. +func (e *Engine) DeleteGrantsByPrincipalsInScope(ctx context.Context, scopeHash string, principalIDs map[string]struct{}) (int64, error) { + if len(principalIDs) == 0 { + return 0, nil + } + prefix := encodeGrantBySourceScopePrefix(scopeHash) + var deleted int64 + + err := e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return err + } + defer iter.Close() + + opts := writeOpts(e.opts.durability) + if e.IsFreshSync() { + opts = pebble.NoSync + } + batch := e.db.NewBatch() + defer func() { _ = batch.Close() }() + + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return err + } + key := iter.Key() + tail := key[len(prefix):] + // Tail layout: ent_rt | ent_rid | flag | ent_tail | prin_rt | prin_id + // (identical to the grant primary key tail; decoder shared with + // the primary-prefix scan paths in grants.go). + id, ok := decodeGrantIdentityTail(key, prefix) + if !ok { + continue // malformed index key — defensive skip + } + if _, hit := principalIDs[id.principalID]; !hit { + continue + } + // Primary key = grant header + the identity tail verbatim. + priKey := make([]byte, 0, 3+len(tail)) + priKey = append(priKey, versionV3, typeGrant) + priKey = codec.AppendTupleSeparator(priKey) + priKey = append(priKey, tail...) + if err := batch.Delete(priKey, nil); err != nil { + return err + } + if err := batch.Delete(encodeGrantByPrincipalIdentityIndexKey(id), nil); err != nil { + return err + } + if err := batch.Delete(encodeGrantByNeedsExpansionIdentityIndexKey(id), nil); err != nil { + return err + } + // The scope index entry itself (the key under the iterator — + // safe: the iterator reads a snapshot). + if err := batch.Delete(key, nil); err != nil { + return err + } + deleted++ + } + if err := iter.Error(); err != nil { + return err + } + return batch.Commit(opts) + }) + if err != nil { + return 0, err + } + return deleted, nil +} + +// DeleteGrantsByExternalIDsInScope deletes every grant row in the CURRENT +// store stamped with scopeHash whose STORED grant id (external id, which +// may be a connector-custom shape) is in ids. One scan of the scope's +// index, loading each candidate's primary row to compare the stored id — +// bounded by the scope's row count, never the whole keyspace. This is the +// tombstone path for connectors with custom grant ids whose scopes span +// multiple resources (so principal-scoped deletes would over-delete). +func (e *Engine) DeleteGrantsByExternalIDsInScope(ctx context.Context, scopeHash string, ids map[string]struct{}) (int64, error) { + if len(ids) == 0 { + return 0, nil + } + prefix := encodeGrantBySourceScopePrefix(scopeHash) + var deleted int64 + + err := e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return err + } + defer iter.Close() + + opts := writeOpts(e.opts.durability) + if e.IsFreshSync() { + opts = pebble.NoSync + } + batch := e.db.NewBatch() + defer func() { _ = batch.Close() }() + + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return err + } + key := iter.Key() + tail := key[len(prefix):] + id, ok := decodeGrantIdentityTail(key, prefix) + if !ok { + continue // malformed index key — defensive skip + } + // Primary key = grant header + the identity tail verbatim. + priKey := make([]byte, 0, 3+len(tail)) + priKey = append(priKey, versionV3, typeGrant) + priKey = codec.AppendTupleSeparator(priKey) + priKey = append(priKey, tail...) + + val, closer, err := e.db.Get(priKey) + if err != nil { + if errors.Is(err, pebble.ErrNotFound) { + continue // index ahead of primary — defensive skip + } + return err + } + rec := &v3.GrantRecord{} + uerr := unmarshalRecord(val, rec) + _ = closer.Close() + if uerr != nil { + return fmt.Errorf("DeleteGrantsByExternalIDsInScope: unmarshal: %w", uerr) + } + if _, hit := ids[rec.GetExternalId()]; !hit { + continue + } + if err := batch.Delete(priKey, nil); err != nil { + return err + } + if err := batch.Delete(encodeGrantByPrincipalIdentityIndexKey(id), nil); err != nil { + return err + } + if err := batch.Delete(encodeGrantByNeedsExpansionIdentityIndexKey(id), nil); err != nil { + return err + } + if err := batch.Delete(key, nil); err != nil { + return err + } + deleted++ + } + if err := iter.Error(); err != nil { + return err + } + return batch.Commit(opts) + }) + if err != nil { + return 0, err + } + return deleted, nil +} + +// DeleteResourcesByIDsInScope deletes every resource row in the CURRENT +// store stamped with scopeHash whose resource id is in resourceIDs (any +// resource type) — principal-scoped tombstones for RowKindResources. +func (e *Engine) DeleteResourcesByIDsInScope(ctx context.Context, scopeHash string, resourceIDs map[string]struct{}) (int64, error) { + if len(resourceIDs) == 0 { + return 0, nil + } + prefix := encodeResourceBySourceScopePrefix(scopeHash) + var deleted int64 + + err := e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return err + } + defer iter.Close() + + opts := writeOpts(e.opts.durability) + if e.IsFreshSync() { + opts = pebble.NoSync + } + batch := e.db.NewBatch() + defer func() { _ = batch.Close() }() + + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return err + } + key := iter.Key() + tail := key[len(prefix):] + // Tail layout: resource_type_id | resource_id. + rtBytes, next, ok := codec.DecodeTupleStringAlias(tail, 0) + if !ok || next >= len(tail) { + continue + } + ridBytes, _, ok := codec.DecodeTupleStringAlias(tail, next+1) + if !ok { + continue + } + if _, hit := resourceIDs[string(ridBytes)]; !hit { + continue + } + rt, rid := string(rtBytes), string(ridBytes) + priKey := make([]byte, 0, 3+len(tail)) + priKey = append(priKey, versionV3, typeResource) + priKey = codec.AppendTupleSeparator(priKey) + priKey = append(priKey, tail...) + // by_parent cleanup needs the parent ref from the value. + if val, closer, getErr := e.db.Get(priKey); getErr == nil { + err := e.deleteResourceIndexesRaw(batch, rt, rid, val) + closer.Close() + if err != nil { + return err + } + } else if !errors.Is(getErr, pebble.ErrNotFound) { + return getErr + } + if err := batch.Delete(priKey, nil); err != nil { + return err + } + // deleteResourceIndexesRaw already covered the scope entry + // (source_scope_hash is in the value), but delete the iterated + // key too in case the value read missed (orphan entry). + if err := batch.Delete(key, nil); err != nil { + return err + } + deleted++ + } + if err := iter.Error(); err != nil { + return err + } + return batch.Commit(opts) + }) + if err != nil { + return 0, err + } + return deleted, nil +} + +// grantValueHasSourcesRaw reports whether a marshaled GrantRecord carries +// at least one sources entry (field 9), without unmarshaling. +func grantValueHasSourcesRaw(value []byte) (bool, error) { + for len(value) > 0 { + num, typ, n := protowire.ConsumeTag(value) + if n < 0 { + return false, protowire.ParseError(n) + } + value = value[n:] + if num == 9 { + return true, nil + } + n = protowire.ConsumeFieldValue(num, typ, value) + if n < 0 { + return false, protowire.ParseError(n) + } + value = value[n:] + } + return false, nil +} + +// stripExpanderSourcesRaw clears a replayed grant's Sources map when it is +// expander-written, so the current sync's expansion recomputes it from +// true state instead of inheriting contributions that may have been +// removed upstream. Classification mirrors RollbackExpansion: a Sources +// map containing a self-source entry (keyed by the grant's own entitlement +// id) was written by the expander; one without a self-source is +// connector-set public data and is preserved. Returns (newValue, true) when +// the record was rewritten, (nil, false) when the original bytes should be +// copied verbatim. +func stripExpanderSourcesRaw(value []byte, ownEntitlementID string) ([]byte, bool, error) { + r := &v3.GrantRecord{} + if err := unmarshalRecord(value, r); err != nil { + return nil, false, fmt.Errorf("source cache replay: unmarshal grant for sources strip: %w", err) + } + sources := r.GetSources() + if len(sources) == 0 { + return nil, false, nil + } + if _, hasSelf := sources[ownEntitlementID]; !hasSelf { + // No self-source: connector-set Sources. Preserve verbatim. + return nil, false, nil + } + r.SetSources(nil) + stripped, err := marshalRecord(r) + if err != nil { + return nil, false, fmt.Errorf("source cache replay: re-marshal grant after sources strip: %w", err) + } + return stripped, true, nil +} + +// decodeResourcePrimaryTail decodes (resource_type_id, resource_id) +// from a resource primary key (v3 | typeResource | 0x00 | rt | 0x00 | rid). +func decodeResourcePrimaryTail(priKey []byte) (string, string, error) { + const headerLen = 3 // versionV3, typeResource, separator + if len(priKey) <= headerLen { + return "", "", fmt.Errorf("source cache replay: malformed resource primary key %x", priKey) + } + tail := priKey[headerLen:] + rtBytes, next, err := codec.DecodeTupleStringTo(nil, tail, 0) + if err != nil { + return "", "", err + } + if next >= len(tail) { + return "", "", fmt.Errorf("source cache replay: resource primary key missing resource_id: %x", priKey) + } + ridBytes, _, err := codec.DecodeTupleStringTo(nil, tail, next+1) + if err != nil { + return "", "", err + } + return string(rtBytes), string(ridBytes), nil +} + +// replayPrimaryFromIndexKey derives a record's primary key from its +// by_source_scope index key. The index prefix (header|0x00|scope|0x00) +// is followed by exactly the identity tuple that forms the primary +// key's tail, so the primary is header' + 0x00-separated remainder. +func replayPrimaryFromIndexKey(indexKey, indexPrefix []byte, primaryHeader [2]byte) ([]byte, error) { + if len(indexKey) <= len(indexPrefix) { + return nil, fmt.Errorf("source cache replay: malformed index key %x", indexKey) + } + tail := indexKey[len(indexPrefix):] + key := make([]byte, 0, 3+len(tail)) + key = append(key, primaryHeader[0], primaryHeader[1], 0x00) + return append(key, tail...), nil +} + +// ReplaySourceCacheGrants copies every grant stamped with scopeHash from +// prev into the receiver: raw primary copy plus index synthesis from the +// raw value (principal, needs_expansion, source-scope families). Mirrors +// PutGrantRecords' read-before-write index cleanup when the receiver +// already holds a record at the same identity. +func (e *Engine) ReplaySourceCacheGrants(ctx context.Context, prev *Engine, scopeHash string) (SourceCacheReplayResult, error) { + var res SourceCacheReplayResult + prefix := encodeGrantBySourceScopePrefix(scopeHash) + primaryHeader := [2]byte{versionV3, typeGrant} + + err := e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + iter, err := prev.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return err + } + defer iter.Close() + + opts := writeOpts(e.opts.durability) + if e.IsFreshSync() { + opts = pebble.NoSync + } + batch := e.db.NewBatch() + defer func() { _ = batch.Close() }() + rowsInBatch := 0 + + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return err + } + priKey, err := replayPrimaryFromIndexKey(iter.Key(), prefix, primaryHeader) + if err != nil { + return err + } + val, closer, getErr := prev.db.Get(priKey) + if getErr != nil { + if errors.Is(getErr, pebble.ErrNotFound) { + // Orphan index entry in the previous file — skip, + // matching the defensive-skip semantic of the other + // index read paths. + continue + } + return fmt.Errorf("source cache replay: get prev grant: %w", getErr) + } + + entRT, entRID, entID, principalRT, principalID, needsExpansion, srcScope, scanErr := scanGrantIndexFieldsRaw(val) + if scanErr != nil { + closer.Close() + return scanErr + } + // Stale-index defense: only copy rows whose VALUE stamp matches + // the queried scope. An index entry pointing at a row stamped + // differently (or not at all) is left over from a path that + // replaced the row without cleaning the index — e.g. a fold + // compaction predating the source-cache bucket plans, or an + // in-sync same-identity rewrite under a different scope. Copying + // it would inject rows upstream never returned for this scope. + if srcScope != scopeHash { + closer.Close() + continue + } + + // Clean up index entries for any record the current sync + // already wrote at this identity (same discipline as + // PutGrantRecords' read-before-write). + if oldVal, oldCloser, oldErr := e.db.Get(priKey); oldErr == nil { + if err := e.deleteGrantIndexesRaw(batch, "", oldVal); err != nil { + oldCloser.Close() + closer.Close() + return err + } + oldCloser.Close() + } else if !errors.Is(oldErr, pebble.ErrNotFound) { + closer.Close() + return fmt.Errorf("source cache replay: get current grant: %w", oldErr) + } + + // Replay-equivalence: a cached sync must reproduce what a full + // resync would produce. The one field where a verbatim copy + // diverges is expander-written Sources — the previous sync's + // expansion baked contributions into direct grants, and + // re-expansion only ADDS, so a contribution removed this sync + // (via a delta tombstone or a refetched page) would survive + // forever. Strip expander-written Sources so the current sync's + // expansion recomputes them from true state; connector-set + // Sources (no self-source entry — same classification as + // RollbackExpansion) are connector data and are preserved + // verbatim. The probe is a cheap protowire scan; the vast + // majority of rows carry no Sources and stay on the raw-copy + // path. + writeVal := val + if hasSources, probeErr := grantValueHasSourcesRaw(val); probeErr != nil { + closer.Close() + return probeErr + } else if hasSources { + stripped, strippedOK, stripErr := stripExpanderSourcesRaw(val, entID) + if stripErr != nil { + closer.Close() + return stripErr + } + if strippedOK { + writeVal = stripped + } + } + if err := batch.Set(priKey, writeVal, nil); err != nil { + closer.Close() + return err + } + closer.Close() + + if entID != "" && entRT != "" && entRID != "" && principalRT != "" && principalID != "" { + id := grantIdentity{ + entitlement: entitlementIdentityFromParts(entRT, entRID, entID), + principalTypeID: principalRT, + principalID: principalID, + } + if _, err := e.writeGrantIndexesForIdentityScratch(batch, id, needsExpansion, srcScope, nil); err != nil { + return err + } + } + if needsExpansion { + res.NeedsExpansion = true + } + res.Rows++ + rowsInBatch++ + if rowsInBatch >= replayBatchRows { + if err := batch.Commit(opts); err != nil { + return err + } + _ = batch.Close() + batch = e.db.NewBatch() + rowsInBatch = 0 + } + } + if err := iter.Error(); err != nil { + return err + } + return batch.Commit(opts) + }) + if err != nil { + return SourceCacheReplayResult{}, err + } + return res, nil +} + +// ReplaySourceCacheEntitlements copies every entitlement stamped with +// scopeHash from prev into the receiver. +func (e *Engine) ReplaySourceCacheEntitlements(ctx context.Context, prev *Engine, scopeHash string) (SourceCacheReplayResult, error) { + var res SourceCacheReplayResult + prefix := encodeEntitlementBySourceScopePrefix(scopeHash) + primaryHeader := [2]byte{versionV3, typeEntitlement} + + err := e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + iter, err := prev.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return err + } + defer iter.Close() + + opts := writeOpts(e.opts.durability) + if e.IsFreshSync() { + opts = pebble.NoSync + } + batch := e.db.NewBatch() + defer func() { _ = batch.Close() }() + rowsInBatch := 0 + + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return err + } + priKey, err := replayPrimaryFromIndexKey(iter.Key(), prefix, primaryHeader) + if err != nil { + return err + } + val, closer, getErr := prev.db.Get(priKey) + if getErr != nil { + if errors.Is(getErr, pebble.ErrNotFound) { + continue + } + return fmt.Errorf("source cache replay: get prev entitlement: %w", getErr) + } + // Stale-index defense: only copy rows whose VALUE stamp matches + // the queried scope (see the grants replay for rationale). + prevScope, prevScanErr := scanEntitlementSourceScopeRaw(val) + if prevScanErr != nil { + closer.Close() + return prevScanErr + } + if prevScope != scopeHash { + closer.Close() + continue + } + // The replayed row's source-scope index key is byte-identical + // to prev's — the only entitlement secondary index. Clean up a + // differing stamp on any record the current sync already wrote + // at this identity. + if oldVal, oldCloser, oldErr := e.db.Get(priKey); oldErr == nil { + oldScope, scanErr := scanEntitlementSourceScopeRaw(oldVal) + oldCloser.Close() + if scanErr != nil { + closer.Close() + return scanErr + } + if oldScope != "" && oldScope != scopeHash { + // The old index key's tail equals the primary key's + // tail (identity tuple), so rebuild it byte-wise. + oldIdxKey := make([]byte, 0, 8+len(oldScope)+len(priKey)) + oldIdxKey = append(oldIdxKey, versionV3, typeIndex, idxEntitlementBySourceScope) + oldIdxKey = codec.AppendTupleSeparator(oldIdxKey) + oldIdxKey = codec.AppendTupleStrings(oldIdxKey, oldScope) + oldIdxKey = codec.AppendTupleSeparator(oldIdxKey) + oldIdxKey = append(oldIdxKey, priKey[3:]...) + if err := batch.Delete(oldIdxKey, nil); err != nil { + closer.Close() + return err + } + } + } else if !errors.Is(oldErr, pebble.ErrNotFound) { + closer.Close() + return fmt.Errorf("source cache replay: get current entitlement: %w", oldErr) + } + if err := batch.Set(priKey, val, nil); err != nil { + closer.Close() + return err + } + closer.Close() + if err := batch.Set(iter.Key(), nil, nil); err != nil { + return err + } + res.Rows++ + rowsInBatch++ + if rowsInBatch >= replayBatchRows { + if err := batch.Commit(opts); err != nil { + return err + } + _ = batch.Close() + batch = e.db.NewBatch() + rowsInBatch = 0 + } + } + if err := iter.Error(); err != nil { + return err + } + return batch.Commit(opts) + }) + if err != nil { + return SourceCacheReplayResult{}, err + } + return res, nil +} + +// ReplaySourceCacheResources copies every resource stamped with scopeHash +// from prev into the receiver, synthesizing by_parent and by_source_scope +// index entries from the raw value. +func (e *Engine) ReplaySourceCacheResources(ctx context.Context, prev *Engine, scopeHash string) (SourceCacheReplayResult, error) { + var res SourceCacheReplayResult + prefix := encodeResourceBySourceScopePrefix(scopeHash) + primaryHeader := [2]byte{versionV3, typeResource} + + err := e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + iter, err := prev.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return err + } + defer iter.Close() + + opts := writeOpts(e.opts.durability) + if e.IsFreshSync() { + opts = pebble.NoSync + } + batch := e.db.NewBatch() + defer func() { _ = batch.Close() }() + rowsInBatch := 0 + + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return err + } + priKey, err := replayPrimaryFromIndexKey(iter.Key(), prefix, primaryHeader) + if err != nil { + return err + } + val, closer, getErr := prev.db.Get(priKey) + if getErr != nil { + if errors.Is(getErr, pebble.ErrNotFound) { + continue + } + return fmt.Errorf("source cache replay: get prev resource: %w", getErr) + } + + rt, rid, decodeErr := decodeResourcePrimaryTail(priKey) + if decodeErr != nil { + closer.Close() + return decodeErr + } + parentRT, parentID, srcScope, scanErr := scanResourceIndexFieldsRaw(val) + if scanErr != nil { + closer.Close() + return scanErr + } + // Stale-index defense: only copy rows whose VALUE stamp matches + // the queried scope (see the grants replay for rationale). + if srcScope != scopeHash { + closer.Close() + continue + } + if oldVal, oldCloser, oldErr := e.db.Get(priKey); oldErr == nil { + if err := e.deleteResourceIndexesRaw(batch, rt, rid, oldVal); err != nil { + oldCloser.Close() + closer.Close() + return err + } + oldCloser.Close() + } else if !errors.Is(oldErr, pebble.ErrNotFound) { + closer.Close() + return fmt.Errorf("source cache replay: get current resource: %w", oldErr) + } + + if err := batch.Set(priKey, val, nil); err != nil { + closer.Close() + return err + } + closer.Close() + + if parentID != "" { + if err := batch.Set(encodeResourceByParentIndexKey(parentRT, parentID, rt, rid), nil, nil); err != nil { + return err + } + } + if srcScope != "" { + if err := batch.Set(encodeResourceBySourceScopeIndexKey(srcScope, rt, rid), nil, nil); err != nil { + return err + } + } + res.Rows++ + rowsInBatch++ + if rowsInBatch >= replayBatchRows { + if err := batch.Commit(opts); err != nil { + return err + } + _ = batch.Close() + batch = e.db.NewBatch() + rowsInBatch = 0 + } + } + if err := iter.Error(); err != nil { + return err + } + return batch.Commit(opts) + }) + if err != nil { + return SourceCacheReplayResult{}, err + } + return res, nil +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/sync_runs.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/sync_runs.go index b3d2b267..8894a719 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/sync_runs.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/sync_runs.go @@ -142,7 +142,7 @@ func (e *Engine) IterateAllSyncRuns(ctx context.Context, yield func(*v3.SyncRunR // This is the single source of truth for "pick the latest finished // sync" on the Pebble engine; all three external entry points // (connectorstore.LatestFinishedSyncIDFetcher, -// c1zstore.SyncMeta.LatestFullSync / LatestFinishedSyncOfAnyType, and +// dotc1z.SyncMeta.LatestFullSync / LatestFinishedSyncOfAnyType, and // reader_v2.SyncsReaderService.GetLatestFinishedSync) call here so // the tiebreaker and predicate semantics stay consistent. // diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine_registry.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine_registry.go index 403f1683..bc438b43 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine_registry.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine_registry.go @@ -10,7 +10,6 @@ import ( "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" "go.uber.org/zap" - "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" formatv3 "github.com/conductorone/baton-sdk/pkg/dotc1z/format/v3" ) @@ -32,13 +31,13 @@ type StoreOptions struct { SyncLimit int SkipCleanup bool V2GrantsWriter bool - Engine c1zstore.Engine + Engine Engine // PayloadEncoding selects the v3 envelope payload framing for // engines that produce a v3 envelope (currently Pebble). Zero // value means "engine default" (PayloadEncodingIndexedZstd for // Pebble). - PayloadEncoding c1zstore.PayloadEncoding + PayloadEncoding PayloadEncoding // DecoderPool optionally scopes v3 payload-decoder reuse to the // caller's operation (see WithDecoderPool). Nil means a one-shot @@ -85,20 +84,20 @@ func WithDecoderPool(p *EnvelopeDecoderPool) C1ZOption { // and Pebble drivers are both registered statically by this package; // RegisterEngine exists for additional engines. type EngineDriver interface { - Engine() c1zstore.Engine + Engine() Engine Format() C1ZFormat - OpenStore(ctx context.Context, outputFilePath string, opts StoreOptions) (c1zstore.Store, error) + OpenStore(ctx context.Context, outputFilePath string, opts StoreOptions) (C1ZStore, error) } type engineRegistry struct { mu sync.RWMutex - byEngine map[c1zstore.Engine]EngineDriver + byEngine map[Engine]EngineDriver } var defaultEngineRegistry = &engineRegistry{ - byEngine: map[c1zstore.Engine]EngineDriver{ - c1zstore.EngineSQLite: sqliteDriver{}, - c1zstore.EnginePebble: pebbleDriver{}, + byEngine: map[Engine]EngineDriver{ + EngineSQLite: sqliteDriver{}, + EnginePebble: pebbleDriver{}, }, } @@ -109,7 +108,7 @@ func RegisterEngine(driver EngineDriver) error { } // EngineDriverFor returns the registered driver for engine. -func EngineDriverFor(engine c1zstore.Engine) (EngineDriver, bool) { +func EngineDriverFor(engine Engine) (EngineDriver, bool) { return defaultEngineRegistry.driverForEngine(engine) } @@ -135,7 +134,7 @@ func (r *engineRegistry) register(driver EngineDriver) error { return nil } -func (r *engineRegistry) driverForEngine(engine c1zstore.Engine) (EngineDriver, bool) { +func (r *engineRegistry) driverForEngine(engine Engine) (EngineDriver, bool) { r.mu.RLock() defer r.mu.RUnlock() driver, ok := r.byEngine[engine] @@ -146,7 +145,7 @@ func (r *engineRegistry) driverForEngine(engine c1zstore.Engine) (EngineDriver, // the engine-neutral constructor for callers that may opt into non-default // engines. NewC1ZFile remains the concrete SQLite constructor for legacy // callers that need *C1File. -func NewStore(ctx context.Context, outputFilePath string, opts ...C1ZOption) (c1zstore.Store, error) { +func NewStore(ctx context.Context, outputFilePath string, opts ...C1ZOption) (C1ZStore, error) { options, err := buildC1ZOptions(opts...) if err != nil { return nil, err @@ -194,7 +193,7 @@ func storeOptionsFromC1ZOptions(options *c1zOptions) StoreOptions { MaxDecoderMemoryBytes: maxDecoderMemoryBytes, } if out.Engine == "" { - out.Engine = c1zstore.EngineSQLite + out.Engine = EngineSQLite } out.Pragmas = make([]StorePragma, 0, len(options.pragmas)) for _, p := range options.pragmas { @@ -225,7 +224,7 @@ func selectStoreDriver(ctx context.Context, outputFilePath string, options *c1zO l := ctxzap.Extract(ctx) requested := options.engine if requested == "" { - requested = c1zstore.EngineSQLite + requested = EngineSQLite } stat, err := os.Stat(outputFilePath) // #nosec G703 -- c1z path is caller-controlled by API design. @@ -253,11 +252,11 @@ func selectStoreDriver(ctx context.Context, outputFilePath string, options *c1zO return nil, err } - var fileEngine c1zstore.Engine + var fileEngine Engine switch format { case C1ZFormatV1: // Maybe error if the file is read-only? - if requested == c1zstore.EnginePebble && !options.readOnly { + if requested == EnginePebble && !options.readOnly { // Close our header-read handle before converting: the conversion // renames a temp file over outputFilePath, which fails on Windows // if any handle to the destination is still open. Nil out f so @@ -272,9 +271,9 @@ func selectStoreDriver(ctx context.Context, outputFilePath string, options *c1zO return nil, fmt.Errorf("select-store-driver: convert existing v1 c1z to pebble: %w", err) } l.Debug("converted existing v1 c1z to pebble", zap.String("output_file_path", outputFilePath)) - return requireEngineDriver(c1zstore.EnginePebble) + return requireEngineDriver(EnginePebble) } - fileEngine = c1zstore.EngineSQLite + fileEngine = EngineSQLite case C1ZFormatV3: if _, err := f.Seek(0, 0); err != nil { return nil, err @@ -289,13 +288,13 @@ func selectStoreDriver(ctx context.Context, outputFilePath string, options *c1zO if err != nil { return nil, err } - fileEngine = c1zstore.Engine(m.GetEngine()) + fileEngine = Engine(m.GetEngine()) // Current and legacy pebble manifest names all dispatch to the same // driver; legacy interiors are re-keyed by the on-open id-index // migration. Unknown (newer) names fall through and fail loudly in // requireEngineDriver. - if fileEngine == c1zstore.PebbleManifestEngine || fileEngine == c1zstore.PebbleManifestEngineV2 { - fileEngine = c1zstore.EnginePebble + if fileEngine == PebbleManifestEngine || fileEngine == PebbleManifestEngineV2 { + fileEngine = EnginePebble } default: return nil, ErrInvalidFile @@ -312,7 +311,7 @@ func selectStoreDriver(ctx context.Context, outputFilePath string, options *c1zO return requireEngineDriver(fileEngine) } -func requireEngineDriver(engine c1zstore.Engine) (EngineDriver, error) { +func requireEngineDriver(engine Engine) (EngineDriver, error) { driver, ok := EngineDriverFor(engine) if !ok { return nil, fmt.Errorf("require-engine-driver: %w: %s", ErrEngineNotAvailable, engine) @@ -322,12 +321,12 @@ func requireEngineDriver(engine c1zstore.Engine) (EngineDriver, error) { type sqliteDriver struct{} -func (sqliteDriver) Engine() c1zstore.Engine { return c1zstore.EngineSQLite } -func (sqliteDriver) Format() C1ZFormat { return C1ZFormatV1 } +func (sqliteDriver) Engine() Engine { return EngineSQLite } +func (sqliteDriver) Format() C1ZFormat { return C1ZFormatV1 } -func (sqliteDriver) OpenStore(ctx context.Context, outputFilePath string, opts StoreOptions) (c1zstore.Store, error) { +func (sqliteDriver) OpenStore(ctx context.Context, outputFilePath string, opts StoreOptions) (C1ZStore, error) { c1zOpts := []C1ZOption{ - WithEngine(c1zstore.EngineSQLite), + WithEngine(EngineSQLite), WithEncoderConcurrency(opts.EncoderConcurrency), } if opts.TmpDir != "" { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/file_ops.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/file_ops.go new file mode 100644 index 00000000..d3210586 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/file_ops.go @@ -0,0 +1,25 @@ +package dotc1z + +import "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + +// The file-operations contract lives in pkg/dotc1z/c1zstore so storage +// engines can implement it without importing this package. These aliases +// preserve the historical dotc1z names. + +// FileOps is the file-level operations sub-store of C1ZStore. See +// c1zstore.FileOps for the full contract. +type FileOps = c1zstore.FileOps + +// CloneSyncOption configures a FileOps.CloneSync call. See +// c1zstore.CloneSyncOption. +type CloneSyncOption = c1zstore.CloneSyncOption + +// CloneSyncOptions carries the engine-neutral knobs for FileOps.CloneSync. +// See c1zstore.CloneSyncOptions. +type CloneSyncOptions = c1zstore.CloneSyncOptions + +// WithCloneTmpDir sets the temporary directory used while assembling the +// cloned c1z. Replaces WithC1FTmpDir at FileOps.CloneSync call sites. +func WithCloneTmpDir(dir string) CloneSyncOption { + return c1zstore.WithCloneTmpDir(dir) +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/format.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/format.go index bda658a8..9ca49197 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/format.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/format.go @@ -4,6 +4,8 @@ import ( "bytes" "fmt" "io" + + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" ) // C1ZFormat identifies the on-disk format of a .c1z file. The format byte @@ -40,10 +42,64 @@ func (f C1ZFormat) String() string { // C1Z3FileHeader is the magic byte sequence for v3 files. var C1Z3FileHeader = []byte("C1Z3\x00") +// Engine identifies a storage engine implementation. The engine is +// chosen by callers via WithEngine(...) on write; on read, the engine +// is dictated by the file's magic byte and (for v3) the manifest's +// engine field. The type lives in pkg/dotc1z/c1zstore so engine +// packages can name it without importing dotc1z. +type Engine = c1zstore.Engine + +const ( + // EngineSQLite is the default engine: the v1 .c1z format backed by + // a zstd-compressed SQLite database. Connectors use this; backend + // infra can opt out. + EngineSQLite = c1zstore.EngineSQLite + + // EnginePebble is the v3 engine: a Pebble LSM wrapped in the v3 + // envelope. + EnginePebble = c1zstore.EnginePebble + + // PebbleManifestEngine is the engine name recorded in a single-sync + // Pebble v3 manifest. It deliberately differs from EnginePebble so + // pre-single-sync readers reject the file at dispatch instead of + // reading its keys as empty. See c1zstore.PebbleManifestEngine. + PebbleManifestEngine = c1zstore.PebbleManifestEngine + PebbleManifestEngineV2 = c1zstore.PebbleManifestEngineV2 +) + // ErrEngineNotAvailable is returned when a caller requests an engine // that the binary does not support. var ErrEngineNotAvailable = fmt.Errorf("dotc1z: engine not available") +// PayloadEncoding selects the v3 envelope payload framing. Only the +// Pebble engine consults this; SQLite engines ignore it. See +// c1zstore.PayloadEncoding. +type PayloadEncoding = c1zstore.PayloadEncoding + +const ( + // PayloadEncodingUnspecified is the zero value. Means "use the + // engine's default" — IndexedZstd for Pebble. + PayloadEncodingUnspecified = c1zstore.PayloadEncodingUnspecified + + // PayloadEncodingTarZstd is the default Pebble v3 envelope + // encoding: tar of the Pebble directory, compressed with zstd. + PayloadEncodingTarZstd = c1zstore.PayloadEncodingTarZstd + + // PayloadEncodingTar is uncompressed tar. Useful when Pebble's + // L5/L6 SSTs are already zstd-compressed at the engine layer + // (avoids double-compression CPU), or when the storage target + // compresses in transit. + PayloadEncodingTar = c1zstore.PayloadEncodingTar + + // PayloadEncodingIndexedZstd stores each payload file as an + // independent zstd frame with a self-describing header. Opens + // decode frames in parallel, and rewrites of a store opened from + // an indexed file splice unchanged frames verbatim instead of + // re-compressing them (incremental fold compaction relies on + // this). Readers older than this encoding reject the file. + PayloadEncodingIndexedZstd = c1zstore.PayloadEncodingIndexedZstd +) + // ReadHeaderFormat reads the first 5 bytes of reader and returns the // detected format. On return, the reader is positioned immediately // after the header bytes. If reader is also an io.Seeker, it is diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/format/v3/envelope.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/format/v3/envelope.go index 042577c5..4188c5fd 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/format/v3/envelope.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/format/v3/envelope.go @@ -7,7 +7,6 @@ import ( "errors" "fmt" "io" - "math" "os" "path/filepath" "strconv" @@ -41,6 +40,7 @@ var ErrEnvelopeTruncated = errors.New("c1z v3: envelope truncated") // usage and protects against a malicious file claiming a billion-byte // manifest length. const maxManifestBytes = 16 << 20 +const maxTarEntryBytes int64 = 4 << 30 // Tar entries larger than this are streamed straight to disk on the // reader goroutine instead of being buffered in memory for the writer @@ -69,63 +69,19 @@ var fcsFailFastDisabled = os.Getenv(fcsFailFastDisableEnvVar) == "1" // bombs in untrusted v3 c1z files. var ErrMaxSizeExceeded = fmt.Errorf("c1z v3: max decoded payload size exceeded, increase via the %s environment variable", maxDecodedSizeEnvVar) -// envSizeBytesExplicit reads an env var holding a size in MiB and -// converts it to bytes. ok is false when the var is unset, unparsable, -// zero, or large enough to overflow the MiB→bytes conversion. -func envSizeBytesExplicit(envVar string) (uint64, bool) { +// envSizeBytes reads an env var holding a size in MiB and converts it +// to bytes, falling back to def when unset, unparsable, zero, or large +// enough to overflow the MiB→bytes conversion. +func envSizeBytes(envVar string, def uint64) uint64 { v := os.Getenv(envVar) if v == "" { - return 0, false + return def } mb, err := strconv.ParseUint(v, 10, 64) if err != nil || mb == 0 || mb > (1<<63)>>20 { - return 0, false + return def } - return mb << 20, true -} - -// envSizeBytes is envSizeBytesExplicit with a fallback default. -func envSizeBytes(envVar string, def uint64) uint64 { - if n, ok := envSizeBytesExplicit(envVar); ok { - return n - } - return def -} - -// maxPayloadCompressionRatio scales the automatic decoded-payload -// budget with the size of the envelope file itself. The budget's job -// is to bound how much disk a hostile envelope can consume at extract -// relative to what was actually stored; a fixed cap can't do that job -// without also refusing legitimate large files (a whale c1z's payload -// decodes to well past any constant that is still meaningful against -// bombs). Real Pebble payloads compress ~2-5x under zstd, so 100x is -// an order of magnitude of headroom while still capping a bomb at -// 100 bytes of output per byte of input. -const maxPayloadCompressionRatio = 100 - -// payloadBudgetForFileSize resolves the decoded-payload budget for an -// envelope of fileSize bytes when the caller configured nothing -// explicit: the env var when set, otherwise the LARGER of the flat -// default and fileSize × maxPayloadCompressionRatio. This is what -// keeps a legitimately huge envelope openable with default settings — -// the flat default alone would reject any file whose raw payload -// exceeds it, even though the file was written by us moments earlier. -func payloadBudgetForFileSize(fileSize int64) uint64 { - if n, ok := envSizeBytesExplicit(maxDecodedSizeEnvVar); ok { - return n - } - budget := defaultMaxDecodedPayloadBytes - if fileSize > 0 { - scaled := uint64(fileSize) - if scaled > math.MaxUint64/maxPayloadCompressionRatio { - return math.MaxUint64 - } - scaled *= maxPayloadCompressionRatio - if scaled > budget { - budget = scaled - } - } - return budget + return mb << 20 } func maxDecodedPayloadBytes() uint64 { @@ -138,14 +94,9 @@ func decoderMaxMemoryBytes() uint64 { type payloadOptions struct { maxDecodedPayloadBytes uint64 - // budgetExplicit is true when the decoded-payload budget came from - // the caller (WithMaxDecodedPayloadBytes) rather than defaults; - // only non-explicit budgets are rescaled by the envelope file size - // (see payloadBudgetForFileSize). - budgetExplicit bool - maxDecoderMemoryBytes uint64 - disableSizeFailFast bool - pool *DecoderPool + maxDecoderMemoryBytes uint64 + disableSizeFailFast bool + pool *DecoderPool } type PayloadOption func(*payloadOptions) @@ -156,7 +107,6 @@ type PayloadOption func(*payloadOptions) func WithMaxDecodedPayloadBytes(n uint64) PayloadOption { return func(o *payloadOptions) { o.maxDecodedPayloadBytes = n - o.budgetExplicit = n > 0 } } @@ -543,15 +493,6 @@ func readEnvelope(r io.Reader, headerOnly bool, pool *DecoderPool) (*Envelope, e return nil, err } // 4. Payload. The reader is positioned at the first payload byte. - // When the reader is a real file, scale the decoded-byte budget - // with its size (see payloadBudgetForFileSize); a non-stat-able - // stream falls back to the flat default. - budget := maxDecodedPayloadBytes() - if st, ok := r.(interface{ Stat() (os.FileInfo, error) }); ok { - if fi, err := st.Stat(); err == nil { - budget = payloadBudgetForFileSize(fi.Size()) - } - } env := &Envelope{Manifest: m} switch m.GetPayloadEncoding() { case c1zv3.PayloadEncoding_PAYLOAD_ENCODING_TAR_ZSTD: @@ -561,9 +502,9 @@ func readEnvelope(r io.Reader, headerOnly bool, pool *DecoderPool) (*Envelope, e } env.zstdReader = zr env.pool = pool - env.PayloadReader = &limitedPayloadReader{r: zr, limit: budget} + env.PayloadReader = &limitedPayloadReader{r: zr, limit: maxDecodedPayloadBytes()} case c1zv3.PayloadEncoding_PAYLOAD_ENCODING_TAR: - env.PayloadReader = &limitedPayloadReader{r: r, limit: budget} + env.PayloadReader = &limitedPayloadReader{r: r, limit: maxDecodedPayloadBytes()} case c1zv3.PayloadEncoding_PAYLOAD_ENCODING_INDEXED_ZSTD: // Indexed payloads are not a tar stream; extraction goes // through ExtractEnvelopePayload (random access over the @@ -808,7 +749,7 @@ func writeTar(w io.Writer, dir string) error { // writer worker pool; workers perform the per-file open/write/close // syscalls in parallel. Larger entries are streamed straight to disk // on this goroutine so a hostile archive full of multi-GiB entries -// can't drive memory up with the worker fan-out. Memory +// can't drive memory to extractWorkerCount × maxTarEntryBytes. Memory // peak is bounded by (extractWorkerCount + channel buffer) × // inlineCopyThresholdBytes; at Pebble's typical 2 MiB FlushSplitBytes // nearly every entry takes the parallel path — the per-entry @@ -892,13 +833,8 @@ entryLoop: break entryLoop } case tar.TypeReg: - // No per-entry size cap: a single Pebble SST can legitimately - // exceed any fixed bound. Aggregate extraction is bounded by - // the caller's decoded-byte budget (limitedPayloadReader), and - // memory by inlineCopyThresholdBytes — larger entries stream - // straight to disk. - if hdr.Size < 0 { - readErr = fmt.Errorf("c1z v3: tar entry %q has negative size %d", hdr.Name, hdr.Size) + if hdr.Size < 0 || hdr.Size > maxTarEntryBytes { + readErr = fmt.Errorf("c1z v3: tar entry %q size %d exceeds cap %d", hdr.Name, hdr.Size, maxTarEntryBytes) break entryLoop } if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/format/v3/indexed.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/format/v3/indexed.go index 1d051df4..6063c38e 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/format/v3/indexed.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/format/v3/indexed.go @@ -464,27 +464,12 @@ func readIndexedTrailer(f *os.File, payloadStart int64) (*c1zv3.IndexedFrameInde // positive: WithZeroFrames guarantees even an empty file encodes // to a complete zstd frame, so a zero-length frame range can // only come from a corrupt or hand-mangled index. - // - // There is deliberately NO upper bound on either size: a single - // Pebble SST can legitimately exceed any fixed cap (whale-scale - // grants buckets are written as one whole-bucket SST), and the - // bomb protections live elsewhere — compSize is bounds-checked - // against the real file layout just below, and total decoded - // output is enforced by the extraction budget - // (BATON_DECODER_MAX_DECODED_SIZE_MB) regardless of what - // raw_size claims. rawSize, compSize := e.GetRawSize(), e.GetCompressedSize() - if rawSize < 0 || compSize <= 0 { + if rawSize < 0 || rawSize > maxTarEntryBytes || compSize <= 0 || compSize > maxTarEntryBytes { return nil, nil, fmt.Errorf("c1z v3: trailer index entry %q sizes out of range (raw=%d comp=%d)", name, rawSize, compSize) } - // Overflow-safe form of off+compSize > indexOff: with no upper - // bound on compSize, the addition could wrap negative for a - // hostile compressed_size near MaxInt64 and slip past the - // comparison. Subtraction can't wrap here (off >= payloadStart - // >= 0 and indexOff fits the file), and when off > indexOff the - // negative difference rejects too, as it must. off := e.GetFrameOffset() - if off < payloadStart || compSize > indexOff-off { + if off < payloadStart || off+compSize > indexOff { return nil, nil, fmt.Errorf("c1z v3: trailer index entry %q frame range out of bounds (off=%d comp=%d)", name, off, compSize) } if len(e.GetRawSha256()) != sha256.Size { @@ -680,17 +665,6 @@ func extractOneFrame(f *os.File, e *ReuseEntry, dec *zstd.Decoder, budget *decod // owned by the caller. func ExtractEnvelopePayload(f *os.File, destDir string, opts ...PayloadOption) (*c1zv3.C1ZManifestV3, *PayloadReuse, error) { cfg := resolvePayloadOptions(opts...) - // With no explicit budget from the caller, scale the decoded-byte - // budget with the envelope's own size: the flat default would - // refuse any legitimately large file (one WE wrote), while the - // scaled budget still bounds a hostile envelope's disk consumption - // proportionally to its actual size. The env var, when set, wins - // inside payloadBudgetForFileSize. - if !cfg.budgetExplicit { - if st, err := f.Stat(); err == nil { - cfg.maxDecodedPayloadBytes = payloadBudgetForFileSize(st.Size()) - } - } if _, err := f.Seek(0, io.SeekStart); err != nil { return nil, nil, err } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/grant_store.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/grant_store.go new file mode 100644 index 00000000..9aad68b8 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/grant_store.go @@ -0,0 +1,19 @@ +package dotc1z + +import "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + +// The grant-store contract lives in pkg/dotc1z/c1zstore so storage engines +// can implement it without importing this package. These aliases preserve +// the historical dotc1z names. + +// GrantStore is the grant-specific slice of C1ZStore. See +// c1zstore.GrantStore for the full contract. +type GrantStore = c1zstore.GrantStore + +// PendingExpansion is a lightweight row yielded by +// GrantStore.PendingExpansion. See c1zstore.PendingExpansion. +type PendingExpansion = c1zstore.PendingExpansion + +// GrantAnnotation is a row yielded by GrantStore.ListWithAnnotations. See +// c1zstore.GrantAnnotation. +type GrantAnnotation = c1zstore.GrantAnnotation diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/pebble_store.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/pebble_store.go index f30f25b0..72789c26 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/pebble_store.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/pebble_store.go @@ -18,7 +18,6 @@ import ( v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" "github.com/conductorone/baton-sdk/pkg/connectorstore" - "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble" formatv3 "github.com/conductorone/baton-sdk/pkg/dotc1z/format/v3" batonGrant "github.com/conductorone/baton-sdk/pkg/types/grant" @@ -27,7 +26,7 @@ import ( // pebbleDriver is the EngineDriver for the Pebble v3 engine. type pebbleDriver struct{} -var _ c1zstore.Store = (*pebbleStore)(nil) +var _ C1ZStore = (*pebbleStore)(nil) var _ connectorstore.Writer = (*pebbleStore)(nil) // Local mirrors of the optional capabilities the c1z sanitizer probes on @@ -42,7 +41,7 @@ type sanitizeSupportsDiffWriter interface { SetSupportsDiff(ctx context.Context, syncID string) error } type sanitizeSyncRunMetadataReader interface { - ListSyncRuns(ctx context.Context, pageToken string, pageSize uint32) ([]*c1zstore.SyncRun, string, error) + ListSyncRuns(ctx context.Context, pageToken string, pageSize uint32) ([]*SyncRun, string, error) } var ( @@ -54,10 +53,10 @@ var ( _ sanitizeSyncRunMetadataReader = (*C1File)(nil) ) -func (pebbleDriver) Engine() c1zstore.Engine { return c1zstore.EnginePebble } -func (pebbleDriver) Format() C1ZFormat { return C1ZFormatV3 } +func (pebbleDriver) Engine() Engine { return EnginePebble } +func (pebbleDriver) Format() C1ZFormat { return C1ZFormatV3 } -func (pebbleDriver) OpenStore(ctx context.Context, outputFilePath string, opts StoreOptions) (c1zstore.Store, error) { +func (pebbleDriver) OpenStore(ctx context.Context, outputFilePath string, opts StoreOptions) (C1ZStore, error) { tmpDir, err := os.MkdirTemp(opts.TmpDir, "c1z-pebble") if err != nil { return nil, err @@ -103,7 +102,7 @@ func (pebbleDriver) OpenStore(ctx context.Context, outputFilePath string, opts S return nil, cleanupOnError(err) } encoding := opts.PayloadEncoding - if encoding == c1zstore.PayloadEncodingUnspecified { + if encoding == PayloadEncodingUnspecified { encoding = fileEncoding } @@ -144,32 +143,32 @@ func unpackExistingPebbleC1Z( maxDecodedPayloadBytes uint64, maxDecoderMemoryBytes uint64, pool *EnvelopeDecoderPool, -) (*formatv3.PayloadReuse, c1zstore.PayloadEncoding, int64, error) { +) (*formatv3.PayloadReuse, PayloadEncoding, int64, error) { stat, err := os.Stat(outputFilePath) switch { case errors.Is(err, os.ErrNotExist): - return nil, c1zstore.PayloadEncodingUnspecified, 0, nil + return nil, PayloadEncodingUnspecified, 0, nil case err != nil: - return nil, c1zstore.PayloadEncodingUnspecified, 0, err + return nil, PayloadEncodingUnspecified, 0, err case stat.Size() == 0: - return nil, c1zstore.PayloadEncodingUnspecified, 0, nil + return nil, PayloadEncodingUnspecified, 0, nil } f, err := os.Open(outputFilePath) if err != nil { - return nil, c1zstore.PayloadEncodingUnspecified, 0, err + return nil, PayloadEncodingUnspecified, 0, err } defer f.Close() header, err := formatv3.ReadManifestHeader(f) if err != nil { - return nil, c1zstore.PayloadEncodingUnspecified, 0, err + return nil, PayloadEncodingUnspecified, 0, err } - if e := c1zstore.Engine(header.GetEngine()); e != c1zstore.EnginePebble && e != c1zstore.PebbleManifestEngine && e != c1zstore.PebbleManifestEngineV2 { - return nil, c1zstore.PayloadEncodingUnspecified, 0, fmt.Errorf("%w: %s", pebble.ErrUnknownEngine, header.GetEngine()) + if e := Engine(header.GetEngine()); e != EnginePebble && e != PebbleManifestEngine && e != PebbleManifestEngineV2 { + return nil, PayloadEncodingUnspecified, 0, fmt.Errorf("%w: %s", pebble.ErrUnknownEngine, header.GetEngine()) } if err := os.MkdirAll(dbDir, 0o755); err != nil { - return nil, c1zstore.PayloadEncodingUnspecified, 0, err + return nil, PayloadEncodingUnspecified, 0, err } manifest, reuse, err := formatv3.ExtractEnvelopePayload(f, dbDir, formatv3.WithMaxDecodedPayloadBytes(maxDecodedPayloadBytes), @@ -177,7 +176,7 @@ func unpackExistingPebbleC1Z( formatv3.WithPayloadDecoderPool(pool), ) if err != nil { - return nil, c1zstore.PayloadEncodingUnspecified, 0, err + return nil, PayloadEncodingUnspecified, 0, err } // fold_dead_bytes is inherited from the source file so the waste // accounting survives arbitrary open/save cycles, not just fold @@ -185,18 +184,18 @@ func unpackExistingPebbleC1Z( return reuse, payloadEncodingFromProto(manifest.GetPayloadEncoding()), header.GetFoldDeadBytes(), nil } -func payloadEncodingFromProto(enc c1zv3.PayloadEncoding) c1zstore.PayloadEncoding { +func payloadEncodingFromProto(enc c1zv3.PayloadEncoding) PayloadEncoding { switch enc { case c1zv3.PayloadEncoding_PAYLOAD_ENCODING_TAR: - return c1zstore.PayloadEncodingTar + return PayloadEncodingTar case c1zv3.PayloadEncoding_PAYLOAD_ENCODING_INDEXED_ZSTD: - return c1zstore.PayloadEncodingIndexedZstd + return PayloadEncodingIndexedZstd case c1zv3.PayloadEncoding_PAYLOAD_ENCODING_TAR_ZSTD: - return c1zstore.PayloadEncodingTarZstd + return PayloadEncodingTarZstd case c1zv3.PayloadEncoding_PAYLOAD_ENCODING_UNSPECIFIED: - return c1zstore.PayloadEncodingUnspecified + return PayloadEncodingUnspecified default: - return c1zstore.PayloadEncodingUnspecified + return PayloadEncodingUnspecified } } @@ -206,7 +205,7 @@ type pebbleStore struct { outputFilePath string tmpDir string readOnly bool - payloadEncoding c1zstore.PayloadEncoding + payloadEncoding PayloadEncoding payloadReuse *formatv3.PayloadReuse // foldDeadBytes is the cumulative fold-waste counter carried in // the envelope manifest (C1ZManifestV3.fold_dead_bytes): seeded @@ -234,7 +233,7 @@ type pebbleStore struct { // Close(ctx) signature. Lets callers route Pebble stores through // pkg/sync.NewSyncer's WithConnectorStore option the same way they // route SQLite *C1File handles today. -var _ c1zstore.Store = (*pebbleStore)(nil) +var _ C1ZStore = (*pebbleStore)(nil) // FileOps overrides the Adapter-level FileOps for two reasons: // @@ -245,7 +244,7 @@ var _ c1zstore.Store = (*pebbleStore)(nil) // flip the dirty bit — without it, Close would skip the envelope // save and the diff sync would exist only in the discarded temp // directory. -func (s *pebbleStore) FileOps() c1zstore.FileOps { +func (s *pebbleStore) FileOps() FileOps { return pebbleStoreFileOps{inner: s.FileOpsWithEncoding(s.payloadEncoding), store: s} } @@ -254,15 +253,15 @@ func (s *pebbleStore) FileOps() c1zstore.FileOps { // dirty-marking path. CloneSync writes a separate file and passes // through unchanged. type pebbleStoreFileOps struct { - inner c1zstore.FileOps + inner FileOps store *pebbleStore } -func (f pebbleStoreFileOps) CloneSync(ctx context.Context, outPath string, syncID string, opts ...c1zstore.CloneSyncOption) error { +func (f pebbleStoreFileOps) CloneSync(ctx context.Context, outPath string, syncID string, opts ...CloneSyncOption) error { return f.inner.CloneSync(ctx, outPath, syncID, opts...) } -func (f pebbleStoreFileOps) CopyIsolateSync(ctx context.Context, outPath string, syncID string, opts ...c1zstore.CloneSyncOption) error { +func (f pebbleStoreFileOps) CopyIsolateSync(ctx context.Context, outPath string, syncID string, opts ...CloneSyncOption) error { return f.inner.CopyIsolateSync(ctx, outPath, syncID, opts...) } @@ -285,8 +284,8 @@ func (f pebbleStoreFileOps) GenerateSyncDiff(ctx context.Context, baseSyncID, ap func (s *pebbleStore) Metadata() connectorstore.StoreMetadata { md := s.Adapter.Metadata() enc := s.payloadEncoding - if enc == c1zstore.PayloadEncodingUnspecified { - enc = c1zstore.PayloadEncodingIndexedZstd + if enc == PayloadEncodingUnspecified { + enc = PayloadEncodingIndexedZstd } md.PayloadEncoding = enc.String() return md @@ -507,7 +506,7 @@ func (s *pebbleStore) DeleteGrantByRefs(ctx context.Context, grant *v2.Grant) er // routes StoreExpandedGrants through the pebbleStore's dirty-marking // path. The Adapter-level wrapper calls Adapter.PutGrants directly, // which skips the dirty flag. -func (s *pebbleStore) Grants() c1zstore.GrantStore { +func (s *pebbleStore) Grants() GrantStore { return pebbleStoreGrants{inner: s.Adapter.Grants(), store: s} } @@ -515,7 +514,7 @@ func (s *pebbleStore) Grants() c1zstore.GrantStore { // only StoreExpandedGrants (the lone mutating method) to flip the // dirty bit. Read-only methods pass through. type pebbleStoreGrants struct { - inner c1zstore.GrantStore + inner GrantStore store *pebbleStore } @@ -642,25 +641,25 @@ func newPebbleStoreExpandedGrant(dest *v2.Entitlement, principal *v2.Resource, s }.Build(), nil } -func (g pebbleStoreGrants) PendingExpansionPage(ctx context.Context, pageToken string) ([]c1zstore.PendingExpansion, string, error) { +func (g pebbleStoreGrants) PendingExpansionPage(ctx context.Context, pageToken string) ([]PendingExpansion, string, error) { return g.inner.PendingExpansionPage(ctx, pageToken) } -func (g pebbleStoreGrants) PendingExpansion(ctx context.Context) iter.Seq2[c1zstore.PendingExpansion, error] { +func (g pebbleStoreGrants) PendingExpansion(ctx context.Context) iter.Seq2[PendingExpansion, error] { return g.inner.PendingExpansion(ctx) } -func (g pebbleStoreGrants) ListWithAnnotationsPage(ctx context.Context, pageToken string) ([]c1zstore.GrantAnnotation, string, error) { +func (g pebbleStoreGrants) ListWithAnnotationsPage(ctx context.Context, pageToken string) ([]GrantAnnotation, string, error) { return g.inner.ListWithAnnotationsPage(ctx, pageToken) } func (g pebbleStoreGrants) ListWithAnnotationsForResourcePage( ctx context.Context, resource *v2.Resource, syncID string, pageToken string, pageSize uint32, -) ([]c1zstore.GrantAnnotation, string, error) { +) ([]GrantAnnotation, string, error) { return g.inner.ListWithAnnotationsForResourcePage(ctx, resource, syncID, pageToken, pageSize) } -func (g pebbleStoreGrants) ListWithAnnotations(ctx context.Context) iter.Seq2[c1zstore.GrantAnnotation, error] { +func (g pebbleStoreGrants) ListWithAnnotations(ctx context.Context) iter.Seq2[GrantAnnotation, error] { return g.inner.ListWithAnnotations(ctx) } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/source_cache.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/source_cache.go new file mode 100644 index 00000000..80171981 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/source_cache.go @@ -0,0 +1,213 @@ +package dotc1z + +import ( + "context" + "errors" + "fmt" + + cdbpebble "github.com/cockroachdb/pebble/v2" + + "github.com/conductorone/baton-sdk/pkg/bid" + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble" + "github.com/conductorone/baton-sdk/pkg/sourcecache" +) + +// SourceCacheReplayResult reports what one scope's replay copied. +type SourceCacheReplayResult = pebble.SourceCacheReplayResult + +// SourceCacheStore is the optional store capability backing source-cache +// replay (see proto/c1/connector/v2/annotation_source_cache.proto). It is +// implemented ONLY by the Pebble engine; the syncer type-asserts for it and +// treats a store without it as "source cache unsupported" (no-op lookup, +// no replay). It is deliberately NOT part of c1zstore.Store. +type SourceCacheStore interface { + // LookupSourceCacheEntry returns this store's manifest entry for + // (kind, scopeHash). Backs the connector-facing lookup when this + // store is the previous sync. + LookupSourceCacheEntry(ctx context.Context, kind sourcecache.RowKind, scopeHash string) (sourcecache.Entry, bool, error) + + // PutSourceCacheEntry writes the current sync's manifest entry for + // (kind, scopeHash). Zero-row scopes still get entries. + PutSourceCacheEntry(ctx context.Context, kind sourcecache.RowKind, scopeHash string, etag string) error + + // ReplaySourceCache copies every row stamped with scopeHash from prev + // (the previous sync's store, opened read-only) into this store. prev + // must be a Pebble store. Does NOT write the manifest entry — the + // caller writes it after the scope's overlay/deletes complete, so a + // failed replay can't leave a phantom hit for the next sync. + ReplaySourceCache(ctx context.Context, prev connectorstore.Reader, kind sourcecache.RowKind, scopeHash string) (SourceCacheReplayResult, error) + + // DeleteSourceCacheRows removes rows by public canonical ID from the + // current sync, after replay + overlay (delta-query tombstones). + // ID formats per kind: grants and entitlements use their canonical + // IDs; resources use Baton resource BIDs ("bid:r:..."). + // + // Grant resolution is BOUNDED: candidate probing only, never the + // O(all grants) stored-external-id scan. Grants stored under + // connector-custom ids are unreachable here (delete no-ops); such + // connectors use DeleteSourceCacheRowsInScope instead. + DeleteSourceCacheRows(ctx context.Context, kind sourcecache.RowKind, ids []string) error + + // DeleteSourceCacheRowsInScope removes rows stamped with scopeHash by + // bare object id — grants by principal id (no principal type, no + // canonical-id reconstruction), resources by resource id (any type). + // One index scan of the scope per call; a page's tombstones are + // batched into one call. Ids with no matching rows are no-ops. + // Not supported for entitlements. + DeleteSourceCacheRowsInScope(ctx context.Context, kind sourcecache.RowKind, scopeHash string, ids []string) (int64, error) + + // DeleteSourceCacheGrantsByIDInScope removes grant rows stamped with + // scopeHash whose STORED grant id is in ids — works for + // connector-custom grant-id shapes that the global bounded delete + // cannot resolve, and stays bounded by the scope's row count. Ids with + // no matching rows are no-ops. + DeleteSourceCacheGrantsByIDInScope(ctx context.Context, scopeHash string, ids []string) (int64, error) +} + +var _ SourceCacheStore = (*pebbleStore)(nil) + +// sourceCacheEngine recovers the Pebble engine from an arbitrary store, +// nil-safe. Mirrors pebble.AsEngine but accepts any value so the syncer +// can probe its previous-sync reader without caring about its static type. +func sourceCacheEngine(store any) (*pebble.Engine, bool) { + a, ok := store.(interface{ PebbleEngine() *pebble.Engine }) + if !ok { + return nil, false + } + e := a.PebbleEngine() + return e, e != nil +} + +func (s *pebbleStore) LookupSourceCacheEntry(ctx context.Context, kind sourcecache.RowKind, scopeHash string) (sourcecache.Entry, bool, error) { + if err := sourcecache.ValidateRowKind(kind); err != nil { + return sourcecache.Entry{}, false, err + } + rec, err := s.engine.GetSourceCacheEntry(ctx, string(kind), scopeHash) + if err != nil { + if errors.Is(err, cdbpebble.ErrNotFound) { + return sourcecache.Entry{}, false, nil + } + return sourcecache.Entry{}, false, err + } + return sourcecache.Entry{ + ETag: rec.GetEtag(), + DiscoveredAt: rec.GetDiscoveredAt().AsTime(), + }, true, nil +} + +func (s *pebbleStore) PutSourceCacheEntry(ctx context.Context, kind sourcecache.RowKind, scopeHash string, etag string) error { + if err := sourcecache.ValidateRowKind(kind); err != nil { + return err + } + return s.markDirty(s.engine.PutSourceCacheEntry(ctx, string(kind), scopeHash, etag)) +} + +func (s *pebbleStore) ReplaySourceCache(ctx context.Context, prev connectorstore.Reader, kind sourcecache.RowKind, scopeHash string) (SourceCacheReplayResult, error) { + prevEngine, ok := sourceCacheEngine(prev) + if !ok { + return SourceCacheReplayResult{}, errors.New("source cache replay: previous sync store is not a pebble store") + } + var res SourceCacheReplayResult + var err error + switch kind { + case sourcecache.RowKindResources: + res, err = s.engine.ReplaySourceCacheResources(ctx, prevEngine, scopeHash) + case sourcecache.RowKindEntitlements: + res, err = s.engine.ReplaySourceCacheEntitlements(ctx, prevEngine, scopeHash) + case sourcecache.RowKindGrants: + res, err = s.engine.ReplaySourceCacheGrants(ctx, prevEngine, scopeHash) + default: + return SourceCacheReplayResult{}, fmt.Errorf("source cache replay: invalid row kind %q", kind) + } + if err != nil { + return SourceCacheReplayResult{}, err + } + if res.Rows > 0 { + s.MarkDirty() + } + return res, nil +} + +// DeleteSourceCacheRows deletes delta tombstones by public id string. +// +// NOTE on the bare-id lookup safety contract (engine/pebble/lookup.go): +// sync paths normally must not resolve grants by string. This path is a +// deliberate, narrow exception: tombstone ids are strings the connector +// itself emitted for these rows, volumes are delta-sized (not O(rows)), +// and resolution keeps the exactly-one rule — an ambiguous id fails the +// sync loudly rather than guessing a delete, which matches the +// source-cache replay-phase error policy. +func (s *pebbleStore) DeleteSourceCacheRows(ctx context.Context, kind sourcecache.RowKind, ids []string) error { + for _, id := range ids { + switch kind { + case sourcecache.RowKindGrants: + if err := s.markDirty(s.engine.DeleteGrantRecordBounded(ctx, id)); err != nil { + return fmt.Errorf("source cache delete grant %q: %w", id, err) + } + case sourcecache.RowKindEntitlements: + if err := s.markDirty(s.engine.DeleteEntitlementRecord(ctx, id)); err != nil { + return fmt.Errorf("source cache delete entitlement %q: %w", id, err) + } + case sourcecache.RowKindResources: + r, err := bid.ParseResourceBid(id) + if err != nil { + return fmt.Errorf("source cache delete resource: invalid resource bid %q: %w", id, err) + } + rid := r.GetId() + if err := s.markDirty(s.engine.DeleteResourceRecord(ctx, rid.GetResourceType(), rid.GetResource())); err != nil { + return fmt.Errorf("source cache delete resource %q: %w", id, err) + } + default: + return fmt.Errorf("source cache delete: invalid row kind %q", kind) + } + } + return nil +} + +func (s *pebbleStore) DeleteSourceCacheGrantsByIDInScope(ctx context.Context, scopeHash string, ids []string) (int64, error) { + if len(ids) == 0 { + return 0, nil + } + idSet := make(map[string]struct{}, len(ids)) + for _, id := range ids { + idSet[id] = struct{}{} + } + deleted, err := s.engine.DeleteGrantsByExternalIDsInScope(ctx, scopeHash, idSet) + if err != nil { + return 0, fmt.Errorf("source cache grant-id delete for scope %q: %w", scopeHash, err) + } + if deleted > 0 { + s.MarkDirty() + } + return deleted, nil +} + +func (s *pebbleStore) DeleteSourceCacheRowsInScope(ctx context.Context, kind sourcecache.RowKind, scopeHash string, ids []string) (int64, error) { + if len(ids) == 0 { + return 0, nil + } + idSet := make(map[string]struct{}, len(ids)) + for _, id := range ids { + idSet[id] = struct{}{} + } + var deleted int64 + var err error + switch kind { + case sourcecache.RowKindGrants: + deleted, err = s.engine.DeleteGrantsByPrincipalsInScope(ctx, scopeHash, idSet) + case sourcecache.RowKindResources: + deleted, err = s.engine.DeleteResourcesByIDsInScope(ctx, scopeHash, idSet) + case sourcecache.RowKindEntitlements: + return 0, fmt.Errorf("source cache scoped delete: not supported for entitlements") + default: + return 0, fmt.Errorf("source cache scoped delete: invalid row kind %q", kind) + } + if err != nil { + return 0, fmt.Errorf("source cache scoped delete for scope %q: %w", scopeHash, err) + } + if deleted > 0 { + s.MarkDirty() + } + return deleted, nil +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sql_helpers.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sql_helpers.go index b4da1e81..301983e3 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sql_helpers.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sql_helpers.go @@ -574,7 +574,7 @@ func (c *C1File) getResourceObject(ctx context.Context, resourceID *v2.ResourceI case c.viewSyncID != "": q = q.Where(goqu.C("sync_id").Eq(c.viewSyncID)) default: - var latestSyncRun *c1zstore.SyncRun + var latestSyncRun *SyncRun var err error latestSyncRun, err = c.getFinishedSync(ctx, 0, connectorstore.SyncTypeFull) if err != nil { @@ -634,7 +634,7 @@ func (c *C1File) getConnectorObject(ctx context.Context, tableName string, id st case c.viewSyncID != "": q = q.Where(goqu.C("sync_id").Eq(c.viewSyncID)) default: - var latestSyncRun *c1zstore.SyncRun + var latestSyncRun *SyncRun var err error latestSyncRun, err = c.getFinishedSync(ctx, 0, connectorstore.SyncTypeAny) if err != nil { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/store.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/store.go index 1d7ee54d..7d12cb5e 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/store.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/store.go @@ -4,16 +4,33 @@ import ( "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" ) -// AsSQLiteStore type-asserts a c1zstore.Store to the concrete *C1File. It is an +// C1ZStore is the internal contract used by the sync pipeline, compactor, and +// related infrastructure to read and write a .c1z file. The interface lives +// in pkg/dotc1z/c1zstore (as c1zstore.Store) so storage engines can +// implement it without importing this package; this alias preserves the +// historical dotc1z name. +// +// Implementations: +// +// - *C1File — the original SQLite-backed implementation +// (pkg/dotc1z/c1file.go). +// - *pebbleStore — the Pebble v3 engine implementation opened via +// NewStore(WithEngine(EnginePebble)) (pkg/dotc1z/pebble_store.go). +// +// Both engines are registered statically; no extra imports are needed to +// open either format. +type C1ZStore = c1zstore.Store + +// AsSQLiteStore type-asserts a C1ZStore to the concrete *C1File. It is an // escape hatch for callers that legitimately need SQLite-specific primitives // (today: the attached compactor in pkg/synccompactor/attached, which uses // SQL ATTACH for cross-file merge). Returns (nil, false) when the store is // not backed by *C1File OR when the underlying *C1File is nil. // // Avoid using this outside pkg/synccompactor. If you find yourself reaching -// for it, prefer adding a named method to c1zstore.Store that expresses what you +// for it, prefer adding a named method to C1ZStore that expresses what you // need; sqlite-specific leak-through is a smell. See RFC 0002 §4.4. -func AsSQLiteStore(s c1zstore.Store) (*C1File, bool) { +func AsSQLiteStore(s C1ZStore) (*C1File, bool) { cf, ok := s.(*C1File) if !ok || cf == nil { return nil, false diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sync_meta.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sync_meta.go new file mode 100644 index 00000000..bea6f88a --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sync_meta.go @@ -0,0 +1,14 @@ +package dotc1z + +import "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + +// The sync-metadata contract lives in pkg/dotc1z/c1zstore so storage +// engines can implement it without importing this package. These aliases +// preserve the historical dotc1z names. + +// SyncMeta is the sync-run-metadata sub-store of C1ZStore. See +// c1zstore.SyncMeta for the full contract. +type SyncMeta = c1zstore.SyncMeta + +// SyncRun is the exported shape of a sync run. See c1zstore.SyncRun. +type SyncRun = c1zstore.SyncRun diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sync_runs.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sync_runs.go index 9171ac44..afdbbac1 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sync_runs.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sync_runs.go @@ -154,7 +154,7 @@ func (r *syncRunsTable) Migrations(ctx context.Context, db *goqu.Database) (bool // getCachedViewSyncRun returns the cached sync run for read operations. // This avoids N+1 queries when paginating through listConnectorObjects. // The cache is invalidated when a sync starts or ends. -func (c *C1File) getCachedViewSyncRun(ctx context.Context) (*c1zstore.SyncRun, error) { +func (c *C1File) getCachedViewSyncRun(ctx context.Context) (*SyncRun, error) { ctx, span := tracer.Start(ctx, "C1File.getCachedViewSyncRun") var err error defer func() { uotel.EndSpanWithError(span, err) }() @@ -188,7 +188,7 @@ func (c *C1File) invalidateCachedViewSyncRun() { c.cachedViewSyncErr = nil } -func (c *C1File) getLatestUnfinishedSync(ctx context.Context, syncType connectorstore.SyncType) (*c1zstore.SyncRun, error) { +func (c *C1File) getLatestUnfinishedSync(ctx context.Context, syncType connectorstore.SyncType) (*SyncRun, error) { ctx, span := tracer.Start(ctx, "C1File.getLatestUnfinishedSync") var err error defer func() { uotel.EndSpanWithError(span, err) }() @@ -200,7 +200,7 @@ func (c *C1File) getLatestUnfinishedSync(ctx context.Context, syncType connector // Don't resume syncs that started over a week ago oneWeekAgo := time.Now().AddDate(0, 0, -7) - ret := &c1zstore.SyncRun{} + ret := &SyncRun{} q := c.db.From(syncRuns.Name()) q = q.Select("sync_id", "started_at", "ended_at", "sync_token", "sync_type", "parent_sync_id", "linked_sync_id", "supports_diff", "stats") q = q.Where(goqu.C("ended_at").IsNull()) @@ -231,7 +231,7 @@ func (c *C1File) getLatestUnfinishedSync(ctx context.Context, syncType connector return ret, nil } -func (c *C1File) getFinishedSync(ctx context.Context, offset uint, syncType connectorstore.SyncType) (*c1zstore.SyncRun, error) { +func (c *C1File) getFinishedSync(ctx context.Context, offset uint, syncType connectorstore.SyncType) (*SyncRun, error) { ctx, span := tracer.Start(ctx, "C1File.getFinishedSync") var err error defer func() { uotel.EndSpanWithError(span, err) }() @@ -246,7 +246,7 @@ func (c *C1File) getFinishedSync(ctx context.Context, offset uint, syncType conn return nil, status.Errorf(codes.InvalidArgument, "invalid sync type: %s", syncType) } - ret := &c1zstore.SyncRun{} + ret := &SyncRun{} q := c.db.From(syncRuns.Name()) q = q.Select("sync_id", "started_at", "ended_at", "sync_token", "sync_type", "parent_sync_id", "linked_sync_id", "supports_diff", "stats") q = q.Where(goqu.C("ended_at").IsNotNull()) @@ -301,7 +301,7 @@ func parseStats(ctx context.Context, statsBytes *[]byte) *reader_v2.SyncStats { return ret } -func (c *C1File) ListSyncRuns(ctx context.Context, pageToken string, pageSize uint32) ([]*c1zstore.SyncRun, string, error) { +func (c *C1File) ListSyncRuns(ctx context.Context, pageToken string, pageSize uint32) ([]*SyncRun, string, error) { ctx, span := tracer.Start(ctx, "C1File.ListSyncRuns") var err error defer func() { uotel.EndSpanWithError(span, err) }() @@ -325,7 +325,7 @@ func (c *C1File) ListSyncRuns(ctx context.Context, pageToken string, pageSize ui q = q.Order(goqu.C("id").Asc()) q = q.Limit(uint(pageSize + 1)) - var ret []*c1zstore.SyncRun + var ret []*SyncRun query, args, err := q.ToSQL() if err != nil { @@ -347,7 +347,7 @@ func (c *C1File) ListSyncRuns(ctx context.Context, pageToken string, pageSize ui } statsBytes := &[]byte{} rowId := 0 - data := &c1zstore.SyncRun{} + data := &SyncRun{} err := rows.Scan(&rowId, &data.ID, &data.StartedAt, &data.EndedAt, &data.SyncToken, &data.Type, &data.ParentSyncID, &data.LinkedSyncID, &data.SupportsDiff, &statsBytes) if err != nil { return nil, "", err @@ -432,7 +432,7 @@ func (c *C1File) LatestFinishedSyncID(ctx context.Context, syncType connectorsto return s.ID, nil } -func (c *C1File) getSync(ctx context.Context, syncID string) (*c1zstore.SyncRun, error) { +func (c *C1File) getSync(ctx context.Context, syncID string) (*SyncRun, error) { ctx, span := tracer.Start(ctx, "C1File.getSync") var err error defer func() { uotel.EndSpanWithError(span, err) }() @@ -442,7 +442,7 @@ func (c *C1File) getSync(ctx context.Context, syncID string) (*c1zstore.SyncRun, return nil, err } - ret := &c1zstore.SyncRun{} + ret := &SyncRun{} q := c.db.From(syncRuns.Name()) q = q.Select("sync_id", "started_at", "ended_at", "sync_token", "sync_type", "parent_sync_id", "linked_sync_id", "supports_diff", "stats") @@ -464,7 +464,7 @@ func (c *C1File) getSync(ctx context.Context, syncID string) (*c1zstore.SyncRun, return ret, nil } -func (c *C1File) getCurrentSync(ctx context.Context) (*c1zstore.SyncRun, error) { +func (c *C1File) getCurrentSync(ctx context.Context) (*SyncRun, error) { ctx, span := tracer.Start(ctx, "C1File.getCurrentSync") var err error defer func() { uotel.EndSpanWithError(span, err) }() @@ -885,7 +885,7 @@ func (c *C1File) Cleanup(ctx context.Context) error { return err } - var candidates []c1zstore.SyncRun + var candidates []SyncRun pageToken := "" for { runs, nextPageToken, err := c.ListSyncRuns(ctx, pageToken, 100) diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/to_pebble.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/to_pebble.go index 0b251e3e..c97d6240 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/to_pebble.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/to_pebble.go @@ -22,7 +22,6 @@ import ( v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" "github.com/conductorone/baton-sdk/pkg/connectorstore" - "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble" "github.com/conductorone/baton-sdk/pkg/uotel" ) @@ -169,7 +168,7 @@ func (c *C1File) ToPebble(ctx context.Context, outPath string, syncID string, op start := time.Now() l := ctxzap.Extract(ctx) - dest, err := NewStore(ctx, outPath, WithEngine(c1zstore.EnginePebble), WithTmpDir(cfg.tmpDir)) + dest, err := NewStore(ctx, outPath, WithEngine(EnginePebble), WithTmpDir(cfg.tmpDir)) if err != nil { return nil, fmt.Errorf("to-pebble: open destination: %w", err) } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/field/defaults.go b/vendor/github.com/conductorone/baton-sdk/pkg/field/defaults.go index e4572664..a3271a5c 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/field/defaults.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/field/defaults.go @@ -264,6 +264,10 @@ var ( WithDescription("The path to the c1z file to sync external baton resources with"), WithPersistent(true), WithExportTarget(ExportTargetNone)) + PreviousSyncC1ZField = StringField("previous-sync-c1z", + WithDescription("The path to the previous sync c1z file to use as a source-cache replay input"), + WithPersistent(true), + WithExportTarget(ExportTargetNone)) externalResourceEntitlementIdFilter = StringField("external-resource-entitlement-id-filter", WithDescription("The entitlement that external users, groups must have access to sync external baton resources"), WithPersistent(true), @@ -431,6 +435,7 @@ var DefaultFields = append([]SchemaField{ skipEntitlementsAndGrants, skipGrants, externalResourceC1ZField, + PreviousSyncC1ZField, externalResourceEntitlementIdFilter, KeepPreviousSyncC1ZField, diffSyncsField, diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/lambda/grpc/client.go b/vendor/github.com/conductorone/baton-sdk/pkg/lambda/grpc/client.go index df25813c..81206188 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/lambda/grpc/client.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/lambda/grpc/client.go @@ -11,8 +11,6 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/lambda" "github.com/aws/aws-sdk-go-v2/service/lambda/types" - "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" - "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" @@ -26,18 +24,10 @@ type lambdaTransport struct { } func (l *lambdaTransport) RoundTrip(ctx context.Context, req *Request) (*Response, error) { - payload, frameOnly, err := req.marshalPayload() + payload, err := req.MarshalJSON() if err != nil { return nil, fmt.Errorf("lambda_transport: failed to marshal frame: %w", err) } - if frameOnly != nil { - ctxzap.Extract(ctx).Warn( - "lambda_transport: request has no legacy encoding, sending v2 frame only; a connector on a pre-frame SDK cannot process this call", - zap.String("method", req.Method()), - zap.String("function_name", l.functionName), - zap.NamedError("legacy_encoding_error", frameOnly), - ) - } input := &lambda.InvokeInput{ LogType: types.LogTypeTail, diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/lambda/grpc/server.go b/vendor/github.com/conductorone/baton-sdk/pkg/lambda/grpc/server.go index 2a3da875..a8746242 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/lambda/grpc/server.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/lambda/grpc/server.go @@ -236,18 +236,7 @@ func TimeoutForRequest(req *Request) (time.Duration, bool, error) { return 0, false, nil } -// Handler serves one transport request. The response echoes the request's -// wire version so v2 invokers get lossless frames and legacy invokers get -// protojson (see Response.MarshalJSON). func (s *Server) Handler(ctx context.Context, req *Request) (*Response, error) { - resp, err := s.handle(ctx, req) - if resp != nil { - resp.wireV2 = req.wireV2 - } - return resp, err -} - -func (s *Server) handle(ctx context.Context, req *Request) (*Response, error) { serviceName, methodName, err := parseMethod(req.Method()) if err != nil { return ErrorResponse(err), nil diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/lambda/grpc/transport.go b/vendor/github.com/conductorone/baton-sdk/pkg/lambda/grpc/transport.go index 7da6f05d..28d1b9c0 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/lambda/grpc/transport.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/lambda/grpc/transport.go @@ -20,12 +20,10 @@ import ( const annotationsFieldName = "annotations" /* -unmarshalTransportJSON unmarshals transport JSON into msg. It reports whether -the payload carried a v2 wire frame (binary proto, see wireFrame), which is -decoded losslessly with no type resolution. +unmarshalTransportJSON unmarshals transport JSON into msg, discarding any +unknown fields. -Legacy payloads are protojson, unmarshaled discarding any unknown fields. -When a legacy payload fails to unmarshal, it retries after filtering out any +When the payload fails to unmarshal, it retries after filtering out any annotations whose types are not known to the global registry. Annotation type skew happens frequently for new features and would otherwise require rolling every lambda function (and, in the response direction, would let an old @@ -41,11 +39,7 @@ payloads that already failed, where the alternative is a hard error. Our payloads are small relative to the work of the connector, so the performance impact is negligible. */ -func unmarshalTransportJSON(b []byte, msg proto.Message) (bool, error) { - if ok, err := decodeWireFrame(b, msg); ok { - return true, err - } - +func unmarshalTransportJSON(b []byte, msg proto.Message) error { unmarshalOptions := protojson.UnmarshalOptions{ DiscardUnknown: true, } @@ -53,19 +47,19 @@ func unmarshalTransportJSON(b []byte, msg proto.Message) (bool, error) { // so any failure falls through to the annotation filter. originalErr := unmarshalOptions.Unmarshal(b, msg) if originalErr == nil { - return false, nil + return nil } filtered, changed := filterUnknownAnnotations(b) if !changed { - return false, originalErr + return originalErr } if err := unmarshalOptions.Unmarshal(filtered, msg); err != nil { - return false, errors.Join(originalErr, err) + return errors.Join(originalErr, err) } - return false, nil + return nil } // filterUnknownAnnotations recursively walks raw JSON and prunes entries from @@ -187,61 +181,18 @@ func filterAnnotationsArray(raw json.RawMessage) (json.RawMessage, bool) { type Request struct { msg *pbtransport.Request - - // wireV2 records that the request arrived as a v2 wire frame, proving - // the invoker can read one back. The server stamps it onto the Response. - wireV2 bool } -// UnmarshalJSON unmarshals the JSON into a Request. v2 wire frames decode -// losslessly; legacy payloads are protojson, discarding unknown fields and -// filtering annotations with unresolvable types. See unmarshalTransportJSON. +// UnmarshalJSON unmarshals the JSON into a Request, discarding unknown fields +// and filtering annotations with unresolvable types. See +// unmarshalTransportJSON. func (f *Request) UnmarshalJSON(b []byte) error { f.msg = &pbtransport.Request{} - wireV2, err := unmarshalTransportJSON(b, f.msg) - if err != nil { - return err - } - f.wireV2 = wireV2 - return nil + return unmarshalTransportJSON(b, f.msg) } -// MarshalJSON dual-encodes the request: the legacy protojson fields and the -// v2 wire frame share one JSON object, so legacy connectors keep working -// (they discard the unknown frame fields) while v2 connectors decode the -// frame and see annotations whose types this process cannot resolve. When no -// legacy view can be produced — protojson cannot represent an Any whose type -// is not linked into this process — the frame is sent alone: a legacy -// connector would have failed on that payload anyway. Oversized dual -// payloads fall back to legacy-only to stay under the Lambda invoke limit; -// v2 connectors accept those too. func (f *Request) MarshalJSON() ([]byte, error) { - payload, _, err := f.marshalPayload() - return payload, err -} - -// marshalPayload builds the invoke payload. The middle return reports the -// frame-only condition: when non-nil, the payload carries only the v2 frame -// and the value is the reason the legacy view could not be produced — -// callers with a context should surface it, since a legacy connector cannot -// process a frame-only payload. -func (f *Request) marshalPayload() ([]byte, error, error) { - legacy, legacyErr := protojson.Marshal(f.msg) - if legacyErr != nil { - payload, err := encodeWireFrame(f.msg) - if err != nil { - return nil, nil, errors.Join(legacyErr, err) - } - return payload, legacyErr, nil - } - dual, err := spliceWireFrame(legacy, f.msg) - if err != nil { - return nil, nil, err - } - if len(dual) > maxDualEncodedPayload { - return legacy, nil, nil - } - return dual, nil, nil + return protojson.Marshal(f.msg) } func (f *Request) Method() string { @@ -285,40 +236,20 @@ func NewRequest(method string, req proto.Message, headers metadata.MD) (*Request type Response struct { msg *pbtransport.Response - - // wireV2 selects the v2 wire frame encoding. The server sets it from - // the request: a frame in the request proves the invoker reads frames. - wireV2 bool } -// UnmarshalJSON unmarshals the JSON into a Response. v2 wire frames decode -// losslessly with no type resolution. Legacy payloads are protojson, -// discarding unknown fields and filtering annotations with unresolvable -// types: responses carry annotations at the response level and nested inside -// rows (grants embed resources, etc.), so this protects an invoker from -// annotation types it does not know about — for example an older invoker -// receiving annotations from a connector built with a newer SDK. See -// unmarshalTransportJSON. +// UnmarshalJSON unmarshals the JSON into a Response, discarding unknown +// fields and filtering annotations with unresolvable types. Responses carry +// annotations at the response level and nested inside rows (grants embed +// resources, etc.), so this protects an invoker from annotation types it +// does not know about — for example an older invoker receiving annotations +// from a connector built with a newer SDK. See unmarshalTransportJSON. func (f *Response) UnmarshalJSON(b []byte) error { f.msg = &pbtransport.Response{} - wireV2, err := unmarshalTransportJSON(b, f.msg) - if err != nil { - return err - } - f.wireV2 = wireV2 - return nil + return unmarshalTransportJSON(b, f.msg) } -// MarshalJSON encodes a v2 wire frame when the invoker proved it reads them -// (see wireV2), preserving annotations whose types this process cannot -// resolve. Legacy invokers get plain protojson, which fails on an Any whose -// type is not linked into this process — deliberately: the sender's registry -// is no authority on what the receiver understands or needs, so degrading -// the payload by silently dropping data is worse than failing loudly. func (f *Response) MarshalJSON() ([]byte, error) { - if f.wireV2 { - return encodeWireFrame(f.msg) - } return protojson.Marshal(f.msg) } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/lambda/grpc/wire.go b/vendor/github.com/conductorone/baton-sdk/pkg/lambda/grpc/wire.go deleted file mode 100644 index 8be74902..00000000 --- a/vendor/github.com/conductorone/baton-sdk/pkg/lambda/grpc/wire.go +++ /dev/null @@ -1,88 +0,0 @@ -package grpc - -import ( - "bytes" - "encoding/json" - "fmt" - - "google.golang.org/protobuf/proto" -) - -const transportWireVersion = 2 - -// maxDualEncodedPayload caps dual-encoded requests below the 6MiB Lambda -// invoke payload limit. Past it the frame is dropped and the request goes -// out legacy-only, which v2 peers also accept. -var maxDualEncodedPayload = 5 << 20 - -/* -wireFrame is the v2 transport encoding: the binary proto bytes of a -transport Request or Response, carried base64-encoded in the JSON Lambda -payload. Binary proto copies google.protobuf.Any payloads verbatim instead -of resolving their type URLs the way protojson must, so annotation types -that are not linked into a process survive the transport intact — the fix -for connector-specific annotations (e.g. baton-jira's CustomField) being -dropped or crashing the marshal in runtimes that don't register them. - -Version skew is handled without negotiation state: - - - Requests are dual-encoded: the legacy protojson fields and the frame - share one JSON object. Legacy peers unmarshal with DiscardUnknown and - never see the frame; v2 peers prefer it. - - Responses carry the frame alone, but only when the request carried - one — a frame in the request proves the invoker can read it. Legacy - requests get legacy responses. - -The field names cannot collide with the legacy encoding: protojson emits -only "method"/"req"/"headers" for Requests and -"resp"/"status"/"headers"/"trailers" for Responses. -*/ -type wireFrame struct { - V int `json:"v"` - Frame []byte `json:"frame"` -} - -// decodeWireFrame reports whether raw carries a v2 wire frame and, if so, -// decodes it into msg. A false return means raw is a legacy payload: either -// it isn't shaped like a frame, or it doesn't parse as JSON at all — the -// legacy path owns reporting that error. -func decodeWireFrame(raw []byte, msg proto.Message) (bool, error) { - var wf wireFrame - if err := json.Unmarshal(raw, &wf); err != nil || len(wf.Frame) == 0 { - return false, nil //nolint:nilerr // not a v2 frame; the legacy path owns error reporting - } - if wf.V != transportWireVersion { - return true, fmt.Errorf("transport: unsupported wire frame version %d", wf.V) - } - return true, proto.Unmarshal(wf.Frame, msg) -} - -func encodeWireFrame(msg proto.Message) ([]byte, error) { - frame, err := proto.Marshal(msg) - if err != nil { - return nil, err - } - return json.Marshal(wireFrame{V: transportWireVersion, Frame: frame}) -} - -// spliceWireFrame appends the v2 frame fields to a legacy protojson object, -// producing the dual-encoded request payload. -func spliceWireFrame(legacy []byte, msg proto.Message) ([]byte, error) { - suffix, err := encodeWireFrame(msg) - if err != nil { - return nil, err - } - legacy = bytes.TrimSpace(legacy) - if len(legacy) < 2 || legacy[0] != '{' || legacy[len(legacy)-1] != '}' { - return nil, fmt.Errorf("transport: legacy payload is not a JSON object") - } - if len(legacy) == 2 { - return suffix, nil - } - var buf bytes.Buffer - buf.Grow(len(legacy) + len(suffix)) - buf.Write(legacy[:len(legacy)-1]) - buf.WriteByte(',') - buf.Write(suffix[1:]) - return buf.Bytes(), nil -} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sdk/version.go b/vendor/github.com/conductorone/baton-sdk/pkg/sdk/version.go index e9dc1322..c3bf2e37 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sdk/version.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sdk/version.go @@ -1,3 +1,3 @@ package sdk -const Version = "v0.18.1" +const Version = "v0.17.0" diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/context.go b/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/context.go new file mode 100644 index 00000000..266dd5c0 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/context.go @@ -0,0 +1,20 @@ +package sourcecache + +import "context" + +type scopeContextKey struct{} + +// WithScope returns a context carrying the source-cache scope hash for +// rows written under it. The syncer wraps a page's store writes in this +// context when the page carried a SourceCacheScope annotation; the Pebble +// write path stamps the record's source_scope_hash from it. +func WithScope(ctx context.Context, scopeHash string) context.Context { + return context.WithValue(ctx, scopeContextKey{}, scopeHash) +} + +// ScopeFromContext returns the scope hash set by WithScope, or "" when +// the context carries none (the common, unstamped case). +func ScopeFromContext(ctx context.Context) string { + s, _ := ctx.Value(scopeContextKey{}).(string) + return s +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/grpc_lookup.go b/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/grpc_lookup.go new file mode 100644 index 00000000..199ecda9 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/grpc_lookup.go @@ -0,0 +1,54 @@ +package sourcecache + +import ( + "context" + "fmt" + + v1 "github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1" +) + +// GRPCLookup is the connector-side Lookup implementation that talks to +// BatonSourceCacheService on the parent SDK. +// +// This is deliberately not routed through the session store: session data +// passes through the connector's local MemorySessionCache (otter), which +// would apply generic TTL/eviction policies to sync-scoped validator state. +// The dedicated service keeps the path uncached and the message shape +// explicit. +// +// The parent has exactly one active Lookup registered at a time (set per +// sync via SetSourceCache on the server), so the wire format carries no +// sync_id; routing is implicit. +type GRPCLookup struct { + client v1.BatonSourceCacheServiceClient +} + +// NewGRPCLookup returns a Lookup backed by the given client. A nil client +// yields NoopLookup so callers can configure the client optionally without +// nil checks at every call site. +func NewGRPCLookup(client v1.BatonSourceCacheServiceClient) Lookup { + if client == nil { + return NoopLookup{} + } + return &GRPCLookup{client: client} +} + +func (g *GRPCLookup) LookupPreviousSourceCache(ctx context.Context, rowKind RowKind, scopeHash string) (Entry, bool, error) { + if err := ValidateRowKind(rowKind); err != nil { + return Entry{}, false, err + } + if err := ValidateScopeHash(scopeHash); err != nil { + return Entry{}, false, err + } + resp, err := g.client.Lookup(ctx, v1.LookupRequest_builder{ + RowKind: string(rowKind), + ScopeHash: scopeHash, + }.Build()) + if err != nil { + return Entry{}, false, fmt.Errorf("source cache rpc lookup: %w", err) + } + if !resp.GetFound() { + return Entry{}, false, nil + } + return Entry{ETag: resp.GetEtag()}, true, nil +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/grpc_server.go b/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/grpc_server.go new file mode 100644 index 00000000..950aaf7c --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/grpc_server.go @@ -0,0 +1,71 @@ +package sourcecache + +import ( + "context" + "fmt" + "sync/atomic" + + v1 "github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1" +) + +// GRPCServer is the parent-side BatonSourceCacheService implementation. +// +// The parent SDK holds a single GRPCServer for the lifetime of the connector +// subprocess and swaps the active Lookup via SetSourceCache as syncs come +// and go. The syncer installs a real lookup once it has resolved a usable +// previous sync, and clears it when the sync ends so a late RPC can't serve +// from a store the syncer no longer owns. +// +// Until the first SetSourceCache call the server answers every lookup with +// found=false, which the connector treats as "no previous sync" and falls +// back to an unconditional fetch. +type GRPCServer struct { + v1.UnimplementedBatonSourceCacheServiceServer + lookup atomic.Pointer[Lookup] +} + +var _ v1.BatonSourceCacheServiceServer = (*GRPCServer)(nil) +var _ SetLookup = (*GRPCServer)(nil) + +// NewGRPCServer returns a GRPCServer with no active Lookup registered. +func NewGRPCServer() *GRPCServer { + return &GRPCServer{} +} + +// SetSourceCache replaces the active lookup. Safe to call concurrently with +// in-flight RPCs: existing RPCs continue against the value they read at +// entry; new RPCs see the swapped value. +func (s *GRPCServer) SetSourceCache(ctx context.Context, lookup Lookup) { + if lookup == nil { + s.lookup.Store(nil) + return + } + s.lookup.Store(&lookup) +} + +func (s *GRPCServer) Lookup(ctx context.Context, req *v1.LookupRequest) (*v1.LookupResponse, error) { + rowKind := RowKind(req.GetRowKind()) + if err := ValidateRowKind(rowKind); err != nil { + return nil, err + } + scopeHash := req.GetScopeHash() + if err := ValidateScopeHash(scopeHash); err != nil { + return nil, err + } + + lookupPtr := s.lookup.Load() + if lookupPtr == nil { + return v1.LookupResponse_builder{Found: false}.Build(), nil + } + entry, found, err := (*lookupPtr).LookupPreviousSourceCache(ctx, rowKind, scopeHash) + if err != nil { + return nil, fmt.Errorf("source cache lookup: %w", err) + } + if !found { + return v1.LookupResponse_builder{Found: false}.Build(), nil + } + return v1.LookupResponse_builder{ + Found: true, + Etag: entry.ETag, + }.Build(), nil +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/sourcecache.go b/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/sourcecache.go new file mode 100644 index 00000000..6b40fdca --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/sourcecache.go @@ -0,0 +1,135 @@ +// Package sourcecache defines the connector-facing surface of source-cache +// replay (see proto/c1/connector/v2/annotation_source_cache.proto). +// +// A connector that can cheaply revalidate upstream data — HTTP conditional +// requests (GitHub), delta queries (Microsoft Graph) — opts in by attaching +// SourceCacheCapability MODE_READ_WRITE to its Validate response. During a +// sync it looks up the previous validator for a scope via the Lookup the SDK +// provides on SyncOpAttrs, revalidates upstream, and either emits fresh rows +// tagged with SourceCacheScope or asks the SDK to replay the previous rows +// with SourceCacheReplay. +// +// The connector owns scope computation; the SDK only keys storage by the +// connector-supplied scope hash. The validator (etag, delta token) is opaque +// to the SDK. +// +// Invariant that keeps replay safe: a connector must only emit +// SourceCacheReplay for a scope whose validator it received from THIS sync's +// Lookup. When source cache is disabled or degraded (no capability, no +// usable previous sync, unsupported storage engine) the SDK installs +// NoopLookup, every lookup misses, and a well-behaved connector naturally +// falls back to full fetch. +// +// Replay equivalence: a cached sync must reproduce what a full resync +// would produce. Replayed rows are verbatim copies of the previous sync's +// rows with one deliberate exception — expander-written Sources on direct +// grants (classified by a self-source entry, mirroring RollbackExpansion) +// are stripped at copy time so the current sync's expansion recomputes +// them from true state; re-expansion only adds contributions, so carrying +// them verbatim would immortalize contributions removed upstream. +// Connector-set Sources (no self-source) are public connector data and +// survive replay byte-for-byte. +package sourcecache + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "time" +) + +// RowKind partitions source-cache scopes by the row type they produce. +// It doubles as the row_kind value stored in SourceCacheEntryRecord. +type RowKind string + +const ( + RowKindResources RowKind = "resources" + RowKindEntitlements RowKind = "entitlements" + RowKindGrants RowKind = "grants" +) + +// Valid reports whether k is one of the defined row kinds. +func (k RowKind) Valid() bool { + switch k { + case RowKindResources, RowKindEntitlements, RowKindGrants: + return true + } + return false +} + +// ValidateRowKind returns an error if rowKind is not one of the known +// RowKind* constants. +func ValidateRowKind(rowKind RowKind) error { + if !rowKind.Valid() { + return fmt.Errorf("invalid source cache row kind: %q", rowKind) + } + return nil +} + +// maxScopeHashLen bounds scope identifiers on the wire and in storage +// keys. Deliberately generous: the shape is a connector convention +// (HashScope produces 64 hex chars) and is not enforced beyond +// non-emptiness and this cap while the model is being proven out against +// real providers. +const maxScopeHashLen = 256 + +// ValidateScopeHash returns an error when scopeHash is empty or +// unreasonably long. Connectors conventionally use HashScope, but any +// stable identifier is accepted. +func ValidateScopeHash(scopeHash string) error { + if scopeHash == "" { + return fmt.Errorf("source cache scope hash is required") + } + if len(scopeHash) > maxScopeHashLen { + return fmt.Errorf("source cache scope hash too long: %d bytes (max %d)", len(scopeHash), maxScopeHashLen) + } + return nil +} + +// Entry is a previous sync's persisted validator for one scope. +type Entry struct { + // ETag is the opaque upstream validator: a literal HTTP ETag, a delta + // token, etc. Never interpreted by the SDK. + ETag string + + // DiscoveredAt is when the entry was written. + DiscoveredAt time.Time +} + +// Lookup resolves a scope's previous-sync validator. The SDK provides an +// implementation on SyncOpAttrs; connectors call it before revalidating +// upstream. +type Lookup interface { + // LookupPreviousSourceCache returns the previous sync's entry for + // (rowKind, scopeHash). found=false means no entry: fetch fresh. + // Implementations must treat internal read errors that leave fresh + // fetch available as misses rather than failing the connector call. + LookupPreviousSourceCache(ctx context.Context, rowKind RowKind, scopeHash string) (entry Entry, found bool, err error) +} + +// NoopLookup is the Lookup installed when source cache is disabled or +// degraded. Every lookup misses. +type NoopLookup struct{} + +var _ Lookup = NoopLookup{} + +func (NoopLookup) LookupPreviousSourceCache(context.Context, RowKind, string) (Entry, bool, error) { + return Entry{}, false, nil +} + +// SetLookup is implemented by connector clients/servers that can receive a +// source-cache lookup implementation from the sync runner. The SDK calls +// SetSourceCache(lookup) at the start of each sync and SetSourceCache(nil) +// when the sync ends so a late RPC can't read stale state. +type SetLookup interface { + SetSourceCache(ctx context.Context, lookup Lookup) +} + +// HashScope returns the lowercase-hex sha256 of a canonical scope string. +// Convenience for connectors; any stable identifier is acceptable as a +// scope hash (only non-emptiness and a length cap are enforced). +func HashScope(canonicalScope string) string { + sum := sha256.Sum256([]byte(canonicalScope)) + return hex.EncodeToString(sum[:]) +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/expander.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/expander.go index c54e25b4..1eb3a4cd 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/expander.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/expander.go @@ -63,12 +63,12 @@ var ErrMaxDepthExceeded = errors.New("max depth exceeded") // ExpanderStore defines the minimal store interface needed for grant expansion. // Implementations: -// - *dotc1z.C1File (via c1zstore.Store) for production syncs +// - *dotc1z.C1File (via dotc1z.C1ZStore) for production syncs // - mocks for unit tests // // StoreExpandedGrants writes a batch of expanded grants back to storage, // preserving existing expansion metadata columns on the underlying rows. -// See c1zstore.GrantStore.StoreExpandedGrants for the full contract. +// See dotc1z.GrantStore.StoreExpandedGrants for the full contract. type ExpanderStore interface { GetEntitlement(ctx context.Context, req *reader_v2.EntitlementsReaderServiceGetEntitlementRequest) (*reader_v2.EntitlementsReaderServiceGetEntitlementResponse, error) ListGrantsForEntitlement(ctx context.Context, req *reader_v2.GrantsReaderServiceListGrantsForEntitlementRequest) (*reader_v2.GrantsReaderServiceListGrantsForEntitlementResponse, error) diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/source_cache.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/source_cache.go new file mode 100644 index 00000000..8eccd160 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/source_cache.go @@ -0,0 +1,344 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +import ( + "context" + "fmt" + stdsync "sync" + + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "go.uber.org/zap" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/sourcecache" +) + +// Source-cache replay, syncer side. See +// proto/c1/connector/v2/annotation_source_cache.proto for the contract. +// +// Setup degrades, replay fails loudly. Any setup problem (capability +// absent, store engine unsupported, no usable previous sync) installs the +// no-op lookup: the connector never sees a previous validator, never gets +// a conditional-request hit, and therefore never emits SourceCacheReplay +// — which is what makes it safe to treat a replay annotation arriving +// while degraded as a hard error (the connector already skipped row +// generation; there is nothing to fall back to). + +// syncerSourceCache is the per-sync source-cache state resolved by +// configureSourceCache. +// +// Write side and read side enable independently: the FIRST sync of a chain +// has no previous sync but must still stamp rows and write manifest +// entries, or the second sync would have nothing to replay. enabled covers +// the write side (capability declared + current store supports it); prev +// is non-nil only when a usable previous sync exists (read side — lookup +// hits and replay). +type syncerSourceCache struct { + enabled bool + // current is the writable output store's source-cache capability. + current dotc1z.SourceCacheStore + // prev is the previous sync's lookup/replay source (read-only). Nil + // when no usable previous sync exists; lookups then miss and replay + // annotations are hard errors. + prev dotc1z.SourceCacheStore + // prevReader is the same store as prev, typed for ReplaySourceCache. + prevReader connectorstore.Reader +} + +// prevStoreLookup adapts the previous store's manifest to the +// connector-facing Lookup. Mid-sync read errors are logged once and +// treated as misses: at lookup time the connector can still fetch fresh, +// so degrading beats failing the sync. +type prevStoreLookup struct { + prev dotc1z.SourceCacheStore + logOnce *stdsync.Once +} + +var _ sourcecache.Lookup = prevStoreLookup{} + +func (p prevStoreLookup) LookupPreviousSourceCache(ctx context.Context, kind sourcecache.RowKind, scopeHash string) (sourcecache.Entry, bool, error) { + entry, found, err := p.prev.LookupSourceCacheEntry(ctx, kind, scopeHash) + if err != nil { + p.logOnce.Do(func() { + ctxzap.Extract(ctx).Warn("source cache lookup failed; treating as miss", zap.Error(err)) + }) + return sourcecache.Entry{}, false, nil //nolint:nilerr // intentional: a failed lookup degrades to a miss (connector fetches fresh) rather than failing the connector call + } + return entry, found, nil +} + +// configureSourceCache resolves per-sync source-cache state from the +// connector's Validate response and installs the connector-facing lookup. +// Called once per Sync, after Validate. +func (s *syncer) configureSourceCache(ctx context.Context, resp *v2.ConnectorServiceValidateResponse) error { + l := ctxzap.Extract(ctx) + s.sourceCache = syncerSourceCache{} + + setLookup, canSetLookup := s.connector.(sourcecache.SetLookup) + degrade := func(reason string) error { + if canSetLookup { + setLookup.SetSourceCache(ctx, sourcecache.NoopLookup{}) + } + if reason != "" { + l.Info("source cache disabled", zap.String("reason", reason)) + } + return nil + } + + capability := &v2.SourceCacheCapability{} + annos := annotations.Annotations(resp.GetAnnotations()) + ok, err := annos.Pick(capability) + if err != nil { + return fmt.Errorf("error parsing source cache capability annotation: %w", err) + } + if !ok || capability.GetMode() != v2.SourceCacheCapability_MODE_READ_WRITE { + // The common case; stay quiet. + return degrade("") + } + current, ok := s.store.(dotc1z.SourceCacheStore) + if !ok { + return degrade("storage engine does not support source cache") + } + + // Write side enabled: rows produced under a scope get stamped and + // manifest entries get written, so this sync is usable as the NEXT + // sync's replay source even when this one has nothing to replay from. + s.sourceCache = syncerSourceCache{enabled: true, current: current} + + // Read side: a usable previous sync makes lookups hit and replay legal. + var readReason string + if s.previousSyncReader == nil { + readReason = "no previous-sync c1z configured" + } else if prev, ok := s.previousSyncReader.(dotc1z.SourceCacheStore); !ok { + readReason = "previous-sync store engine does not support source cache" + } else { + s.sourceCache.prev = prev + s.sourceCache.prevReader = s.previousSyncReader + } + + lookup := sourcecache.Lookup(sourcecache.NoopLookup{}) + if s.sourceCache.prev != nil { + lookup = prevStoreLookup{prev: s.sourceCache.prev, logOnce: &stdsync.Once{}} + } + if canSetLookup { + setLookup.SetSourceCache(ctx, lookup) + } else { + // The connector declared the capability but the client offers no + // way to deliver lookups. Its own lookup stays no-op, so every + // scope misses and no replay annotations can legally arrive. + l.Warn("source cache capability declared but connector client cannot receive lookups") + } + l.Info("source cache enabled", + zap.Bool("replay_available", s.sourceCache.prev != nil), + zap.String("replay_unavailable_reason", readReason), + ) + return nil +} + +// clearSourceCacheLookup detaches the per-sync lookup so a late RPC from +// the connector cannot read a store the syncer no longer owns. +func (s *syncer) clearSourceCacheLookup(ctx context.Context) { + if setLookup, ok := s.connector.(sourcecache.SetLookup); ok { + setLookup.SetSourceCache(ctx, nil) + } +} + +// sourceCachePage carries one list response's source-cache instructions +// from beginSourceCachePage (before rows are written) to +// finishSourceCachePage (after rows are written). +type sourceCachePage struct { + kind sourcecache.RowKind + scopeHash string + etag string + // replayed reports that beginSourceCachePage copied the previous + // sync's rows for this scope into the current sync BEFORE the page's + // own rows commit. Consumers that dedupe against "already synced this + // sync" state (the resources path) must not skip this page's rows: + // they are the overlay, and the already-present row is the stale + // replayed base they exist to overwrite. + replayed bool + // deletedIDs are canonical-id tombstones (grant/entitlement ids, + // resource BIDs); deletedPrincipalIDs are bare-object-id tombstones + // applied scope-relatively. Both may arrive on any page of a scope + // (replay annotation on the first page, scope annotation on every + // page) and apply after the page's rows commit. + deletedIDs []string + deletedPrincipalIDs []string +} + +// beginSourceCachePage inspects a list response's annotations, performs +// any requested replay, and returns the context to write the page's rows +// under (stamped with the scope when one is present). A nil page means the +// response carried no source-cache instructions. +// +// rowCount is the number of rows in the response; a non-overlay replay +// that also returned rows gets a warning (the rows are upserted anyway). +func (s *syncer) beginSourceCachePage( + ctx context.Context, + kind sourcecache.RowKind, + respAnnos annotations.Annotations, + rowCount int, +) (context.Context, *sourceCachePage, error) { + replay := &v2.SourceCacheReplay{} + hasReplay, err := respAnnos.Pick(replay) + if err != nil { + return ctx, nil, fmt.Errorf("source cache: error parsing replay annotation: %w", err) + } + scope := &v2.SourceCacheScope{} + hasScope, err := respAnnos.Pick(scope) + if err != nil { + return ctx, nil, fmt.Errorf("source cache: error parsing scope annotation: %w", err) + } + if !hasReplay && !hasScope { + return ctx, nil, nil + } + + if !s.sourceCache.enabled { + if hasReplay { + // The connector skipped row generation expecting a replay; with + // source cache disabled there is nothing to replay from. This is + // a connector bug (replay for a scope it never got a lookup hit + // on), not a degradable condition. + return ctx, nil, fmt.Errorf("source cache: connector requested replay for scope %q but source cache is disabled", replay.GetScopeHash()) + } + // Scope annotations without the capability handshake are ignored. + return ctx, nil, nil + } + + page := &sourceCachePage{kind: kind} + switch { + case hasReplay && hasScope: + if replay.GetScopeHash() != scope.GetScopeHash() { + return ctx, nil, fmt.Errorf("source cache: replay scope %q and page scope %q disagree", replay.GetScopeHash(), scope.GetScopeHash()) + } + page.scopeHash = replay.GetScopeHash() + case hasReplay: + page.scopeHash = replay.GetScopeHash() + default: + page.scopeHash = scope.GetScopeHash() + } + if err := sourcecache.ValidateScopeHash(page.scopeHash); err != nil { + return ctx, nil, fmt.Errorf("source cache: %w", err) + } + // Prefer the scope annotation's etag (the freshest validator on + // overlay pages); fall back to the replay's. + page.etag = scope.GetEtag() + if page.etag == "" { + page.etag = replay.GetEtag() + } + // Tombstones may ride either annotation — the replay annotation on a + // round's first page, the scope annotation on every page (so a + // multi-page delta round never buffers deletions). + page.deletedIDs = append(replay.GetDeletedIds(), scope.GetDeletedIds()...) + page.deletedPrincipalIDs = append(replay.GetDeletedPrincipalIds(), scope.GetDeletedPrincipalIds()...) + + if hasReplay { + if s.sourceCache.prev == nil { + // Same invariant violation as the disabled case: the connector + // can only have gotten a lookup hit if a previous source exists. + return ctx, nil, fmt.Errorf("source cache: connector requested replay for scope %q but no previous sync is available", replay.GetScopeHash()) + } + page.replayed = true + if !replay.GetOverlay() && rowCount > 0 { + // The contract says a 304-style replay page is empty, but rows + // arriving here are more data, not less — upsert them on top of + // the replayed base (overlay semantics) rather than failing the + // sync. Kept lenient while the model is proven against real + // providers. + ctxzap.Extract(ctx).Warn("source cache: non-overlay replay returned rows; treating them as an overlay", + zap.String("scope_hash", page.scopeHash), + zap.Int("rows", rowCount), + ) + } + // Advisory check: a well-behaved connector only replays a scope + // whose validator came from this sync's lookup, so a missing + // previous manifest entry is suspicious — but not by itself data + // loss (the stamped rows may still exist, e.g. a partially carried + // file). The hard error below is reserved for a replay that + // produces nothing. + _, entryFound, err := s.sourceCache.prev.LookupSourceCacheEntry(ctx, kind, page.scopeHash) + if err != nil { + return ctx, nil, fmt.Errorf("source cache: error reading previous manifest for scope %q: %w", page.scopeHash, err) + } + if !entryFound { + ctxzap.Extract(ctx).Warn("source cache: replay requested for scope with no previous manifest entry", + zap.String("scope_hash", page.scopeHash)) + } + res, err := s.sourceCache.current.ReplaySourceCache(ctx, s.sourceCache.prevReader, kind, page.scopeHash) + if err != nil { + return ctx, nil, fmt.Errorf("source cache: replay for scope %q failed: %w", page.scopeHash, err) + } + if res.Rows == 0 && !entryFound { + // The connector skipped row generation expecting a base that + // does not exist anywhere in the previous file — this sync + // would silently drop the scope's rows. + return ctx, nil, fmt.Errorf("source cache: replay for scope %q found no previous rows and no manifest entry; the connector replayed a scope it never looked up", page.scopeHash) + } + // Replay bypasses the connector-response path that normally arms + // grant expansion (seeing GrantExpandable on returned rows), so a + // sync whose expandable pages all replay would silently skip the + // expansion phase without this. + if kind == sourcecache.RowKindGrants && res.NeedsExpansion && !s.dontExpandGrants { + s.state.SetNeedsExpansion() + } + ctxzap.Extract(ctx).Debug("source cache replayed scope", + zap.String("row_kind", string(kind)), + zap.String("scope_hash", page.scopeHash), + zap.Int64("rows", res.Rows), + zap.Bool("needs_expansion", res.NeedsExpansion), + zap.Int("deleted_ids", len(page.deletedIDs)), + zap.Int("deleted_principal_ids", len(page.deletedPrincipalIDs)), + ) + } + + return sourcecache.WithScope(ctx, page.scopeHash), page, nil +} + +// finishSourceCachePage runs after the page's rows committed: applies +// delta tombstones and, when the page carried a validator, writes the +// current sync's manifest entry. The entry write is last so a failed page +// can never leave a phantom hit for the next sync. +func (s *syncer) finishSourceCachePage(ctx context.Context, page *sourceCachePage) error { + if page == nil { + return nil + } + if len(page.deletedIDs) > 0 { + if page.kind == sourcecache.RowKindGrants { + // Grant-id tombstones resolve within the scope's own rows so + // connector-custom grant-id shapes (unreachable by the global + // bounded delete) work, and the cost stays bounded by the + // scope's size. + deleted, err := s.sourceCache.current.DeleteSourceCacheGrantsByIDInScope(ctx, page.scopeHash, page.deletedIDs) + if err != nil { + return fmt.Errorf("source cache: error applying grant deletions for scope %q: %w", page.scopeHash, err) + } + ctxzap.Extract(ctx).Debug("source cache applied grant-id deletions", + zap.String("scope_hash", page.scopeHash), + zap.Int("tombstones", len(page.deletedIDs)), + zap.Int64("rows_deleted", deleted), + ) + } else if err := s.sourceCache.current.DeleteSourceCacheRows(ctx, page.kind, page.deletedIDs); err != nil { + return fmt.Errorf("source cache: error applying deletions for scope %q: %w", page.scopeHash, err) + } + } + if len(page.deletedPrincipalIDs) > 0 { + deleted, err := s.sourceCache.current.DeleteSourceCacheRowsInScope(ctx, page.kind, page.scopeHash, page.deletedPrincipalIDs) + if err != nil { + return fmt.Errorf("source cache: error applying scoped deletions for scope %q: %w", page.scopeHash, err) + } + ctxzap.Extract(ctx).Debug("source cache applied scoped deletions", + zap.String("row_kind", string(page.kind)), + zap.String("scope_hash", page.scopeHash), + zap.Int("tombstones", len(page.deletedPrincipalIDs)), + zap.Int64("rows_deleted", deleted), + ) + } + if page.etag != "" { + if err := s.sourceCache.current.PutSourceCacheEntry(ctx, page.kind, page.scopeHash, page.etag); err != nil { + return fmt.Errorf("source cache: error writing manifest entry for scope %q: %w", page.scopeHash, err) + } + } + return nil +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go index 205f4297..86b20673 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go @@ -19,7 +19,7 @@ import ( storage_v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" "github.com/conductorone/baton-sdk/pkg/bid" "github.com/conductorone/baton-sdk/pkg/dotc1z" - "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/sourcecache" "github.com/conductorone/baton-sdk/pkg/sync/expand" "github.com/conductorone/baton-sdk/pkg/types/entitlement" batonGrant "github.com/conductorone/baton-sdk/pkg/types/grant" @@ -116,7 +116,7 @@ type syncer struct { externalResourceEntitlementIdFilter string previousSyncC1ZPath string previousSyncC1ZPathOptional bool - store c1zstore.Store + store dotc1z.C1ZStore externalResourceReader connectorstore.Reader previousSyncReader connectorstore.Reader connector types.ConnectorClient @@ -125,7 +125,7 @@ type syncer struct { transitionHandler func(s Action) progressHandler func(p *Progress) tmpDir string - storageEngine c1zstore.Engine + storageEngine dotc1z.Engine skipFullSync bool lastCheckPointTime time.Time counts *progresslog.ProgressLog @@ -135,6 +135,7 @@ type syncer struct { syncID string skipEGForResourceType syncMap[string, bool] skipEntitlementsForResourceType syncMap[string, bool] + typeScopedGrantsForResourceType syncMap[string, bool] skipEntitlementsAndGrants bool skipGrants bool resourceTypeTraits syncMap[string, []v2.ResourceType_Trait] @@ -145,6 +146,7 @@ type syncer struct { workerCount int // If 1, sync is sequential (default). If > 1, sync operations are done in parallel. metricsHandler metrics.Handler syncIdentity uotel.SyncIdentity + sourceCache syncerSourceCache } var _ Syncer = (*syncer)(nil) @@ -153,7 +155,7 @@ var _ Syncer = (*syncer)(nil) // GrantStore.StoreExpandedGrants so the expander package can depend on // a single narrow interface without knowing about C1ZStore. type expanderStoreAdapter struct { - store c1zstore.Store + store dotc1z.C1ZStore } func (a expanderStoreAdapter) GetEntitlement(ctx context.Context, req *reader_v2.EntitlementsReaderServiceGetEntitlementRequest) (*reader_v2.EntitlementsReaderServiceGetEntitlementResponse, error) { @@ -562,6 +564,11 @@ func (s *syncer) Sync(ctx context.Context) error { } } + err = s.configureSourceCache(ctx, resp) + if err != nil { + return err + } + syncResourceTypeMap := make(map[string]bool) if len(s.syncResourceTypes) > 0 { for _, rt := range s.syncResourceTypes { @@ -1059,6 +1066,25 @@ func (s *syncer) syncResources(ctx context.Context, action *Action) error { return err } + putCtx, scPage, err := s.beginSourceCachePage(ctx, sourcecache.RowKindResources, annotations.Annotations(resp.GetAnnotations()), len(resp.GetList())) + if err != nil { + return err + } + + // On any source-cache-scoped page the "already synced this sync" + // dedupe below is wrong — scoped pages are upsert streams whose rows + // are always authoritative: + // - Replayed rounds copy the previous sync's rows in on the round's + // FIRST page, so an overlay row's identity ALWAYS hits the store, + // on every page of the round — but the stored row is the stale + // base and the overlay row is the update that must overwrite it. + // Keying off the replay annotation alone would only protect page + // one (the annotation fires once per round). + // - Cold delta enumerations may legally return the same object + // multiple times (changed mid-walk), later occurrences + // authoritative; deduping would keep the STALE first occurrence. + pageScoped := scPage != nil + bulkPutResoruces := []*v2.Resource{} for _, r := range resp.GetList() { validatedResource := false @@ -1075,7 +1101,7 @@ func (s *syncer) syncResources(ctx context.Context, action *Action) error { validatedResource = true // We must *ALSO* check if we have any child resources. - if !s.hasChildResources(r) { + if !pageScoped && !s.hasChildResources(r) { // Since we only have the resource type IDs of child resources, // we can't tell if we already have synced those child resources. // Those children may also have their own child resources, @@ -1104,12 +1130,16 @@ func (s *syncer) syncResources(ctx context.Context, action *Action) error { } if len(bulkPutResoruces) > 0 { - err = s.store.PutResources(ctx, bulkPutResoruces...) + err = s.store.PutResources(putCtx, bulkPutResoruces...) if err != nil { return err } } + if err := s.finishSourceCachePage(ctx, scPage); err != nil { + return err + } + s.handleProgress(ctx, action, len(resp.GetList())) s.counts.AddResources(action.ResourceTypeID, len(resp.GetList())) if resp.GetNextPageToken() == "" { @@ -1328,11 +1358,21 @@ func (s *syncer) syncEntitlementsForResource(ctx context.Context, action *Action if err := s.validateEntitlementExclusionGroups(resp.GetList()); err != nil { return err } - err = s.store.PutEntitlements(ctx, resp.GetList()...) + + putCtx, scPage, err := s.beginSourceCachePage(ctx, sourcecache.RowKindEntitlements, annotations.Annotations(resp.GetAnnotations()), len(resp.GetList())) + if err != nil { + return err + } + + err = s.store.PutEntitlements(putCtx, resp.GetList()...) if err != nil { return err } + if err := s.finishSourceCachePage(ctx, scPage); err != nil { + return err + } + s.handleProgress(ctx, action, len(resp.GetList())) if resp.GetNextPageToken() == "" { s.counts.AddEntitlementsProgress(resourceID.ResourceType, 1) @@ -1758,6 +1798,9 @@ func (s *syncer) fixEntitlementGraphCycles(ctx context.Context, graph *expand.En // SyncGrants fetches the grants for each resource from the connector. It iterates each resource // from the datastore, and pushes a new action to sync the grants for each individual resource. +// Resource types annotated with TypeScopedGrants are excluded from the per-resource fan-out and +// get a single type-scoped action instead (empty ResourceID); the connector enumerates the whole +// type, optionally spawning additional cursors via the SpawnCursors annotation. func (s *syncer) SyncGrants(ctx context.Context, action *Action) error { ctx, span := uotel.StartWithLink(ctx, tracer, "syncer.SyncGrants") uotel.SetSyncIdentityAttrs(ctx, span) @@ -1765,9 +1808,20 @@ func (s *syncer) SyncGrants(ctx context.Context, action *Action) error { defer func() { uotel.EndSpanWithError(span, err) }() if action.ResourceTypeID == "" && action.ResourceID == "" { + actions := make([]Action, 0) if action.PageToken == "" { ctxzap.Extract(ctx).Info("Syncing grants...") s.handleInitialActionForStep(ctx, *action) + + // One type-scoped action per annotated resource type, enqueued + // exactly once (the planner's first page). + typeScoped, err := s.typeScopedGrantsResourceTypes(ctx) + if err != nil { + return fmt.Errorf("sync-grants: error listing type-scoped resource types: %w", err) + } + for _, rtID := range typeScoped { + actions = append(actions, Action{Op: SyncGrantsOp, ResourceTypeID: rtID}) + } } resp, err := s.store.ListResources(ctx, v2.ResourcesServiceListResourcesRequest_builder{ @@ -1778,16 +1832,25 @@ func (s *syncer) SyncGrants(ctx context.Context, action *Action) error { return fmt.Errorf("sync-grants: error listing resources: %w", err) } - actions := make([]Action, 0) for _, r := range resp.GetList() { shouldSkip, err := s.shouldSkipGrants(ctx, r) if err != nil { return err } - if shouldSkip { continue } + + // Types with type-scoped grants are excluded from the + // per-resource fan-out; their single action is enqueued above. + typeScoped, err := s.resourceTypeHasTypeScopedGrants(ctx, r.GetId().GetResourceType()) + if err != nil { + return err + } + if typeScoped { + continue + } + actions = append(actions, Action{Op: SyncGrantsOp, ResourceID: r.GetId().GetResource(), ResourceTypeID: r.GetId().GetResourceType()}) } @@ -1801,26 +1864,91 @@ func (s *syncer) SyncGrants(ctx context.Context, action *Action) error { return nil } +// resourceTypeHasTypeScopedGrants reports (cached per sync) whether the +// resource type carries the TypeScopedGrants annotation. +func (s *syncer) resourceTypeHasTypeScopedGrants(ctx context.Context, resourceTypeID string) (bool, error) { + if v, ok := s.typeScopedGrantsForResourceType.Load(resourceTypeID); ok { + return v, nil + } + rt, err := s.store.GetResourceType(ctx, reader_v2.ResourceTypesReaderServiceGetResourceTypeRequest_builder{ + ResourceTypeId: resourceTypeID, + }.Build()) + if err != nil { + return false, err + } + rtAnnos := annotations.Annotations(rt.GetResourceType().GetAnnotations()) + typeScoped := rtAnnos.Contains(&v2.TypeScopedGrants{}) + s.typeScopedGrantsForResourceType.Store(resourceTypeID, typeScoped) + return typeScoped, nil +} + +// typeScopedGrantsResourceTypes lists every synced resource type annotated +// with TypeScopedGrants. +func (s *syncer) typeScopedGrantsResourceTypes(ctx context.Context) ([]string, error) { + var out []string + pageToken := "" + for { + resp, err := s.store.ListResourceTypes(ctx, v2.ResourceTypesServiceListResourceTypesRequest_builder{ + PageToken: pageToken, + }.Build()) + if err != nil { + return nil, err + } + for _, rt := range resp.GetList() { + rtAnnos := annotations.Annotations(rt.GetAnnotations()) + typeScoped := rtAnnos.Contains(&v2.TypeScopedGrants{}) + s.typeScopedGrantsForResourceType.Store(rt.GetId(), typeScoped) + if typeScoped { + out = append(out, rt.GetId()) + } + } + pageToken = resp.GetNextPageToken() + if pageToken == "" { + return out, nil + } + } +} + // syncGrantsForResource fetches the grants for a specific resource from the connector. +// An action with an empty ResourceID is a TYPE-SCOPED grants cursor: the connector +// enumerates grants for the whole resource type (no single resource backs the call), +// and may spawn sibling cursors via the SpawnCursors response annotation. // No span here: only call site is SyncGrants, which already owns a span. func (s *syncer) syncGrantsForResource(ctx context.Context, action *Action) error { + typeScoped := action.ResourceID == "" resourceID := v2.ResourceId_builder{ ResourceType: action.ResourceTypeID, Resource: action.ResourceID, }.Build() - resourceResponse, err := s.store.GetResource(ctx, reader_v2.ResourcesReaderServiceGetResourceRequest_builder{ - ResourceId: resourceID, - }.Build()) - if err != nil { - return fmt.Errorf("sync-grants-for-resource: error getting resource: %w", err) - } - resource := resourceResponse.GetResource() + var resource *v2.Resource + var reqAnnos annotations.Annotations + if typeScoped { + // Wire validation requires a non-empty resource id, so the stub is + // self-referential ({type, type}) and the request carries the + // TypeScopedGrants annotation as the routing marker. + resource = v2.Resource_builder{ + Id: v2.ResourceId_builder{ + ResourceType: action.ResourceTypeID, + Resource: action.ResourceTypeID, + }.Build(), + }.Build() + reqAnnos.Update(&v2.TypeScopedGrants{}) + } else { + resourceResponse, err := s.store.GetResource(ctx, reader_v2.ResourcesReaderServiceGetResourceRequest_builder{ + ResourceId: resourceID, + }.Build()) + if err != nil { + return fmt.Errorf("sync-grants-for-resource: error getting resource: %w", err) + } + resource = resourceResponse.GetResource() + } resp, err := s.connector.ListGrants(ctx, v2.GrantsServiceListGrantsRequest_builder{ Resource: resource, PageToken: action.PageToken, ActiveSyncId: s.getActiveSyncID(), + Annotations: reqAnnos, }.Build()) if err != nil { return fmt.Errorf("sync-grants-for-resource: error listing grants: %w", err) @@ -1833,6 +1961,15 @@ func (s *syncer) syncGrantsForResource(ctx context.Context, action *Action) erro respAnnos := annotations.Annotations(resp.GetAnnotations()) insertResourceGrants := respAnnos.Contains(&v2.InsertResourceGrants{}) + // Source-cache replay/stamping for this grants page. putCtx applies + // ONLY to the PutGrants call below — the related-resource PutResources + // writes in this function are resource rows and must not inherit a + // grants-scope stamp. + putCtx, scPage, err := s.beginSourceCachePage(ctx, sourcecache.RowKindGrants, respAnnos, len(grants)) + if err != nil { + return fmt.Errorf("sync-grants-for-resource: %w", err) + } + // Stamp InsertResourceGrants per-grant so the slim-blob writer's // gate sees it. The annotation is response-level, but the writer // needs it per-row to avoid stripping the Resource this path @@ -1915,11 +2052,15 @@ func (s *syncer) syncGrantsForResource(ctx context.Context, action *Action) erro } } - err = s.store.PutGrants(ctx, grants...) + err = s.store.PutGrants(putCtx, grants...) if err != nil { return fmt.Errorf("sync-grants-for-resource: error putting grants: %w", err) } + if err := s.finishSourceCachePage(ctx, scPage); err != nil { + return fmt.Errorf("sync-grants-for-resource: %w", err) + } + s.handleProgress(ctx, action, len(grants)) if resp.GetNextPageToken() == "" { @@ -1927,7 +2068,37 @@ func (s *syncer) syncGrantsForResource(ctx context.Context, action *Action) erro s.counts.LogGrantsProgress(ctx, resourceID.GetResourceType()) } - return s.nextPageOrFinishAction(ctx, action, resp.GetNextPageToken()) + // SpawnCursors: a type-scoped response may enqueue sibling cursors for + // the same resource type (e.g. one per connector-defined shard). Each + // runs as its own action — scheduled, rate-limited, and checkpointed + // like any other pagination. Only meaningful on type-scoped calls; + // per-resource responses carrying it are a connector bug. + spawn := &v2.SpawnCursors{} + hasSpawn, err := respAnnos.Pick(spawn) + if err != nil { + return fmt.Errorf("sync-grants-for-resource: error parsing spawn-cursors annotation: %w", err) + } + var spawned []Action + if hasSpawn { + if !typeScoped { + l.Warn("sync-grants-for-resource: SpawnCursors on a per-resource grants response; ignored", + zap.String("resource_type_id", action.ResourceTypeID), + zap.String("resource_id", action.ResourceID)) + } else { + for _, tok := range spawn.GetPageTokens() { + if tok == "" { + continue + } + spawned = append(spawned, Action{Op: SyncGrantsOp, ResourceTypeID: action.ResourceTypeID, PageToken: tok}) + } + l.Debug("sync-grants-for-resource: spawned type-scoped grant cursors", + zap.String("resource_type_id", action.ResourceTypeID), + zap.Int("cursors", len(spawned)), + zap.Int64("estimated_total", spawn.GetEstimatedTotal())) + } + } + + return s.nextPageOrFinishAction(ctx, action, resp.GetNextPageToken(), spawned...) } func (s *syncer) SyncExternalResources(ctx context.Context, action *Action) error { @@ -2735,6 +2906,10 @@ func (s *syncer) Close(ctx context.Context) error { var errs []error + // Detach the source-cache lookup before the stores go away so a late + // connector RPC can't read a store the syncer no longer owns. + s.clearSourceCacheLookup(ctx) + var storeCloseErr error if s.store != nil { storeCloseErr = s.store.Close(finalizeCtx) @@ -2800,7 +2975,7 @@ func WithProgressHandler(f func(s *Progress)) SyncOpt { // WithConnectorStore sets the connector store to use. This is the preferred option. // Either this or WithC1ZPath must be provided to create a new syncer. -func WithConnectorStore(store c1zstore.Store) SyncOpt { +func WithConnectorStore(store dotc1z.C1ZStore) SyncOpt { return func(s *syncer) { s.store = store } @@ -2822,7 +2997,7 @@ func WithTmpDir(path string) SyncOpt { // WithStorageEngine selects the dotc1z storage engine when opening the c1z // file via WithC1ZPath. Empty uses the baton-sdk default. -func WithStorageEngine(engine c1zstore.Engine) SyncOpt { +func WithStorageEngine(engine dotc1z.Engine) SyncOpt { return func(s *syncer) { s.storageEngine = engine } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/attached/attached.go b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/attached/attached.go index 97c1e50b..d1b904f7 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/attached/attached.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/attached/attached.go @@ -8,7 +8,6 @@ import ( reader_v2 "github.com/conductorone/baton-sdk/pb/c1/reader/v2" "github.com/conductorone/baton-sdk/pkg/connectorstore" "github.com/conductorone/baton-sdk/pkg/dotc1z" - "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" "go.uber.org/zap" ) @@ -26,7 +25,7 @@ type Compactor struct { // Both arguments are C1ZStore; the constructor type-asserts to *dotc1z.C1File // and returns an error on mismatch. This keeps the public entry point clean // while confining the SQLite-specific concern to the attached package. -func NewAttachedCompactor(base, applied c1zstore.Store) (*Compactor, error) { +func NewAttachedCompactor(base, applied dotc1z.C1ZStore) (*Compactor, error) { baseFile, ok := dotc1z.AsSQLiteStore(base) if !ok { return nil, fmt.Errorf("attached compactor requires SQLite-backed base store, got %T", base) diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/compactor.go b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/compactor.go index 48514230..6a298f37 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/compactor.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/compactor.go @@ -13,7 +13,6 @@ import ( reader_v2 "github.com/conductorone/baton-sdk/pb/c1/reader/v2" "github.com/conductorone/baton-sdk/pkg/connectorstore" "github.com/conductorone/baton-sdk/pkg/dotc1z" - "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" "github.com/conductorone/baton-sdk/pkg/sdk" "github.com/conductorone/baton-sdk/pkg/sync" "github.com/conductorone/baton-sdk/pkg/synccompactor/attached" @@ -36,7 +35,7 @@ const ( type Compactor struct { compactorType CompactorType entries []*CompactableSync - compactedC1z c1zstore.Store + compactedC1z dotc1z.C1ZStore tmpDir string destDir string @@ -48,7 +47,7 @@ type Compactor struct { // Empty means EngineSQLite (the default; behavior is unchanged and // the output is byte-identical to the pre-engine-option compactor). // EnginePebble produces a v3 Pebble c1z via a native record merge. - engine c1zstore.Engine + engine dotc1z.Engine // pebbleMode optionally forces the Pebble merge strategy; the zero // value (Auto) lets the compactor choose. See WithPebbleCompactorMode. pebbleMode PebbleCompactorMode @@ -77,9 +76,9 @@ type Compactor struct { // resolvedEngine returns the configured engine, treating the zero value // as EngineSQLite. Compact calls inferEngineFromInputs first so the zero // value can follow existing c1z inputs instead of always producing SQLite. -func (c *Compactor) resolvedEngine() c1zstore.Engine { +func (c *Compactor) resolvedEngine() dotc1z.Engine { if c.engine == "" { - return c1zstore.EngineSQLite + return dotc1z.EngineSQLite } return c.engine } @@ -104,7 +103,7 @@ var ErrEnginePolicyConflict = errors.New("compactor: engine policy conflict: can // // Constraint: an explicit SQLite request (WithEngine(EngineSQLite)) when any // input is Pebble/v3 returns ErrEnginePolicyConflict. -func (c *Compactor) inferEngineFromInputs() (c1zstore.Engine, error) { +func (c *Compactor) inferEngineFromInputs() (dotc1z.Engine, error) { hasPebble := false hasSQLite := false for _, entry := range c.entries { @@ -135,7 +134,7 @@ func (c *Compactor) inferEngineFromInputs() (c1zstore.Engine, error) { // Explicit engine: validate and return. if c.engine != "" { - if c.engine == c1zstore.EngineSQLite && hasPebble { + if c.engine == dotc1z.EngineSQLite && hasPebble { return "", fmt.Errorf("%w: caller requested SQLite but at least one input is Pebble/v3", ErrEnginePolicyConflict) } return c.engine, nil @@ -143,13 +142,13 @@ func (c *Compactor) inferEngineFromInputs() (c1zstore.Engine, error) { // Auto-select: any Pebble input → Pebble output. if hasPebble { - return c1zstore.EnginePebble, nil + return dotc1z.EnginePebble, nil } if hasSQLite { - return c1zstore.EngineSQLite, nil + return dotc1z.EngineSQLite, nil } // No readable inputs: default to SQLite to preserve historical behavior. - return c1zstore.EngineSQLite, nil + return dotc1z.EngineSQLite, nil } type CompactableSync struct { @@ -311,7 +310,7 @@ func (c *Compactor) Compact(ctx context.Context) (*CompactableSync, error) { // and partials are merged into the base keyspace via keep-newer // writes; the folded output is then re-keyed to a fresh sync id. // The original base file is never mutated. See compactPebbleFold. - if c.resolvedEngine() == c1zstore.EnginePebble { + if c.resolvedEngine() == dotc1z.EnginePebble { c.pebbleMode = c.resolvePebbleMode(ctx) } foldMode := c.pebbleMode == PebbleCompactorModeFold @@ -337,7 +336,7 @@ func (c *Compactor) Compact(ctx context.Context) (*CompactableSync, error) { } } - if c.resolvedEngine() == c1zstore.EnginePebble { + if c.resolvedEngine() == dotc1z.EnginePebble { // One payload-decoder pool for the whole compaction: the merge // opens every source's envelope (selection + per-chunk unpack), // and reusing one decoder across those opens avoids a fresh @@ -348,10 +347,10 @@ func (c *Compactor) Compact(ctx context.Context) (*CompactableSync, error) { opts = append(opts, dotc1z.WithDecoderPool(c.decoderPool)) } - if c.resolvedEngine() == c1zstore.EnginePebble { + if c.resolvedEngine() == dotc1z.EnginePebble { // Force the resolved engine last so a stray engine passed via // WithC1ZOptions cannot mislabel the artifact. - c.compactedC1z, err = dotc1z.NewStore(ctx, destFilePath, append(opts, dotc1z.WithEngine(c1zstore.EnginePebble))...) + c.compactedC1z, err = dotc1z.NewStore(ctx, destFilePath, append(opts, dotc1z.WithEngine(dotc1z.EnginePebble))...) } else { c.compactedC1z, err = dotc1z.NewStore(ctx, destFilePath, opts...) } @@ -381,7 +380,7 @@ func (c *Compactor) Compact(ctx context.Context) (*CompactableSync, error) { } return nil, fmt.Errorf("failed to compact (pebble fold): %w", err) } - case c.resolvedEngine() == c1zstore.EnginePebble: + case c.resolvedEngine() == dotc1z.EnginePebble: newSyncId, err = c.runPebbleRebuild(ctx, runCtx) if err != nil { if cause := context.Cause(runCtx); errors.Is(cause, context.DeadlineExceeded) && c.runDuration > 0 && ctx.Err() == nil { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/compactor_pebble.go b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/compactor_pebble.go index 00fc64a4..792e7ce3 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/compactor_pebble.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/compactor_pebble.go @@ -19,7 +19,6 @@ import ( v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" "github.com/conductorone/baton-sdk/pkg/connectorstore" "github.com/conductorone/baton-sdk/pkg/dotc1z" - "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" enginepkg "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble" formatv3 "github.com/conductorone/baton-sdk/pkg/dotc1z/format/v3" mergepkg "github.com/conductorone/baton-sdk/pkg/synccompactor/pebble" @@ -34,7 +33,7 @@ import ( // This is the only supported way to choose the engine; an engine // passed through WithC1ZOptions does not select the compaction // strategy and is overridden. -func WithEngine(engine c1zstore.Engine) Option { +func WithEngine(engine dotc1z.Engine) Option { return func(c *Compactor) { c.engine = engine } @@ -340,7 +339,7 @@ func fileSizeOrZero(path string) int64 { // pre-static-registration era. Pebble is now registered by dotc1z init, so this // is a cheap sanity check. func ensurePebbleRegistered() error { - if _, ok := dotc1z.EngineDriverFor(c1zstore.EnginePebble); ok { + if _, ok := dotc1z.EngineDriverFor(dotc1z.EnginePebble); ok { return nil } return dotc1z.ErrEngineNotAvailable diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/bucket_plans.go b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/bucket_plans.go index 5b5b1d94..2cd44d8e 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/bucket_plans.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/bucket_plans.go @@ -60,6 +60,34 @@ func buildBucketPlans() []bucketPlan { lower: enginepkg.GrantByNeedsExpansionLowerBound(), upper: enginepkg.GrantByNeedsExpansionUpperBound(), }, + // Source-cache state MUST fold with its rows. Excising these + // ranges alongside the record buckets replaces the base sync's + // manifest/index entries with the applied sync's: scopes the + // applied sync touched keep a consistent (entry, rows) pair, and + // scopes it never touched become clean lookup misses. Leaving + // them out would strand the BASE sync's validators and index + // entries on top of the applied sync's rows — a stale manifest + // hit could then replay rows that no longer belong to the scope. + { + name: "grant_by_source_scope", + lower: enginepkg.GrantBySourceScopeLowerBound(), + upper: enginepkg.GrantBySourceScopeUpperBound(), + }, + { + name: "entitlement_by_source_scope", + lower: enginepkg.EntitlementBySourceScopeLowerBound(), + upper: enginepkg.EntitlementBySourceScopeUpperBound(), + }, + { + name: "resource_by_source_scope", + lower: enginepkg.ResourceBySourceScopeLowerBound(), + upper: enginepkg.ResourceBySourceScopeUpperBound(), + }, + { + name: "source_cache_entry", + lower: enginepkg.SourceCacheEntryLowerBound(), + upper: enginepkg.SourceCacheEntryUpperBound(), + }, { name: "asset", lower: enginepkg.AssetLowerBound(), diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/tasks/c1api/full_sync.go b/vendor/github.com/conductorone/baton-sdk/pkg/tasks/c1api/full_sync.go index ed2fe9fc..7b4c56d8 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/tasks/c1api/full_sync.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/tasks/c1api/full_sync.go @@ -16,7 +16,7 @@ import ( v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" v1 "github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1" "github.com/conductorone/baton-sdk/pkg/annotations" - "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z" formatv3 "github.com/conductorone/baton-sdk/pkg/dotc1z/format/v3" "github.com/conductorone/baton-sdk/pkg/session" sdkSync "github.com/conductorone/baton-sdk/pkg/sync" @@ -42,7 +42,7 @@ type fullSyncTaskHandler struct { targetedSyncResources []*v2.Resource syncResourceTypeIDs []string workerCount int - storageEngine c1zstore.Engine + storageEngine dotc1z.Engine // previousSyncSparePath is the connector's ETag-replay opt-in: when // non-empty, the handler retains one spare c1z (the last successfully @@ -160,7 +160,7 @@ func (c *fullSyncTaskHandler) sync(ctx context.Context, c1zPath string) error { } engine := c.storageEngine if engine == "" && c.task.GetSyncFull().GetStorageEngine() != "" { - engine = c1zstore.Engine(c.task.GetSyncFull().GetStorageEngine()) + engine = dotc1z.Engine(c.task.GetSyncFull().GetStorageEngine()) } if engine != "" { syncOpts = append(syncOpts, sdkSync.WithStorageEngine(engine)) @@ -359,7 +359,7 @@ func newFullSyncTaskHandler( targetedSyncResources []*v2.Resource, syncResourceTypeIDs []string, workerCount int, - storageEngine c1zstore.Engine, + storageEngine dotc1z.Engine, previousSyncSparePath string, ) tasks.TaskHandler { return &fullSyncTaskHandler{ diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/tasks/c1api/manager.go b/vendor/github.com/conductorone/baton-sdk/pkg/tasks/c1api/manager.go index c0df2348..5ed25a90 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/tasks/c1api/manager.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/tasks/c1api/manager.go @@ -14,7 +14,7 @@ import ( "google.golang.org/protobuf/types/known/anypb" "github.com/conductorone/baton-sdk/pkg/annotations" - "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z" "github.com/conductorone/baton-sdk/pkg/uotel" "github.com/conductorone/baton-sdk/pkg/uotel/uotelzap" @@ -68,7 +68,7 @@ type c1ApiTaskManager struct { targetedSyncResources []*v2.Resource syncResourceTypeIDs []string workerCount int - storageEngine c1zstore.Engine + storageEngine dotc1z.Engine // previousSyncSparePath is non-empty when the connector opted into // ETag replay (keepPreviousSyncC1Z): the fixed, client-id-namespaced @@ -500,7 +500,7 @@ func NewC1TaskManager( targetedSyncResources []*v2.Resource, syncResourceTypeIDs []string, workerCount int, - storageEngine c1zstore.Engine, + storageEngine dotc1z.Engine, taskConcurrency int, keepPreviousSyncC1Z bool, ) (BootstrappingTaskManager, error) { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/tasks/local/compactor.go b/vendor/github.com/conductorone/baton-sdk/pkg/tasks/local/compactor.go index f9ee3c04..2c55874e 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/tasks/local/compactor.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/tasks/local/compactor.go @@ -6,7 +6,7 @@ import ( "time" v1 "github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1" - "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z" "github.com/conductorone/baton-sdk/pkg/synccompactor" "github.com/conductorone/baton-sdk/pkg/tasks" "github.com/conductorone/baton-sdk/pkg/types" @@ -23,12 +23,12 @@ type localCompactor struct { compactableSyncs []*synccompactor.CompactableSync outputPath string tmpDir string - storageEngine c1zstore.Engine + storageEngine dotc1z.Engine } type CompactorOption func(*localCompactor) -func WithCompactorStorageEngine(engine c1zstore.Engine) CompactorOption { +func WithCompactorStorageEngine(engine dotc1z.Engine) CompactorOption { return func(m *localCompactor) { m.storageEngine = engine } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/tasks/local/syncer.go b/vendor/github.com/conductorone/baton-sdk/pkg/tasks/local/syncer.go index 5d469ca5..4342c8ca 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/tasks/local/syncer.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/tasks/local/syncer.go @@ -10,7 +10,7 @@ import ( v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" v1 "github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1" - "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z" "github.com/conductorone/baton-sdk/pkg/session" sdkSync "github.com/conductorone/baton-sdk/pkg/sync" "github.com/conductorone/baton-sdk/pkg/tasks" @@ -24,13 +24,14 @@ type localSyncer struct { o sync.Once tmpDir string externalResourceC1Z string + previousSyncC1Z string externalResourceEntitlementIdFilter string targetedSyncResources []*v2.Resource skipEntitlementsAndGrants bool skipGrants bool syncResourceTypeIDs []string workerCount int - storageEngine c1zstore.Engine + storageEngine dotc1z.Engine } type Option func(*localSyncer) @@ -47,6 +48,12 @@ func WithExternalResourceC1Z(externalResourceC1Z string) Option { } } +func WithPreviousSyncC1Z(previousSyncC1Z string) Option { + return func(m *localSyncer) { + m.previousSyncC1Z = previousSyncC1Z + } +} + func WithExternalResourceEntitlementIdFilter(entitlementId string) Option { return func(m *localSyncer) { m.externalResourceEntitlementIdFilter = entitlementId @@ -83,7 +90,7 @@ func WithWorkerCount(workerCount int) Option { } } -func WithStorageEngine(engine c1zstore.Engine) Option { +func WithStorageEngine(engine dotc1z.Engine) Option { return func(m *localSyncer) { m.storageEngine = engine } @@ -122,6 +129,7 @@ func (m *localSyncer) Process(ctx context.Context, task *v1.Task, cc types.Conne sdkSync.WithC1ZPath(m.dbPath), sdkSync.WithTmpDir(m.tmpDir), sdkSync.WithExternalResourceC1ZPath(m.externalResourceC1Z), + sdkSync.WithPreviousSyncC1ZPath(m.previousSyncC1Z), sdkSync.WithExternalResourceEntitlementIdFilter(m.externalResourceEntitlementIdFilter), sdkSync.WithTargetedSyncResources(m.targetedSyncResources), sdkSync.WithSkipEntitlementsAndGrants(m.skipEntitlementsAndGrants), diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/types/resource/resource.go b/vendor/github.com/conductorone/baton-sdk/pkg/types/resource/resource.go index 16585ed3..49a01e0f 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/types/resource/resource.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/types/resource/resource.go @@ -8,6 +8,7 @@ import ( v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" "github.com/conductorone/baton-sdk/pkg/annotations" "github.com/conductorone/baton-sdk/pkg/pagination" + "github.com/conductorone/baton-sdk/pkg/sourcecache" "github.com/conductorone/baton-sdk/pkg/types/sessions" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -524,9 +525,14 @@ func NewManagedDeviceResource( } type SyncOpAttrs struct { - Session sessions.SessionStore - SyncID string - PageToken pagination.Token + Session sessions.SessionStore + // SourceCache resolves a scope's previous-sync validator (etag / + // delta token) for source-cache replay. Never nil for framework-built + // connectors: when source cache is disabled or degraded the SDK + // supplies sourcecache.NoopLookup, which always misses. + SourceCache sourcecache.Lookup + SyncID string + PageToken pagination.Token } type SyncOpResults struct { diff --git a/vendor/modules.txt b/vendor/modules.txt index 48afa10c..ffcf1ce8 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -273,7 +273,7 @@ github.com/cockroachdb/swiss # github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 ## explicit; go 1.19 github.com/cockroachdb/tokenbucket -# github.com/conductorone/baton-sdk v0.18.2 +# github.com/conductorone/baton-sdk v0.18.2 => ../baton-sdk-2 ## explicit; go 1.25.2 github.com/conductorone/baton-sdk/internal/connector github.com/conductorone/baton-sdk/pb/c1/c1z/v1 @@ -294,6 +294,7 @@ github.com/conductorone/baton-sdk/pkg/bid github.com/conductorone/baton-sdk/pkg/cli github.com/conductorone/baton-sdk/pkg/config github.com/conductorone/baton-sdk/pkg/connectorbuilder +github.com/conductorone/baton-sdk/pkg/connectorclient github.com/conductorone/baton-sdk/pkg/connectorrunner github.com/conductorone/baton-sdk/pkg/connectorstore github.com/conductorone/baton-sdk/pkg/crypto @@ -317,6 +318,7 @@ github.com/conductorone/baton-sdk/pkg/ratelimit github.com/conductorone/baton-sdk/pkg/retry github.com/conductorone/baton-sdk/pkg/sdk github.com/conductorone/baton-sdk/pkg/session +github.com/conductorone/baton-sdk/pkg/sourcecache github.com/conductorone/baton-sdk/pkg/sync github.com/conductorone/baton-sdk/pkg/sync/expand github.com/conductorone/baton-sdk/pkg/sync/expand/scc @@ -1053,3 +1055,4 @@ modernc.org/memory modernc.org/sqlite modernc.org/sqlite/lib modernc.org/sqlite/vtab +# github.com/conductorone/baton-sdk => ../baton-sdk-2 From f7afea5a5043f39b8a4b25cbed1501c1c2b348f6 Mon Sep 17 00:00:00 2001 From: Matt Kaniaris Date: Sun, 12 Jul 2026 20:23:18 -0600 Subject: [PATCH 2/3] wip; --- pkg/connector/group.go | 6 ++- pkg/connector/group_type_scoped.go | 2 +- pkg/connector/request_log.go | 2 +- pkg/connector/sourcecache_fuzz_test.go | 36 ++++++------- pkg/connector/sourcecache_sync_test.go | 53 +++++++++++-------- .../pkg/sync/progresslog/progresslog.go | 23 +++++++- .../conductorone/baton-sdk/pkg/sync/syncer.go | 11 +++- 7 files changed, 88 insertions(+), 45 deletions(-) diff --git a/pkg/connector/group.go b/pkg/connector/group.go index 060e6c05..e8f01c6d 100644 --- a/pkg/connector/group.go +++ b/pkg/connector/group.go @@ -33,6 +33,10 @@ const appGroupType = "APP_GROUP" const oktaGroupType = "OKTA_GROUP" const apiPathGetGroupFmt = "/api/v1/groups/%s" +// groupRoleAssignmentType is the assignmentType value on role assignments +// held by a group (vs "USER"). +const groupRoleAssignmentType = "GROUP" + type groupResourceType struct { resourceType *v2.ResourceType connector *Okta @@ -209,7 +213,7 @@ func (o *groupResourceType) Grants( } for _, role := range roles { - if role.Status == roleStatusInactive || role.AssignmentType != "GROUP" { + if role.Status == roleStatusInactive || role.AssignmentType != groupRoleAssignmentType { continue } diff --git a/pkg/connector/group_type_scoped.go b/pkg/connector/group_type_scoped.go index f13e1709..99a626b2 100644 --- a/pkg/connector/group_type_scoped.go +++ b/pkg/connector/group_type_scoped.go @@ -407,7 +407,7 @@ func (o *groupResourceType) groupRolesPage( shouldExpand := tok.UsersCount == nil || *tok.UsersCount > 0 var rv []*v2.Grant for _, role := range roles { - if role.Status == roleStatusInactive || role.AssignmentType != "GROUP" { + if role.Status == roleStatusInactive || role.AssignmentType != groupRoleAssignmentType { continue } if !o.connector.SyncCustomRoles && role.Type == roleTypeCustom { diff --git a/pkg/connector/request_log.go b/pkg/connector/request_log.go index 03ee326b..935ac549 100644 --- a/pkg/connector/request_log.go +++ b/pkg/connector/request_log.go @@ -38,7 +38,7 @@ func wrapRequestCounting(httpClient *http.Client) (*http.Client, error) { if logPath == "" { return httpClient, nil } - f, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644) + f, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644) //nolint:gosec // path comes from the operator's own env var, not untrusted input if err != nil { return nil, fmt.Errorf("okta-connectorv2: failed to open request log %s: %w", logPath, err) } diff --git a/pkg/connector/sourcecache_fuzz_test.go b/pkg/connector/sourcecache_fuzz_test.go index 00d42d56..b18e7d5d 100644 --- a/pkg/connector/sourcecache_fuzz_test.go +++ b/pkg/connector/sourcecache_fuzz_test.go @@ -47,7 +47,7 @@ func (m *mockOkta) fuzzView() fuzzOrgView { } for _, id := range m.userOrder { v.allUsers = append(v.allUsers, id) - if m.users[id].Status == "ACTIVE" { + if m.users[id].Status == mockStatusActive { v.activeUsers = append(v.activeUsers, id) } } @@ -86,13 +86,13 @@ func (f *fuzzRun) pick(items []string) string { return items[f.rng.Intn(len(items))] } -func (f *fuzzRun) note(format string, args ...any) { +func (f *fuzzRun) notef(format string, args ...any) { f.log = append(f.log, fmt.Sprintf(format, args...)) } // fuzzableRoleTypes are standard org roles the fuzzer assigns to groups // (drawn from standardRoleTypes so the role resources exist). -var fuzzableRoleTypes = []string{"USER_ADMIN", "HELP_DESK_ADMIN", "APP_ADMIN", "REPORT_ADMIN"} +var fuzzableRoleTypes = []string{roleTypeUserAdmin, roleTypeHelpDesk, "APP_ADMIN", "REPORT_ADMIN"} func fuzzOps() []fuzzOp { return []fuzzOp{ @@ -104,7 +104,7 @@ func fuzzOps() []fuzzOp { f.m.addUser(&mockOktaUser{ ID: id, FirstName: "Fuzz", LastName: id, Email: id + "@x.test", }) - f.note("add-user %s", id) + f.notef("add-user %s", id) }, }, { @@ -113,7 +113,7 @@ func fuzzOps() []fuzzOp { apply: func(f *fuzzRun, v fuzzOrgView) { uid := f.pick(v.activeUsers) f.m.deactivateUser(uid) - f.note("deactivate-user %s", uid) + f.notef("deactivate-user %s", uid) }, }, { @@ -122,7 +122,7 @@ func fuzzOps() []fuzzOp { apply: func(f *fuzzRun, v fuzzOrgView) { uid := f.pick(v.allUsers) f.m.deleteUser(uid) - f.note("delete-user %s", uid) + f.notef("delete-user %s", uid) }, }, { @@ -135,7 +135,7 @@ func fuzzOps() []fuzzOp { g.Members = []string{f.pick(v.activeUsers)} } f.m.addGroup(g) - f.note("create-group %s (members=%v)", gid, g.Members) + f.notef("create-group %s (members=%v)", gid, g.Members) }, }, { @@ -144,7 +144,7 @@ func fuzzOps() []fuzzOp { apply: func(f *fuzzRun, v fuzzOrgView) { gid := f.pick(v.groups) f.m.deleteGroup(gid) - f.note("delete-group %s", gid) + f.notef("delete-group %s", gid) }, }, { @@ -169,7 +169,7 @@ func fuzzOps() []fuzzOp { } uid := f.pick(cands) f.m.addMember(gid, uid) - f.note("add-member %s -> %s", uid, gid) + f.notef("add-member %s -> %s", uid, gid) }, }, { @@ -192,7 +192,7 @@ func fuzzOps() []fuzzOp { gid := f.pick(withMembers) uid := f.pick(v.members[gid]) f.m.removeMember(gid, uid) - f.note("remove-member %s <- %s", uid, gid) + f.notef("remove-member %s <- %s", uid, gid) }, }, { @@ -201,7 +201,7 @@ func fuzzOps() []fuzzOp { apply: func(f *fuzzRun, v fuzzOrgView) { gid := f.pick(v.groups) f.m.renameGroup(gid, "Renamed "+f.id("nm")) - f.note("rename-group %s", gid) + f.notef("rename-group %s", gid) }, }, { @@ -210,7 +210,7 @@ func fuzzOps() []fuzzOp { apply: func(f *fuzzRun, v fuzzOrgView) { gid := f.pick(v.groups) f.m.touchGroup(gid) - f.note("touch-group %s", gid) + f.notef("touch-group %s", gid) }, }, { @@ -243,7 +243,7 @@ func fuzzOps() []fuzzOp { } rt := f.pick(free) f.m.assignGroupRole(gid, mockOktaGroupRole{AssignmentID: f.id("gra"), Type: rt, Label: rt}) - f.note("assign-group-role %s -> %s", rt, gid) + f.notef("assign-group-role %s -> %s", rt, gid) }, }, { @@ -266,7 +266,7 @@ func fuzzOps() []fuzzOp { gid := f.pick(withRoles) rt := f.pick(v.roles[gid]) f.m.revokeGroupRole(gid, rt) - f.note("revoke-group-role %s <- %s", rt, gid) + f.notef("revoke-group-role %s <- %s", rt, gid) }, }, } @@ -295,17 +295,17 @@ func TestSourceCacheChurnFuzz(t *testing.T) { // round zero. for i := 1; i <= 4; i++ { mock.addUser(&mockOktaUser{ - ID: fmt.Sprintf("u%d", i), FirstName: "User", LastName: fmt.Sprintf("N%d", i), + ID: fmt.Sprintf("u%d", i), FirstName: "Member", LastName: fmt.Sprintf("N%d", i), Email: fmt.Sprintf("u%d@x.test", i), }) } mock.addGroup(&mockOktaGroup{ID: "g1", Name: "Group One", Members: []string{"u1", "u2"}}) mock.addGroup(&mockOktaGroup{ID: "g2", Name: "Group Two", Members: []string{"u3"}}) mock.addGroup(&mockOktaGroup{ID: "g3", Name: "Empty"}) - mock.assignGroupRole("g1", mockOktaGroupRole{AssignmentID: "gra-seed", Type: "USER_ADMIN", Label: "Group Administrator"}) + mock.assignGroupRole("g1", mockOktaGroupRole{AssignmentID: "gra-seed", Type: roleTypeUserAdmin, Label: roleLabelGroupAdmin}) h := newSyncHarness(ctx, t, mock) - f := &fuzzRun{t: t, m: mock, rng: rand.New(rand.NewSource(seed))} + f := &fuzzRun{t: t, m: mock, rng: rand.New(rand.NewSource(seed))} //nolint:gosec // deterministic replayable fuzzing requires seeded math/rand ops := fuzzOps() prev := h.runSync("fuzz-cold", "") @@ -334,7 +334,7 @@ func TestSourceCacheChurnFuzz(t *testing.T) { for _, gid := range mock.fuzzView().groups { mock.touchGroup(gid) } - f.note("mass-invalidation") + f.notef("mass-invalidation") } warm := h.runSync(fmt.Sprintf("fuzz-warm-%02d", round), prev) diff --git a/pkg/connector/sourcecache_sync_test.go b/pkg/connector/sourcecache_sync_test.go index 52854a9e..debbfbaa 100644 --- a/pkg/connector/sourcecache_sync_test.go +++ b/pkg/connector/sourcecache_sync_test.go @@ -56,6 +56,17 @@ const mockPageSize = 2 var mockTimeBase = time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) +// Test-local constants for repeated literals (keeps package-wide goconst +// counts at their pre-existing baseline). +const ( + mockStatusActive = "ACTIVE" + mockErrNotFound = "E0000007" + roleTypeUserAdmin = "USER_ADMIN" + roleTypeHelpDesk = "HELP_DESK_ADMIN" + roleLabelGroupAdmin = "Group Administrator" + mockKeyStatus = "status" +) + type mockOktaUser struct { ID string FirstName string @@ -120,7 +131,7 @@ func (m *mockOkta) addUser(u *mockOktaUser) { m.mu.Lock() defer m.mu.Unlock() if u.Status == "" { - u.Status = "ACTIVE" + u.Status = mockStatusActive } m.users[u.ID] = u m.userOrder = append(m.userOrder, u.ID) @@ -261,7 +272,7 @@ func (m *mockOkta) snapshotCounts() map[string]int { func (m *mockOkta) userJSON(u *mockOktaUser) map[string]any { return map[string]any{ "id": u.ID, - "status": u.Status, + mockKeyStatus: u.Status, "created": mockTS(0), "lastUpdated": mockTS(0), "profile": map[string]any{ @@ -276,7 +287,7 @@ func (m *mockOkta) userJSON(u *mockOktaUser) map[string]any { func (m *mockOkta) groupJSON(g *mockOktaGroup, withStats bool) map[string]any { obj := map[string]any{ "id": g.ID, - "type": g.Type, + groupTypeProfileKey: g.Type, "created": mockTS(0), "lastUpdated": mockTS(g.lastUpdated), "lastMembershipUpdated": mockTS(g.lastMembershipUpdated), @@ -299,7 +310,7 @@ func (m *mockOkta) groupJSON(g *mockOktaGroup, withStats bool) map[string]any { // pageOf slices order after the given cursor. The cursor is the last id of // the previous page (Okta's after-cursor is opaque; ids work fine). -func pageOf(order []string, after string, size int) (ids []string, next string) { +func pageOf(order []string, after string, size int) ([]string, string) { start := 0 if after != "" { for i, id := range order { @@ -406,7 +417,7 @@ func (m *mockOkta) handler() http.HandlerFunc { g, ok := m.groups[gid] if !ok { w.WriteHeader(http.StatusNotFound) - mockWriteJSON(w, map[string]any{"errorCode": "E0000007", "errorSummary": "Not found: " + gid}) + mockWriteJSON(w, map[string]any{"errorCode": mockErrNotFound, "errorSummary": "Not found: " + gid}) return } ids, next := pageOf(g.Members, q.Get("after"), mockPageSize) @@ -424,17 +435,17 @@ func (m *mockOkta) handler() http.HandlerFunc { g, ok := m.groups[gid] if !ok { w.WriteHeader(http.StatusNotFound) - mockWriteJSON(w, map[string]any{"errorCode": "E0000007", "errorSummary": "Not found: " + gid}) + mockWriteJSON(w, map[string]any{"errorCode": mockErrNotFound, "errorSummary": "Not found: " + gid}) return } out := make([]map[string]any, 0, len(g.Roles)) for _, role := range g.Roles { out = append(out, map[string]any{ - "id": role.AssignmentID, - "type": role.Type, - "label": role.Label, - "status": "ACTIVE", - "assignmentType": "GROUP", + "id": role.AssignmentID, + groupTypeProfileKey: role.Type, + "label": role.Label, + mockKeyStatus: mockStatusActive, + "assignmentType": "GROUP", }) } mockWriteJSON(w, out) @@ -446,14 +457,14 @@ func (m *mockOkta) handler() http.HandlerFunc { default: m.t.Errorf("mock okta: unexpected request %s %s", r.Method, r.URL.String()) w.WriteHeader(http.StatusNotFound) - mockWriteJSON(w, map[string]any{"errorCode": "E0000007", "errorSummary": "unhandled: " + path}) + mockWriteJSON(w, map[string]any{"errorCode": mockErrNotFound, "errorSummary": "unhandled: " + path}) } } } // --- harness ----------------------------------------------------------------- -var harnessSyncResourceTypes = []string{"user", "group", "role"} +var harnessSyncResourceTypes = []string{resourceTypeUser.Id, resourceTypeGroup.Id, resourceTypeRole.Id} type syncHarness struct { t *testing.T @@ -512,7 +523,7 @@ func newSyncHarness(ctx context.Context, t *testing.T, mock *mockOkta) *syncHarn v2.RegisterAccountManagerServiceServer(gs, srv) v2.RegisterCredentialManagerServiceServer(gs, srv) - lis, err := net.Listen("tcp", "127.0.0.1:0") + lis, err := net.Listen("tcp", "127.0.0.1:0") //nolint:noctx // test-scoped loopback listener require.NoError(t, err) go func() { _ = gs.Serve(lis) }() t.Cleanup(gs.Stop) @@ -710,7 +721,7 @@ func TestSourceCacheReplayEndToEnd(t *testing.T) { for i := 1; i <= 5; i++ { mock.addUser(&mockOktaUser{ ID: fmt.Sprintf("u%d", i), - FirstName: "User", + FirstName: "Member", LastName: fmt.Sprintf("Number%d", i), Email: fmt.Sprintf("u%d@x.test", i), }) @@ -718,7 +729,7 @@ func TestSourceCacheReplayEndToEnd(t *testing.T) { mock.addGroup(&mockOktaGroup{ID: "g1", Name: "Engineering", Members: []string{"u1", "u2", "u3"}}) mock.addGroup(&mockOktaGroup{ID: "g2", Name: "Sales", Members: []string{"u4"}}) mock.addGroup(&mockOktaGroup{ID: "g3", Name: "Empty"}) - mock.assignGroupRole("g1", mockOktaGroupRole{AssignmentID: "gra1", Type: "USER_ADMIN", Label: "Group Administrator"}) + mock.assignGroupRole("g1", mockOktaGroupRole{AssignmentID: "gra1", Type: roleTypeUserAdmin, Label: roleLabelGroupAdmin}) h := newSyncHarness(ctx, t, mock) @@ -740,7 +751,7 @@ func TestSourceCacheReplayEndToEnd(t *testing.T) { require.Contains(t, snap1, "grant:"+memberGrantID("g1", "u1")) require.Contains(t, snap1, "grant:"+memberGrantID("g1", "u3")) require.Contains(t, snap1, "grant:"+memberGrantID("g2", "u4")) - require.Contains(t, snap1, "grant:"+roleGroupGrantID("USER_ADMIN", "g1"), "group role grant from the fresh roles leg") + require.Contains(t, snap1, "grant:"+roleGroupGrantID(roleTypeUserAdmin, "g1"), "group role grant from the fresh roles leg") // Grant expansion: g1's members must hold derived USER_ADMIN grants. foundDerived := false for k := range snap1 { @@ -847,17 +858,17 @@ func TestSourceCacheReplayEndToEnd(t *testing.T) { h.requireEquivalent(sync11, control11, "user deletion") // --- Scenario 10: role assignment changes ride the fresh leg --------------- - mock.assignGroupRole("g2", mockOktaGroupRole{AssignmentID: "gra2", Type: "HELP_DESK_ADMIN", Label: "Help Desk Administrator"}) + mock.assignGroupRole("g2", mockOktaGroupRole{AssignmentID: "gra2", Type: roleTypeHelpDesk, Label: "Help Desk Administrator"}) sync12 := h.runSync("role-assign", sync11) mc12 := memberListingCalls(mock.snapshotCounts()) require.Empty(t, sortedKeys(mc12), "role assignment must not dirty the member scope, got %v", mc12) - require.Contains(t, h.snapshot(sync12), "grant:"+roleGroupGrantID("HELP_DESK_ADMIN", "g2"), "new role grant arrives on a fully-warm round via the fresh leg") + require.Contains(t, h.snapshot(sync12), "grant:"+roleGroupGrantID(roleTypeHelpDesk, "g2"), "new role grant arrives on a fully-warm round via the fresh leg") control12 := h.runControlSync("role-assign-control") h.requireEquivalent(sync12, control12, "group role assignment") - mock.revokeGroupRole("g2", "HELP_DESK_ADMIN") + mock.revokeGroupRole("g2", roleTypeHelpDesk) sync13 := h.runSync("role-revoke", sync12) - require.NotContains(t, h.snapshot(sync13), "grant:"+roleGroupGrantID("HELP_DESK_ADMIN", "g2")) + require.NotContains(t, h.snapshot(sync13), "grant:"+roleGroupGrantID(roleTypeHelpDesk, "g2")) control13 := h.runControlSync("role-revoke-control") h.requireEquivalent(sync13, control13, "group role revocation") diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/progresslog/progresslog.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/progresslog/progresslog.go index 734ccd54..747e6036 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sync/progresslog/progresslog.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/progresslog/progresslog.go @@ -38,6 +38,7 @@ type ProgressLog struct { lastEntitlementLog map[string]time.Time grantsProgress map[string]int lastGrantLog map[string]time.Time + grantsCountOnly map[string]bool mu sync.RWMutex l *zap.Logger maxLogFrequency time.Duration @@ -150,6 +151,7 @@ func NewProgressCounts(ctx context.Context, opts ...Option) *ProgressLog { lastEntitlementLog: make(map[string]time.Time), grantsProgress: make(map[string]int), lastGrantLog: make(map[string]time.Time), + grantsCountOnly: make(map[string]bool), l: ctxzap.Extract(ctx), maxLogFrequency: defaultMaxLogFrequency, mu: sync.RWMutex{}, @@ -256,18 +258,35 @@ func (p *ProgressLog) LogEntitlementsProgress(ctx context.Context, resourceType } } +// SetGrantsCountOnly marks a resource type's grant progress as a plain +// count with no resources-covered denominator. Used for type-scoped grant +// enumeration (v2.TypeScopedGrants), where cursors don't map 1:1 to +// resources: the per-cursor accounting would exceed the type's resource +// total and trip the "more grant resources than resources" warning for a +// perfectly healthy sync. Count-only types log synced row counts +// periodically and never compute the ratio. +func (p *ProgressLog) SetGrantsCountOnly(resourceType string) { + p.mu.Lock() + defer p.mu.Unlock() + p.grantsCountOnly[resourceType] = true +} + func (p *ProgressLog) LogGrantsProgress(ctx context.Context, resourceType string) { var grantsProgress, resources int var lastLogTime time.Time + var countOnly bool p.mu.RLock() grantsProgress = p.grantsProgress[resourceType] resources = p.resources[resourceType] lastLogTime = p.lastGrantLog[resourceType] + countOnly = p.grantsCountOnly[resourceType] p.mu.RUnlock() - if resources == 0 { - // if resuming sync, resource counts will be zero, so don't calculate percentage. just log every 10 seconds. + if resources == 0 || countOnly { + // Count-only: either a resumed sync (resource counts are zero) or a + // type-scoped grants type (no meaningful denominator). Log the raw + // synced count every log window; never compute a percentage. if time.Since(lastLogTime) > p.maxLogFrequency { p.l.Info("Syncing grants", zap.String("resource_type_id", resourceType), diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go index 86b20673..84c86001 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go @@ -2063,7 +2063,16 @@ func (s *syncer) syncGrantsForResource(ctx context.Context, action *Action) erro s.handleProgress(ctx, action, len(grants)) - if resp.GetNextPageToken() == "" { + if typeScoped { + // Cursors don't map 1:1 to resources (one cursor can cover many + // groups and may also emit cross-type grants), so the per-resource + // "N of M resources covered" accounting is meaningless here and + // would trip the "more grant resources than resources" warning on + // healthy syncs. Count raw grant rows instead. + s.counts.SetGrantsCountOnly(resourceID.GetResourceType()) + s.counts.AddGrantsProgress(resourceID.GetResourceType(), len(grants)) + s.counts.LogGrantsProgress(ctx, resourceID.GetResourceType()) + } else if resp.GetNextPageToken() == "" { s.counts.AddGrantsProgress(resourceID.GetResourceType(), 1) s.counts.LogGrantsProgress(ctx, resourceID.GetResourceType()) } From bdfac9849a42572608ac9c8eee71814e7d3bf270 Mon Sep 17 00:00:00 2001 From: Matt Kaniaris Date: Mon, 13 Jul 2026 11:23:51 -0600 Subject: [PATCH 3/3] update to work with sdk changes --- pkg/connector/sourcecache_sync_test.go | 134 +++- .../v2/annotation_source_cache.pb.go | 411 ++++++++++- .../v2/annotation_source_cache.pb.validate.go | 651 ++++++++++++++++++ .../annotation_source_cache_protoopaque.pb.go | 411 ++++++++++- .../v2/annotation_type_scoped_grants.pb.go | 42 +- ...ation_type_scoped_grants_protoopaque.pb.go | 42 +- .../baton-sdk/pkg/cli/commands.go | 21 +- .../baton-sdk/pkg/config/config.go | 20 + .../pkg/connectorbuilder/resource_syncer.go | 116 +++- .../baton-sdk/pkg/connectorrunner/runner.go | 25 +- .../pkg/connectorstore/connectorstore.go | 4 +- .../baton-sdk/pkg/dotc1z/c1file.go | 38 +- .../baton-sdk/pkg/dotc1z/c1file_store.go | 56 +- .../baton-sdk/pkg/dotc1z/c1zstore/c1zstore.go | 5 +- .../baton-sdk/pkg/dotc1z/cleanup_policy.go | 2 +- .../baton-sdk/pkg/dotc1z/clone_sync.go | 5 +- .../baton-sdk/pkg/dotc1z/convert_open.go | 3 +- .../pkg/dotc1z/engine/pebble/adapter.go | 2 +- .../pkg/dotc1z/engine/pebble/engine_stub.go | 2 +- .../pkg/dotc1z/engine/pebble/sync_runs.go | 2 +- .../baton-sdk/pkg/dotc1z/engine_registry.go | 51 +- .../baton-sdk/pkg/dotc1z/file_ops.go | 25 - .../baton-sdk/pkg/dotc1z/format.go | 56 -- .../pkg/dotc1z/format/v3/envelope.go | 96 ++- .../baton-sdk/pkg/dotc1z/format/v3/indexed.go | 30 +- .../baton-sdk/pkg/dotc1z/grant_store.go | 19 - .../baton-sdk/pkg/dotc1z/pebble_store.go | 75 +- .../baton-sdk/pkg/dotc1z/sql_helpers.go | 4 +- .../baton-sdk/pkg/dotc1z/store.go | 23 +- .../baton-sdk/pkg/dotc1z/sync_meta.go | 14 - .../baton-sdk/pkg/dotc1z/sync_runs.go | 24 +- .../baton-sdk/pkg/dotc1z/to_pebble.go | 3 +- .../baton-sdk/pkg/field/defaults.go | 10 + .../baton-sdk/pkg/field/fields.go | 28 + .../baton-sdk/pkg/lambda/grpc/client.go | 12 +- .../baton-sdk/pkg/lambda/grpc/server.go | 11 + .../baton-sdk/pkg/lambda/grpc/transport.go | 109 ++- .../baton-sdk/pkg/lambda/grpc/wire.go | 88 +++ .../conductorone/baton-sdk/pkg/sdk/version.go | 2 +- .../baton-sdk/pkg/sourcecache/continuation.go | 232 +++++++ .../baton-sdk/pkg/sourcecache/sourcecache.go | 13 +- .../baton-sdk/pkg/sync/expand/expander.go | 4 +- .../pkg/sync/progresslog/progresslog.go | 11 + .../baton-sdk/pkg/sync/source_cache.go | 10 + .../pkg/sync/source_cache_continuation.go | 248 +++++++ .../conductorone/baton-sdk/pkg/sync/state.go | 9 + .../conductorone/baton-sdk/pkg/sync/syncer.go | 159 +++-- .../pkg/synccompactor/attached/attached.go | 3 +- .../baton-sdk/pkg/synccompactor/compactor.go | 29 +- .../pkg/synccompactor/compactor_pebble.go | 5 +- .../baton-sdk/pkg/tasks/c1api/full_sync.go | 8 +- .../baton-sdk/pkg/tasks/c1api/manager.go | 6 +- .../baton-sdk/pkg/tasks/local/compactor.go | 6 +- .../baton-sdk/pkg/tasks/local/syncer.go | 6 +- 54 files changed, 2950 insertions(+), 471 deletions(-) delete mode 100644 vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/file_ops.go delete mode 100644 vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/grant_store.go delete mode 100644 vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sync_meta.go create mode 100644 vendor/github.com/conductorone/baton-sdk/pkg/lambda/grpc/wire.go create mode 100644 vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/continuation.go create mode 100644 vendor/github.com/conductorone/baton-sdk/pkg/sync/source_cache_continuation.go diff --git a/pkg/connector/sourcecache_sync_test.go b/pkg/connector/sourcecache_sync_test.go index debbfbaa..bff7d6dc 100644 --- a/pkg/connector/sourcecache_sync_test.go +++ b/pkg/connector/sourcecache_sync_test.go @@ -35,6 +35,7 @@ import ( "github.com/conductorone/baton-sdk/pkg/connectorbuilder" "github.com/conductorone/baton-sdk/pkg/connectorclient" "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" "github.com/conductorone/baton-sdk/pkg/logging" "github.com/conductorone/baton-sdk/pkg/sourcecache" sdkSync "github.com/conductorone/baton-sdk/pkg/sync" @@ -476,6 +477,27 @@ type syncHarness struct { } func newSyncHarness(ctx context.Context, t *testing.T, mock *mockOkta) *syncHarness { + return newSyncHarnessTopology(ctx, t, mock, false) +} + +// newSyncHarnessTopology builds the harness in one of two lookup +// topologies: +// +// - direct (deferredLookup=false): the syncer's per-sync lookup is wired +// into the builder via SetSourceCacheSetter, mirroring in-process and +// subprocess runtimes — connector lookups are answered inline. +// - deferred (deferredLookup=true): the setter wiring is OMITTED, so the +// builder has no direct lookup and the ask/answer continuation runs +// for real over the gRPC loopback, mirroring single-shot transports +// (gRPC-over-Lambda): the syncer attaches SourceCacheLookupOffer on +// warm syncs, the connector's phase-1 lookup defers with +// ErrLookupDeferred, the builder answers with a SourceCacheLookupAsk, +// and the syncer re-invokes with SourceCacheLookupAnswers. +// +// Every scenario assertion is topology-independent: a warm round that +// spends zero member-listing requests in deferred mode proves the bounce +// delivered the validator (a broken continuation would miss and go cold). +func newSyncHarnessTopology(ctx context.Context, t *testing.T, mock *mockOkta, deferredLookup bool) *syncHarness { t.Helper() server := httptest.NewServer(mock.handler()) @@ -534,16 +556,21 @@ func newSyncHarness(ctx context.Context, t *testing.T, mock *mockOkta) *syncHarn cc := connectorclient.NewConnectorClient(ctx, conn) - // In-process lookup delivery: the syncer installs its per-sync lookup on + // Direct lookup delivery: the syncer installs its per-sync lookup on // the client, which forwards it to the builder (the CLI wrapper does - // this same wiring in internal/connector). - setter, ok := cc.(interface { - SetSourceCacheSetter(sourcecache.SetLookup) - }) - require.True(t, ok, "connector client must accept a source-cache setter") - lookupSink, ok := srv.(sourcecache.SetLookup) - require.True(t, ok, "connectorbuilder server must implement sourcecache.SetLookup") - setter.SetSourceCacheSetter(lookupSink) + // this same wiring in internal/connector). Skipped in the deferred + // topology: with no direct lookup, the builder installs a per-request + // ContinuationLookup and the ask/answer protocol carries validators + // across the loopback instead. + if !deferredLookup { + setter, ok := cc.(interface { + SetSourceCacheSetter(sourcecache.SetLookup) + }) + require.True(t, ok, "connector client must accept a source-cache setter") + lookupSink, ok := srv.(sourcecache.SetLookup) + require.True(t, ok, "connectorbuilder server must implement sourcecache.SetLookup") + setter.SetSourceCacheSetter(lookupSink) + } return &syncHarness{t: t, ctx: ctx, mock: mock, cc: cc, tmpDir: t.TempDir()} } @@ -556,7 +583,7 @@ func (h *syncHarness) runSync(name string, prevPath string) string { path := filepath.Join(h.tmpDir, fmt.Sprintf("%02d-%s.c1z", h.syncN, name)) store, err := dotc1z.NewStore(h.ctx, path, - dotc1z.WithEngine(dotc1z.EnginePebble), + dotc1z.WithEngine(c1zstore.EnginePebble), dotc1z.WithTmpDir(h.tmpDir), ) require.NoError(h.t, err) @@ -591,7 +618,7 @@ func (h *syncHarness) runControlSync(name string) string { func (h *syncHarness) snapshot(path string) map[string]string { h.t.Helper() store, err := dotc1z.NewStore(h.ctx, path, - dotc1z.WithEngine(dotc1z.EnginePebble), + dotc1z.WithEngine(c1zstore.EnginePebble), dotc1z.WithReadOnly(true), dotc1z.WithTmpDir(h.tmpDir), ) @@ -889,3 +916,88 @@ func TestSourceCacheReplayEndToEnd(t *testing.T) { control15 := h.runControlSync("recovery-control") h.requireEquivalent(sync15, control15, "post-invalidation recovery") } + +// TestSourceCacheReplayDeferredLookup runs the replay flow through the +// ask/answer lookup continuation — the topology of single-shot transports +// (gRPC-over-Lambda), where the connector cannot reach the syncer's lookup +// service mid-request. The harness omits the direct-lookup wiring, so on +// warm syncs every group cursor's first call genuinely bounces: phase 1 +// defers at the scope lookup (before any upstream request — the mock's +// strict counters prove no phase-1 double work), the builder answers with +// a SourceCacheLookupAsk, and the syncer re-invokes with answers resolved +// from the previous c1z. +// +// The assertions are deliberately the same ones the direct-topology test +// makes: zero member listings on clean groups is only possible if the +// bounce delivered the stored validator (a broken continuation degrades to +// cold and fails the count ceilings), and reader-surface equivalence +// proves phase-2 re-execution changed nothing. +func TestSourceCacheReplayDeferredLookup(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + + mock := newMockOkta(t) + for i := 1; i <= 4; i++ { + mock.addUser(&mockOktaUser{ + ID: fmt.Sprintf("u%d", i), + FirstName: "Member", + LastName: fmt.Sprintf("Number%d", i), + Email: fmt.Sprintf("u%d@x.test", i), + }) + } + mock.addGroup(&mockOktaGroup{ID: "g1", Name: "Engineering", Members: []string{"u1", "u2", "u3"}}) + mock.addGroup(&mockOktaGroup{ID: "g2", Name: "Sales", Members: []string{"u4"}}) + mock.addGroup(&mockOktaGroup{ID: "g3", Name: "Empty"}) + mock.assignGroupRole("g1", mockOktaGroupRole{AssignmentID: "gra1", Type: roleTypeUserAdmin, Label: roleLabelGroupAdmin}) + + h := newSyncHarnessTopology(ctx, t, mock, true) + + // Cold: no previous sync means no offer, no deferral — plain cold + // enumeration that seeds the scope manifest. + sync1 := h.runSync("deferred-cold", "") + c1 := mock.snapshotCounts() + mc1 := memberListingCalls(c1) + require.Equal(t, 2, mc1["g1"], "3 members at page size 2 = 2 requests") + require.Equal(t, 1, mc1["g2"]) + require.Zero(t, mc1["g3"], "users_count==0 skips the member listing even cold") + + // Warm no-op: every cursor bounces (ask → answers) and replays. + sync2 := h.runSync("deferred-noop", sync1) + c2 := mock.snapshotCounts() + mc2 := memberListingCalls(c2) + require.Empty(t, sortedKeys(mc2), "warm round over the continuation must spend ZERO member-listing requests, got %v", mc2) + require.Equal(t, 1, c2["group-roles:g1"], "phase-2 re-execution must not double the roles leg") + require.Equal(t, 1, c2["group-roles:g2"]) + require.Equal(t, 1, c2["group-roles:g3"]) + control2 := h.runControlSync("deferred-noop-control") + h.requireEquivalent(sync2, control2, "deferred no-op round") + + // Member REMOVE: the bounce answers with a stale validator, the dirty + // group re-enumerates cold in phase 2, clean groups still replay. + mock.removeMember("g1", "u2") + sync3 := h.runSync("deferred-remove", sync2) + mc3 := memberListingCalls(mock.snapshotCounts()) + require.Equal(t, []string{"g1"}, sortedKeys(mc3), "only the dirty group re-enumerates, got %v", mc3) + snap3 := h.snapshot(sync3) + require.NotContains(t, snap3, "grant:"+memberGrantID("g1", "u2")) + require.Contains(t, snap3, "grant:"+memberGrantID("g1", "u1")) + control3 := h.runControlSync("deferred-remove-control") + h.requireEquivalent(sync3, control3, "deferred member remove") + + // Mass invalidation + recovery: every scope misses on the bounce, all + // non-empty groups go cold, and the next round is fully warm again. + for _, gid := range []string{"g1", "g2", "g3"} { + mock.touchGroup(gid) + } + sync4 := h.runSync("deferred-mass-invalidation", sync3) + mc4 := memberListingCalls(mock.snapshotCounts()) + require.Equal(t, []string{"g1", "g2"}, sortedKeys(mc4), "non-empty groups re-enumerate, got %v", mc4) + control4 := h.runControlSync("deferred-mass-invalidation-control") + h.requireEquivalent(sync4, control4, "deferred mass invalidation") + + sync5 := h.runSync("deferred-recovery", sync4) + mc5 := memberListingCalls(mock.snapshotCounts()) + require.Empty(t, sortedKeys(mc5), "post-invalidation round must be fully warm again, got %v", mc5) + control5 := h.runControlSync("deferred-recovery-control") + h.requireEquivalent(sync5, control5, "deferred recovery") +} diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_source_cache.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_source_cache.pb.go index 8796df98..437dfd89 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_source_cache.pb.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_source_cache.pb.go @@ -9,6 +9,7 @@ package v2 import ( + _ "github.com/envoyproxy/protoc-gen-validate/validate" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -459,11 +460,371 @@ func (b0 SourceCacheReplay_builder) Build() *SourceCacheReplay { return m0 } +// SourceCacheLookupOffer is attached by the SDK to list REQUESTS when the +// syncer can answer source-cache lookup asks: the connector declared +// SourceCacheCapability and a warm previous-sync lookup is installed. +// Its presence is the connector's permission to answer with +// SourceCacheLookupAsk instead of rows. Connectors with a direct lookup +// (in-process, subprocess) never need to defer and may ignore it. +type SourceCacheLookupOffer struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupOffer) Reset() { + *x = SourceCacheLookupOffer{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupOffer) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupOffer) ProtoMessage() {} + +func (x *SourceCacheLookupOffer) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +type SourceCacheLookupOffer_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + +} + +func (b0 SourceCacheLookupOffer_builder) Build() *SourceCacheLookupOffer { + m0 := &SourceCacheLookupOffer{} + b, x := &b0, m0 + _, _ = b, x + return m0 +} + +// SourceCacheLookupAsk is attached to a list RESPONSE in place of rows: +// the connector needs previous-sync validators before it can serve the +// page. An ask response must carry NO rows, NO next page token, and no +// other source-cache annotations; the syncer consumes it (it never +// reaches page handling), resolves every query, and re-invokes the same +// request with SourceCacheLookupAnswers attached. +// +// Only legal on responses to requests that carried +// SourceCacheLookupOffer. Bounces are capped per action; a connector +// that keeps asking past the cap fails the sync loudly. +type SourceCacheLookupAsk struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + Queries []*SourceCacheLookupAsk_Query `protobuf:"bytes,1,rep,name=queries,proto3" json:"queries,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupAsk) Reset() { + *x = SourceCacheLookupAsk{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupAsk) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupAsk) ProtoMessage() {} + +func (x *SourceCacheLookupAsk) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheLookupAsk) GetQueries() []*SourceCacheLookupAsk_Query { + if x != nil { + return x.Queries + } + return nil +} + +func (x *SourceCacheLookupAsk) SetQueries(v []*SourceCacheLookupAsk_Query) { + x.Queries = v +} + +type SourceCacheLookupAsk_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + Queries []*SourceCacheLookupAsk_Query +} + +func (b0 SourceCacheLookupAsk_builder) Build() *SourceCacheLookupAsk { + m0 := &SourceCacheLookupAsk{} + b, x := &b0, m0 + _, _ = b, x + x.Queries = b.Queries + return m0 +} + +// SourceCacheLookupAnswers is attached by the SDK to the re-invoked +// REQUEST, carrying the resolution of a prior ask's queries. +// +// An ABSENT query (asked but not answered — e.g. dropped to the answer +// size budget) is distinct from found=false: not-found means the previous +// sync has no entry (fetch fresh); absent means unresolved (the connector +// may ask again, subject to the bounce cap). Not-found answers are always +// complete for the queried set — only found answers with large etags are +// ever dropped to budget. +type SourceCacheLookupAnswers struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + Answers []*SourceCacheLookupAnswers_Answer `protobuf:"bytes,1,rep,name=answers,proto3" json:"answers,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupAnswers) Reset() { + *x = SourceCacheLookupAnswers{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupAnswers) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupAnswers) ProtoMessage() {} + +func (x *SourceCacheLookupAnswers) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheLookupAnswers) GetAnswers() []*SourceCacheLookupAnswers_Answer { + if x != nil { + return x.Answers + } + return nil +} + +func (x *SourceCacheLookupAnswers) SetAnswers(v []*SourceCacheLookupAnswers_Answer) { + x.Answers = v +} + +type SourceCacheLookupAnswers_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + Answers []*SourceCacheLookupAnswers_Answer +} + +func (b0 SourceCacheLookupAnswers_builder) Build() *SourceCacheLookupAnswers { + m0 := &SourceCacheLookupAnswers{} + b, x := &b0, m0 + _, _ = b, x + x.Answers = b.Answers + return m0 +} + +type SourceCacheLookupAsk_Query struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + // One of the sourcecache.RowKind values: "resources", + // "entitlements", "grants". + RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3" json:"row_kind,omitempty"` + ScopeHash string `protobuf:"bytes,2,opt,name=scope_hash,json=scopeHash,proto3" json:"scope_hash,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupAsk_Query) Reset() { + *x = SourceCacheLookupAsk_Query{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupAsk_Query) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupAsk_Query) ProtoMessage() {} + +func (x *SourceCacheLookupAsk_Query) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheLookupAsk_Query) GetRowKind() string { + if x != nil { + return x.RowKind + } + return "" +} + +func (x *SourceCacheLookupAsk_Query) GetScopeHash() string { + if x != nil { + return x.ScopeHash + } + return "" +} + +func (x *SourceCacheLookupAsk_Query) SetRowKind(v string) { + x.RowKind = v +} + +func (x *SourceCacheLookupAsk_Query) SetScopeHash(v string) { + x.ScopeHash = v +} + +type SourceCacheLookupAsk_Query_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // One of the sourcecache.RowKind values: "resources", + // "entitlements", "grants". + RowKind string + ScopeHash string +} + +func (b0 SourceCacheLookupAsk_Query_builder) Build() *SourceCacheLookupAsk_Query { + m0 := &SourceCacheLookupAsk_Query{} + b, x := &b0, m0 + _, _ = b, x + x.RowKind = b.RowKind + x.ScopeHash = b.ScopeHash + return m0 +} + +type SourceCacheLookupAnswers_Answer struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3" json:"row_kind,omitempty"` + ScopeHash string `protobuf:"bytes,2,opt,name=scope_hash,json=scopeHash,proto3" json:"scope_hash,omitempty"` + Found bool `protobuf:"varint,3,opt,name=found,proto3" json:"found,omitempty"` + // The previous sync's validator; empty when found is false. Cap + // matches the lookup RPC (Graph delta tokens run long). + Etag string `protobuf:"bytes,4,opt,name=etag,proto3" json:"etag,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupAnswers_Answer) Reset() { + *x = SourceCacheLookupAnswers_Answer{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupAnswers_Answer) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupAnswers_Answer) ProtoMessage() {} + +func (x *SourceCacheLookupAnswers_Answer) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheLookupAnswers_Answer) GetRowKind() string { + if x != nil { + return x.RowKind + } + return "" +} + +func (x *SourceCacheLookupAnswers_Answer) GetScopeHash() string { + if x != nil { + return x.ScopeHash + } + return "" +} + +func (x *SourceCacheLookupAnswers_Answer) GetFound() bool { + if x != nil { + return x.Found + } + return false +} + +func (x *SourceCacheLookupAnswers_Answer) GetEtag() string { + if x != nil { + return x.Etag + } + return "" +} + +func (x *SourceCacheLookupAnswers_Answer) SetRowKind(v string) { + x.RowKind = v +} + +func (x *SourceCacheLookupAnswers_Answer) SetScopeHash(v string) { + x.ScopeHash = v +} + +func (x *SourceCacheLookupAnswers_Answer) SetFound(v bool) { + x.Found = v +} + +func (x *SourceCacheLookupAnswers_Answer) SetEtag(v string) { + x.Etag = v +} + +type SourceCacheLookupAnswers_Answer_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + RowKind string + ScopeHash string + Found bool + // The previous sync's validator; empty when found is false. Cap + // matches the lookup RPC (Graph delta tokens run long). + Etag string +} + +func (b0 SourceCacheLookupAnswers_Answer_builder) Build() *SourceCacheLookupAnswers_Answer { + m0 := &SourceCacheLookupAnswers_Answer{} + b, x := &b0, m0 + _, _ = b, x + x.RowKind = b.RowKind + x.ScopeHash = b.ScopeHash + x.Found = b.Found + x.Etag = b.Etag + return m0 +} + var File_c1_connector_v2_annotation_source_cache_proto protoreflect.FileDescriptor const file_c1_connector_v2_annotation_source_cache_proto_rawDesc = "" + "\n" + - "-c1/connector/v2/annotation_source_cache.proto\x12\x0fc1.connector.v2\"\x9e\x01\n" + + "-c1/connector/v2/annotation_source_cache.proto\x12\x0fc1.connector.v2\x1a\x17validate/validate.proto\"\x9e\x01\n" + "\x15SourceCacheCapability\x12?\n" + "\x04mode\x18\x01 \x01(\x0e2+.c1.connector.v2.SourceCacheCapability.ModeR\x04mode\"D\n" + "\x04Mode\x12\x14\n" + @@ -484,23 +845,47 @@ const file_c1_connector_v2_annotation_source_cache_proto_rawDesc = "" + "\aoverlay\x18\x03 \x01(\bR\aoverlay\x12\x1f\n" + "\vdeleted_ids\x18\x04 \x03(\tR\n" + "deletedIds\x122\n" + - "\x15deleted_principal_ids\x18\x05 \x03(\tR\x13deletedPrincipalIdsB6Z4github.com/conductorone/baton-sdk/pb/c1/connector/v2b\x06proto3" + "\x15deleted_principal_ids\x18\x05 \x03(\tR\x13deletedPrincipalIds\"\x18\n" + + "\x16SourceCacheLookupOffer\"\xc4\x01\n" + + "\x14SourceCacheLookupAsk\x12R\n" + + "\aqueries\x18\x01 \x03(\v2+.c1.connector.v2.SourceCacheLookupAsk.QueryB\v\xfaB\b\x92\x01\x05\b\x01\x10\x80 R\aqueries\x1aX\n" + + "\x05Query\x12$\n" + + "\brow_kind\x18\x01 \x01(\tB\t\xfaB\x06r\x04 \x01(@R\arowKind\x12)\n" + + "\n" + + "scope_hash\x18\x02 \x01(\tB\n" + + "\xfaB\ar\x05 \x01(\x80\x02R\tscopeHash\"\xfa\x01\n" + + "\x18SourceCacheLookupAnswers\x12J\n" + + "\aanswers\x18\x01 \x03(\v20.c1.connector.v2.SourceCacheLookupAnswers.AnswerR\aanswers\x1a\x91\x01\n" + + "\x06Answer\x12$\n" + + "\brow_kind\x18\x01 \x01(\tB\t\xfaB\x06r\x04 \x01(@R\arowKind\x12)\n" + + "\n" + + "scope_hash\x18\x02 \x01(\tB\n" + + "\xfaB\ar\x05 \x01(\x80\x02R\tscopeHash\x12\x14\n" + + "\x05found\x18\x03 \x01(\bR\x05found\x12 \n" + + "\x04etag\x18\x04 \x01(\tB\f\xfaB\tr\a(\x80\x80\x04\xd0\x01\x01R\x04etagB6Z4github.com/conductorone/baton-sdk/pb/c1/connector/v2b\x06proto3" var file_c1_connector_v2_annotation_source_cache_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_c1_connector_v2_annotation_source_cache_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_c1_connector_v2_annotation_source_cache_proto_msgTypes = make([]protoimpl.MessageInfo, 8) var file_c1_connector_v2_annotation_source_cache_proto_goTypes = []any{ - (SourceCacheCapability_Mode)(0), // 0: c1.connector.v2.SourceCacheCapability.Mode - (*SourceCacheCapability)(nil), // 1: c1.connector.v2.SourceCacheCapability - (*SourceCacheScope)(nil), // 2: c1.connector.v2.SourceCacheScope - (*SourceCacheReplay)(nil), // 3: c1.connector.v2.SourceCacheReplay + (SourceCacheCapability_Mode)(0), // 0: c1.connector.v2.SourceCacheCapability.Mode + (*SourceCacheCapability)(nil), // 1: c1.connector.v2.SourceCacheCapability + (*SourceCacheScope)(nil), // 2: c1.connector.v2.SourceCacheScope + (*SourceCacheReplay)(nil), // 3: c1.connector.v2.SourceCacheReplay + (*SourceCacheLookupOffer)(nil), // 4: c1.connector.v2.SourceCacheLookupOffer + (*SourceCacheLookupAsk)(nil), // 5: c1.connector.v2.SourceCacheLookupAsk + (*SourceCacheLookupAnswers)(nil), // 6: c1.connector.v2.SourceCacheLookupAnswers + (*SourceCacheLookupAsk_Query)(nil), // 7: c1.connector.v2.SourceCacheLookupAsk.Query + (*SourceCacheLookupAnswers_Answer)(nil), // 8: c1.connector.v2.SourceCacheLookupAnswers.Answer } var file_c1_connector_v2_annotation_source_cache_proto_depIdxs = []int32{ 0, // 0: c1.connector.v2.SourceCacheCapability.mode:type_name -> c1.connector.v2.SourceCacheCapability.Mode - 1, // [1:1] is the sub-list for method output_type - 1, // [1:1] is the sub-list for method input_type - 1, // [1:1] is the sub-list for extension type_name - 1, // [1:1] is the sub-list for extension extendee - 0, // [0:1] is the sub-list for field type_name + 7, // 1: c1.connector.v2.SourceCacheLookupAsk.queries:type_name -> c1.connector.v2.SourceCacheLookupAsk.Query + 8, // 2: c1.connector.v2.SourceCacheLookupAnswers.answers:type_name -> c1.connector.v2.SourceCacheLookupAnswers.Answer + 3, // [3:3] is the sub-list for method output_type + 3, // [3:3] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name } func init() { file_c1_connector_v2_annotation_source_cache_proto_init() } @@ -514,7 +899,7 @@ func file_c1_connector_v2_annotation_source_cache_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_connector_v2_annotation_source_cache_proto_rawDesc), len(file_c1_connector_v2_annotation_source_cache_proto_rawDesc)), NumEnums: 1, - NumMessages: 3, + NumMessages: 8, NumExtensions: 0, NumServices: 0, }, diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_source_cache.pb.validate.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_source_cache.pb.validate.go index e35ec1ac..a301fbcb 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_source_cache.pb.validate.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_source_cache.pb.validate.go @@ -350,3 +350,654 @@ var _ interface { Cause() error ErrorName() string } = SourceCacheReplayValidationError{} + +// Validate checks the field values on SourceCacheLookupOffer with the rules +// defined in the proto definition for this message. If any rules are +// violated, the first error encountered is returned, or nil if there are no violations. +func (m *SourceCacheLookupOffer) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SourceCacheLookupOffer with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// SourceCacheLookupOfferMultiError, or nil if none found. +func (m *SourceCacheLookupOffer) ValidateAll() error { + return m.validate(true) +} + +func (m *SourceCacheLookupOffer) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + if len(errors) > 0 { + return SourceCacheLookupOfferMultiError(errors) + } + + return nil +} + +// SourceCacheLookupOfferMultiError is an error wrapping multiple validation +// errors returned by SourceCacheLookupOffer.ValidateAll() if the designated +// constraints aren't met. +type SourceCacheLookupOfferMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SourceCacheLookupOfferMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SourceCacheLookupOfferMultiError) AllErrors() []error { return m } + +// SourceCacheLookupOfferValidationError is the validation error returned by +// SourceCacheLookupOffer.Validate if the designated constraints aren't met. +type SourceCacheLookupOfferValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SourceCacheLookupOfferValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SourceCacheLookupOfferValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SourceCacheLookupOfferValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SourceCacheLookupOfferValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SourceCacheLookupOfferValidationError) ErrorName() string { + return "SourceCacheLookupOfferValidationError" +} + +// Error satisfies the builtin error interface +func (e SourceCacheLookupOfferValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSourceCacheLookupOffer.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SourceCacheLookupOfferValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SourceCacheLookupOfferValidationError{} + +// Validate checks the field values on SourceCacheLookupAsk with the rules +// defined in the proto definition for this message. If any rules are +// violated, the first error encountered is returned, or nil if there are no violations. +func (m *SourceCacheLookupAsk) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SourceCacheLookupAsk with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// SourceCacheLookupAskMultiError, or nil if none found. +func (m *SourceCacheLookupAsk) ValidateAll() error { + return m.validate(true) +} + +func (m *SourceCacheLookupAsk) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + if l := len(m.GetQueries()); l < 1 || l > 4096 { + err := SourceCacheLookupAskValidationError{ + field: "Queries", + reason: "value must contain between 1 and 4096 items, inclusive", + } + if !all { + return err + } + errors = append(errors, err) + } + + for idx, item := range m.GetQueries() { + _, _ = idx, item + + if all { + switch v := interface{}(item).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, SourceCacheLookupAskValidationError{ + field: fmt.Sprintf("Queries[%v]", idx), + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, SourceCacheLookupAskValidationError{ + field: fmt.Sprintf("Queries[%v]", idx), + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(item).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return SourceCacheLookupAskValidationError{ + field: fmt.Sprintf("Queries[%v]", idx), + reason: "embedded message failed validation", + cause: err, + } + } + } + + } + + if len(errors) > 0 { + return SourceCacheLookupAskMultiError(errors) + } + + return nil +} + +// SourceCacheLookupAskMultiError is an error wrapping multiple validation +// errors returned by SourceCacheLookupAsk.ValidateAll() if the designated +// constraints aren't met. +type SourceCacheLookupAskMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SourceCacheLookupAskMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SourceCacheLookupAskMultiError) AllErrors() []error { return m } + +// SourceCacheLookupAskValidationError is the validation error returned by +// SourceCacheLookupAsk.Validate if the designated constraints aren't met. +type SourceCacheLookupAskValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SourceCacheLookupAskValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SourceCacheLookupAskValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SourceCacheLookupAskValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SourceCacheLookupAskValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SourceCacheLookupAskValidationError) ErrorName() string { + return "SourceCacheLookupAskValidationError" +} + +// Error satisfies the builtin error interface +func (e SourceCacheLookupAskValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSourceCacheLookupAsk.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SourceCacheLookupAskValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SourceCacheLookupAskValidationError{} + +// Validate checks the field values on SourceCacheLookupAnswers with the rules +// defined in the proto definition for this message. If any rules are +// violated, the first error encountered is returned, or nil if there are no violations. +func (m *SourceCacheLookupAnswers) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SourceCacheLookupAnswers with the +// rules defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// SourceCacheLookupAnswersMultiError, or nil if none found. +func (m *SourceCacheLookupAnswers) ValidateAll() error { + return m.validate(true) +} + +func (m *SourceCacheLookupAnswers) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + for idx, item := range m.GetAnswers() { + _, _ = idx, item + + if all { + switch v := interface{}(item).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, SourceCacheLookupAnswersValidationError{ + field: fmt.Sprintf("Answers[%v]", idx), + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, SourceCacheLookupAnswersValidationError{ + field: fmt.Sprintf("Answers[%v]", idx), + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(item).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return SourceCacheLookupAnswersValidationError{ + field: fmt.Sprintf("Answers[%v]", idx), + reason: "embedded message failed validation", + cause: err, + } + } + } + + } + + if len(errors) > 0 { + return SourceCacheLookupAnswersMultiError(errors) + } + + return nil +} + +// SourceCacheLookupAnswersMultiError is an error wrapping multiple validation +// errors returned by SourceCacheLookupAnswers.ValidateAll() if the designated +// constraints aren't met. +type SourceCacheLookupAnswersMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SourceCacheLookupAnswersMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SourceCacheLookupAnswersMultiError) AllErrors() []error { return m } + +// SourceCacheLookupAnswersValidationError is the validation error returned by +// SourceCacheLookupAnswers.Validate if the designated constraints aren't met. +type SourceCacheLookupAnswersValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SourceCacheLookupAnswersValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SourceCacheLookupAnswersValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SourceCacheLookupAnswersValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SourceCacheLookupAnswersValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SourceCacheLookupAnswersValidationError) ErrorName() string { + return "SourceCacheLookupAnswersValidationError" +} + +// Error satisfies the builtin error interface +func (e SourceCacheLookupAnswersValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSourceCacheLookupAnswers.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SourceCacheLookupAnswersValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SourceCacheLookupAnswersValidationError{} + +// Validate checks the field values on SourceCacheLookupAsk_Query with the +// rules defined in the proto definition for this message. If any rules are +// violated, the first error encountered is returned, or nil if there are no violations. +func (m *SourceCacheLookupAsk_Query) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SourceCacheLookupAsk_Query with the +// rules defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// SourceCacheLookupAsk_QueryMultiError, or nil if none found. +func (m *SourceCacheLookupAsk_Query) ValidateAll() error { + return m.validate(true) +} + +func (m *SourceCacheLookupAsk_Query) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + if l := len(m.GetRowKind()); l < 1 || l > 64 { + err := SourceCacheLookupAsk_QueryValidationError{ + field: "RowKind", + reason: "value length must be between 1 and 64 bytes, inclusive", + } + if !all { + return err + } + errors = append(errors, err) + } + + if l := len(m.GetScopeHash()); l < 1 || l > 256 { + err := SourceCacheLookupAsk_QueryValidationError{ + field: "ScopeHash", + reason: "value length must be between 1 and 256 bytes, inclusive", + } + if !all { + return err + } + errors = append(errors, err) + } + + if len(errors) > 0 { + return SourceCacheLookupAsk_QueryMultiError(errors) + } + + return nil +} + +// SourceCacheLookupAsk_QueryMultiError is an error wrapping multiple +// validation errors returned by SourceCacheLookupAsk_Query.ValidateAll() if +// the designated constraints aren't met. +type SourceCacheLookupAsk_QueryMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SourceCacheLookupAsk_QueryMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SourceCacheLookupAsk_QueryMultiError) AllErrors() []error { return m } + +// SourceCacheLookupAsk_QueryValidationError is the validation error returned +// by SourceCacheLookupAsk_Query.Validate if the designated constraints aren't met. +type SourceCacheLookupAsk_QueryValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SourceCacheLookupAsk_QueryValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SourceCacheLookupAsk_QueryValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SourceCacheLookupAsk_QueryValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SourceCacheLookupAsk_QueryValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SourceCacheLookupAsk_QueryValidationError) ErrorName() string { + return "SourceCacheLookupAsk_QueryValidationError" +} + +// Error satisfies the builtin error interface +func (e SourceCacheLookupAsk_QueryValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSourceCacheLookupAsk_Query.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SourceCacheLookupAsk_QueryValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SourceCacheLookupAsk_QueryValidationError{} + +// Validate checks the field values on SourceCacheLookupAnswers_Answer with the +// rules defined in the proto definition for this message. If any rules are +// violated, the first error encountered is returned, or nil if there are no violations. +func (m *SourceCacheLookupAnswers_Answer) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SourceCacheLookupAnswers_Answer with +// the rules defined in the proto definition for this message. If any rules +// are violated, the result is a list of violation errors wrapped in +// SourceCacheLookupAnswers_AnswerMultiError, or nil if none found. +func (m *SourceCacheLookupAnswers_Answer) ValidateAll() error { + return m.validate(true) +} + +func (m *SourceCacheLookupAnswers_Answer) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + if l := len(m.GetRowKind()); l < 1 || l > 64 { + err := SourceCacheLookupAnswers_AnswerValidationError{ + field: "RowKind", + reason: "value length must be between 1 and 64 bytes, inclusive", + } + if !all { + return err + } + errors = append(errors, err) + } + + if l := len(m.GetScopeHash()); l < 1 || l > 256 { + err := SourceCacheLookupAnswers_AnswerValidationError{ + field: "ScopeHash", + reason: "value length must be between 1 and 256 bytes, inclusive", + } + if !all { + return err + } + errors = append(errors, err) + } + + // no validation rules for Found + + if m.GetEtag() != "" { + + if len(m.GetEtag()) > 65536 { + err := SourceCacheLookupAnswers_AnswerValidationError{ + field: "Etag", + reason: "value length must be at most 65536 bytes", + } + if !all { + return err + } + errors = append(errors, err) + } + + } + + if len(errors) > 0 { + return SourceCacheLookupAnswers_AnswerMultiError(errors) + } + + return nil +} + +// SourceCacheLookupAnswers_AnswerMultiError is an error wrapping multiple +// validation errors returned by SourceCacheLookupAnswers_Answer.ValidateAll() +// if the designated constraints aren't met. +type SourceCacheLookupAnswers_AnswerMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SourceCacheLookupAnswers_AnswerMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SourceCacheLookupAnswers_AnswerMultiError) AllErrors() []error { return m } + +// SourceCacheLookupAnswers_AnswerValidationError is the validation error +// returned by SourceCacheLookupAnswers_Answer.Validate if the designated +// constraints aren't met. +type SourceCacheLookupAnswers_AnswerValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SourceCacheLookupAnswers_AnswerValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SourceCacheLookupAnswers_AnswerValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SourceCacheLookupAnswers_AnswerValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SourceCacheLookupAnswers_AnswerValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SourceCacheLookupAnswers_AnswerValidationError) ErrorName() string { + return "SourceCacheLookupAnswers_AnswerValidationError" +} + +// Error satisfies the builtin error interface +func (e SourceCacheLookupAnswers_AnswerValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSourceCacheLookupAnswers_Answer.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SourceCacheLookupAnswers_AnswerValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SourceCacheLookupAnswers_AnswerValidationError{} diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_source_cache_protoopaque.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_source_cache_protoopaque.pb.go index 08ab696f..23ad8c5b 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_source_cache_protoopaque.pb.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_source_cache_protoopaque.pb.go @@ -9,6 +9,7 @@ package v2 import ( + _ "github.com/envoyproxy/protoc-gen-validate/validate" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -405,11 +406,371 @@ func (b0 SourceCacheReplay_builder) Build() *SourceCacheReplay { return m0 } +// SourceCacheLookupOffer is attached by the SDK to list REQUESTS when the +// syncer can answer source-cache lookup asks: the connector declared +// SourceCacheCapability and a warm previous-sync lookup is installed. +// Its presence is the connector's permission to answer with +// SourceCacheLookupAsk instead of rows. Connectors with a direct lookup +// (in-process, subprocess) never need to defer and may ignore it. +type SourceCacheLookupOffer struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupOffer) Reset() { + *x = SourceCacheLookupOffer{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupOffer) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupOffer) ProtoMessage() {} + +func (x *SourceCacheLookupOffer) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +type SourceCacheLookupOffer_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + +} + +func (b0 SourceCacheLookupOffer_builder) Build() *SourceCacheLookupOffer { + m0 := &SourceCacheLookupOffer{} + b, x := &b0, m0 + _, _ = b, x + return m0 +} + +// SourceCacheLookupAsk is attached to a list RESPONSE in place of rows: +// the connector needs previous-sync validators before it can serve the +// page. An ask response must carry NO rows, NO next page token, and no +// other source-cache annotations; the syncer consumes it (it never +// reaches page handling), resolves every query, and re-invokes the same +// request with SourceCacheLookupAnswers attached. +// +// Only legal on responses to requests that carried +// SourceCacheLookupOffer. Bounces are capped per action; a connector +// that keeps asking past the cap fails the sync loudly. +type SourceCacheLookupAsk struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Queries *[]*SourceCacheLookupAsk_Query `protobuf:"bytes,1,rep,name=queries,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupAsk) Reset() { + *x = SourceCacheLookupAsk{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupAsk) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupAsk) ProtoMessage() {} + +func (x *SourceCacheLookupAsk) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheLookupAsk) GetQueries() []*SourceCacheLookupAsk_Query { + if x != nil { + if x.xxx_hidden_Queries != nil { + return *x.xxx_hidden_Queries + } + } + return nil +} + +func (x *SourceCacheLookupAsk) SetQueries(v []*SourceCacheLookupAsk_Query) { + x.xxx_hidden_Queries = &v +} + +type SourceCacheLookupAsk_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + Queries []*SourceCacheLookupAsk_Query +} + +func (b0 SourceCacheLookupAsk_builder) Build() *SourceCacheLookupAsk { + m0 := &SourceCacheLookupAsk{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_Queries = &b.Queries + return m0 +} + +// SourceCacheLookupAnswers is attached by the SDK to the re-invoked +// REQUEST, carrying the resolution of a prior ask's queries. +// +// An ABSENT query (asked but not answered — e.g. dropped to the answer +// size budget) is distinct from found=false: not-found means the previous +// sync has no entry (fetch fresh); absent means unresolved (the connector +// may ask again, subject to the bounce cap). Not-found answers are always +// complete for the queried set — only found answers with large etags are +// ever dropped to budget. +type SourceCacheLookupAnswers struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Answers *[]*SourceCacheLookupAnswers_Answer `protobuf:"bytes,1,rep,name=answers,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupAnswers) Reset() { + *x = SourceCacheLookupAnswers{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupAnswers) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupAnswers) ProtoMessage() {} + +func (x *SourceCacheLookupAnswers) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheLookupAnswers) GetAnswers() []*SourceCacheLookupAnswers_Answer { + if x != nil { + if x.xxx_hidden_Answers != nil { + return *x.xxx_hidden_Answers + } + } + return nil +} + +func (x *SourceCacheLookupAnswers) SetAnswers(v []*SourceCacheLookupAnswers_Answer) { + x.xxx_hidden_Answers = &v +} + +type SourceCacheLookupAnswers_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + Answers []*SourceCacheLookupAnswers_Answer +} + +func (b0 SourceCacheLookupAnswers_builder) Build() *SourceCacheLookupAnswers { + m0 := &SourceCacheLookupAnswers{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_Answers = &b.Answers + return m0 +} + +type SourceCacheLookupAsk_Query struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3"` + xxx_hidden_ScopeHash string `protobuf:"bytes,2,opt,name=scope_hash,json=scopeHash,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupAsk_Query) Reset() { + *x = SourceCacheLookupAsk_Query{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupAsk_Query) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupAsk_Query) ProtoMessage() {} + +func (x *SourceCacheLookupAsk_Query) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheLookupAsk_Query) GetRowKind() string { + if x != nil { + return x.xxx_hidden_RowKind + } + return "" +} + +func (x *SourceCacheLookupAsk_Query) GetScopeHash() string { + if x != nil { + return x.xxx_hidden_ScopeHash + } + return "" +} + +func (x *SourceCacheLookupAsk_Query) SetRowKind(v string) { + x.xxx_hidden_RowKind = v +} + +func (x *SourceCacheLookupAsk_Query) SetScopeHash(v string) { + x.xxx_hidden_ScopeHash = v +} + +type SourceCacheLookupAsk_Query_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // One of the sourcecache.RowKind values: "resources", + // "entitlements", "grants". + RowKind string + ScopeHash string +} + +func (b0 SourceCacheLookupAsk_Query_builder) Build() *SourceCacheLookupAsk_Query { + m0 := &SourceCacheLookupAsk_Query{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_RowKind = b.RowKind + x.xxx_hidden_ScopeHash = b.ScopeHash + return m0 +} + +type SourceCacheLookupAnswers_Answer struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3"` + xxx_hidden_ScopeHash string `protobuf:"bytes,2,opt,name=scope_hash,json=scopeHash,proto3"` + xxx_hidden_Found bool `protobuf:"varint,3,opt,name=found,proto3"` + xxx_hidden_Etag string `protobuf:"bytes,4,opt,name=etag,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupAnswers_Answer) Reset() { + *x = SourceCacheLookupAnswers_Answer{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupAnswers_Answer) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupAnswers_Answer) ProtoMessage() {} + +func (x *SourceCacheLookupAnswers_Answer) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheLookupAnswers_Answer) GetRowKind() string { + if x != nil { + return x.xxx_hidden_RowKind + } + return "" +} + +func (x *SourceCacheLookupAnswers_Answer) GetScopeHash() string { + if x != nil { + return x.xxx_hidden_ScopeHash + } + return "" +} + +func (x *SourceCacheLookupAnswers_Answer) GetFound() bool { + if x != nil { + return x.xxx_hidden_Found + } + return false +} + +func (x *SourceCacheLookupAnswers_Answer) GetEtag() string { + if x != nil { + return x.xxx_hidden_Etag + } + return "" +} + +func (x *SourceCacheLookupAnswers_Answer) SetRowKind(v string) { + x.xxx_hidden_RowKind = v +} + +func (x *SourceCacheLookupAnswers_Answer) SetScopeHash(v string) { + x.xxx_hidden_ScopeHash = v +} + +func (x *SourceCacheLookupAnswers_Answer) SetFound(v bool) { + x.xxx_hidden_Found = v +} + +func (x *SourceCacheLookupAnswers_Answer) SetEtag(v string) { + x.xxx_hidden_Etag = v +} + +type SourceCacheLookupAnswers_Answer_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + RowKind string + ScopeHash string + Found bool + // The previous sync's validator; empty when found is false. Cap + // matches the lookup RPC (Graph delta tokens run long). + Etag string +} + +func (b0 SourceCacheLookupAnswers_Answer_builder) Build() *SourceCacheLookupAnswers_Answer { + m0 := &SourceCacheLookupAnswers_Answer{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_RowKind = b.RowKind + x.xxx_hidden_ScopeHash = b.ScopeHash + x.xxx_hidden_Found = b.Found + x.xxx_hidden_Etag = b.Etag + return m0 +} + var File_c1_connector_v2_annotation_source_cache_proto protoreflect.FileDescriptor const file_c1_connector_v2_annotation_source_cache_proto_rawDesc = "" + "\n" + - "-c1/connector/v2/annotation_source_cache.proto\x12\x0fc1.connector.v2\"\x9e\x01\n" + + "-c1/connector/v2/annotation_source_cache.proto\x12\x0fc1.connector.v2\x1a\x17validate/validate.proto\"\x9e\x01\n" + "\x15SourceCacheCapability\x12?\n" + "\x04mode\x18\x01 \x01(\x0e2+.c1.connector.v2.SourceCacheCapability.ModeR\x04mode\"D\n" + "\x04Mode\x12\x14\n" + @@ -430,23 +791,47 @@ const file_c1_connector_v2_annotation_source_cache_proto_rawDesc = "" + "\aoverlay\x18\x03 \x01(\bR\aoverlay\x12\x1f\n" + "\vdeleted_ids\x18\x04 \x03(\tR\n" + "deletedIds\x122\n" + - "\x15deleted_principal_ids\x18\x05 \x03(\tR\x13deletedPrincipalIdsB6Z4github.com/conductorone/baton-sdk/pb/c1/connector/v2b\x06proto3" + "\x15deleted_principal_ids\x18\x05 \x03(\tR\x13deletedPrincipalIds\"\x18\n" + + "\x16SourceCacheLookupOffer\"\xc4\x01\n" + + "\x14SourceCacheLookupAsk\x12R\n" + + "\aqueries\x18\x01 \x03(\v2+.c1.connector.v2.SourceCacheLookupAsk.QueryB\v\xfaB\b\x92\x01\x05\b\x01\x10\x80 R\aqueries\x1aX\n" + + "\x05Query\x12$\n" + + "\brow_kind\x18\x01 \x01(\tB\t\xfaB\x06r\x04 \x01(@R\arowKind\x12)\n" + + "\n" + + "scope_hash\x18\x02 \x01(\tB\n" + + "\xfaB\ar\x05 \x01(\x80\x02R\tscopeHash\"\xfa\x01\n" + + "\x18SourceCacheLookupAnswers\x12J\n" + + "\aanswers\x18\x01 \x03(\v20.c1.connector.v2.SourceCacheLookupAnswers.AnswerR\aanswers\x1a\x91\x01\n" + + "\x06Answer\x12$\n" + + "\brow_kind\x18\x01 \x01(\tB\t\xfaB\x06r\x04 \x01(@R\arowKind\x12)\n" + + "\n" + + "scope_hash\x18\x02 \x01(\tB\n" + + "\xfaB\ar\x05 \x01(\x80\x02R\tscopeHash\x12\x14\n" + + "\x05found\x18\x03 \x01(\bR\x05found\x12 \n" + + "\x04etag\x18\x04 \x01(\tB\f\xfaB\tr\a(\x80\x80\x04\xd0\x01\x01R\x04etagB6Z4github.com/conductorone/baton-sdk/pb/c1/connector/v2b\x06proto3" var file_c1_connector_v2_annotation_source_cache_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_c1_connector_v2_annotation_source_cache_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_c1_connector_v2_annotation_source_cache_proto_msgTypes = make([]protoimpl.MessageInfo, 8) var file_c1_connector_v2_annotation_source_cache_proto_goTypes = []any{ - (SourceCacheCapability_Mode)(0), // 0: c1.connector.v2.SourceCacheCapability.Mode - (*SourceCacheCapability)(nil), // 1: c1.connector.v2.SourceCacheCapability - (*SourceCacheScope)(nil), // 2: c1.connector.v2.SourceCacheScope - (*SourceCacheReplay)(nil), // 3: c1.connector.v2.SourceCacheReplay + (SourceCacheCapability_Mode)(0), // 0: c1.connector.v2.SourceCacheCapability.Mode + (*SourceCacheCapability)(nil), // 1: c1.connector.v2.SourceCacheCapability + (*SourceCacheScope)(nil), // 2: c1.connector.v2.SourceCacheScope + (*SourceCacheReplay)(nil), // 3: c1.connector.v2.SourceCacheReplay + (*SourceCacheLookupOffer)(nil), // 4: c1.connector.v2.SourceCacheLookupOffer + (*SourceCacheLookupAsk)(nil), // 5: c1.connector.v2.SourceCacheLookupAsk + (*SourceCacheLookupAnswers)(nil), // 6: c1.connector.v2.SourceCacheLookupAnswers + (*SourceCacheLookupAsk_Query)(nil), // 7: c1.connector.v2.SourceCacheLookupAsk.Query + (*SourceCacheLookupAnswers_Answer)(nil), // 8: c1.connector.v2.SourceCacheLookupAnswers.Answer } var file_c1_connector_v2_annotation_source_cache_proto_depIdxs = []int32{ 0, // 0: c1.connector.v2.SourceCacheCapability.mode:type_name -> c1.connector.v2.SourceCacheCapability.Mode - 1, // [1:1] is the sub-list for method output_type - 1, // [1:1] is the sub-list for method input_type - 1, // [1:1] is the sub-list for extension type_name - 1, // [1:1] is the sub-list for extension extendee - 0, // [0:1] is the sub-list for field type_name + 7, // 1: c1.connector.v2.SourceCacheLookupAsk.queries:type_name -> c1.connector.v2.SourceCacheLookupAsk.Query + 8, // 2: c1.connector.v2.SourceCacheLookupAnswers.answers:type_name -> c1.connector.v2.SourceCacheLookupAnswers.Answer + 3, // [3:3] is the sub-list for method output_type + 3, // [3:3] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name } func init() { file_c1_connector_v2_annotation_source_cache_proto_init() } @@ -460,7 +845,7 @@ func file_c1_connector_v2_annotation_source_cache_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_connector_v2_annotation_source_cache_proto_rawDesc), len(file_c1_connector_v2_annotation_source_cache_proto_rawDesc)), NumEnums: 1, - NumMessages: 3, + NumMessages: 8, NumExtensions: 0, NumServices: 0, }, diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_type_scoped_grants.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_type_scoped_grants.pb.go index 189ef309..52a9383b 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_type_scoped_grants.pb.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_type_scoped_grants.pb.go @@ -90,24 +90,40 @@ func (b0 TypeScopedGrants_builder) Build() *TypeScopedGrants { return m0 } -// SpawnCursors is attached to a type-scoped ListGrants response to enqueue -// additional independent cursors for the same resource type. Each token is -// delivered back to the connector as the page token of its own action — -// scheduled by the syncer's worker pool, rate-limited, and checkpointed -// like any other pagination. +// SpawnCursors is attached to a ListGrants response to enqueue additional +// independent sibling cursors. Each token is delivered back to the +// connector as the page token of its own action — scheduled by the +// syncer's worker pool, rate-limited, and checkpointed like any other +// pagination. Honored on BOTH type-scoped and per-resource ListGrants +// responses; the spawned actions inherit the response's resource identity +// (type only for type-scoped calls, type+resource for per-resource calls). // -// Typical use: the first (planning) call computes shard assignments — e.g. -// one 50-id delta filter chunk per cursor — returns no rows, and spawns -// one cursor per shard. Cold enumeration then parallelizes across shards -// instead of serializing through one stream. +// Typical uses: +// +// - Type-scoped planning: the first call computes shard assignments — +// e.g. one 50-id delta filter chunk per cursor — returns no rows, and +// spawns one cursor per shard. Cold enumeration then parallelizes +// across shards instead of serializing through one stream. +// - Per-resource parallel warm revalidation (page-numbered APIs): on the +// first page of a collection the connector already knows every other +// page's URL and stored validator, so it answers page one and spawns +// pages 2..N; the worker pool revalidates them concurrently instead +// of paying one round trip per page serially. +// +// Spawned cursors are ORDINARY pages that happen to be enqueued eagerly. +// The SDK assumes nothing about how they resolve: a spawned page may hit +// its source-cache lookup and replay, miss and fetch cold (e.g. a page +// boundary shifted since the last sync), and may continue a chain via its +// response's next page token. Any page of a fan-out may itself spawn. +// +// Progress accounting counts a resource as covered when its ORIGIN +// action's chain ends; spawned siblings never double-count it. // // Tokens are opaque to the SDK. They must be self-contained: a cursor may // execute after a suspend/resume, on a different worker, so anything the // connector needs to serve the cursor must be inside the token (or -// re-derivable from it). -// -// Honored only on responses to type-scoped ListGrants calls; ignored (with -// a warning) elsewhere. +// re-derivable from it) — for per-resource spawns the resource identity +// rides the action, so tokens only need the page coordinate. type SpawnCursors struct { state protoimpl.MessageState `protogen:"hybrid.v1"` // Page tokens for the sibling cursors to enqueue, one action each. diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_type_scoped_grants_protoopaque.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_type_scoped_grants_protoopaque.pb.go index d49107b0..e9b3571b 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_type_scoped_grants_protoopaque.pb.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_type_scoped_grants_protoopaque.pb.go @@ -90,24 +90,40 @@ func (b0 TypeScopedGrants_builder) Build() *TypeScopedGrants { return m0 } -// SpawnCursors is attached to a type-scoped ListGrants response to enqueue -// additional independent cursors for the same resource type. Each token is -// delivered back to the connector as the page token of its own action — -// scheduled by the syncer's worker pool, rate-limited, and checkpointed -// like any other pagination. +// SpawnCursors is attached to a ListGrants response to enqueue additional +// independent sibling cursors. Each token is delivered back to the +// connector as the page token of its own action — scheduled by the +// syncer's worker pool, rate-limited, and checkpointed like any other +// pagination. Honored on BOTH type-scoped and per-resource ListGrants +// responses; the spawned actions inherit the response's resource identity +// (type only for type-scoped calls, type+resource for per-resource calls). // -// Typical use: the first (planning) call computes shard assignments — e.g. -// one 50-id delta filter chunk per cursor — returns no rows, and spawns -// one cursor per shard. Cold enumeration then parallelizes across shards -// instead of serializing through one stream. +// Typical uses: +// +// - Type-scoped planning: the first call computes shard assignments — +// e.g. one 50-id delta filter chunk per cursor — returns no rows, and +// spawns one cursor per shard. Cold enumeration then parallelizes +// across shards instead of serializing through one stream. +// - Per-resource parallel warm revalidation (page-numbered APIs): on the +// first page of a collection the connector already knows every other +// page's URL and stored validator, so it answers page one and spawns +// pages 2..N; the worker pool revalidates them concurrently instead +// of paying one round trip per page serially. +// +// Spawned cursors are ORDINARY pages that happen to be enqueued eagerly. +// The SDK assumes nothing about how they resolve: a spawned page may hit +// its source-cache lookup and replay, miss and fetch cold (e.g. a page +// boundary shifted since the last sync), and may continue a chain via its +// response's next page token. Any page of a fan-out may itself spawn. +// +// Progress accounting counts a resource as covered when its ORIGIN +// action's chain ends; spawned siblings never double-count it. // // Tokens are opaque to the SDK. They must be self-contained: a cursor may // execute after a suspend/resume, on a different worker, so anything the // connector needs to serve the cursor must be inside the token (or -// re-derivable from it). -// -// Honored only on responses to type-scoped ListGrants calls; ignored (with -// a warning) elsewhere. +// re-derivable from it) — for per-resource spawns the resource identity +// rides the action, so tokens only need the page coordinate. type SpawnCursors struct { state protoimpl.MessageState `protogen:"opaque.v1"` xxx_hidden_PageTokens []string `protobuf:"bytes,1,rep,name=page_tokens,json=pageTokens,proto3"` diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/cli/commands.go b/vendor/github.com/conductorone/baton-sdk/pkg/cli/commands.go index a221be47..7bde72bd 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/cli/commands.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/cli/commands.go @@ -13,6 +13,7 @@ import ( "time" "github.com/conductorone/baton-sdk/pkg/connectorbuilder" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" "github.com/conductorone/baton-sdk/pkg/types" "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" "github.com/maypok86/otter/v2" @@ -32,7 +33,6 @@ import ( baton_v1 "github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1" "github.com/conductorone/baton-sdk/pkg/connectorrunner" "github.com/conductorone/baton-sdk/pkg/crypto" - "github.com/conductorone/baton-sdk/pkg/dotc1z" "github.com/conductorone/baton-sdk/pkg/field" "github.com/conductorone/baton-sdk/pkg/logging" "github.com/conductorone/baton-sdk/pkg/session" @@ -438,13 +438,20 @@ func MakeMainCommand[T field.Configurable]( } } - if v.GetBool(field.ParallelSyncField.GetName()) { - opts = append(opts, connectorrunner.WithWorkerCount(-1)) - } - + // Worker count resolves through viper with no zero sentinel in the + // default chain: explicit flag > env (BATON_WORKERS) > config file > + // connector default (field.WithConnectorDefault on WorkerCountField) + // > shared default (0). A resolved 0 — explicit or defaulted — means + // sequential, which is the runner's zero-value behavior (normalized + // to one worker downstream), so no option is appended; -1 means + // auto-detect. The deprecated --parallel-sync only applies when + // workers resolved to 0. workers := v.GetInt(field.WorkerCountField.GetName()) - if workers != 0 { + switch { + case workers != 0: opts = append(opts, connectorrunner.WithWorkerCount(workers)) + case v.GetBool(field.ParallelSyncField.GetName()): + opts = append(opts, connectorrunner.WithWorkerCount(-1)) } c1zTmpDir := tempdir.Resolve(v.GetString("c1z-temp-dir")) @@ -509,7 +516,7 @@ func MakeMainCommand[T field.Configurable]( return err } if storageEngine != "" { - opts = append(opts, connectorrunner.WithStorageEngine(dotc1z.Engine(storageEngine))) + opts = append(opts, connectorrunner.WithStorageEngine(c1zstore.Engine(storageEngine))) } taskConcurrency := v.GetInt(field.TaskConcurrencyField.GetName()) diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/config/config.go b/vendor/github.com/conductorone/baton-sdk/pkg/config/config.go index d937dd0b..f250a20a 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/config/config.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/config/config.go @@ -161,6 +161,20 @@ func DefineConfigurationV2[T field.Configurable]( } confschema.Fields = fields + // The replay flags are hidden by default (source-cache replay is + // author-opt-in functionality); surface them in help only for + // connectors whose author baked the capability into their runner + // options. Hidden flags still parse, so this is a visibility decision, + // not a behavioral one. + if connectorrunner.DeclaresPreviousSyncCapability(ctx, options...) { + for i, f := range confschema.Fields { + switch f.FieldName { + case field.PreviousSyncC1ZField.FieldName, field.KeepPreviousSyncC1ZField.FieldName: + confschema.Fields[i].SyncerConfig.Hidden = false + } + } + } + // setup CLI with cobra mainCMD := &cobra.Command{ Use: connectorName, @@ -293,6 +307,12 @@ func verifyStructFields[T field.Configurable](schema field.Configuration) error return fmt.Errorf("T must be a struct type, got %v", configType.Kind()) //nolint:staticcheck // we want to capital letter here } for _, field := range schema.Fields { + if field.WasReExported { + // Re-exported shared SDK fields (field.WithConnectorDefault) + // are parsed by the SDK's own flag handling, not the + // connector's configuration struct. + continue + } fieldFound := false for i := 0; i < configType.NumField(); i++ { structField := configType.Field(i) diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/resource_syncer.go b/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/resource_syncer.go index 37d7764a..3193ed84 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/resource_syncer.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/resource_syncer.go @@ -2,6 +2,7 @@ package connectorbuilder import ( "context" + "errors" "fmt" v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" @@ -106,6 +107,71 @@ func (b *builder) syncOpAttrs(activeSyncID string, token pagination.Token) resou } } +// continuationOpAttrs builds SyncOpAttrs for a list RPC, selecting the +// source-cache lookup by topology: +// +// - a runner-supplied lookup (in-process interface, subprocess loopback +// gRPC) always wins: those topologies answer lookups directly and +// never bounce; +// - otherwise, when the request carries SourceCacheLookupOffer (and/or +// SourceCacheLookupAnswers from a previous bounce), a per-request +// ContinuationLookup is installed — it serves answered scopes and +// defers the rest via ErrLookupDeferred; +// - otherwise NoopLookup (today's behavior). +// +// The returned ContinuationLookup is nil when the continuation is not in +// play for this request. +func (b *builder) continuationOpAttrs(activeSyncID string, token pagination.Token, reqAnnos annotations.Annotations) (resource.SyncOpAttrs, *sourcecache.ContinuationLookup, error) { + attrs := b.syncOpAttrs(activeSyncID, token) + if b.sourceCache != nil { + return attrs, nil, nil + } + answersMsg := &v2.SourceCacheLookupAnswers{} + hasAnswers, err := reqAnnos.Pick(answersMsg) + if err != nil { + return attrs, nil, fmt.Errorf("error parsing source-cache lookup answers annotation: %w", err) + } + if !hasAnswers && !reqAnnos.Contains(&v2.SourceCacheLookupOffer{}) { + return attrs, nil, nil + } + var answers []sourcecache.Answer + if hasAnswers { + answers = sourcecache.AnswersFromProto(answersMsg) + } + cl := sourcecache.NewContinuationLookup(answers) + attrs.SourceCache = cl + return attrs, cl, nil +} + +// continuationOutcome inspects a list handler's result when a +// ContinuationLookup was installed. +// +// Returns (askAnnos, deferred, misuseErr): +// - deferred=true: the handler deferred on recorded scopes. Respond with +// askAnnos (the SourceCacheLookupAsk), NO rows, NO next page token, +// and no failure metrics — this is a protocol turn, not a failure. +// - misuseErr != nil: the handler swallowed a deferred lookup (scopes +// were recorded but no ErrLookupDeferred propagated). Loud failure: +// silently continuing past a deferral re-asks forever and dies at the +// bounce cap in production, so it must fail here, visibly. +// - all-zero: the handler resolved normally; use its result as-is. +func continuationOutcome(cl *sourcecache.ContinuationLookup, handlerErr error) (annotations.Annotations, bool, error) { + if cl == nil { + return nil, false, nil + } + asked := cl.Asked() + if errors.Is(handlerErr, sourcecache.ErrLookupDeferred) && len(asked) > 0 { + annos := annotations.Annotations{} + annos.Update(sourcecache.AskProto(asked)) + return annos, true, nil + } + if handlerErr == nil && len(asked) > 0 { + return nil, false, status.Errorf(codes.Internal, + "connector swallowed a deferred source-cache lookup (%d scope(s) asked, no error propagated): ErrLookupDeferred must be returned — wrap with %%w, never swallow", len(asked)) + } + return nil, false, nil +} + // ResourceTargetedSyncer extends ResourceSyncer to add capabilities for directly syncing an individual resource // // Implementing this interface indicates the connector supports calling "get" on a resource @@ -171,12 +237,26 @@ func (b *builder) ListResources(ctx context.Context, request *v2.ResourcesServic Size: int(request.GetPageSize()), Token: request.GetPageToken(), } - opts := b.syncOpAttrs(request.GetActiveSyncId(), token) + opts, contLookup, err := b.continuationOpAttrs(request.GetActiveSyncId(), token, annotations.Annotations(request.GetAnnotations())) + if err != nil { + b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start), err) + return nil, err + } out, retOptions, err := rb.List(ctx, request.GetParentResourceId(), opts) if retOptions == nil { retOptions = &resource.SyncOpResults{} } + askAnnos, deferred, misuseErr := continuationOutcome(contLookup, err) + if misuseErr != nil { + b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start), misuseErr) + return nil, misuseErr + } + if deferred { + b.m.RecordTaskSuccess(ctx, tt, b.nowFunc().Sub(start)) + return v2.ResourcesServiceListResourcesResponse_builder{Annotations: askAnnos}.Build(), nil + } + resp := v2.ResourcesServiceListResourcesResponse_builder{ List: out, NextPageToken: retOptions.NextPageToken, @@ -307,12 +387,26 @@ func (b *builder) ListEntitlements(ctx context.Context, request *v2.Entitlements Size: int(request.GetPageSize()), Token: request.GetPageToken(), } - opts := b.syncOpAttrs(request.GetActiveSyncId(), token) + opts, contLookup, err := b.continuationOpAttrs(request.GetActiveSyncId(), token, annotations.Annotations(request.GetAnnotations())) + if err != nil { + b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start), err) + return nil, err + } out, retOptions, err := rb.Entitlements(ctx, request.GetResource(), opts) if retOptions == nil { retOptions = &resource.SyncOpResults{} } + askAnnos, deferred, misuseErr := continuationOutcome(contLookup, err) + if misuseErr != nil { + b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start), misuseErr) + return nil, misuseErr + } + if deferred { + b.m.RecordTaskSuccess(ctx, tt, b.nowFunc().Sub(start)) + return v2.EntitlementsServiceListEntitlementsResponse_builder{Annotations: askAnnos}.Build(), nil + } + resp := v2.EntitlementsServiceListEntitlementsResponse_builder{ List: out, NextPageToken: retOptions.NextPageToken, @@ -364,9 +458,13 @@ func (b *builder) ListGrants(ctx context.Context, request *v2.GrantsServiceListG Size: int(request.GetPageSize()), Token: request.GetPageToken(), } - opts := b.syncOpAttrs(request.GetActiveSyncId(), token) - reqAnnos := annotations.Annotations(request.GetAnnotations()) + opts, contLookup, err := b.continuationOpAttrs(request.GetActiveSyncId(), token, reqAnnos) + if err != nil { + b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start), err) + return nil, err + } + typeScoped := reqAnnos.Contains(&v2.TypeScopedGrants{}) var out []*v2.Grant @@ -391,6 +489,16 @@ func (b *builder) ListGrants(ctx context.Context, request *v2.GrantsServiceListG retOptions = &resource.SyncOpResults{} } + askAnnos, deferred, misuseErr := continuationOutcome(contLookup, err) + if misuseErr != nil { + b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start), misuseErr) + return nil, misuseErr + } + if deferred { + b.m.RecordTaskSuccess(ctx, tt, b.nowFunc().Sub(start)) + return v2.GrantsServiceListGrantsResponse_builder{Annotations: askAnnos}.Build(), nil + } + resp := v2.GrantsServiceListGrantsResponse_builder{ List: out, Annotations: retOptions.Annotations, diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/connectorrunner/runner.go b/vendor/github.com/conductorone/baton-sdk/pkg/connectorrunner/runner.go index dd6ec4ee..4db9cf18 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/connectorrunner/runner.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/connectorrunner/runner.go @@ -13,7 +13,7 @@ import ( "github.com/conductorone/baton-sdk/pkg/bid" "github.com/conductorone/baton-sdk/pkg/connectorbuilder" - "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" "github.com/conductorone/baton-sdk/pkg/field" "github.com/conductorone/baton-sdk/pkg/healthcheck" "github.com/conductorone/baton-sdk/pkg/synccompactor" @@ -422,7 +422,7 @@ type runnerConfig struct { syncDifferConfig *syncDifferConfig syncCompactorConfig *syncCompactorConfig skipFullSync bool - storageEngine dotc1z.Engine + storageEngine c1zstore.Engine workerCount int targetedSyncResourceIDs []string externalResourceC1Z string @@ -670,7 +670,7 @@ func WithWorkerCount(workerCount int) Option { } } -func WithStorageEngine(engine dotc1z.Engine) Option { +func WithStorageEngine(engine c1zstore.Engine) Option { return func(ctx context.Context, cfg *runnerConfig) error { cfg.storageEngine = engine return nil @@ -810,6 +810,25 @@ func WithKeepPreviousSyncC1ZRuntimeOptIn() Option { } } +// DeclaresPreviousSyncCapability reports whether opts include the connector +// author's replay declaration (WithKeepPreviousSyncC1Z). Used at +// command-definition time to decide whether the replay CLI flags +// (--previous-sync-c1z / --keep-previous-sync-c1z) appear in help: they are +// hidden for the overwhelming majority of connectors that don't support +// replay, and surfaced only for the ones whose author baked the capability +// into their RunConnector options. Options are applied to a scratch config; +// they are plain setters, so this is side-effect free. +func DeclaresPreviousSyncCapability(ctx context.Context, opts ...Option) bool { + cfg := &runnerConfig{} + for _, o := range opts { + if o == nil { + continue + } + _ = o(ctx, cfg) + } + return cfg.keepPreviousSyncC1ZCapable +} + func WithDiffSyncs(c1zPath string, baseSyncID string, newSyncID string) Option { return func(ctx context.Context, cfg *runnerConfig) error { cfg.onDemand = true diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/connectorstore/connectorstore.go b/vendor/github.com/conductorone/baton-sdk/pkg/connectorstore/connectorstore.go index 17045488..fa1f03fe 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/connectorstore/connectorstore.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/connectorstore/connectorstore.go @@ -39,7 +39,7 @@ var AllSyncTypes = []SyncType{ // unless the value is recognized. type StoreMetadata struct { // Engine identifies the storage backend. Values match - // dotc1z.Engine string values; using string here keeps + // c1zstore.Engine string values; using string here keeps // connectorstore from depending on dotc1z (avoids an import // cycle). // "sqlite" — original .c1z, v1 magic + zstd-compressed SQLite @@ -56,7 +56,7 @@ type StoreMetadata struct { // PayloadEncoding identifies the v3 envelope payload framing. // Empty for v1 / SQLite. Values match - // dotc1z.PayloadEncoding.String(): + // c1zstore.PayloadEncoding.String(): // "tar_zstd" — Pebble checkpoint as zstd-compressed tar // "tar" — Pebble checkpoint as uncompressed tar // "" — N/A or unset diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file.go index d6d3a8f1..2845ae7f 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file.go @@ -31,6 +31,7 @@ import ( v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" reader_v2 "github.com/conductorone/baton-sdk/pb/c1/reader/v2" "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" "github.com/conductorone/baton-sdk/pkg/uotel" ) @@ -73,7 +74,7 @@ type C1File struct { deferredIndexTables []tableDescriptor // Cached sync run for listConnectorObjects (avoids N+1 queries) - cachedViewSyncRun *SyncRun + cachedViewSyncRun *c1zstore.SyncRun cachedViewSyncMu sync.Mutex cachedViewSyncErr error @@ -94,22 +95,22 @@ type C1File struct { // engine is the storage engine to use for newly created files. // Reads dispatch on magic byte regardless of this value. Default // is EngineSQLite (v1 .c1z format). - engine Engine + engine c1zstore.Engine // payloadEncoding selects the v3 envelope payload framing for // Pebble-written files. Zero value = PayloadEncodingTarZstd // (default). Ignored by the SQLite engine. - payloadEncoding PayloadEncoding + payloadEncoding c1zstore.PayloadEncoding } // *C1File satisfies connectorstore.Writer (the connector-facing contract), // connectorstore.LatestFinishedSyncIDFetcher (narrow optional capability -// added in PR #774), and dotc1z.C1ZStore (the internal sync-pipeline +// added in PR #774), and c1zstore.Store (the internal sync-pipeline // contract asserted in c1file_store.go alongside the sub-store assertions). var ( _ connectorstore.Writer = (*C1File)(nil) _ connectorstore.LatestFinishedSyncIDFetcher = (*C1File)(nil) - _ C1ZStore = (*C1File)(nil) + _ c1zstore.Store = (*C1File)(nil) ) type C1FOption func(*C1File) @@ -210,7 +211,7 @@ func WithC1FSyncCountLimit(limit int) C1FOption { // Engine selection only affects newly created files. Existing files // dispatch on their magic byte; readers handle both v1 and v3 // regardless of this option. -func WithC1FEngine(engine Engine) C1FOption { +func WithC1FEngine(engine c1zstore.Engine) C1FOption { return func(o *C1File) { o.engine = engine } @@ -218,7 +219,7 @@ func WithC1FEngine(engine Engine) C1FOption { // WithC1FPayloadEncoding selects the v3 envelope payload encoding // (TAR_ZSTD default, TAR uncompressed). No-op for SQLite engines. -func WithC1FPayloadEncoding(enc PayloadEncoding) C1FOption { +func WithC1FPayloadEncoding(enc c1zstore.PayloadEncoding) C1FOption { return func(o *C1File) { o.payloadEncoding = enc } @@ -293,7 +294,7 @@ func NewC1File(ctx context.Context, dbFilePath string, opts ...C1FOption) (*C1Fi // engine manages its own storage and does not use those indexes, so bulk // load does not apply there. Make the combination an explicit, logged // no-op rather than leaving it silently unspecified. - if c1File.bulkLoad && c1File.engine == EnginePebble { + if c1File.bulkLoad && c1File.engine == c1zstore.EnginePebble { l.Info("new-c1-file: bulk load ignored for the pebble engine; the deferred-index optimization applies only to the sqlite engine") c1File.bulkLoad = false } @@ -311,7 +312,7 @@ func NewC1File(ctx context.Context, dbFilePath string, opts ...C1FOption) (*C1Fi // Normalize the engine zero value so downstream switch/if-eq // checks treat an unset engine as EngineSQLite. if c1File.engine == "" { - c1File.engine = EngineSQLite + c1File.engine = c1zstore.EngineSQLite } err = c1File.validateDb(ctx) @@ -341,13 +342,13 @@ type c1zOptions struct { // engine is the storage engine to use for newly created files. // Reads dispatch on magic byte regardless. Default EngineSQLite. - engine Engine + engine c1zstore.Engine // payloadEncoding controls the v3 envelope payload framing. Only // honored when engine == EnginePebble (the v3 path). Allowed // values: PayloadEncodingTarZstd (default), PayloadEncodingTar. // Zero value means PayloadEncodingTarZstd. - payloadEncoding PayloadEncoding + payloadEncoding c1zstore.PayloadEncoding // decoderPool optionally scopes v3 payload-decoder reuse to the // caller's operation. See WithDecoderPool. @@ -430,7 +431,7 @@ func WithSyncLimit(limit int) C1ZOption { // // Reading existing files dispatches on the file's magic byte and is // independent of this option. -func WithEngine(engine Engine) C1ZOption { +func WithEngine(engine c1zstore.Engine) C1ZOption { return func(o *c1zOptions) { o.engine = engine } @@ -465,7 +466,7 @@ func WithBulkLoad(enabled bool) C1ZOption { // // No-op for SQLite engines; the encoding selector applies only to // the v3 envelope written by Pebble. -func WithPayloadEncoding(enc PayloadEncoding) C1ZOption { +func WithPayloadEncoding(enc c1zstore.PayloadEncoding) C1ZOption { return func(o *c1zOptions) { o.payloadEncoding = enc } @@ -491,8 +492,11 @@ func NewC1ZFile(ctx context.Context, outputFilePath string, opts ...C1ZOption) ( return nil, err } - if options.engine == EnginePebble && !options.readOnly { - err = fmt.Errorf("new-c1z-file: %s is a v1/sqlite c1z and engine %q was requested; NewC1ZFile cannot return a *C1File for it — open with NewStore to convert", outputFilePath, EnginePebble) + if options.engine == c1zstore.EnginePebble && !options.readOnly { + err = fmt.Errorf( + "new-c1z-file: %s is a v1/sqlite c1z and engine %q was requested; "+ + "NewC1ZFile cannot return a *C1File for it — open with NewStore to convert", + outputFilePath, c1zstore.EnginePebble) return nil, err } @@ -535,7 +539,7 @@ func NewC1ZFile(ctx context.Context, outputFilePath string, opts ...C1ZOption) ( if options.engine != "" { c1fopts = append(c1fopts, WithC1FEngine(options.engine)) } - if options.payloadEncoding != PayloadEncodingUnspecified { + if options.payloadEncoding != c1zstore.PayloadEncodingUnspecified { c1fopts = append(c1fopts, WithC1FPayloadEncoding(options.payloadEncoding)) } @@ -1420,7 +1424,7 @@ func (c *C1File) OutputFilepath() (string, error) { func (c *C1File) Metadata() connectorstore.StoreMetadata { engine := c.engine if engine == "" { - engine = EngineSQLite + engine = c1zstore.EngineSQLite } return connectorstore.StoreMetadata{ Engine: string(engine), diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file_store.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file_store.go index 226a1783..dbc4cb35 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file_store.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file_store.go @@ -17,21 +17,21 @@ import ( // wrapper structs satisfy each sub-interface. These assertions catch // signature drift at build time rather than at the first runtime call. var ( - _ C1ZStore = (*C1File)(nil) - _ GrantStore = c1FileGrantStore{} - _ SyncMeta = c1FileSyncMeta{} - _ FileOps = c1FileFileOps{} - _ SessionStore = c1FileSessionStore{} + _ c1zstore.Store = (*C1File)(nil) + _ c1zstore.GrantStore = c1FileGrantStore{} + _ c1zstore.SyncMeta = c1FileSyncMeta{} + _ c1zstore.FileOps = c1FileFileOps{} + _ SessionStore = c1FileSessionStore{} ) // Grants returns the grant-store slice of this c1z. -func (c *C1File) Grants() GrantStore { return c1FileGrantStore{c} } +func (c *C1File) Grants() c1zstore.GrantStore { return c1FileGrantStore{c} } // SyncMeta returns the sync-metadata slice of this c1z. -func (c *C1File) SyncMeta() SyncMeta { return c1FileSyncMeta{c} } +func (c *C1File) SyncMeta() c1zstore.SyncMeta { return c1FileSyncMeta{c} } // FileOps returns the file-operations slice of this c1z. -func (c *C1File) FileOps() FileOps { return c1FileFileOps{c} } +func (c *C1File) FileOps() c1zstore.FileOps { return c1FileFileOps{c} } // SessionStore returns the session-store slice of this c1z. func (c *C1File) SessionStore() sessions.SessionStore { return c1FileSessionStore{c} } @@ -97,7 +97,7 @@ func (c *C1File) StoreExpandedGrants(ctx context.Context, grants ...*v2.Grant) e // PendingExpansionPage implements GrantStore. Thin wrapper over // listExpandableGrantsInternal(Mode: ExpansionNeedsOnly) that reshapes // the internal row struct into the exported PendingExpansion shape. -func (g c1FileGrantStore) PendingExpansionPage(ctx context.Context, pageToken string) ([]PendingExpansion, string, error) { +func (g c1FileGrantStore) PendingExpansionPage(ctx context.Context, pageToken string) ([]c1zstore.PendingExpansion, string, error) { defs, nextPageToken, err := g.c.listExpandableGrantsInternal(ctx, grantListOptions{ Mode: grantListModeExpansionNeedsOnly, PageToken: pageToken, @@ -105,12 +105,12 @@ func (g c1FileGrantStore) PendingExpansionPage(ctx context.Context, pageToken st if err != nil { return nil, "", err } - out := make([]PendingExpansion, 0, len(defs)) + out := make([]c1zstore.PendingExpansion, 0, len(defs)) for _, def := range defs { if def == nil { continue } - out = append(out, PendingExpansion{ + out = append(out, c1zstore.PendingExpansion{ GrantExternalID: def.GrantExternalID, TargetEntitlementID: def.TargetEntitlementID, PrincipalResourceTypeID: def.PrincipalResourceTypeID, @@ -136,17 +136,17 @@ func (g c1FileGrantStore) PendingExpansionPage(ctx context.Context, pageToken st // walks terminate promptly when the caller's deadline/cancel fires; // rows within a single page are still delivered (they are already in // memory), so cancellation responsiveness is page-grained, not row-grained. -func (g c1FileGrantStore) PendingExpansion(ctx context.Context) iter.Seq2[PendingExpansion, error] { - return func(yield func(PendingExpansion, error) bool) { +func (g c1FileGrantStore) PendingExpansion(ctx context.Context) iter.Seq2[c1zstore.PendingExpansion, error] { + return func(yield func(c1zstore.PendingExpansion, error) bool) { pageToken := "" for { if err := ctx.Err(); err != nil { - _ = yield(PendingExpansion{}, err) + _ = yield(c1zstore.PendingExpansion{}, err) return } page, nextPageToken, err := g.PendingExpansionPage(ctx, pageToken) if err != nil { - _ = yield(PendingExpansion{}, err) + _ = yield(c1zstore.PendingExpansion{}, err) return } for _, pe := range page { @@ -172,7 +172,7 @@ func (g c1FileGrantStore) ListWithAnnotationsForResourcePage( syncID string, pageToken string, pageSize uint32, -) ([]GrantAnnotation, string, error) { +) ([]c1zstore.GrantAnnotation, string, error) { resp, err := g.c.listGrantsWithExpansionInternal(ctx, grantListOptions{ Mode: grantListModePayloadWithExpansion, Resource: resource, @@ -189,13 +189,13 @@ func (g c1FileGrantStore) ListWithAnnotationsForResourcePage( // grantAnnotationRowsFromInternal converts the internal row shape into // the exported GrantAnnotation shape, unifying the code path between // ListWithAnnotationsPage and ListWithAnnotationsForResourcePage. -func grantAnnotationRowsFromInternal(rows []*internalGrantRow) []GrantAnnotation { - out := make([]GrantAnnotation, 0, len(rows)) +func grantAnnotationRowsFromInternal(rows []*internalGrantRow) []c1zstore.GrantAnnotation { + out := make([]c1zstore.GrantAnnotation, 0, len(rows)) for _, row := range rows { if row == nil { continue } - ga := GrantAnnotation{ + ga := c1zstore.GrantAnnotation{ Grant: row.Grant, GrantExternalID: row.Grant.GetId(), TargetEntitlementID: row.Grant.GetEntitlement().GetId(), @@ -223,7 +223,7 @@ func grantAnnotationRowsFromInternal(rows []*internalGrantRow) []GrantAnnotation // from the underlying grant proto, regardless of whether the grant has // an expansion annotation, so callers don't need to branch on // Annotation-nil to get identity. -func (g c1FileGrantStore) ListWithAnnotationsPage(ctx context.Context, pageToken string) ([]GrantAnnotation, string, error) { +func (g c1FileGrantStore) ListWithAnnotationsPage(ctx context.Context, pageToken string) ([]c1zstore.GrantAnnotation, string, error) { resp, err := g.c.listGrantsWithExpansionInternal(ctx, grantListOptions{ Mode: grantListModePayloadWithExpansion, PageToken: pageToken, @@ -237,17 +237,17 @@ func (g c1FileGrantStore) ListWithAnnotationsPage(ctx context.Context, pageToken // ListWithAnnotations implements GrantStore. Convenience iterator that // walks every page via ListWithAnnotationsPage. Cancellation behavior is // identical to PendingExpansion (page-grained). -func (g c1FileGrantStore) ListWithAnnotations(ctx context.Context) iter.Seq2[GrantAnnotation, error] { - return func(yield func(GrantAnnotation, error) bool) { +func (g c1FileGrantStore) ListWithAnnotations(ctx context.Context) iter.Seq2[c1zstore.GrantAnnotation, error] { + return func(yield func(c1zstore.GrantAnnotation, error) bool) { pageToken := "" for { if err := ctx.Err(); err != nil { - _ = yield(GrantAnnotation{}, err) + _ = yield(c1zstore.GrantAnnotation{}, err) return } page, nextPageToken, err := g.ListWithAnnotationsPage(ctx, pageToken) if err != nil { - _ = yield(GrantAnnotation{}, err) + _ = yield(c1zstore.GrantAnnotation{}, err) return } for _, ga := range page { @@ -276,7 +276,7 @@ func (s c1FileSyncMeta) MarkSyncSupportsDiff(ctx context.Context, syncID string) // LatestFullSync implements SyncMeta. Returns the most-recent finished // SyncTypeFull run, or nil if none. -func (s c1FileSyncMeta) LatestFullSync(ctx context.Context) (*SyncRun, error) { +func (s c1FileSyncMeta) LatestFullSync(ctx context.Context) (*c1zstore.SyncRun, error) { run, err := s.c.getFinishedSync(ctx, 0, connectorstore.SyncTypeFull) if err != nil { return nil, err @@ -286,7 +286,7 @@ func (s c1FileSyncMeta) LatestFullSync(ctx context.Context) (*SyncRun, error) { // LatestFinishedSyncOfAnyType implements SyncMeta. Returns the most-recent // finished sync of any type (including diff types), or nil if none. -func (s c1FileSyncMeta) LatestFinishedSyncOfAnyType(ctx context.Context) (*SyncRun, error) { +func (s c1FileSyncMeta) LatestFinishedSyncOfAnyType(ctx context.Context) (*c1zstore.SyncRun, error) { run, err := s.c.getFinishedSync(ctx, 0, connectorstore.SyncTypeAny) if err != nil { return nil, err @@ -314,7 +314,7 @@ type c1FileFileOps struct{ c *C1File } // CloneSync implements FileOps. Translates the engine-neutral // CloneSyncOptions into the SQLite-specific C1FOptions applied to the // destination file. -func (f c1FileFileOps) CloneSync(ctx context.Context, outPath string, syncID string, opts ...CloneSyncOption) error { +func (f c1FileFileOps) CloneSync(ctx context.Context, outPath string, syncID string, opts ...c1zstore.CloneSyncOption) error { cloneOpts := c1zstore.NewCloneSyncOptions(opts...) var c1fOpts []C1FOption if cloneOpts.TmpDir != "" { @@ -326,7 +326,7 @@ func (f c1FileFileOps) CloneSync(ctx context.Context, outPath string, syncID str // CopyIsolateSync implements FileOps. Translates the engine-neutral // CloneSyncOptions into the SQLite-specific C1FOptions applied to the // destination file. -func (f c1FileFileOps) CopyIsolateSync(ctx context.Context, outPath string, syncID string, opts ...CloneSyncOption) error { +func (f c1FileFileOps) CopyIsolateSync(ctx context.Context, outPath string, syncID string, opts ...c1zstore.CloneSyncOption) error { cloneOpts := c1zstore.NewCloneSyncOptions(opts...) var c1fOpts []C1FOption if cloneOpts.TmpDir != "" { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore/c1zstore.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore/c1zstore.go index 5ab9d50f..8aa50210 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore/c1zstore.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore/c1zstore.go @@ -7,9 +7,8 @@ // pkg/connectorstore. Storage engines (pkg/dotc1z's SQLite C1File, // pkg/dotc1z/engine/pebble's Adapter) implement these interfaces without // importing pkg/dotc1z, which lets dotc1z import the engines and register -// them statically. pkg/dotc1z re-exports every type here under its -// historical name (dotc1z.C1ZStore = c1zstore.Store, etc.), so callers -// outside the engine packages can keep using the dotc1z names. +// them statically. Callers reference these types directly through this +// package (c1zstore.Store, c1zstore.Engine, etc.). package c1zstore import ( diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/cleanup_policy.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/cleanup_policy.go index e9aa16b6..8f1b07e8 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/cleanup_policy.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/cleanup_policy.go @@ -9,7 +9,7 @@ import "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" // SelectSyncsToDelete applies the SDK retention policy to a snapshot of sync // runs and returns the IDs whose data should be deleted. See // c1zstore.SelectSyncsToDelete for the policy details. -func SelectSyncsToDelete(candidates []SyncRun, currentSyncID string, syncLimit int) []string { +func SelectSyncsToDelete(candidates []c1zstore.SyncRun, currentSyncID string, syncLimit int) []string { return c1zstore.SelectSyncsToDelete(candidates, currentSyncID, syncLimit) } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/clone_sync.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/clone_sync.go index 429df6b7..9c34ed97 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/clone_sync.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/clone_sync.go @@ -12,6 +12,7 @@ import ( "strings" "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" "github.com/conductorone/baton-sdk/pkg/uotel" "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" "go.uber.org/zap" @@ -195,8 +196,8 @@ func (c *C1File) SnapshotTo(ctx context.Context, outPath string, opts ...C1FOpti return err } - if c.engine == EnginePebble { - err = fmt.Errorf("snapshot-to: unsupported for the %q engine; it manages its own storage", EnginePebble) + if c.engine == c1zstore.EnginePebble { + err = fmt.Errorf("snapshot-to: unsupported for the %q engine; it manages its own storage", c1zstore.EnginePebble) return err } if c.readOnly { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/convert_open.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/convert_open.go index b69b0f81..c6d5086b 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/convert_open.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/convert_open.go @@ -9,6 +9,7 @@ import ( "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" "go.uber.org/zap" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" "github.com/conductorone/baton-sdk/pkg/uotel" ) @@ -22,7 +23,7 @@ type pebbleOpenOptions struct { skipCleanup bool skipVacuum bool v2GrantsWriter bool - payloadEncoding PayloadEncoding + payloadEncoding c1zstore.PayloadEncoding } func pebbleOpenOptionsFromC1Z(options *c1zOptions) pebbleOpenOptions { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter.go index 28b37a91..b2c38511 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter.go @@ -985,7 +985,7 @@ func (a *Adapter) CurrentDBSizeBytes() (int64, error) { // // Strings are inlined rather than referencing dotc1z constants // because this subpackage is imported by dotc1z, so the reverse -// import would cycle. The values match dotc1z.EnginePebble.String() +// import would cycle. The values match c1zstore.EnginePebble.String() // and dotc1z.C1ZFormatV3.String() — see connectorstore.StoreMetadata // docs for the canonical value list. func (a *Adapter) Metadata() connectorstore.StoreMetadata { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/engine_stub.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/engine_stub.go index 2ac92fbc..fd15e349 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/engine_stub.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/engine_stub.go @@ -1,5 +1,5 @@ // Package pebble is the v3 storage engine for baton-sdk. It is the -// implementation behind dotc1z.EnginePebble and the v3 envelope. +// implementation behind c1zstore.EnginePebble and the v3 envelope. package pebble import ( diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/sync_runs.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/sync_runs.go index 8894a719..b3d2b267 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/sync_runs.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/sync_runs.go @@ -142,7 +142,7 @@ func (e *Engine) IterateAllSyncRuns(ctx context.Context, yield func(*v3.SyncRunR // This is the single source of truth for "pick the latest finished // sync" on the Pebble engine; all three external entry points // (connectorstore.LatestFinishedSyncIDFetcher, -// dotc1z.SyncMeta.LatestFullSync / LatestFinishedSyncOfAnyType, and +// c1zstore.SyncMeta.LatestFullSync / LatestFinishedSyncOfAnyType, and // reader_v2.SyncsReaderService.GetLatestFinishedSync) call here so // the tiebreaker and predicate semantics stay consistent. // diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine_registry.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine_registry.go index bc438b43..403f1683 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine_registry.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine_registry.go @@ -10,6 +10,7 @@ import ( "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" "go.uber.org/zap" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" formatv3 "github.com/conductorone/baton-sdk/pkg/dotc1z/format/v3" ) @@ -31,13 +32,13 @@ type StoreOptions struct { SyncLimit int SkipCleanup bool V2GrantsWriter bool - Engine Engine + Engine c1zstore.Engine // PayloadEncoding selects the v3 envelope payload framing for // engines that produce a v3 envelope (currently Pebble). Zero // value means "engine default" (PayloadEncodingIndexedZstd for // Pebble). - PayloadEncoding PayloadEncoding + PayloadEncoding c1zstore.PayloadEncoding // DecoderPool optionally scopes v3 payload-decoder reuse to the // caller's operation (see WithDecoderPool). Nil means a one-shot @@ -84,20 +85,20 @@ func WithDecoderPool(p *EnvelopeDecoderPool) C1ZOption { // and Pebble drivers are both registered statically by this package; // RegisterEngine exists for additional engines. type EngineDriver interface { - Engine() Engine + Engine() c1zstore.Engine Format() C1ZFormat - OpenStore(ctx context.Context, outputFilePath string, opts StoreOptions) (C1ZStore, error) + OpenStore(ctx context.Context, outputFilePath string, opts StoreOptions) (c1zstore.Store, error) } type engineRegistry struct { mu sync.RWMutex - byEngine map[Engine]EngineDriver + byEngine map[c1zstore.Engine]EngineDriver } var defaultEngineRegistry = &engineRegistry{ - byEngine: map[Engine]EngineDriver{ - EngineSQLite: sqliteDriver{}, - EnginePebble: pebbleDriver{}, + byEngine: map[c1zstore.Engine]EngineDriver{ + c1zstore.EngineSQLite: sqliteDriver{}, + c1zstore.EnginePebble: pebbleDriver{}, }, } @@ -108,7 +109,7 @@ func RegisterEngine(driver EngineDriver) error { } // EngineDriverFor returns the registered driver for engine. -func EngineDriverFor(engine Engine) (EngineDriver, bool) { +func EngineDriverFor(engine c1zstore.Engine) (EngineDriver, bool) { return defaultEngineRegistry.driverForEngine(engine) } @@ -134,7 +135,7 @@ func (r *engineRegistry) register(driver EngineDriver) error { return nil } -func (r *engineRegistry) driverForEngine(engine Engine) (EngineDriver, bool) { +func (r *engineRegistry) driverForEngine(engine c1zstore.Engine) (EngineDriver, bool) { r.mu.RLock() defer r.mu.RUnlock() driver, ok := r.byEngine[engine] @@ -145,7 +146,7 @@ func (r *engineRegistry) driverForEngine(engine Engine) (EngineDriver, bool) { // the engine-neutral constructor for callers that may opt into non-default // engines. NewC1ZFile remains the concrete SQLite constructor for legacy // callers that need *C1File. -func NewStore(ctx context.Context, outputFilePath string, opts ...C1ZOption) (C1ZStore, error) { +func NewStore(ctx context.Context, outputFilePath string, opts ...C1ZOption) (c1zstore.Store, error) { options, err := buildC1ZOptions(opts...) if err != nil { return nil, err @@ -193,7 +194,7 @@ func storeOptionsFromC1ZOptions(options *c1zOptions) StoreOptions { MaxDecoderMemoryBytes: maxDecoderMemoryBytes, } if out.Engine == "" { - out.Engine = EngineSQLite + out.Engine = c1zstore.EngineSQLite } out.Pragmas = make([]StorePragma, 0, len(options.pragmas)) for _, p := range options.pragmas { @@ -224,7 +225,7 @@ func selectStoreDriver(ctx context.Context, outputFilePath string, options *c1zO l := ctxzap.Extract(ctx) requested := options.engine if requested == "" { - requested = EngineSQLite + requested = c1zstore.EngineSQLite } stat, err := os.Stat(outputFilePath) // #nosec G703 -- c1z path is caller-controlled by API design. @@ -252,11 +253,11 @@ func selectStoreDriver(ctx context.Context, outputFilePath string, options *c1zO return nil, err } - var fileEngine Engine + var fileEngine c1zstore.Engine switch format { case C1ZFormatV1: // Maybe error if the file is read-only? - if requested == EnginePebble && !options.readOnly { + if requested == c1zstore.EnginePebble && !options.readOnly { // Close our header-read handle before converting: the conversion // renames a temp file over outputFilePath, which fails on Windows // if any handle to the destination is still open. Nil out f so @@ -271,9 +272,9 @@ func selectStoreDriver(ctx context.Context, outputFilePath string, options *c1zO return nil, fmt.Errorf("select-store-driver: convert existing v1 c1z to pebble: %w", err) } l.Debug("converted existing v1 c1z to pebble", zap.String("output_file_path", outputFilePath)) - return requireEngineDriver(EnginePebble) + return requireEngineDriver(c1zstore.EnginePebble) } - fileEngine = EngineSQLite + fileEngine = c1zstore.EngineSQLite case C1ZFormatV3: if _, err := f.Seek(0, 0); err != nil { return nil, err @@ -288,13 +289,13 @@ func selectStoreDriver(ctx context.Context, outputFilePath string, options *c1zO if err != nil { return nil, err } - fileEngine = Engine(m.GetEngine()) + fileEngine = c1zstore.Engine(m.GetEngine()) // Current and legacy pebble manifest names all dispatch to the same // driver; legacy interiors are re-keyed by the on-open id-index // migration. Unknown (newer) names fall through and fail loudly in // requireEngineDriver. - if fileEngine == PebbleManifestEngine || fileEngine == PebbleManifestEngineV2 { - fileEngine = EnginePebble + if fileEngine == c1zstore.PebbleManifestEngine || fileEngine == c1zstore.PebbleManifestEngineV2 { + fileEngine = c1zstore.EnginePebble } default: return nil, ErrInvalidFile @@ -311,7 +312,7 @@ func selectStoreDriver(ctx context.Context, outputFilePath string, options *c1zO return requireEngineDriver(fileEngine) } -func requireEngineDriver(engine Engine) (EngineDriver, error) { +func requireEngineDriver(engine c1zstore.Engine) (EngineDriver, error) { driver, ok := EngineDriverFor(engine) if !ok { return nil, fmt.Errorf("require-engine-driver: %w: %s", ErrEngineNotAvailable, engine) @@ -321,12 +322,12 @@ func requireEngineDriver(engine Engine) (EngineDriver, error) { type sqliteDriver struct{} -func (sqliteDriver) Engine() Engine { return EngineSQLite } -func (sqliteDriver) Format() C1ZFormat { return C1ZFormatV1 } +func (sqliteDriver) Engine() c1zstore.Engine { return c1zstore.EngineSQLite } +func (sqliteDriver) Format() C1ZFormat { return C1ZFormatV1 } -func (sqliteDriver) OpenStore(ctx context.Context, outputFilePath string, opts StoreOptions) (C1ZStore, error) { +func (sqliteDriver) OpenStore(ctx context.Context, outputFilePath string, opts StoreOptions) (c1zstore.Store, error) { c1zOpts := []C1ZOption{ - WithEngine(EngineSQLite), + WithEngine(c1zstore.EngineSQLite), WithEncoderConcurrency(opts.EncoderConcurrency), } if opts.TmpDir != "" { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/file_ops.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/file_ops.go deleted file mode 100644 index d3210586..00000000 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/file_ops.go +++ /dev/null @@ -1,25 +0,0 @@ -package dotc1z - -import "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" - -// The file-operations contract lives in pkg/dotc1z/c1zstore so storage -// engines can implement it without importing this package. These aliases -// preserve the historical dotc1z names. - -// FileOps is the file-level operations sub-store of C1ZStore. See -// c1zstore.FileOps for the full contract. -type FileOps = c1zstore.FileOps - -// CloneSyncOption configures a FileOps.CloneSync call. See -// c1zstore.CloneSyncOption. -type CloneSyncOption = c1zstore.CloneSyncOption - -// CloneSyncOptions carries the engine-neutral knobs for FileOps.CloneSync. -// See c1zstore.CloneSyncOptions. -type CloneSyncOptions = c1zstore.CloneSyncOptions - -// WithCloneTmpDir sets the temporary directory used while assembling the -// cloned c1z. Replaces WithC1FTmpDir at FileOps.CloneSync call sites. -func WithCloneTmpDir(dir string) CloneSyncOption { - return c1zstore.WithCloneTmpDir(dir) -} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/format.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/format.go index 9ca49197..bda658a8 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/format.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/format.go @@ -4,8 +4,6 @@ import ( "bytes" "fmt" "io" - - "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" ) // C1ZFormat identifies the on-disk format of a .c1z file. The format byte @@ -42,64 +40,10 @@ func (f C1ZFormat) String() string { // C1Z3FileHeader is the magic byte sequence for v3 files. var C1Z3FileHeader = []byte("C1Z3\x00") -// Engine identifies a storage engine implementation. The engine is -// chosen by callers via WithEngine(...) on write; on read, the engine -// is dictated by the file's magic byte and (for v3) the manifest's -// engine field. The type lives in pkg/dotc1z/c1zstore so engine -// packages can name it without importing dotc1z. -type Engine = c1zstore.Engine - -const ( - // EngineSQLite is the default engine: the v1 .c1z format backed by - // a zstd-compressed SQLite database. Connectors use this; backend - // infra can opt out. - EngineSQLite = c1zstore.EngineSQLite - - // EnginePebble is the v3 engine: a Pebble LSM wrapped in the v3 - // envelope. - EnginePebble = c1zstore.EnginePebble - - // PebbleManifestEngine is the engine name recorded in a single-sync - // Pebble v3 manifest. It deliberately differs from EnginePebble so - // pre-single-sync readers reject the file at dispatch instead of - // reading its keys as empty. See c1zstore.PebbleManifestEngine. - PebbleManifestEngine = c1zstore.PebbleManifestEngine - PebbleManifestEngineV2 = c1zstore.PebbleManifestEngineV2 -) - // ErrEngineNotAvailable is returned when a caller requests an engine // that the binary does not support. var ErrEngineNotAvailable = fmt.Errorf("dotc1z: engine not available") -// PayloadEncoding selects the v3 envelope payload framing. Only the -// Pebble engine consults this; SQLite engines ignore it. See -// c1zstore.PayloadEncoding. -type PayloadEncoding = c1zstore.PayloadEncoding - -const ( - // PayloadEncodingUnspecified is the zero value. Means "use the - // engine's default" — IndexedZstd for Pebble. - PayloadEncodingUnspecified = c1zstore.PayloadEncodingUnspecified - - // PayloadEncodingTarZstd is the default Pebble v3 envelope - // encoding: tar of the Pebble directory, compressed with zstd. - PayloadEncodingTarZstd = c1zstore.PayloadEncodingTarZstd - - // PayloadEncodingTar is uncompressed tar. Useful when Pebble's - // L5/L6 SSTs are already zstd-compressed at the engine layer - // (avoids double-compression CPU), or when the storage target - // compresses in transit. - PayloadEncodingTar = c1zstore.PayloadEncodingTar - - // PayloadEncodingIndexedZstd stores each payload file as an - // independent zstd frame with a self-describing header. Opens - // decode frames in parallel, and rewrites of a store opened from - // an indexed file splice unchanged frames verbatim instead of - // re-compressing them (incremental fold compaction relies on - // this). Readers older than this encoding reject the file. - PayloadEncodingIndexedZstd = c1zstore.PayloadEncodingIndexedZstd -) - // ReadHeaderFormat reads the first 5 bytes of reader and returns the // detected format. On return, the reader is positioned immediately // after the header bytes. If reader is also an io.Seeker, it is diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/format/v3/envelope.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/format/v3/envelope.go index 4188c5fd..042577c5 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/format/v3/envelope.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/format/v3/envelope.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "io" + "math" "os" "path/filepath" "strconv" @@ -40,7 +41,6 @@ var ErrEnvelopeTruncated = errors.New("c1z v3: envelope truncated") // usage and protects against a malicious file claiming a billion-byte // manifest length. const maxManifestBytes = 16 << 20 -const maxTarEntryBytes int64 = 4 << 30 // Tar entries larger than this are streamed straight to disk on the // reader goroutine instead of being buffered in memory for the writer @@ -69,19 +69,63 @@ var fcsFailFastDisabled = os.Getenv(fcsFailFastDisableEnvVar) == "1" // bombs in untrusted v3 c1z files. var ErrMaxSizeExceeded = fmt.Errorf("c1z v3: max decoded payload size exceeded, increase via the %s environment variable", maxDecodedSizeEnvVar) -// envSizeBytes reads an env var holding a size in MiB and converts it -// to bytes, falling back to def when unset, unparsable, zero, or large -// enough to overflow the MiB→bytes conversion. -func envSizeBytes(envVar string, def uint64) uint64 { +// envSizeBytesExplicit reads an env var holding a size in MiB and +// converts it to bytes. ok is false when the var is unset, unparsable, +// zero, or large enough to overflow the MiB→bytes conversion. +func envSizeBytesExplicit(envVar string) (uint64, bool) { v := os.Getenv(envVar) if v == "" { - return def + return 0, false } mb, err := strconv.ParseUint(v, 10, 64) if err != nil || mb == 0 || mb > (1<<63)>>20 { - return def + return 0, false } - return mb << 20 + return mb << 20, true +} + +// envSizeBytes is envSizeBytesExplicit with a fallback default. +func envSizeBytes(envVar string, def uint64) uint64 { + if n, ok := envSizeBytesExplicit(envVar); ok { + return n + } + return def +} + +// maxPayloadCompressionRatio scales the automatic decoded-payload +// budget with the size of the envelope file itself. The budget's job +// is to bound how much disk a hostile envelope can consume at extract +// relative to what was actually stored; a fixed cap can't do that job +// without also refusing legitimate large files (a whale c1z's payload +// decodes to well past any constant that is still meaningful against +// bombs). Real Pebble payloads compress ~2-5x under zstd, so 100x is +// an order of magnitude of headroom while still capping a bomb at +// 100 bytes of output per byte of input. +const maxPayloadCompressionRatio = 100 + +// payloadBudgetForFileSize resolves the decoded-payload budget for an +// envelope of fileSize bytes when the caller configured nothing +// explicit: the env var when set, otherwise the LARGER of the flat +// default and fileSize × maxPayloadCompressionRatio. This is what +// keeps a legitimately huge envelope openable with default settings — +// the flat default alone would reject any file whose raw payload +// exceeds it, even though the file was written by us moments earlier. +func payloadBudgetForFileSize(fileSize int64) uint64 { + if n, ok := envSizeBytesExplicit(maxDecodedSizeEnvVar); ok { + return n + } + budget := defaultMaxDecodedPayloadBytes + if fileSize > 0 { + scaled := uint64(fileSize) + if scaled > math.MaxUint64/maxPayloadCompressionRatio { + return math.MaxUint64 + } + scaled *= maxPayloadCompressionRatio + if scaled > budget { + budget = scaled + } + } + return budget } func maxDecodedPayloadBytes() uint64 { @@ -94,9 +138,14 @@ func decoderMaxMemoryBytes() uint64 { type payloadOptions struct { maxDecodedPayloadBytes uint64 - maxDecoderMemoryBytes uint64 - disableSizeFailFast bool - pool *DecoderPool + // budgetExplicit is true when the decoded-payload budget came from + // the caller (WithMaxDecodedPayloadBytes) rather than defaults; + // only non-explicit budgets are rescaled by the envelope file size + // (see payloadBudgetForFileSize). + budgetExplicit bool + maxDecoderMemoryBytes uint64 + disableSizeFailFast bool + pool *DecoderPool } type PayloadOption func(*payloadOptions) @@ -107,6 +156,7 @@ type PayloadOption func(*payloadOptions) func WithMaxDecodedPayloadBytes(n uint64) PayloadOption { return func(o *payloadOptions) { o.maxDecodedPayloadBytes = n + o.budgetExplicit = n > 0 } } @@ -493,6 +543,15 @@ func readEnvelope(r io.Reader, headerOnly bool, pool *DecoderPool) (*Envelope, e return nil, err } // 4. Payload. The reader is positioned at the first payload byte. + // When the reader is a real file, scale the decoded-byte budget + // with its size (see payloadBudgetForFileSize); a non-stat-able + // stream falls back to the flat default. + budget := maxDecodedPayloadBytes() + if st, ok := r.(interface{ Stat() (os.FileInfo, error) }); ok { + if fi, err := st.Stat(); err == nil { + budget = payloadBudgetForFileSize(fi.Size()) + } + } env := &Envelope{Manifest: m} switch m.GetPayloadEncoding() { case c1zv3.PayloadEncoding_PAYLOAD_ENCODING_TAR_ZSTD: @@ -502,9 +561,9 @@ func readEnvelope(r io.Reader, headerOnly bool, pool *DecoderPool) (*Envelope, e } env.zstdReader = zr env.pool = pool - env.PayloadReader = &limitedPayloadReader{r: zr, limit: maxDecodedPayloadBytes()} + env.PayloadReader = &limitedPayloadReader{r: zr, limit: budget} case c1zv3.PayloadEncoding_PAYLOAD_ENCODING_TAR: - env.PayloadReader = &limitedPayloadReader{r: r, limit: maxDecodedPayloadBytes()} + env.PayloadReader = &limitedPayloadReader{r: r, limit: budget} case c1zv3.PayloadEncoding_PAYLOAD_ENCODING_INDEXED_ZSTD: // Indexed payloads are not a tar stream; extraction goes // through ExtractEnvelopePayload (random access over the @@ -749,7 +808,7 @@ func writeTar(w io.Writer, dir string) error { // writer worker pool; workers perform the per-file open/write/close // syscalls in parallel. Larger entries are streamed straight to disk // on this goroutine so a hostile archive full of multi-GiB entries -// can't drive memory to extractWorkerCount × maxTarEntryBytes. Memory +// can't drive memory up with the worker fan-out. Memory // peak is bounded by (extractWorkerCount + channel buffer) × // inlineCopyThresholdBytes; at Pebble's typical 2 MiB FlushSplitBytes // nearly every entry takes the parallel path — the per-entry @@ -833,8 +892,13 @@ entryLoop: break entryLoop } case tar.TypeReg: - if hdr.Size < 0 || hdr.Size > maxTarEntryBytes { - readErr = fmt.Errorf("c1z v3: tar entry %q size %d exceeds cap %d", hdr.Name, hdr.Size, maxTarEntryBytes) + // No per-entry size cap: a single Pebble SST can legitimately + // exceed any fixed bound. Aggregate extraction is bounded by + // the caller's decoded-byte budget (limitedPayloadReader), and + // memory by inlineCopyThresholdBytes — larger entries stream + // straight to disk. + if hdr.Size < 0 { + readErr = fmt.Errorf("c1z v3: tar entry %q has negative size %d", hdr.Name, hdr.Size) break entryLoop } if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/format/v3/indexed.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/format/v3/indexed.go index 6063c38e..1d051df4 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/format/v3/indexed.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/format/v3/indexed.go @@ -464,12 +464,27 @@ func readIndexedTrailer(f *os.File, payloadStart int64) (*c1zv3.IndexedFrameInde // positive: WithZeroFrames guarantees even an empty file encodes // to a complete zstd frame, so a zero-length frame range can // only come from a corrupt or hand-mangled index. + // + // There is deliberately NO upper bound on either size: a single + // Pebble SST can legitimately exceed any fixed cap (whale-scale + // grants buckets are written as one whole-bucket SST), and the + // bomb protections live elsewhere — compSize is bounds-checked + // against the real file layout just below, and total decoded + // output is enforced by the extraction budget + // (BATON_DECODER_MAX_DECODED_SIZE_MB) regardless of what + // raw_size claims. rawSize, compSize := e.GetRawSize(), e.GetCompressedSize() - if rawSize < 0 || rawSize > maxTarEntryBytes || compSize <= 0 || compSize > maxTarEntryBytes { + if rawSize < 0 || compSize <= 0 { return nil, nil, fmt.Errorf("c1z v3: trailer index entry %q sizes out of range (raw=%d comp=%d)", name, rawSize, compSize) } + // Overflow-safe form of off+compSize > indexOff: with no upper + // bound on compSize, the addition could wrap negative for a + // hostile compressed_size near MaxInt64 and slip past the + // comparison. Subtraction can't wrap here (off >= payloadStart + // >= 0 and indexOff fits the file), and when off > indexOff the + // negative difference rejects too, as it must. off := e.GetFrameOffset() - if off < payloadStart || off+compSize > indexOff { + if off < payloadStart || compSize > indexOff-off { return nil, nil, fmt.Errorf("c1z v3: trailer index entry %q frame range out of bounds (off=%d comp=%d)", name, off, compSize) } if len(e.GetRawSha256()) != sha256.Size { @@ -665,6 +680,17 @@ func extractOneFrame(f *os.File, e *ReuseEntry, dec *zstd.Decoder, budget *decod // owned by the caller. func ExtractEnvelopePayload(f *os.File, destDir string, opts ...PayloadOption) (*c1zv3.C1ZManifestV3, *PayloadReuse, error) { cfg := resolvePayloadOptions(opts...) + // With no explicit budget from the caller, scale the decoded-byte + // budget with the envelope's own size: the flat default would + // refuse any legitimately large file (one WE wrote), while the + // scaled budget still bounds a hostile envelope's disk consumption + // proportionally to its actual size. The env var, when set, wins + // inside payloadBudgetForFileSize. + if !cfg.budgetExplicit { + if st, err := f.Stat(); err == nil { + cfg.maxDecodedPayloadBytes = payloadBudgetForFileSize(st.Size()) + } + } if _, err := f.Seek(0, io.SeekStart); err != nil { return nil, nil, err } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/grant_store.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/grant_store.go deleted file mode 100644 index 9aad68b8..00000000 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/grant_store.go +++ /dev/null @@ -1,19 +0,0 @@ -package dotc1z - -import "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" - -// The grant-store contract lives in pkg/dotc1z/c1zstore so storage engines -// can implement it without importing this package. These aliases preserve -// the historical dotc1z names. - -// GrantStore is the grant-specific slice of C1ZStore. See -// c1zstore.GrantStore for the full contract. -type GrantStore = c1zstore.GrantStore - -// PendingExpansion is a lightweight row yielded by -// GrantStore.PendingExpansion. See c1zstore.PendingExpansion. -type PendingExpansion = c1zstore.PendingExpansion - -// GrantAnnotation is a row yielded by GrantStore.ListWithAnnotations. See -// c1zstore.GrantAnnotation. -type GrantAnnotation = c1zstore.GrantAnnotation diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/pebble_store.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/pebble_store.go index 72789c26..f30f25b0 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/pebble_store.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/pebble_store.go @@ -18,6 +18,7 @@ import ( v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble" formatv3 "github.com/conductorone/baton-sdk/pkg/dotc1z/format/v3" batonGrant "github.com/conductorone/baton-sdk/pkg/types/grant" @@ -26,7 +27,7 @@ import ( // pebbleDriver is the EngineDriver for the Pebble v3 engine. type pebbleDriver struct{} -var _ C1ZStore = (*pebbleStore)(nil) +var _ c1zstore.Store = (*pebbleStore)(nil) var _ connectorstore.Writer = (*pebbleStore)(nil) // Local mirrors of the optional capabilities the c1z sanitizer probes on @@ -41,7 +42,7 @@ type sanitizeSupportsDiffWriter interface { SetSupportsDiff(ctx context.Context, syncID string) error } type sanitizeSyncRunMetadataReader interface { - ListSyncRuns(ctx context.Context, pageToken string, pageSize uint32) ([]*SyncRun, string, error) + ListSyncRuns(ctx context.Context, pageToken string, pageSize uint32) ([]*c1zstore.SyncRun, string, error) } var ( @@ -53,10 +54,10 @@ var ( _ sanitizeSyncRunMetadataReader = (*C1File)(nil) ) -func (pebbleDriver) Engine() Engine { return EnginePebble } -func (pebbleDriver) Format() C1ZFormat { return C1ZFormatV3 } +func (pebbleDriver) Engine() c1zstore.Engine { return c1zstore.EnginePebble } +func (pebbleDriver) Format() C1ZFormat { return C1ZFormatV3 } -func (pebbleDriver) OpenStore(ctx context.Context, outputFilePath string, opts StoreOptions) (C1ZStore, error) { +func (pebbleDriver) OpenStore(ctx context.Context, outputFilePath string, opts StoreOptions) (c1zstore.Store, error) { tmpDir, err := os.MkdirTemp(opts.TmpDir, "c1z-pebble") if err != nil { return nil, err @@ -102,7 +103,7 @@ func (pebbleDriver) OpenStore(ctx context.Context, outputFilePath string, opts S return nil, cleanupOnError(err) } encoding := opts.PayloadEncoding - if encoding == PayloadEncodingUnspecified { + if encoding == c1zstore.PayloadEncodingUnspecified { encoding = fileEncoding } @@ -143,32 +144,32 @@ func unpackExistingPebbleC1Z( maxDecodedPayloadBytes uint64, maxDecoderMemoryBytes uint64, pool *EnvelopeDecoderPool, -) (*formatv3.PayloadReuse, PayloadEncoding, int64, error) { +) (*formatv3.PayloadReuse, c1zstore.PayloadEncoding, int64, error) { stat, err := os.Stat(outputFilePath) switch { case errors.Is(err, os.ErrNotExist): - return nil, PayloadEncodingUnspecified, 0, nil + return nil, c1zstore.PayloadEncodingUnspecified, 0, nil case err != nil: - return nil, PayloadEncodingUnspecified, 0, err + return nil, c1zstore.PayloadEncodingUnspecified, 0, err case stat.Size() == 0: - return nil, PayloadEncodingUnspecified, 0, nil + return nil, c1zstore.PayloadEncodingUnspecified, 0, nil } f, err := os.Open(outputFilePath) if err != nil { - return nil, PayloadEncodingUnspecified, 0, err + return nil, c1zstore.PayloadEncodingUnspecified, 0, err } defer f.Close() header, err := formatv3.ReadManifestHeader(f) if err != nil { - return nil, PayloadEncodingUnspecified, 0, err + return nil, c1zstore.PayloadEncodingUnspecified, 0, err } - if e := Engine(header.GetEngine()); e != EnginePebble && e != PebbleManifestEngine && e != PebbleManifestEngineV2 { - return nil, PayloadEncodingUnspecified, 0, fmt.Errorf("%w: %s", pebble.ErrUnknownEngine, header.GetEngine()) + if e := c1zstore.Engine(header.GetEngine()); e != c1zstore.EnginePebble && e != c1zstore.PebbleManifestEngine && e != c1zstore.PebbleManifestEngineV2 { + return nil, c1zstore.PayloadEncodingUnspecified, 0, fmt.Errorf("%w: %s", pebble.ErrUnknownEngine, header.GetEngine()) } if err := os.MkdirAll(dbDir, 0o755); err != nil { - return nil, PayloadEncodingUnspecified, 0, err + return nil, c1zstore.PayloadEncodingUnspecified, 0, err } manifest, reuse, err := formatv3.ExtractEnvelopePayload(f, dbDir, formatv3.WithMaxDecodedPayloadBytes(maxDecodedPayloadBytes), @@ -176,7 +177,7 @@ func unpackExistingPebbleC1Z( formatv3.WithPayloadDecoderPool(pool), ) if err != nil { - return nil, PayloadEncodingUnspecified, 0, err + return nil, c1zstore.PayloadEncodingUnspecified, 0, err } // fold_dead_bytes is inherited from the source file so the waste // accounting survives arbitrary open/save cycles, not just fold @@ -184,18 +185,18 @@ func unpackExistingPebbleC1Z( return reuse, payloadEncodingFromProto(manifest.GetPayloadEncoding()), header.GetFoldDeadBytes(), nil } -func payloadEncodingFromProto(enc c1zv3.PayloadEncoding) PayloadEncoding { +func payloadEncodingFromProto(enc c1zv3.PayloadEncoding) c1zstore.PayloadEncoding { switch enc { case c1zv3.PayloadEncoding_PAYLOAD_ENCODING_TAR: - return PayloadEncodingTar + return c1zstore.PayloadEncodingTar case c1zv3.PayloadEncoding_PAYLOAD_ENCODING_INDEXED_ZSTD: - return PayloadEncodingIndexedZstd + return c1zstore.PayloadEncodingIndexedZstd case c1zv3.PayloadEncoding_PAYLOAD_ENCODING_TAR_ZSTD: - return PayloadEncodingTarZstd + return c1zstore.PayloadEncodingTarZstd case c1zv3.PayloadEncoding_PAYLOAD_ENCODING_UNSPECIFIED: - return PayloadEncodingUnspecified + return c1zstore.PayloadEncodingUnspecified default: - return PayloadEncodingUnspecified + return c1zstore.PayloadEncodingUnspecified } } @@ -205,7 +206,7 @@ type pebbleStore struct { outputFilePath string tmpDir string readOnly bool - payloadEncoding PayloadEncoding + payloadEncoding c1zstore.PayloadEncoding payloadReuse *formatv3.PayloadReuse // foldDeadBytes is the cumulative fold-waste counter carried in // the envelope manifest (C1ZManifestV3.fold_dead_bytes): seeded @@ -233,7 +234,7 @@ type pebbleStore struct { // Close(ctx) signature. Lets callers route Pebble stores through // pkg/sync.NewSyncer's WithConnectorStore option the same way they // route SQLite *C1File handles today. -var _ C1ZStore = (*pebbleStore)(nil) +var _ c1zstore.Store = (*pebbleStore)(nil) // FileOps overrides the Adapter-level FileOps for two reasons: // @@ -244,7 +245,7 @@ var _ C1ZStore = (*pebbleStore)(nil) // flip the dirty bit — without it, Close would skip the envelope // save and the diff sync would exist only in the discarded temp // directory. -func (s *pebbleStore) FileOps() FileOps { +func (s *pebbleStore) FileOps() c1zstore.FileOps { return pebbleStoreFileOps{inner: s.FileOpsWithEncoding(s.payloadEncoding), store: s} } @@ -253,15 +254,15 @@ func (s *pebbleStore) FileOps() FileOps { // dirty-marking path. CloneSync writes a separate file and passes // through unchanged. type pebbleStoreFileOps struct { - inner FileOps + inner c1zstore.FileOps store *pebbleStore } -func (f pebbleStoreFileOps) CloneSync(ctx context.Context, outPath string, syncID string, opts ...CloneSyncOption) error { +func (f pebbleStoreFileOps) CloneSync(ctx context.Context, outPath string, syncID string, opts ...c1zstore.CloneSyncOption) error { return f.inner.CloneSync(ctx, outPath, syncID, opts...) } -func (f pebbleStoreFileOps) CopyIsolateSync(ctx context.Context, outPath string, syncID string, opts ...CloneSyncOption) error { +func (f pebbleStoreFileOps) CopyIsolateSync(ctx context.Context, outPath string, syncID string, opts ...c1zstore.CloneSyncOption) error { return f.inner.CopyIsolateSync(ctx, outPath, syncID, opts...) } @@ -284,8 +285,8 @@ func (f pebbleStoreFileOps) GenerateSyncDiff(ctx context.Context, baseSyncID, ap func (s *pebbleStore) Metadata() connectorstore.StoreMetadata { md := s.Adapter.Metadata() enc := s.payloadEncoding - if enc == PayloadEncodingUnspecified { - enc = PayloadEncodingIndexedZstd + if enc == c1zstore.PayloadEncodingUnspecified { + enc = c1zstore.PayloadEncodingIndexedZstd } md.PayloadEncoding = enc.String() return md @@ -506,7 +507,7 @@ func (s *pebbleStore) DeleteGrantByRefs(ctx context.Context, grant *v2.Grant) er // routes StoreExpandedGrants through the pebbleStore's dirty-marking // path. The Adapter-level wrapper calls Adapter.PutGrants directly, // which skips the dirty flag. -func (s *pebbleStore) Grants() GrantStore { +func (s *pebbleStore) Grants() c1zstore.GrantStore { return pebbleStoreGrants{inner: s.Adapter.Grants(), store: s} } @@ -514,7 +515,7 @@ func (s *pebbleStore) Grants() GrantStore { // only StoreExpandedGrants (the lone mutating method) to flip the // dirty bit. Read-only methods pass through. type pebbleStoreGrants struct { - inner GrantStore + inner c1zstore.GrantStore store *pebbleStore } @@ -641,25 +642,25 @@ func newPebbleStoreExpandedGrant(dest *v2.Entitlement, principal *v2.Resource, s }.Build(), nil } -func (g pebbleStoreGrants) PendingExpansionPage(ctx context.Context, pageToken string) ([]PendingExpansion, string, error) { +func (g pebbleStoreGrants) PendingExpansionPage(ctx context.Context, pageToken string) ([]c1zstore.PendingExpansion, string, error) { return g.inner.PendingExpansionPage(ctx, pageToken) } -func (g pebbleStoreGrants) PendingExpansion(ctx context.Context) iter.Seq2[PendingExpansion, error] { +func (g pebbleStoreGrants) PendingExpansion(ctx context.Context) iter.Seq2[c1zstore.PendingExpansion, error] { return g.inner.PendingExpansion(ctx) } -func (g pebbleStoreGrants) ListWithAnnotationsPage(ctx context.Context, pageToken string) ([]GrantAnnotation, string, error) { +func (g pebbleStoreGrants) ListWithAnnotationsPage(ctx context.Context, pageToken string) ([]c1zstore.GrantAnnotation, string, error) { return g.inner.ListWithAnnotationsPage(ctx, pageToken) } func (g pebbleStoreGrants) ListWithAnnotationsForResourcePage( ctx context.Context, resource *v2.Resource, syncID string, pageToken string, pageSize uint32, -) ([]GrantAnnotation, string, error) { +) ([]c1zstore.GrantAnnotation, string, error) { return g.inner.ListWithAnnotationsForResourcePage(ctx, resource, syncID, pageToken, pageSize) } -func (g pebbleStoreGrants) ListWithAnnotations(ctx context.Context) iter.Seq2[GrantAnnotation, error] { +func (g pebbleStoreGrants) ListWithAnnotations(ctx context.Context) iter.Seq2[c1zstore.GrantAnnotation, error] { return g.inner.ListWithAnnotations(ctx) } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sql_helpers.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sql_helpers.go index 301983e3..b4da1e81 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sql_helpers.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sql_helpers.go @@ -574,7 +574,7 @@ func (c *C1File) getResourceObject(ctx context.Context, resourceID *v2.ResourceI case c.viewSyncID != "": q = q.Where(goqu.C("sync_id").Eq(c.viewSyncID)) default: - var latestSyncRun *SyncRun + var latestSyncRun *c1zstore.SyncRun var err error latestSyncRun, err = c.getFinishedSync(ctx, 0, connectorstore.SyncTypeFull) if err != nil { @@ -634,7 +634,7 @@ func (c *C1File) getConnectorObject(ctx context.Context, tableName string, id st case c.viewSyncID != "": q = q.Where(goqu.C("sync_id").Eq(c.viewSyncID)) default: - var latestSyncRun *SyncRun + var latestSyncRun *c1zstore.SyncRun var err error latestSyncRun, err = c.getFinishedSync(ctx, 0, connectorstore.SyncTypeAny) if err != nil { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/store.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/store.go index 7d12cb5e..1d7ee54d 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/store.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/store.go @@ -4,33 +4,16 @@ import ( "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" ) -// C1ZStore is the internal contract used by the sync pipeline, compactor, and -// related infrastructure to read and write a .c1z file. The interface lives -// in pkg/dotc1z/c1zstore (as c1zstore.Store) so storage engines can -// implement it without importing this package; this alias preserves the -// historical dotc1z name. -// -// Implementations: -// -// - *C1File — the original SQLite-backed implementation -// (pkg/dotc1z/c1file.go). -// - *pebbleStore — the Pebble v3 engine implementation opened via -// NewStore(WithEngine(EnginePebble)) (pkg/dotc1z/pebble_store.go). -// -// Both engines are registered statically; no extra imports are needed to -// open either format. -type C1ZStore = c1zstore.Store - -// AsSQLiteStore type-asserts a C1ZStore to the concrete *C1File. It is an +// AsSQLiteStore type-asserts a c1zstore.Store to the concrete *C1File. It is an // escape hatch for callers that legitimately need SQLite-specific primitives // (today: the attached compactor in pkg/synccompactor/attached, which uses // SQL ATTACH for cross-file merge). Returns (nil, false) when the store is // not backed by *C1File OR when the underlying *C1File is nil. // // Avoid using this outside pkg/synccompactor. If you find yourself reaching -// for it, prefer adding a named method to C1ZStore that expresses what you +// for it, prefer adding a named method to c1zstore.Store that expresses what you // need; sqlite-specific leak-through is a smell. See RFC 0002 §4.4. -func AsSQLiteStore(s C1ZStore) (*C1File, bool) { +func AsSQLiteStore(s c1zstore.Store) (*C1File, bool) { cf, ok := s.(*C1File) if !ok || cf == nil { return nil, false diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sync_meta.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sync_meta.go deleted file mode 100644 index bea6f88a..00000000 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sync_meta.go +++ /dev/null @@ -1,14 +0,0 @@ -package dotc1z - -import "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" - -// The sync-metadata contract lives in pkg/dotc1z/c1zstore so storage -// engines can implement it without importing this package. These aliases -// preserve the historical dotc1z names. - -// SyncMeta is the sync-run-metadata sub-store of C1ZStore. See -// c1zstore.SyncMeta for the full contract. -type SyncMeta = c1zstore.SyncMeta - -// SyncRun is the exported shape of a sync run. See c1zstore.SyncRun. -type SyncRun = c1zstore.SyncRun diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sync_runs.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sync_runs.go index afdbbac1..9171ac44 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sync_runs.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sync_runs.go @@ -154,7 +154,7 @@ func (r *syncRunsTable) Migrations(ctx context.Context, db *goqu.Database) (bool // getCachedViewSyncRun returns the cached sync run for read operations. // This avoids N+1 queries when paginating through listConnectorObjects. // The cache is invalidated when a sync starts or ends. -func (c *C1File) getCachedViewSyncRun(ctx context.Context) (*SyncRun, error) { +func (c *C1File) getCachedViewSyncRun(ctx context.Context) (*c1zstore.SyncRun, error) { ctx, span := tracer.Start(ctx, "C1File.getCachedViewSyncRun") var err error defer func() { uotel.EndSpanWithError(span, err) }() @@ -188,7 +188,7 @@ func (c *C1File) invalidateCachedViewSyncRun() { c.cachedViewSyncErr = nil } -func (c *C1File) getLatestUnfinishedSync(ctx context.Context, syncType connectorstore.SyncType) (*SyncRun, error) { +func (c *C1File) getLatestUnfinishedSync(ctx context.Context, syncType connectorstore.SyncType) (*c1zstore.SyncRun, error) { ctx, span := tracer.Start(ctx, "C1File.getLatestUnfinishedSync") var err error defer func() { uotel.EndSpanWithError(span, err) }() @@ -200,7 +200,7 @@ func (c *C1File) getLatestUnfinishedSync(ctx context.Context, syncType connector // Don't resume syncs that started over a week ago oneWeekAgo := time.Now().AddDate(0, 0, -7) - ret := &SyncRun{} + ret := &c1zstore.SyncRun{} q := c.db.From(syncRuns.Name()) q = q.Select("sync_id", "started_at", "ended_at", "sync_token", "sync_type", "parent_sync_id", "linked_sync_id", "supports_diff", "stats") q = q.Where(goqu.C("ended_at").IsNull()) @@ -231,7 +231,7 @@ func (c *C1File) getLatestUnfinishedSync(ctx context.Context, syncType connector return ret, nil } -func (c *C1File) getFinishedSync(ctx context.Context, offset uint, syncType connectorstore.SyncType) (*SyncRun, error) { +func (c *C1File) getFinishedSync(ctx context.Context, offset uint, syncType connectorstore.SyncType) (*c1zstore.SyncRun, error) { ctx, span := tracer.Start(ctx, "C1File.getFinishedSync") var err error defer func() { uotel.EndSpanWithError(span, err) }() @@ -246,7 +246,7 @@ func (c *C1File) getFinishedSync(ctx context.Context, offset uint, syncType conn return nil, status.Errorf(codes.InvalidArgument, "invalid sync type: %s", syncType) } - ret := &SyncRun{} + ret := &c1zstore.SyncRun{} q := c.db.From(syncRuns.Name()) q = q.Select("sync_id", "started_at", "ended_at", "sync_token", "sync_type", "parent_sync_id", "linked_sync_id", "supports_diff", "stats") q = q.Where(goqu.C("ended_at").IsNotNull()) @@ -301,7 +301,7 @@ func parseStats(ctx context.Context, statsBytes *[]byte) *reader_v2.SyncStats { return ret } -func (c *C1File) ListSyncRuns(ctx context.Context, pageToken string, pageSize uint32) ([]*SyncRun, string, error) { +func (c *C1File) ListSyncRuns(ctx context.Context, pageToken string, pageSize uint32) ([]*c1zstore.SyncRun, string, error) { ctx, span := tracer.Start(ctx, "C1File.ListSyncRuns") var err error defer func() { uotel.EndSpanWithError(span, err) }() @@ -325,7 +325,7 @@ func (c *C1File) ListSyncRuns(ctx context.Context, pageToken string, pageSize ui q = q.Order(goqu.C("id").Asc()) q = q.Limit(uint(pageSize + 1)) - var ret []*SyncRun + var ret []*c1zstore.SyncRun query, args, err := q.ToSQL() if err != nil { @@ -347,7 +347,7 @@ func (c *C1File) ListSyncRuns(ctx context.Context, pageToken string, pageSize ui } statsBytes := &[]byte{} rowId := 0 - data := &SyncRun{} + data := &c1zstore.SyncRun{} err := rows.Scan(&rowId, &data.ID, &data.StartedAt, &data.EndedAt, &data.SyncToken, &data.Type, &data.ParentSyncID, &data.LinkedSyncID, &data.SupportsDiff, &statsBytes) if err != nil { return nil, "", err @@ -432,7 +432,7 @@ func (c *C1File) LatestFinishedSyncID(ctx context.Context, syncType connectorsto return s.ID, nil } -func (c *C1File) getSync(ctx context.Context, syncID string) (*SyncRun, error) { +func (c *C1File) getSync(ctx context.Context, syncID string) (*c1zstore.SyncRun, error) { ctx, span := tracer.Start(ctx, "C1File.getSync") var err error defer func() { uotel.EndSpanWithError(span, err) }() @@ -442,7 +442,7 @@ func (c *C1File) getSync(ctx context.Context, syncID string) (*SyncRun, error) { return nil, err } - ret := &SyncRun{} + ret := &c1zstore.SyncRun{} q := c.db.From(syncRuns.Name()) q = q.Select("sync_id", "started_at", "ended_at", "sync_token", "sync_type", "parent_sync_id", "linked_sync_id", "supports_diff", "stats") @@ -464,7 +464,7 @@ func (c *C1File) getSync(ctx context.Context, syncID string) (*SyncRun, error) { return ret, nil } -func (c *C1File) getCurrentSync(ctx context.Context) (*SyncRun, error) { +func (c *C1File) getCurrentSync(ctx context.Context) (*c1zstore.SyncRun, error) { ctx, span := tracer.Start(ctx, "C1File.getCurrentSync") var err error defer func() { uotel.EndSpanWithError(span, err) }() @@ -885,7 +885,7 @@ func (c *C1File) Cleanup(ctx context.Context) error { return err } - var candidates []SyncRun + var candidates []c1zstore.SyncRun pageToken := "" for { runs, nextPageToken, err := c.ListSyncRuns(ctx, pageToken, 100) diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/to_pebble.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/to_pebble.go index c97d6240..0b251e3e 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/to_pebble.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/to_pebble.go @@ -22,6 +22,7 @@ import ( v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble" "github.com/conductorone/baton-sdk/pkg/uotel" ) @@ -168,7 +169,7 @@ func (c *C1File) ToPebble(ctx context.Context, outPath string, syncID string, op start := time.Now() l := ctxzap.Extract(ctx) - dest, err := NewStore(ctx, outPath, WithEngine(EnginePebble), WithTmpDir(cfg.tmpDir)) + dest, err := NewStore(ctx, outPath, WithEngine(c1zstore.EnginePebble), WithTmpDir(cfg.tmpDir)) if err != nil { return nil, fmt.Errorf("to-pebble: open destination: %w", err) } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/field/defaults.go b/vendor/github.com/conductorone/baton-sdk/pkg/field/defaults.go index a3271a5c..40c4e551 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/field/defaults.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/field/defaults.go @@ -264,9 +264,16 @@ var ( WithDescription("The path to the c1z file to sync external baton resources with"), WithPersistent(true), WithExportTarget(ExportTargetNone)) + // PreviousSyncC1ZField is hidden by default: source-cache replay is + // author-opt-in functionality (connectorrunner.WithKeepPreviousSyncC1Z), + // and non-replaying connectors shouldn't advertise it to operators. + // DefineConfiguration unhides it for connectors that declare the + // capability; the flag still parses everywhere (hidden ≠ disabled) so + // expert/debug use keeps working. PreviousSyncC1ZField = StringField("previous-sync-c1z", WithDescription("The path to the previous sync c1z file to use as a source-cache replay input"), WithPersistent(true), + WithHidden(true), WithExportTarget(ExportTargetNone)) externalResourceEntitlementIdFilter = StringField("external-resource-entitlement-id-filter", WithDescription("The entitlement that external users, groups must have access to sync external baton resources"), @@ -280,11 +287,14 @@ var ( // connectors whose author also declared ETag-replay support at build // time (connectorrunner.WithKeepPreviousSyncC1Z) — both are // required. Costs one c1z of local disk. + // Hidden by default for the same reason as PreviousSyncC1ZField; unhidden + // when the connector author declares the replay capability. KeepPreviousSyncC1ZField = BoolField("keep-previous-sync-c1z", WithDescription("Keep the previously synced c1z on disk to enable ETag replay across service-mode syncs "+ "(requires a connector that supports ETag replay; costs one c1z of local disk)"), WithDefaultValue(false), WithPersistent(true), + WithHidden(true), WithExportTarget(ExportTargetNone)) LambdaServerClientIDField = StringField("lambda-client-id", WithRequired(true), WithDescription("The oauth client id to use with the configuration endpoint"), diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/field/fields.go b/vendor/github.com/conductorone/baton-sdk/pkg/field/fields.go index 1107861c..738d1ca6 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/field/fields.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/field/fields.go @@ -126,6 +126,34 @@ func (s SchemaField) ExportAs(et ExportTarget) SchemaField { return c } +// WithConnectorDefault returns a copy of a shared/default SDK field carrying +// a connector-specific default value. Include the copy in the connector's +// field.Configuration Fields — DefineConfiguration replaces the SDK's copy +// with it (same re-export mechanism as ExportAs), so --help, flag parsing, +// and exported config schemas all reflect the connector's default. +// +// Value-resolution precedence is unchanged and sentinel-free: an explicit +// flag beats the environment beats the config file beats this default (the +// default lives on the flag itself, so a user-supplied zero value remains +// distinguishable from "unset"). +// +// Example — a connector whose sync fans out well declares its own worker +// default without hiding the shared flag's semantics: +// +// field.NewConfiguration([]field.SchemaField{ +// field.WithConnectorDefault(field.WorkerCountField, 16), +// ... +// }) +// +// The value's type must match the field's variant (int for IntField, etc.); +// a mismatch fails loudly at startup when the flag is registered. +func WithConnectorDefault[T SchemaTypes](s SchemaField, defaultValue T) SchemaField { + c := s + c.DefaultValue = defaultValue + c.WasReExported = true + return c +} + // Go doesn't allow generic methods on a non-generic struct. func ValidateField[T SchemaTypes](s *SchemaField, value T) (bool, error) { return s.validate(value) diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/lambda/grpc/client.go b/vendor/github.com/conductorone/baton-sdk/pkg/lambda/grpc/client.go index 81206188..df25813c 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/lambda/grpc/client.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/lambda/grpc/client.go @@ -11,6 +11,8 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/lambda" "github.com/aws/aws-sdk-go-v2/service/lambda/types" + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" @@ -24,10 +26,18 @@ type lambdaTransport struct { } func (l *lambdaTransport) RoundTrip(ctx context.Context, req *Request) (*Response, error) { - payload, err := req.MarshalJSON() + payload, frameOnly, err := req.marshalPayload() if err != nil { return nil, fmt.Errorf("lambda_transport: failed to marshal frame: %w", err) } + if frameOnly != nil { + ctxzap.Extract(ctx).Warn( + "lambda_transport: request has no legacy encoding, sending v2 frame only; a connector on a pre-frame SDK cannot process this call", + zap.String("method", req.Method()), + zap.String("function_name", l.functionName), + zap.NamedError("legacy_encoding_error", frameOnly), + ) + } input := &lambda.InvokeInput{ LogType: types.LogTypeTail, diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/lambda/grpc/server.go b/vendor/github.com/conductorone/baton-sdk/pkg/lambda/grpc/server.go index a8746242..2a3da875 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/lambda/grpc/server.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/lambda/grpc/server.go @@ -236,7 +236,18 @@ func TimeoutForRequest(req *Request) (time.Duration, bool, error) { return 0, false, nil } +// Handler serves one transport request. The response echoes the request's +// wire version so v2 invokers get lossless frames and legacy invokers get +// protojson (see Response.MarshalJSON). func (s *Server) Handler(ctx context.Context, req *Request) (*Response, error) { + resp, err := s.handle(ctx, req) + if resp != nil { + resp.wireV2 = req.wireV2 + } + return resp, err +} + +func (s *Server) handle(ctx context.Context, req *Request) (*Response, error) { serviceName, methodName, err := parseMethod(req.Method()) if err != nil { return ErrorResponse(err), nil diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/lambda/grpc/transport.go b/vendor/github.com/conductorone/baton-sdk/pkg/lambda/grpc/transport.go index 28d1b9c0..7da6f05d 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/lambda/grpc/transport.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/lambda/grpc/transport.go @@ -20,10 +20,12 @@ import ( const annotationsFieldName = "annotations" /* -unmarshalTransportJSON unmarshals transport JSON into msg, discarding any -unknown fields. +unmarshalTransportJSON unmarshals transport JSON into msg. It reports whether +the payload carried a v2 wire frame (binary proto, see wireFrame), which is +decoded losslessly with no type resolution. -When the payload fails to unmarshal, it retries after filtering out any +Legacy payloads are protojson, unmarshaled discarding any unknown fields. +When a legacy payload fails to unmarshal, it retries after filtering out any annotations whose types are not known to the global registry. Annotation type skew happens frequently for new features and would otherwise require rolling every lambda function (and, in the response direction, would let an old @@ -39,7 +41,11 @@ payloads that already failed, where the alternative is a hard error. Our payloads are small relative to the work of the connector, so the performance impact is negligible. */ -func unmarshalTransportJSON(b []byte, msg proto.Message) error { +func unmarshalTransportJSON(b []byte, msg proto.Message) (bool, error) { + if ok, err := decodeWireFrame(b, msg); ok { + return true, err + } + unmarshalOptions := protojson.UnmarshalOptions{ DiscardUnknown: true, } @@ -47,19 +53,19 @@ func unmarshalTransportJSON(b []byte, msg proto.Message) error { // so any failure falls through to the annotation filter. originalErr := unmarshalOptions.Unmarshal(b, msg) if originalErr == nil { - return nil + return false, nil } filtered, changed := filterUnknownAnnotations(b) if !changed { - return originalErr + return false, originalErr } if err := unmarshalOptions.Unmarshal(filtered, msg); err != nil { - return errors.Join(originalErr, err) + return false, errors.Join(originalErr, err) } - return nil + return false, nil } // filterUnknownAnnotations recursively walks raw JSON and prunes entries from @@ -181,18 +187,61 @@ func filterAnnotationsArray(raw json.RawMessage) (json.RawMessage, bool) { type Request struct { msg *pbtransport.Request + + // wireV2 records that the request arrived as a v2 wire frame, proving + // the invoker can read one back. The server stamps it onto the Response. + wireV2 bool } -// UnmarshalJSON unmarshals the JSON into a Request, discarding unknown fields -// and filtering annotations with unresolvable types. See -// unmarshalTransportJSON. +// UnmarshalJSON unmarshals the JSON into a Request. v2 wire frames decode +// losslessly; legacy payloads are protojson, discarding unknown fields and +// filtering annotations with unresolvable types. See unmarshalTransportJSON. func (f *Request) UnmarshalJSON(b []byte) error { f.msg = &pbtransport.Request{} - return unmarshalTransportJSON(b, f.msg) + wireV2, err := unmarshalTransportJSON(b, f.msg) + if err != nil { + return err + } + f.wireV2 = wireV2 + return nil } +// MarshalJSON dual-encodes the request: the legacy protojson fields and the +// v2 wire frame share one JSON object, so legacy connectors keep working +// (they discard the unknown frame fields) while v2 connectors decode the +// frame and see annotations whose types this process cannot resolve. When no +// legacy view can be produced — protojson cannot represent an Any whose type +// is not linked into this process — the frame is sent alone: a legacy +// connector would have failed on that payload anyway. Oversized dual +// payloads fall back to legacy-only to stay under the Lambda invoke limit; +// v2 connectors accept those too. func (f *Request) MarshalJSON() ([]byte, error) { - return protojson.Marshal(f.msg) + payload, _, err := f.marshalPayload() + return payload, err +} + +// marshalPayload builds the invoke payload. The middle return reports the +// frame-only condition: when non-nil, the payload carries only the v2 frame +// and the value is the reason the legacy view could not be produced — +// callers with a context should surface it, since a legacy connector cannot +// process a frame-only payload. +func (f *Request) marshalPayload() ([]byte, error, error) { + legacy, legacyErr := protojson.Marshal(f.msg) + if legacyErr != nil { + payload, err := encodeWireFrame(f.msg) + if err != nil { + return nil, nil, errors.Join(legacyErr, err) + } + return payload, legacyErr, nil + } + dual, err := spliceWireFrame(legacy, f.msg) + if err != nil { + return nil, nil, err + } + if len(dual) > maxDualEncodedPayload { + return legacy, nil, nil + } + return dual, nil, nil } func (f *Request) Method() string { @@ -236,20 +285,40 @@ func NewRequest(method string, req proto.Message, headers metadata.MD) (*Request type Response struct { msg *pbtransport.Response + + // wireV2 selects the v2 wire frame encoding. The server sets it from + // the request: a frame in the request proves the invoker reads frames. + wireV2 bool } -// UnmarshalJSON unmarshals the JSON into a Response, discarding unknown -// fields and filtering annotations with unresolvable types. Responses carry -// annotations at the response level and nested inside rows (grants embed -// resources, etc.), so this protects an invoker from annotation types it -// does not know about — for example an older invoker receiving annotations -// from a connector built with a newer SDK. See unmarshalTransportJSON. +// UnmarshalJSON unmarshals the JSON into a Response. v2 wire frames decode +// losslessly with no type resolution. Legacy payloads are protojson, +// discarding unknown fields and filtering annotations with unresolvable +// types: responses carry annotations at the response level and nested inside +// rows (grants embed resources, etc.), so this protects an invoker from +// annotation types it does not know about — for example an older invoker +// receiving annotations from a connector built with a newer SDK. See +// unmarshalTransportJSON. func (f *Response) UnmarshalJSON(b []byte) error { f.msg = &pbtransport.Response{} - return unmarshalTransportJSON(b, f.msg) + wireV2, err := unmarshalTransportJSON(b, f.msg) + if err != nil { + return err + } + f.wireV2 = wireV2 + return nil } +// MarshalJSON encodes a v2 wire frame when the invoker proved it reads them +// (see wireV2), preserving annotations whose types this process cannot +// resolve. Legacy invokers get plain protojson, which fails on an Any whose +// type is not linked into this process — deliberately: the sender's registry +// is no authority on what the receiver understands or needs, so degrading +// the payload by silently dropping data is worse than failing loudly. func (f *Response) MarshalJSON() ([]byte, error) { + if f.wireV2 { + return encodeWireFrame(f.msg) + } return protojson.Marshal(f.msg) } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/lambda/grpc/wire.go b/vendor/github.com/conductorone/baton-sdk/pkg/lambda/grpc/wire.go new file mode 100644 index 00000000..8be74902 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/lambda/grpc/wire.go @@ -0,0 +1,88 @@ +package grpc + +import ( + "bytes" + "encoding/json" + "fmt" + + "google.golang.org/protobuf/proto" +) + +const transportWireVersion = 2 + +// maxDualEncodedPayload caps dual-encoded requests below the 6MiB Lambda +// invoke payload limit. Past it the frame is dropped and the request goes +// out legacy-only, which v2 peers also accept. +var maxDualEncodedPayload = 5 << 20 + +/* +wireFrame is the v2 transport encoding: the binary proto bytes of a +transport Request or Response, carried base64-encoded in the JSON Lambda +payload. Binary proto copies google.protobuf.Any payloads verbatim instead +of resolving their type URLs the way protojson must, so annotation types +that are not linked into a process survive the transport intact — the fix +for connector-specific annotations (e.g. baton-jira's CustomField) being +dropped or crashing the marshal in runtimes that don't register them. + +Version skew is handled without negotiation state: + + - Requests are dual-encoded: the legacy protojson fields and the frame + share one JSON object. Legacy peers unmarshal with DiscardUnknown and + never see the frame; v2 peers prefer it. + - Responses carry the frame alone, but only when the request carried + one — a frame in the request proves the invoker can read it. Legacy + requests get legacy responses. + +The field names cannot collide with the legacy encoding: protojson emits +only "method"/"req"/"headers" for Requests and +"resp"/"status"/"headers"/"trailers" for Responses. +*/ +type wireFrame struct { + V int `json:"v"` + Frame []byte `json:"frame"` +} + +// decodeWireFrame reports whether raw carries a v2 wire frame and, if so, +// decodes it into msg. A false return means raw is a legacy payload: either +// it isn't shaped like a frame, or it doesn't parse as JSON at all — the +// legacy path owns reporting that error. +func decodeWireFrame(raw []byte, msg proto.Message) (bool, error) { + var wf wireFrame + if err := json.Unmarshal(raw, &wf); err != nil || len(wf.Frame) == 0 { + return false, nil //nolint:nilerr // not a v2 frame; the legacy path owns error reporting + } + if wf.V != transportWireVersion { + return true, fmt.Errorf("transport: unsupported wire frame version %d", wf.V) + } + return true, proto.Unmarshal(wf.Frame, msg) +} + +func encodeWireFrame(msg proto.Message) ([]byte, error) { + frame, err := proto.Marshal(msg) + if err != nil { + return nil, err + } + return json.Marshal(wireFrame{V: transportWireVersion, Frame: frame}) +} + +// spliceWireFrame appends the v2 frame fields to a legacy protojson object, +// producing the dual-encoded request payload. +func spliceWireFrame(legacy []byte, msg proto.Message) ([]byte, error) { + suffix, err := encodeWireFrame(msg) + if err != nil { + return nil, err + } + legacy = bytes.TrimSpace(legacy) + if len(legacy) < 2 || legacy[0] != '{' || legacy[len(legacy)-1] != '}' { + return nil, fmt.Errorf("transport: legacy payload is not a JSON object") + } + if len(legacy) == 2 { + return suffix, nil + } + var buf bytes.Buffer + buf.Grow(len(legacy) + len(suffix)) + buf.Write(legacy[:len(legacy)-1]) + buf.WriteByte(',') + buf.Write(suffix[1:]) + return buf.Bytes(), nil +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sdk/version.go b/vendor/github.com/conductorone/baton-sdk/pkg/sdk/version.go index c3bf2e37..ff96f32a 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sdk/version.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sdk/version.go @@ -1,3 +1,3 @@ package sdk -const Version = "v0.17.0" +const Version = "v0.18.2" diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/continuation.go b/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/continuation.go new file mode 100644 index 00000000..78058fe2 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/continuation.go @@ -0,0 +1,232 @@ +package sourcecache + +// Lookup continuation (ask/answer): the lookup transport for connector +// runtimes that cannot call back to the syncer mid-request (single-shot +// request/response tunnels, e.g. gRPC-over-Lambda). The connector's first +// execution of a page ("phase 1") records the scopes it needs and fails +// with ErrLookupDeferred; the SDK converts that into a +// SourceCacheLookupAsk response; the syncer resolves the queries against +// its local previous-sync store and re-invokes the same request with +// SourceCacheLookupAnswers attached ("phase 2"), where the same connector +// code gets real answers. See docs/tasks/source-cache-lambda-lookup.md. + +import ( + "context" + "errors" + "fmt" + "sync" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" +) + +// ErrLookupDeferred is returned by a deferring Lookup (phase 1 of the +// ask/answer continuation) when the answer is not yet available. The SDK +// intercepts it and answers the RPC with a SourceCacheLookupAsk. +// +// Propagation contract for connectors: wrap with %w if you must, NEVER +// swallow. The SDK matches with errors.Is, so idiomatic wrapping +// (fmt.Errorf("listing members: %w", err)) is fine. A connector that +// logs-and-continues past this error re-asks for scopes it already +// "handled" and turns every warm sync into a hard failure at the bounce +// cap. +var ErrLookupDeferred = errors.New("source cache lookup deferred: answer arrives on re-invoke") + +// Query identifies one scope to resolve against the previous sync. +type Query struct { + RowKind RowKind + ScopeHash string +} + +// Answer resolves one Query. Found=false means the previous sync has no +// entry for the scope: fetch fresh. (Distinct from a query that got no +// Answer at all, which means unresolved: ask again.) +type Answer struct { + Query + Found bool + ETag string +} + +// BatchLookup is optionally implemented by Lookup implementations that can +// resolve many scopes in one round trip. Connectors should not type-assert +// for it directly; call LookupMany, which falls back to per-query lookups. +type BatchLookup interface { + LookupPreviousSourceCacheMany(ctx context.Context, queries []Query) ([]Answer, error) +} + +// LookupMany resolves a batch of queries through lookup, using one round +// trip when the implementation supports it (BatchLookup) and a loop of +// single lookups otherwise. This is the topology-uniform batch API: +// in-process and subprocess lookups loop over local/loopback point-reads; +// a deferring lookup collects the whole batch into ONE ask. +// +// The returned answers are exact and complete for the queried set — one +// Answer per Query, order preserved, explicit Found per entry. (A +// deferring lookup returns ErrLookupDeferred instead of answers; phase 2 +// then answers the same calls. Answers dropped to the transport size +// budget surface as ErrLookupDeferred again on the affected queries, never +// as silent omissions or false not-founds.) +func LookupMany(ctx context.Context, lookup Lookup, queries []Query) ([]Answer, error) { + if bl, ok := lookup.(BatchLookup); ok { + return bl.LookupPreviousSourceCacheMany(ctx, queries) + } + answers := make([]Answer, 0, len(queries)) + for _, q := range queries { + entry, found, err := lookup.LookupPreviousSourceCache(ctx, q.RowKind, q.ScopeHash) + if err != nil { + return nil, err + } + a := Answer{Query: q, Found: found} + if found { + a.ETag = entry.ETag + } + answers = append(answers, a) + } + return answers, nil +} + +// ContinuationLookup is the per-request Lookup installed for the +// ask/answer continuation. It serves lookups from the answers delivered on +// the request (phase 2) and defers everything else by recording the query +// and returning ErrLookupDeferred (phase 1, or an under-answered phase 2 — +// e.g. answers dropped to the size budget). +// +// It is constructed per RPC by the SDK; connectors only ever see it as +// SyncOpAttrs.SourceCache. +type ContinuationLookup struct { + mu sync.Mutex + answers map[Query]Answer + asked []Query + askedSet map[Query]struct{} +} + +var ( + _ Lookup = (*ContinuationLookup)(nil) + _ BatchLookup = (*ContinuationLookup)(nil) +) + +// NewContinuationLookup builds a ContinuationLookup pre-loaded with the +// request's answers (nil/empty on phase 1). +func NewContinuationLookup(answers []Answer) *ContinuationLookup { + m := make(map[Query]Answer, len(answers)) + for _, a := range answers { + m[a.Query] = a + } + return &ContinuationLookup{ + answers: m, + askedSet: map[Query]struct{}{}, + } +} + +func (c *ContinuationLookup) LookupPreviousSourceCache(_ context.Context, rowKind RowKind, scopeHash string) (Entry, bool, error) { + q := Query{RowKind: rowKind, ScopeHash: scopeHash} + c.mu.Lock() + defer c.mu.Unlock() + if a, ok := c.answers[q]; ok { + if !a.Found { + return Entry{}, false, nil + } + return Entry{ETag: a.ETag}, true, nil + } + c.recordLocked(q) + return Entry{}, false, fmt.Errorf("lookup %s/%s: %w", rowKind, scopeHash, ErrLookupDeferred) +} + +func (c *ContinuationLookup) LookupPreviousSourceCacheMany(_ context.Context, queries []Query) ([]Answer, error) { + c.mu.Lock() + defer c.mu.Unlock() + answers := make([]Answer, 0, len(queries)) + missing := 0 + for _, q := range queries { + a, ok := c.answers[q] + if !ok { + c.recordLocked(q) + missing++ + continue + } + answers = append(answers, a) + } + if missing > 0 { + // Defer the whole batch: phase 2 re-runs the same call with the + // full answer set (already-answered queries remain answered on the + // re-invoked request). + return nil, fmt.Errorf("batch lookup: %d of %d queries unresolved: %w", missing, len(queries), ErrLookupDeferred) + } + return answers, nil +} + +func (c *ContinuationLookup) recordLocked(q Query) { + if _, seen := c.askedSet[q]; seen { + return + } + c.askedSet[q] = struct{}{} + c.asked = append(c.asked, q) +} + +// Asked returns the queries recorded by deferred lookups, deduplicated, in +// first-ask order. Empty means no lookup deferred and the handler's result +// stands. +func (c *ContinuationLookup) Asked() []Query { + c.mu.Lock() + defer c.mu.Unlock() + out := make([]Query, len(c.asked)) + copy(out, c.asked) + return out +} + +// --- proto conversions (shared by connectorbuilder and the syncer) ------ + +// AskProto builds the SourceCacheLookupAsk annotation for a set of +// deferred queries. +func AskProto(queries []Query) *v2.SourceCacheLookupAsk { + qs := make([]*v2.SourceCacheLookupAsk_Query, 0, len(queries)) + for _, q := range queries { + qs = append(qs, v2.SourceCacheLookupAsk_Query_builder{ + RowKind: string(q.RowKind), + ScopeHash: q.ScopeHash, + }.Build()) + } + return v2.SourceCacheLookupAsk_builder{Queries: qs}.Build() +} + +// QueriesFromProto extracts and validates the queries of an ask. +func QueriesFromProto(ask *v2.SourceCacheLookupAsk) ([]Query, error) { + out := make([]Query, 0, len(ask.GetQueries())) + for _, q := range ask.GetQueries() { + kind := RowKind(q.GetRowKind()) + if err := ValidateRowKind(kind); err != nil { + return nil, err + } + if err := ValidateScopeHash(q.GetScopeHash()); err != nil { + return nil, err + } + out = append(out, Query{RowKind: kind, ScopeHash: q.GetScopeHash()}) + } + return out, nil +} + +// AnswersProto builds the SourceCacheLookupAnswers annotation. +func AnswersProto(answers []Answer) *v2.SourceCacheLookupAnswers { + as := make([]*v2.SourceCacheLookupAnswers_Answer, 0, len(answers)) + for _, a := range answers { + as = append(as, v2.SourceCacheLookupAnswers_Answer_builder{ + RowKind: string(a.RowKind), + ScopeHash: a.ScopeHash, + Found: a.Found, + Etag: a.ETag, + }.Build()) + } + return v2.SourceCacheLookupAnswers_builder{Answers: as}.Build() +} + +// AnswersFromProto extracts the answers delivered on a request. +func AnswersFromProto(msg *v2.SourceCacheLookupAnswers) []Answer { + out := make([]Answer, 0, len(msg.GetAnswers())) + for _, a := range msg.GetAnswers() { + out = append(out, Answer{ + Query: Query{RowKind: RowKind(a.GetRowKind()), ScopeHash: a.GetScopeHash()}, + Found: a.GetFound(), + ETag: a.GetEtag(), + }) + } + return out +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/sourcecache.go b/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/sourcecache.go index 6b40fdca..1892cda3 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/sourcecache.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/sourcecache.go @@ -15,10 +15,15 @@ // // Invariant that keeps replay safe: a connector must only emit // SourceCacheReplay for a scope whose validator it received from THIS sync's -// Lookup. When source cache is disabled or degraded (no capability, no -// usable previous sync, unsupported storage engine) the SDK installs -// NoopLookup, every lookup misses, and a well-behaved connector naturally -// falls back to full fetch. +// Lookup. The lookup need not happen in the same call that emits the +// replay: a planning call may batch-resolve many scopes and pass the +// verdicts to sibling cursors through SpawnCursors page tokens — that +// satisfies the invariant, because the validator still originates from the +// consuming sync. What's forbidden is a validator that outlives a sync +// (connector-side caches, config, upstream echoes). When source cache is +// disabled or degraded (no capability, no usable previous sync, unsupported +// storage engine) the SDK installs NoopLookup, every lookup misses, and a +// well-behaved connector naturally falls back to full fetch. // // Replay equivalence: a cached sync must reproduce what a full resync // would produce. Replayed rows are verbatim copies of the previous sync's diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/expander.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/expander.go index 1eb3a4cd..c54e25b4 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/expander.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/expander.go @@ -63,12 +63,12 @@ var ErrMaxDepthExceeded = errors.New("max depth exceeded") // ExpanderStore defines the minimal store interface needed for grant expansion. // Implementations: -// - *dotc1z.C1File (via dotc1z.C1ZStore) for production syncs +// - *dotc1z.C1File (via c1zstore.Store) for production syncs // - mocks for unit tests // // StoreExpandedGrants writes a batch of expanded grants back to storage, // preserving existing expansion metadata columns on the underlying rows. -// See dotc1z.GrantStore.StoreExpandedGrants for the full contract. +// See c1zstore.GrantStore.StoreExpandedGrants for the full contract. type ExpanderStore interface { GetEntitlement(ctx context.Context, req *reader_v2.EntitlementsReaderServiceGetEntitlementRequest) (*reader_v2.EntitlementsReaderServiceGetEntitlementResponse, error) ListGrantsForEntitlement(ctx context.Context, req *reader_v2.GrantsReaderServiceListGrantsForEntitlementRequest) (*reader_v2.GrantsReaderServiceListGrantsForEntitlementResponse, error) diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/progresslog/progresslog.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/progresslog/progresslog.go index 747e6036..51be5a36 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sync/progresslog/progresslog.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/progresslog/progresslog.go @@ -271,6 +271,17 @@ func (p *ProgressLog) SetGrantsCountOnly(resourceType string) { p.grantsCountOnly[resourceType] = true } +// GrantsProgress returns the current grant-coverage counter for a resource +// type. For per-resource types this is "resources covered" and must never +// exceed the type's resource total — spawned sibling cursors +// (v2.SpawnCursors) don't increment it, only the origin action's chain end +// does. Exposed for tests pinning that accounting. +func (p *ProgressLog) GrantsProgress(resourceType string) int { + p.mu.RLock() + defer p.mu.RUnlock() + return p.grantsProgress[resourceType] +} + func (p *ProgressLog) LogGrantsProgress(ctx context.Context, resourceType string) { var grantsProgress, resources int var lastLogTime time.Time diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/source_cache.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/source_cache.go index 8eccd160..7ae42348 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sync/source_cache.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/source_cache.go @@ -45,6 +45,14 @@ type syncerSourceCache struct { prev dotc1z.SourceCacheStore // prevReader is the same store as prev, typed for ReplaySourceCache. prevReader connectorstore.Reader + // lookup is the connector-facing lookup built from prev (NoopLookup + // when prev is nil). The syncer also uses it to answer lookup asks + // from connectors on single-shot transports (the ask/answer + // continuation); both paths see identical results by construction. + lookup sourcecache.Lookup + // contStats accumulates ask/answer continuation counters for the + // sync-complete log line. + contStats *continuationStats } // prevStoreLookup adapts the previous store's manifest to the @@ -122,6 +130,8 @@ func (s *syncer) configureSourceCache(ctx context.Context, resp *v2.ConnectorSer if s.sourceCache.prev != nil { lookup = prevStoreLookup{prev: s.sourceCache.prev, logOnce: &stdsync.Once{}} } + s.sourceCache.lookup = lookup + s.sourceCache.contStats = &continuationStats{} if canSetLookup { setLookup.SetSourceCache(ctx, lookup) } else { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/source_cache_continuation.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/source_cache_continuation.go new file mode 100644 index 00000000..76fb7276 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/source_cache_continuation.go @@ -0,0 +1,248 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// Syncer side of the source-cache lookup continuation (ask/answer). On +// single-shot transports (gRPC-over-Lambda) the connector cannot call the +// lookup service mid-request, so it answers a list RPC with a +// SourceCacheLookupAsk instead of rows; the syncer resolves the queries +// against its LOCAL previous-sync store — the same store replay copies +// from, so lookup and replay can never disagree — and re-invokes the same +// request with SourceCacheLookupAnswers attached. See +// docs/tasks/source-cache-lambda-lookup.md and the annotation contract in +// proto/c1/connector/v2/annotation_source_cache.proto. + +import ( + "context" + "fmt" + stdsync "sync" + + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "go.uber.org/zap" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/sourcecache" +) + +const ( + // sourceCacheBounceCap bounds consecutive asks for the SAME request + // (same page token; only the answers annotation differs between + // re-invokes). A connector that keeps asking without progressing is + // broken (most commonly: swallowing ErrLookupDeferred and re-asking + // for scopes it already "handled"), and silence would be the + // stale-data failure mode — so fail loudly. + // + // Deliberately NOT per action: a multi-page action that asks once per + // page (e.g. a delta planner pre-resolving each planning page's chunk + // scopes) bounces once per page with monotonic progress — every + // NextPageToken advance is a new request and resets the counter. + sourceCacheBounceCap = 4 + + // sourceCacheAnswerBudget caps the FOUND-etag payload attached to one + // re-invoke, keeping the request under single-shot transport payload + // limits (Lambda invokes cap at 6MB; the dual-encoded frame at 5MiB). + // Not-found answers are always complete for the queried set — only + // found answers with large etags are dropped, and a dropped answer is + // ABSENT (re-askable, subject to the cap), never a false not-found. + sourceCacheAnswerBudget = 2 << 20 +) + +// continuationStats accumulates ask/answer counters across a sync for the +// sync-complete log line and bounce-cap diagnostics. Per-op-kind bounce +// counts let a rollout review distinguish planner asks (expected, one per +// planning page) from per-row asks (a batching opportunity). +type continuationStats struct { + mu stdsync.Mutex + requests int // RPCs that bounced at least once + bounces int + bouncesByOp map[string]int + asked int + found int + notFound int + truncated int + capFailures int +} + +func (c *continuationStats) record(op string, bounces, asked, found, notFound, truncated int) { + if c == nil { + return + } + c.mu.Lock() + defer c.mu.Unlock() + if bounces > 0 { + c.requests++ + if c.bouncesByOp == nil { + c.bouncesByOp = map[string]int{} + } + c.bouncesByOp[op] += bounces + } + c.bounces += bounces + c.asked += asked + c.found += found + c.notFound += notFound + c.truncated += truncated +} + +func (c *continuationStats) recordCapFailure() { + if c == nil { + return + } + c.mu.Lock() + defer c.mu.Unlock() + c.capFailures++ +} + +// logTotals emits the continuation counters when any bounces happened. +func (c *continuationStats) logTotals(ctx context.Context) { + if c == nil { + return + } + c.mu.Lock() + defer c.mu.Unlock() + if c.bounces == 0 && c.capFailures == 0 { + return + } + fields := []zap.Field{ + zap.Int("requests_bounced", c.requests), + zap.Int("bounces", c.bounces), + zap.Int("scopes_asked", c.asked), + zap.Int("answered_found", c.found), + zap.Int("answered_not_found", c.notFound), + zap.Int("answers_truncated", c.truncated), + zap.Int("bounce_cap_failures", c.capFailures), + } + for op, n := range c.bouncesByOp { + fields = append(fields, zap.Int("bounces_"+op, n)) + } + ctxzap.Extract(ctx).Info("source-cache lookup continuation totals", fields...) +} + +// listAttempt is one list-RPC attempt as observed by the continuation +// loop: enough of the response to detect and validate an ask. +type listAttempt struct { + annos annotations.Annotations + rows int + nextToken string +} + +// withSourceCacheContinuation drives the ask/answer loop around one list +// RPC. issue performs the RPC with extra request annotations (the lookup +// offer, plus accumulated answers on re-invokes) and reports the response +// surface; the loop returns once a response carries no ask — that final +// response is the one the caller processes. +// +// The offer is attached only when the syncer can actually answer (warm +// previous-sync lookup). Old or cold syncers never send it, and a +// compliant connector never asks without it — that pairing is what makes +// version skew degrade to a cold sync instead of a misread response. +func (s *syncer) withSourceCacheContinuation(ctx context.Context, op string, issue func(extra annotations.Annotations) (listAttempt, error)) error { + l := ctxzap.Extract(ctx) + + warm := s.sourceCache.prev != nil + extra := annotations.Annotations{} + if warm { + extra.Update(&v2.SourceCacheLookupOffer{}) + } + + // Answers accumulate across bounces: the connector re-executes from + // scratch each phase, so every re-invoke must carry the union of all + // resolved queries, in first-resolved order (deterministic requests). + var ordered []sourcecache.Answer + seen := map[sourcecache.Query]bool{} + + asked, found, notFound, truncated := 0, 0, 0, 0 + for bounce := 0; ; bounce++ { + attempt, err := issue(extra) + if err != nil { + return err + } + + ask := &v2.SourceCacheLookupAsk{} + hasAsk, err := attempt.annos.Pick(ask) + if err != nil { + return fmt.Errorf("%s: error parsing source-cache lookup ask: %w", op, err) + } + if !hasAsk { + s.sourceCache.contStats.record(op, bounce, asked, found, notFound, truncated) + return nil + } + + // Ask legality. Failing loudly here is deliberate: every branch + // is a connector bug that would otherwise surface as silently + // wrong data or an unexplained stall. + if !warm { + return fmt.Errorf("%s: connector sent a source-cache lookup ask on a request that carried no offer (connector must gate asks on SourceCacheLookupOffer)", op) + } + if attempt.rows > 0 || attempt.nextToken != "" || + attempt.annos.Contains(&v2.SourceCacheScope{}) || attempt.annos.Contains(&v2.SourceCacheReplay{}) || + attempt.annos.Contains(&v2.SpawnCursors{}) { + return fmt.Errorf("%s: source-cache lookup ask response must carry ONLY the ask: "+ + "no rows, no next page token, no scope/replay annotations, no spawned cursors "+ + "(spawn on the re-invoked request's real response instead)", op) + } + if bounce >= sourceCacheBounceCap { + s.sourceCache.contStats.recordCapFailure() + return fmt.Errorf("%s: source-cache lookup bounce cap (%d) exceeded for one request; "+ + "connector kept asking without progressing (%d scopes still unresolved) — "+ + "check for swallowed ErrLookupDeferred or nondeterministic scope computation", + op, sourceCacheBounceCap, len(ask.GetQueries())) + } + + queries, err := sourcecache.QueriesFromProto(ask) + if err != nil { + return fmt.Errorf("%s: invalid source-cache lookup ask: %w", op, err) + } + + budget := sourceCacheAnswerBudget + for _, a := range ordered { + budget -= len(a.ETag) + } + newAsked, newFound, newNotFound, newTruncated := 0, 0, 0, 0 + for _, q := range queries { + if seen[q] { + continue + } + newAsked++ + entry, ok, err := s.sourceCache.lookup.LookupPreviousSourceCache(ctx, q.RowKind, q.ScopeHash) + if err != nil { + return fmt.Errorf("%s: resolving source-cache lookup ask: %w", op, err) + } + if !ok { + seen[q] = true + ordered = append(ordered, sourcecache.Answer{Query: q, Found: false}) + newNotFound++ + continue + } + if len(entry.ETag) > budget { + // Dropped to budget: the query stays ABSENT from the + // answers (re-askable), never a false not-found. + newTruncated++ + continue + } + budget -= len(entry.ETag) + seen[q] = true + ordered = append(ordered, sourcecache.Answer{Query: q, Found: true, ETag: entry.ETag}) + newFound++ + } + asked += newAsked + found += newFound + notFound += newNotFound + truncated += newTruncated + + if newAsked == 0 { + // Every query was already answered on the request the + // connector just saw; re-invoking cannot make progress. + return fmt.Errorf("%s: connector re-asked only already-answered scopes (%d queries); connector lookup handling is broken", op, len(queries)) + } + + extra.Update(sourcecache.AnswersProto(ordered)) + + l.Debug("source-cache lookup bounce", + zap.String("op", op), + zap.Int("bounce", bounce+1), + zap.Int("asked", newAsked), + zap.Int("found", newFound), + zap.Int("not_found", newNotFound), + zap.Int("truncated_to_budget", newTruncated), + ) + } +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/state.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/state.go index cb1d30ca..bcf89bb7 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sync/state.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/state.go @@ -202,6 +202,15 @@ type Action struct { ResourceID string `json:"resource_id,omitempty"` ParentResourceTypeID string `json:"parent_resource_type_id,omitempty"` ParentResourceID string `json:"parent_resource_id,omitempty"` + + // Spawned marks an action enqueued by a connector's SpawnCursors + // annotation (a sibling cursor) rather than by the syncer's own + // planners. Progress accounting uses it: per-resource grant coverage + // counts a resource once, when its ORIGIN action's chain ends — + // spawned siblings (and their NextPage continuations, which inherit + // the action) don't count, or one resource would count N times. + // omitempty keeps old checkpointed sync tokens decoding unchanged. + Spawned bool `json:"spawned,omitempty"` } var _ State = &state{} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go index 84c86001..d64e99a9 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go @@ -19,6 +19,7 @@ import ( storage_v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" "github.com/conductorone/baton-sdk/pkg/bid" "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" "github.com/conductorone/baton-sdk/pkg/sourcecache" "github.com/conductorone/baton-sdk/pkg/sync/expand" "github.com/conductorone/baton-sdk/pkg/types/entitlement" @@ -116,7 +117,7 @@ type syncer struct { externalResourceEntitlementIdFilter string previousSyncC1ZPath string previousSyncC1ZPathOptional bool - store dotc1z.C1ZStore + store c1zstore.Store externalResourceReader connectorstore.Reader previousSyncReader connectorstore.Reader connector types.ConnectorClient @@ -125,7 +126,7 @@ type syncer struct { transitionHandler func(s Action) progressHandler func(p *Progress) tmpDir string - storageEngine dotc1z.Engine + storageEngine c1zstore.Engine skipFullSync bool lastCheckPointTime time.Time counts *progresslog.ProgressLog @@ -155,7 +156,7 @@ var _ Syncer = (*syncer)(nil) // GrantStore.StoreExpandedGrants so the expander package can depend on // a single narrow interface without knowing about C1ZStore. type expanderStoreAdapter struct { - store dotc1z.C1ZStore + store c1zstore.Store } func (a expanderStoreAdapter) GetEntitlement(ctx context.Context, req *reader_v2.EntitlementsReaderServiceGetEntitlementRequest) (*reader_v2.EntitlementsReaderServiceGetEntitlementResponse, error) { @@ -694,6 +695,7 @@ func (s *syncer) Sync(ctx context.Context) error { return err } + s.sourceCache.contStats.logTotals(ctx) l.Info("Sync complete.") _, err = s.connector.Cleanup(ctx, v2.ConnectorServiceCleanupRequest_builder{ @@ -1049,19 +1051,31 @@ func (s *syncer) SyncResources(ctx context.Context, action *Action) error { // owns a span — the duplicate inflated trace span counts without adding // information. func (s *syncer) syncResources(ctx context.Context, action *Action) error { - req := v2.ResourcesServiceListResourcesRequest_builder{ - ResourceTypeId: action.ResourceTypeID, - PageToken: action.PageToken, - ActiveSyncId: s.getActiveSyncID(), - }.Build() - if action.ParentResourceTypeID != "" && action.ParentResourceID != "" { - req.SetParentResourceId(v2.ResourceId_builder{ - ResourceType: action.ParentResourceTypeID, - Resource: action.ParentResourceID, - }.Build()) - } - - resp, err := s.connector.ListResources(ctx, req) + var resp *v2.ResourcesServiceListResourcesResponse + err := s.withSourceCacheContinuation(ctx, "sync-resources", func(extra annotations.Annotations) (listAttempt, error) { + req := v2.ResourcesServiceListResourcesRequest_builder{ + ResourceTypeId: action.ResourceTypeID, + PageToken: action.PageToken, + ActiveSyncId: s.getActiveSyncID(), + Annotations: extra, + }.Build() + if action.ParentResourceTypeID != "" && action.ParentResourceID != "" { + req.SetParentResourceId(v2.ResourceId_builder{ + ResourceType: action.ParentResourceTypeID, + Resource: action.ParentResourceID, + }.Build()) + } + r, err := s.connector.ListResources(ctx, req) + if err != nil { + return listAttempt{}, err + } + resp = r + return listAttempt{ + annos: annotations.Annotations(r.GetAnnotations()), + rows: len(r.GetList()), + nextToken: r.GetNextPageToken(), + }, nil + }) if err != nil { return err } @@ -1347,11 +1361,24 @@ func (s *syncer) syncEntitlementsForResource(ctx context.Context, action *Action resource := resourceResponse.GetResource() - resp, err := s.connector.ListEntitlements(ctx, v2.EntitlementsServiceListEntitlementsRequest_builder{ - Resource: resource, - PageToken: action.PageToken, - ActiveSyncId: s.getActiveSyncID(), - }.Build()) + var resp *v2.EntitlementsServiceListEntitlementsResponse + err = s.withSourceCacheContinuation(ctx, "sync-entitlements", func(extra annotations.Annotations) (listAttempt, error) { + r, err := s.connector.ListEntitlements(ctx, v2.EntitlementsServiceListEntitlementsRequest_builder{ + Resource: resource, + PageToken: action.PageToken, + ActiveSyncId: s.getActiveSyncID(), + Annotations: extra, + }.Build()) + if err != nil { + return listAttempt{}, err + } + resp = r + return listAttempt{ + annos: annotations.Annotations(r.GetAnnotations()), + rows: len(r.GetList()), + nextToken: r.GetNextPageToken(), + }, nil + }) if err != nil { return err } @@ -1944,12 +1971,27 @@ func (s *syncer) syncGrantsForResource(ctx context.Context, action *Action) erro resource = resourceResponse.GetResource() } - resp, err := s.connector.ListGrants(ctx, v2.GrantsServiceListGrantsRequest_builder{ - Resource: resource, - PageToken: action.PageToken, - ActiveSyncId: s.getActiveSyncID(), - Annotations: reqAnnos, - }.Build()) + var resp *v2.GrantsServiceListGrantsResponse + err := s.withSourceCacheContinuation(ctx, "sync-grants-for-resource", func(extra annotations.Annotations) (listAttempt, error) { + annos := make(annotations.Annotations, 0, len(reqAnnos)+len(extra)) + annos = append(annos, reqAnnos...) + annos = append(annos, extra...) + r, err := s.connector.ListGrants(ctx, v2.GrantsServiceListGrantsRequest_builder{ + Resource: resource, + PageToken: action.PageToken, + ActiveSyncId: s.getActiveSyncID(), + Annotations: annos, + }.Build()) + if err != nil { + return listAttempt{}, err + } + resp = r + return listAttempt{ + annos: annotations.Annotations(r.GetAnnotations()), + rows: len(r.GetList()), + nextToken: r.GetNextPageToken(), + }, nil + }) if err != nil { return fmt.Errorf("sync-grants-for-resource: error listing grants: %w", err) } @@ -2072,16 +2114,31 @@ func (s *syncer) syncGrantsForResource(ctx context.Context, action *Action) erro s.counts.SetGrantsCountOnly(resourceID.GetResourceType()) s.counts.AddGrantsProgress(resourceID.GetResourceType(), len(grants)) s.counts.LogGrantsProgress(ctx, resourceID.GetResourceType()) - } else if resp.GetNextPageToken() == "" { + } else if resp.GetNextPageToken() == "" && !action.Spawned { + // A resource counts as covered exactly once: when its ORIGIN + // action's page chain ends. Spawned sibling cursors for the same + // resource also end with an empty token but must not count, or a + // resource with N spawned pages would count N times and trip the + // progress anomaly warning on healthy syncs. s.counts.AddGrantsProgress(resourceID.GetResourceType(), 1) s.counts.LogGrantsProgress(ctx, resourceID.GetResourceType()) } - // SpawnCursors: a type-scoped response may enqueue sibling cursors for - // the same resource type (e.g. one per connector-defined shard). Each - // runs as its own action — scheduled, rate-limited, and checkpointed - // like any other pagination. Only meaningful on type-scoped calls; - // per-resource responses carrying it are a connector bug. + // SpawnCursors: the response may enqueue sibling cursors — each runs + // as its own action, scheduled by the worker pool, rate-limited, and + // checkpointed like any other pagination. + // + // - Type-scoped calls: one cursor per connector-defined shard (e.g. + // 50-id delta chunks); actions carry only the resource type. + // - Per-resource calls: parallel page fan-out for page-numbered APIs + // (the connector knows every page URL from page one, so a warm + // sync can revalidate all pages concurrently instead of serially); + // actions carry the resource identity. + // + // Spawned cursors are ORDINARY pages that happen to be enqueued + // eagerly: the SDK assumes nothing about replay — a spawned page may + // hit its lookup and replay, miss and fetch cold (page boundary + // shifted since last sync), and may chain further via NextPageToken. spawn := &v2.SpawnCursors{} hasSpawn, err := respAnnos.Pick(spawn) if err != nil { @@ -2089,22 +2146,24 @@ func (s *syncer) syncGrantsForResource(ctx context.Context, action *Action) erro } var spawned []Action if hasSpawn { - if !typeScoped { - l.Warn("sync-grants-for-resource: SpawnCursors on a per-resource grants response; ignored", - zap.String("resource_type_id", action.ResourceTypeID), - zap.String("resource_id", action.ResourceID)) - } else { - for _, tok := range spawn.GetPageTokens() { - if tok == "" { - continue - } - spawned = append(spawned, Action{Op: SyncGrantsOp, ResourceTypeID: action.ResourceTypeID, PageToken: tok}) + for _, tok := range spawn.GetPageTokens() { + if tok == "" { + continue } - l.Debug("sync-grants-for-resource: spawned type-scoped grant cursors", - zap.String("resource_type_id", action.ResourceTypeID), - zap.Int("cursors", len(spawned)), - zap.Int64("estimated_total", spawn.GetEstimatedTotal())) - } + spawned = append(spawned, Action{ + Op: SyncGrantsOp, + ResourceTypeID: action.ResourceTypeID, + ResourceID: action.ResourceID, // empty on type-scoped actions + PageToken: tok, + Spawned: true, + }) + } + l.Debug("sync-grants-for-resource: spawned sibling grant cursors", + zap.String("resource_type_id", action.ResourceTypeID), + zap.String("resource_id", action.ResourceID), + zap.Bool("type_scoped", typeScoped), + zap.Int("cursors", len(spawned)), + zap.Int64("estimated_total", spawn.GetEstimatedTotal())) } return s.nextPageOrFinishAction(ctx, action, resp.GetNextPageToken(), spawned...) @@ -2984,7 +3043,7 @@ func WithProgressHandler(f func(s *Progress)) SyncOpt { // WithConnectorStore sets the connector store to use. This is the preferred option. // Either this or WithC1ZPath must be provided to create a new syncer. -func WithConnectorStore(store dotc1z.C1ZStore) SyncOpt { +func WithConnectorStore(store c1zstore.Store) SyncOpt { return func(s *syncer) { s.store = store } @@ -3006,7 +3065,7 @@ func WithTmpDir(path string) SyncOpt { // WithStorageEngine selects the dotc1z storage engine when opening the c1z // file via WithC1ZPath. Empty uses the baton-sdk default. -func WithStorageEngine(engine dotc1z.Engine) SyncOpt { +func WithStorageEngine(engine c1zstore.Engine) SyncOpt { return func(s *syncer) { s.storageEngine = engine } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/attached/attached.go b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/attached/attached.go index d1b904f7..97c1e50b 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/attached/attached.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/attached/attached.go @@ -8,6 +8,7 @@ import ( reader_v2 "github.com/conductorone/baton-sdk/pb/c1/reader/v2" "github.com/conductorone/baton-sdk/pkg/connectorstore" "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" "go.uber.org/zap" ) @@ -25,7 +26,7 @@ type Compactor struct { // Both arguments are C1ZStore; the constructor type-asserts to *dotc1z.C1File // and returns an error on mismatch. This keeps the public entry point clean // while confining the SQLite-specific concern to the attached package. -func NewAttachedCompactor(base, applied dotc1z.C1ZStore) (*Compactor, error) { +func NewAttachedCompactor(base, applied c1zstore.Store) (*Compactor, error) { baseFile, ok := dotc1z.AsSQLiteStore(base) if !ok { return nil, fmt.Errorf("attached compactor requires SQLite-backed base store, got %T", base) diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/compactor.go b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/compactor.go index 6a298f37..48514230 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/compactor.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/compactor.go @@ -13,6 +13,7 @@ import ( reader_v2 "github.com/conductorone/baton-sdk/pb/c1/reader/v2" "github.com/conductorone/baton-sdk/pkg/connectorstore" "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" "github.com/conductorone/baton-sdk/pkg/sdk" "github.com/conductorone/baton-sdk/pkg/sync" "github.com/conductorone/baton-sdk/pkg/synccompactor/attached" @@ -35,7 +36,7 @@ const ( type Compactor struct { compactorType CompactorType entries []*CompactableSync - compactedC1z dotc1z.C1ZStore + compactedC1z c1zstore.Store tmpDir string destDir string @@ -47,7 +48,7 @@ type Compactor struct { // Empty means EngineSQLite (the default; behavior is unchanged and // the output is byte-identical to the pre-engine-option compactor). // EnginePebble produces a v3 Pebble c1z via a native record merge. - engine dotc1z.Engine + engine c1zstore.Engine // pebbleMode optionally forces the Pebble merge strategy; the zero // value (Auto) lets the compactor choose. See WithPebbleCompactorMode. pebbleMode PebbleCompactorMode @@ -76,9 +77,9 @@ type Compactor struct { // resolvedEngine returns the configured engine, treating the zero value // as EngineSQLite. Compact calls inferEngineFromInputs first so the zero // value can follow existing c1z inputs instead of always producing SQLite. -func (c *Compactor) resolvedEngine() dotc1z.Engine { +func (c *Compactor) resolvedEngine() c1zstore.Engine { if c.engine == "" { - return dotc1z.EngineSQLite + return c1zstore.EngineSQLite } return c.engine } @@ -103,7 +104,7 @@ var ErrEnginePolicyConflict = errors.New("compactor: engine policy conflict: can // // Constraint: an explicit SQLite request (WithEngine(EngineSQLite)) when any // input is Pebble/v3 returns ErrEnginePolicyConflict. -func (c *Compactor) inferEngineFromInputs() (dotc1z.Engine, error) { +func (c *Compactor) inferEngineFromInputs() (c1zstore.Engine, error) { hasPebble := false hasSQLite := false for _, entry := range c.entries { @@ -134,7 +135,7 @@ func (c *Compactor) inferEngineFromInputs() (dotc1z.Engine, error) { // Explicit engine: validate and return. if c.engine != "" { - if c.engine == dotc1z.EngineSQLite && hasPebble { + if c.engine == c1zstore.EngineSQLite && hasPebble { return "", fmt.Errorf("%w: caller requested SQLite but at least one input is Pebble/v3", ErrEnginePolicyConflict) } return c.engine, nil @@ -142,13 +143,13 @@ func (c *Compactor) inferEngineFromInputs() (dotc1z.Engine, error) { // Auto-select: any Pebble input → Pebble output. if hasPebble { - return dotc1z.EnginePebble, nil + return c1zstore.EnginePebble, nil } if hasSQLite { - return dotc1z.EngineSQLite, nil + return c1zstore.EngineSQLite, nil } // No readable inputs: default to SQLite to preserve historical behavior. - return dotc1z.EngineSQLite, nil + return c1zstore.EngineSQLite, nil } type CompactableSync struct { @@ -310,7 +311,7 @@ func (c *Compactor) Compact(ctx context.Context) (*CompactableSync, error) { // and partials are merged into the base keyspace via keep-newer // writes; the folded output is then re-keyed to a fresh sync id. // The original base file is never mutated. See compactPebbleFold. - if c.resolvedEngine() == dotc1z.EnginePebble { + if c.resolvedEngine() == c1zstore.EnginePebble { c.pebbleMode = c.resolvePebbleMode(ctx) } foldMode := c.pebbleMode == PebbleCompactorModeFold @@ -336,7 +337,7 @@ func (c *Compactor) Compact(ctx context.Context) (*CompactableSync, error) { } } - if c.resolvedEngine() == dotc1z.EnginePebble { + if c.resolvedEngine() == c1zstore.EnginePebble { // One payload-decoder pool for the whole compaction: the merge // opens every source's envelope (selection + per-chunk unpack), // and reusing one decoder across those opens avoids a fresh @@ -347,10 +348,10 @@ func (c *Compactor) Compact(ctx context.Context) (*CompactableSync, error) { opts = append(opts, dotc1z.WithDecoderPool(c.decoderPool)) } - if c.resolvedEngine() == dotc1z.EnginePebble { + if c.resolvedEngine() == c1zstore.EnginePebble { // Force the resolved engine last so a stray engine passed via // WithC1ZOptions cannot mislabel the artifact. - c.compactedC1z, err = dotc1z.NewStore(ctx, destFilePath, append(opts, dotc1z.WithEngine(dotc1z.EnginePebble))...) + c.compactedC1z, err = dotc1z.NewStore(ctx, destFilePath, append(opts, dotc1z.WithEngine(c1zstore.EnginePebble))...) } else { c.compactedC1z, err = dotc1z.NewStore(ctx, destFilePath, opts...) } @@ -380,7 +381,7 @@ func (c *Compactor) Compact(ctx context.Context) (*CompactableSync, error) { } return nil, fmt.Errorf("failed to compact (pebble fold): %w", err) } - case c.resolvedEngine() == dotc1z.EnginePebble: + case c.resolvedEngine() == c1zstore.EnginePebble: newSyncId, err = c.runPebbleRebuild(ctx, runCtx) if err != nil { if cause := context.Cause(runCtx); errors.Is(cause, context.DeadlineExceeded) && c.runDuration > 0 && ctx.Err() == nil { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/compactor_pebble.go b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/compactor_pebble.go index 792e7ce3..00fc64a4 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/compactor_pebble.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/compactor_pebble.go @@ -19,6 +19,7 @@ import ( v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" "github.com/conductorone/baton-sdk/pkg/connectorstore" "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" enginepkg "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble" formatv3 "github.com/conductorone/baton-sdk/pkg/dotc1z/format/v3" mergepkg "github.com/conductorone/baton-sdk/pkg/synccompactor/pebble" @@ -33,7 +34,7 @@ import ( // This is the only supported way to choose the engine; an engine // passed through WithC1ZOptions does not select the compaction // strategy and is overridden. -func WithEngine(engine dotc1z.Engine) Option { +func WithEngine(engine c1zstore.Engine) Option { return func(c *Compactor) { c.engine = engine } @@ -339,7 +340,7 @@ func fileSizeOrZero(path string) int64 { // pre-static-registration era. Pebble is now registered by dotc1z init, so this // is a cheap sanity check. func ensurePebbleRegistered() error { - if _, ok := dotc1z.EngineDriverFor(dotc1z.EnginePebble); ok { + if _, ok := dotc1z.EngineDriverFor(c1zstore.EnginePebble); ok { return nil } return dotc1z.ErrEngineNotAvailable diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/tasks/c1api/full_sync.go b/vendor/github.com/conductorone/baton-sdk/pkg/tasks/c1api/full_sync.go index 7b4c56d8..ed2fe9fc 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/tasks/c1api/full_sync.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/tasks/c1api/full_sync.go @@ -16,7 +16,7 @@ import ( v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" v1 "github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1" "github.com/conductorone/baton-sdk/pkg/annotations" - "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" formatv3 "github.com/conductorone/baton-sdk/pkg/dotc1z/format/v3" "github.com/conductorone/baton-sdk/pkg/session" sdkSync "github.com/conductorone/baton-sdk/pkg/sync" @@ -42,7 +42,7 @@ type fullSyncTaskHandler struct { targetedSyncResources []*v2.Resource syncResourceTypeIDs []string workerCount int - storageEngine dotc1z.Engine + storageEngine c1zstore.Engine // previousSyncSparePath is the connector's ETag-replay opt-in: when // non-empty, the handler retains one spare c1z (the last successfully @@ -160,7 +160,7 @@ func (c *fullSyncTaskHandler) sync(ctx context.Context, c1zPath string) error { } engine := c.storageEngine if engine == "" && c.task.GetSyncFull().GetStorageEngine() != "" { - engine = dotc1z.Engine(c.task.GetSyncFull().GetStorageEngine()) + engine = c1zstore.Engine(c.task.GetSyncFull().GetStorageEngine()) } if engine != "" { syncOpts = append(syncOpts, sdkSync.WithStorageEngine(engine)) @@ -359,7 +359,7 @@ func newFullSyncTaskHandler( targetedSyncResources []*v2.Resource, syncResourceTypeIDs []string, workerCount int, - storageEngine dotc1z.Engine, + storageEngine c1zstore.Engine, previousSyncSparePath string, ) tasks.TaskHandler { return &fullSyncTaskHandler{ diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/tasks/c1api/manager.go b/vendor/github.com/conductorone/baton-sdk/pkg/tasks/c1api/manager.go index 5ed25a90..c0df2348 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/tasks/c1api/manager.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/tasks/c1api/manager.go @@ -14,7 +14,7 @@ import ( "google.golang.org/protobuf/types/known/anypb" "github.com/conductorone/baton-sdk/pkg/annotations" - "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" "github.com/conductorone/baton-sdk/pkg/uotel" "github.com/conductorone/baton-sdk/pkg/uotel/uotelzap" @@ -68,7 +68,7 @@ type c1ApiTaskManager struct { targetedSyncResources []*v2.Resource syncResourceTypeIDs []string workerCount int - storageEngine dotc1z.Engine + storageEngine c1zstore.Engine // previousSyncSparePath is non-empty when the connector opted into // ETag replay (keepPreviousSyncC1Z): the fixed, client-id-namespaced @@ -500,7 +500,7 @@ func NewC1TaskManager( targetedSyncResources []*v2.Resource, syncResourceTypeIDs []string, workerCount int, - storageEngine dotc1z.Engine, + storageEngine c1zstore.Engine, taskConcurrency int, keepPreviousSyncC1Z bool, ) (BootstrappingTaskManager, error) { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/tasks/local/compactor.go b/vendor/github.com/conductorone/baton-sdk/pkg/tasks/local/compactor.go index 2c55874e..f9ee3c04 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/tasks/local/compactor.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/tasks/local/compactor.go @@ -6,7 +6,7 @@ import ( "time" v1 "github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1" - "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" "github.com/conductorone/baton-sdk/pkg/synccompactor" "github.com/conductorone/baton-sdk/pkg/tasks" "github.com/conductorone/baton-sdk/pkg/types" @@ -23,12 +23,12 @@ type localCompactor struct { compactableSyncs []*synccompactor.CompactableSync outputPath string tmpDir string - storageEngine dotc1z.Engine + storageEngine c1zstore.Engine } type CompactorOption func(*localCompactor) -func WithCompactorStorageEngine(engine dotc1z.Engine) CompactorOption { +func WithCompactorStorageEngine(engine c1zstore.Engine) CompactorOption { return func(m *localCompactor) { m.storageEngine = engine } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/tasks/local/syncer.go b/vendor/github.com/conductorone/baton-sdk/pkg/tasks/local/syncer.go index 4342c8ca..916b166d 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/tasks/local/syncer.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/tasks/local/syncer.go @@ -10,7 +10,7 @@ import ( v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" v1 "github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1" - "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" "github.com/conductorone/baton-sdk/pkg/session" sdkSync "github.com/conductorone/baton-sdk/pkg/sync" "github.com/conductorone/baton-sdk/pkg/tasks" @@ -31,7 +31,7 @@ type localSyncer struct { skipGrants bool syncResourceTypeIDs []string workerCount int - storageEngine dotc1z.Engine + storageEngine c1zstore.Engine } type Option func(*localSyncer) @@ -90,7 +90,7 @@ func WithWorkerCount(workerCount int) Option { } } -func WithStorageEngine(engine dotc1z.Engine) Option { +func WithStorageEngine(engine c1zstore.Engine) Option { return func(m *localSyncer) { m.storageEngine = engine }