diff --git a/go.mod b/go.mod index e6942672..f3f99788 100644 --- a/go.mod +++ b/go.mod @@ -161,3 +161,5 @@ require ( modernc.org/memory v1.11.0 // indirect modernc.org/sqlite v1.50.0 // indirect ) + +replace github.com/conductorone/baton-sdk => ../baton-sdk-2 diff --git a/go.sum b/go.sum index 63d0a615..1c1cfd68 100644 --- a/go.sum +++ b/go.sum @@ -86,8 +86,6 @@ github.com/cockroachdb/swiss v0.0.0-20251224182025-b0f6560f979b h1:VXvSNzmr8hMj8 github.com/cockroachdb/swiss v0.0.0-20251224182025-b0f6560f979b/go.mod h1:yBRu/cnL4ks9bgy4vAASdjIW+/xMlFwuHKqtmh3GZQg= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= -github.com/conductorone/baton-sdk v0.18.2 h1:2pFlzwSpaFnIv0GbmTmLtIUOqWuNBtlsm28mujt7F5g= -github.com/conductorone/baton-sdk v0.18.2/go.mod h1:xacgmef9cM4dUTdvGN3Qip6fwkRbciqtaZMi5iWnjsY= github.com/conductorone/dpop v0.2.6 h1:fakwai/Xm2b/fcDUwJN41WtcSI/2UhQOyRIVvnnrrNA= github.com/conductorone/dpop v0.2.6/go.mod h1:gyo8TtzB9SCFCsjsICH4IaLZ7y64CcrDXMOPBwfq/3s= github.com/conductorone/dpop/integrations/dpop_grpc v0.2.4 h1:lYxYi9/WTSL9sE96CO0QF2BY3kehs8dTTApI134TGCA= diff --git a/pkg/connector/connector.go b/pkg/connector/connector.go index 9673e30e..a7f910a3 100644 --- a/pkg/connector/connector.go +++ b/pkg/connector/connector.go @@ -54,6 +54,12 @@ func capabilityPermissions(perms ...string) *v2.CapabilityPermissions { return cp } +func groupResourceTypeAnnotations() annotations.Annotations { + annos := v1AnnotationsForResourceType("group", false, capabilityPermissions("okta.groups.read", "okta.groups.manage")) + annos.Update(&v2.TypeScopedGrants{}) + return annos +} + func v1AnnotationsForResourceType(resourceTypeID string, skipEntitlementsAndGrants bool, perms *v2.CapabilityPermissions) annotations.Annotations { annos := annotations.Annotations{} annos.Update(&v2.V1Identifier{ @@ -88,11 +94,15 @@ var ( Traits: []v2.ResourceType_Trait{v2.ResourceType_TRAIT_USER}, Annotations: v1AnnotationsForResourceType(userResourceTypeID, true, capabilityPermissions("okta.users.read", "okta.users.manage")), } + // TypeScopedGrants excludes the group type from the SDK's per-resource + // grants fan-out: full syncs enumerate group grants through the + // planner/cursors in group_type_scoped.go (with source-cache replay for + // clean groups); the per-resource path serves targeted syncs. resourceTypeGroup = &v2.ResourceType{ Id: "group", DisplayName: "Group", Traits: []v2.ResourceType_Trait{v2.ResourceType_TRAIT_GROUP}, - Annotations: v1AnnotationsForResourceType("group", false, capabilityPermissions("okta.groups.read", "okta.groups.manage")), + Annotations: groupResourceTypeAnnotations(), } resourceTypeApp = &v2.ResourceType{ Id: "app", @@ -275,8 +285,14 @@ func (c *Okta) Metadata(ctx context.Context) (*v2.ConnectorMetadata, error) { } func (c *Okta) Validate(ctx context.Context) (annotations.Annotations, error) { + // Source-cache replay opt-in: group member grants are validated by + // lastMembershipUpdated (group_type_scoped.go). + annos := annotations.New(&v2.SourceCacheCapability{ + Mode: v2.SourceCacheCapability_MODE_READ_WRITE, + }) + if c.apiToken == "" { - return nil, nil + return annos, nil } token := newPaginationToken(defaultLimit, "") @@ -296,7 +312,7 @@ func (c *Okta) Validate(ctx context.Context) (annotations.Annotations, error) { return nil, err } - return nil, nil + return annos, nil } func (c *Okta) Asset(ctx context.Context, asset *v2.AssetRef) (string, io.ReadCloser, error) { @@ -320,6 +336,13 @@ func New(ctx context.Context, cc *cfg.Okta, opts *cli.ConnectorOpts) (connectorb return nil, nil, err } + // Demo instrumentation: count/log Okta requests when + // BATON_OKTA_REQUEST_LOG is set (source-cache measurement harness). + client, err = wrapRequestCounting(client) + if err != nil { + return nil, nil, err + } + cacheTTI, err := safeCacheInt32(cc.CacheTti) if err != nil { return nil, nil, err diff --git a/pkg/connector/group.go b/pkg/connector/group.go index 060e6c05..e8f01c6d 100644 --- a/pkg/connector/group.go +++ b/pkg/connector/group.go @@ -33,6 +33,10 @@ const appGroupType = "APP_GROUP" const oktaGroupType = "OKTA_GROUP" const apiPathGetGroupFmt = "/api/v1/groups/%s" +// groupRoleAssignmentType is the assignmentType value on role assignments +// held by a group (vs "USER"). +const groupRoleAssignmentType = "GROUP" + type groupResourceType struct { resourceType *v2.ResourceType connector *Okta @@ -209,7 +213,7 @@ func (o *groupResourceType) Grants( } for _, role := range roles { - if role.Status == roleStatusInactive || role.AssignmentType != "GROUP" { + if role.Status == roleStatusInactive || role.AssignmentType != groupRoleAssignmentType { continue } diff --git a/pkg/connector/group_type_scoped.go b/pkg/connector/group_type_scoped.go new file mode 100644 index 00000000..99a626b2 --- /dev/null +++ b/pkg/connector/group_type_scoped.go @@ -0,0 +1,441 @@ +package connector + +// Type-scoped group grants: source-cache replay validated by Okta's +// lastMembershipUpdated timestamp (dirty-scope model — see +// baton-microsoft-entra/docs/okta-replay-brief.md and the probe results in +// docs/replay-probe-results.md). +// +// Okta has no delta endpoints and no useful ETags, but every group row in +// the ordinary GET /api/v1/groups listing carries lastMembershipUpdated, +// which bumps on every membership change (direct add/remove, rule-driven +// evaluation, user deletion — all probed live). The connector compares +// that value ITSELF against the validator stored by the previous sync: +// +// - equal → the group's member grants are REPLAYED (no overlay, no +// tombstones), spending zero API requests on the group; +// - different → the group is dirty and its members are re-enumerated +// cold (there are no membership deltas to apply); +// - missing/empty → cold. Every surprise fails toward cold. +// +// Comparison is per-group EQUALITY against the group's own previous +// value, never "since T" against a clock: Okta's timestamps are Okta's, +// filter results lag writes, and equality is immune to both. +// +// Shape: the group resource type carries TypeScopedGrants, so the SDK +// issues one planning call instead of a per-resource fan-out. Planning +// pages the groups listing (which returns lastMembershipUpdated for free) +// and spawns one cursor per group via SpawnCursors. Each cursor runs two +// legs: +// +// 1. members — replay or cold enumeration as decided above; +// 2. group role assignments — ALWAYS a fresh enumeration. Role +// assign/revoke changes neither group timestamp (probed live), so +// the API offers nothing to validate with; per the brief the leg is +// not forced into the model. This bounds a fully-warm sync at one +// /groups/{id}/roles request per group. +// +// The per-resource Grants path in group.go remains for targeted syncs. + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "sort" + "strings" + "time" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/connectorbuilder" + "github.com/conductorone/baton-sdk/pkg/sourcecache" + sdkResource "github.com/conductorone/baton-sdk/pkg/types/resource" + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "github.com/okta/okta-sdk-golang/v2/okta" + "go.uber.org/zap" +) + +var _ connectorbuilder.TypeScopedGrantsSyncer = (*groupResourceType)(nil) + +const ( + groupCursorPhaseMembers = "m" + groupCursorPhaseRoles = "r" + + replayModeWarm = "warm" + replayModeCold = "cold" +) + +// groupGrantsCursor rides the SDK page token. Planning pages carry Plan + +// PlanPage (the groups listing's after-cursor); per-group cursors carry +// the group id, the validator read from the planning listing, the +// users_count stat (nil when the listing's expand=stats was absent), and +// the current leg's phase + after-cursor. +type groupGrantsCursor struct { + Plan bool `json:"p,omitempty"` + PlanPage string `json:"pp,omitempty"` + + GroupID string `json:"g,omitempty"` + Validator string `json:"v,omitempty"` + UsersCount *int64 `json:"uc,omitempty"` + + Phase string `json:"ph,omitempty"` + Page string `json:"pg,omitempty"` +} + +func (t *groupGrantsCursor) marshal() (string, error) { + b, err := json.Marshal(t) + if err != nil { + return "", fmt.Errorf("okta-connectorv2: failed to marshal group grants cursor: %w", err) + } + return string(b), nil +} + +// scopeSig hashes the request properties that shape a cached row set. +// Any change yields a different scope — a clean lookup miss and a full +// re-enumeration, exactly what an under- or over-filtered cache requires. +func scopeSig(parts ...string) string { + sum := sha256.Sum256([]byte(strings.Join(parts, "\x00"))) + return hex.EncodeToString(sum[:8]) +} + +// memberScope identifies one group's member-grant rows. The signature +// covers the client-side email-domain filter applied during enumeration +// (shouldIncludeUserAndSetCache): a config change must invalidate every +// group's scope. skipAppGroups is deliberately absent — it decides which +// groups get planned at all, not which rows a planned group's scope holds. +func (o *groupResourceType) memberScope(groupID string) string { + parts := []string{"v1"} + if o.connector.userFilters != nil && len(o.connector.userFilters.includedEmailDomains) > 0 { + domains := make([]string, len(o.connector.userFilters.includedEmailDomains)) + copy(domains, o.connector.userFilters.includedEmailDomains) + sort.Strings(domains) + parts = append(parts, domains...) + } + return fmt.Sprintf("groups/%s/users?sig=%s", groupID, scopeSig(parts...)) +} + +// groupMembershipValidator formats a group's lastMembershipUpdated as the +// scope validator. Empty (never observed live; every group gets the value +// at creation) means the scope cannot be validated and stays cold. +func groupMembershipValidator(group *okta.Group) string { + if group.LastMembershipUpdated == nil { + return "" + } + return group.LastMembershipUpdated.UTC().Format(time.RFC3339Nano) +} + +// GrantsForResourceType implements connectorbuilder.TypeScopedGrantsSyncer +// for the group type: the planning walk pages the groups listing and +// spawns one cursor per group; each cursor replays or re-enumerates that +// group's member grants and freshly enumerates its role assignments. +func (o *groupResourceType) GrantsForResourceType( + ctx context.Context, + resourceTypeID string, + attrs sdkResource.SyncOpAttrs, +) ([]*v2.Grant, *sdkResource.SyncOpResults, error) { + if resourceTypeID != resourceTypeGroup.Id { + return nil, nil, fmt.Errorf("okta-connectorv2: type-scoped grants for unexpected resource type %s", resourceTypeID) + } + + tok := &groupGrantsCursor{Plan: true} + if attrs.PageToken.Token != "" { + tok = &groupGrantsCursor{} + if err := json.Unmarshal([]byte(attrs.PageToken.Token), tok); err != nil { + return nil, nil, fmt.Errorf("okta-connectorv2: invalid group grants cursor: %w", err) + } + } + + if tok.Plan { + return o.planGroupCursorsPage(ctx, tok, attrs) + } + if tok.GroupID == "" { + return nil, nil, fmt.Errorf("okta-connectorv2: malformed group grants cursor: missing group id") + } + + switch tok.Phase { + case "": + return o.startGroupCursor(ctx, tok, attrs) + case groupCursorPhaseMembers: + return o.coldMembersPage(ctx, tok, attrs) + case groupCursorPhaseRoles: + return o.groupRolesPage(ctx, tok, attrs) + default: + return nil, nil, fmt.Errorf("okta-connectorv2: malformed group grants cursor: unknown phase %q", tok.Phase) + } +} + +// planGroupCursorsPage processes ONE page of the groups listing and spawns +// a cursor per group carrying (id, lastMembershipUpdated, users_count). +// Planning state never leaves the page token. The listing is the entire +// fixed cost of a warm sync's member legs: expand=stats rides along so the +// users_count==0 shortcut works exactly like the per-resource path. +func (o *groupResourceType) planGroupCursorsPage( + ctx context.Context, + tok *groupGrantsCursor, + attrs sdkResource.SyncOpAttrs, +) ([]*v2.Grant, *sdkResource.SyncOpResults, error) { + l := ctxzap.Extract(ctx) + + token := newPaginationToken(attrs.PageToken.Size, tok.PlanPage) + qp := queryParamsExpand(token.Size, tok.PlanPage, "stats") + groups, respCtx, err := listGroupsHelper(ctx, o.connector.client, token, qp) + if err != nil { + return nil, nil, fmt.Errorf("okta-connectorv2: group grants planning page failed: %w", err) + } + + nextPage, annos, err := parseResp(respCtx.OktaResponse) + if err != nil { + return nil, nil, fmt.Errorf("okta-connectorv2: failed to parse response: %w", err) + } + + tokens := make([]string, 0, len(groups)) + for _, group := range groups { + if o.connector.skipAppGroups && group.Type == appGroupType { + l.Debug("okta-connectorv2: skipping APP_GROUP type group", zap.String("group_id", group.Id)) + continue + } + cursor := &groupGrantsCursor{ + GroupID: group.Id, + Validator: groupMembershipValidator(group), + } + if usersCount, exists := getGroupUserCount(group); exists { + uc := int64(usersCount) + cursor.UsersCount = &uc + } + ct, err := cursor.marshal() + if err != nil { + return nil, nil, err + } + tokens = append(tokens, ct) + } + + nextToken := "" + if nextPage != "" { + nextToken, err = (&groupGrantsCursor{Plan: true, PlanPage: nextPage}).marshal() + if err != nil { + return nil, nil, err + } + } + + l.Debug("okta-connectorv2: planned group grant cursors (page)", + zap.Int("groups", len(groups)), + zap.Int("cursors_spawned", len(tokens)), + zap.Bool("final_page", nextPage == ""), + ) + + if len(tokens) > 0 { + annos.Update(&v2.SpawnCursors{PageTokens: tokens}) + } + return nil, &sdkResource.SyncOpResults{NextPageToken: nextToken, Annotations: annos}, nil +} + +// startGroupCursor runs one group cursor's first call: source-cache +// lookup, then either a replay of the member scope (zero API calls) or +// the first page of a cold member enumeration. The roles leg follows +// either way. +func (o *groupResourceType) startGroupCursor( + ctx context.Context, + tok *groupGrantsCursor, + attrs sdkResource.SyncOpAttrs, +) ([]*v2.Grant, *sdkResource.SyncOpResults, error) { + l := ctxzap.Extract(ctx) + scope := o.memberScope(tok.GroupID) + + lookup := attrs.SourceCache + if lookup == nil { + lookup = sourcecache.NoopLookup{} + } + entry, found, err := lookup.LookupPreviousSourceCache(ctx, sourcecache.RowKindGrants, scope) + if err != nil { + return nil, nil, fmt.Errorf("okta-connectorv2: group member scope lookup failed: %w", err) + } + + if found && tok.Validator != "" && entry.ETag == tok.Validator { + l.Debug("okta-connectorv2: group members scope", + zap.String("mode", replayModeWarm), + zap.String("group_id", tok.GroupID), + zap.String("scope", scope), + ) + var annos annotations.Annotations + annos.Update(&v2.SourceCacheReplay{ + ScopeHash: scope, + Etag: tok.Validator, + }) + next, err := (&groupGrantsCursor{ + GroupID: tok.GroupID, + UsersCount: tok.UsersCount, + Phase: groupCursorPhaseRoles, + }).marshal() + if err != nil { + return nil, nil, err + } + return nil, &sdkResource.SyncOpResults{NextPageToken: next, Annotations: annos}, nil + } + + l.Debug("okta-connectorv2: group members scope", + zap.String("mode", replayModeCold), + zap.String("group_id", tok.GroupID), + zap.String("scope", scope), + zap.Bool("validator_found", found), + ) + tok.Phase = groupCursorPhaseMembers + tok.Page = "" + return o.coldMembersPage(ctx, tok, attrs) +} + +// coldMembersPage serves one page of a group's full member enumeration, +// stamping rows with the member scope. The validator (read from the +// planning listing) is written on the final page; a membership change +// between planning and enumeration stores a validator OLDER than the rows +// it describes, which the next sync sees as dirty — fails toward cold. +func (o *groupResourceType) coldMembersPage( + ctx context.Context, + tok *groupGrantsCursor, + attrs sdkResource.SyncOpAttrs, +) ([]*v2.Grant, *sdkResource.SyncOpResults, error) { + l := ctxzap.Extract(ctx) + scope := o.memberScope(tok.GroupID) + + rolesToken, err := (&groupGrantsCursor{ + GroupID: tok.GroupID, + UsersCount: tok.UsersCount, + Phase: groupCursorPhaseRoles, + }).marshal() + if err != nil { + return nil, nil, err + } + + // users_count==0 shortcut (parity with the per-resource path): skip + // the members call. A zero-row page still persists the scope entry, + // so the group replays next sync if its validator holds. + if tok.Page == "" && tok.UsersCount != nil && *tok.UsersCount == 0 { + l.Debug("okta-connectorv2: skipping list group users (users_count is 0)", + zap.String("group_id", tok.GroupID)) + var annos annotations.Annotations + annos.Update(&v2.SourceCacheScope{ScopeHash: scope, Etag: tok.Validator}) + return nil, &sdkResource.SyncOpResults{NextPageToken: rolesToken, Annotations: annos}, nil + } + + token := newPaginationToken(attrs.PageToken.Size, tok.Page) + qp := queryParams(token.Size, tok.Page) + users, respCtx, err := o.listGroupUsers(ctx, tok.GroupID, token, qp) + if err != nil { + return nil, nil, convertNotFoundError(err, "okta-connectorv2: failed to list group users") + } + + nextPage, annos, err := parseResp(respCtx.OktaResponse) + if err != nil { + return nil, nil, fmt.Errorf("okta-connectorv2: failed to parse response: %w", err) + } + + groupStub := &v2.Resource{Id: fmtResourceId(resourceTypeGroup.Id, tok.GroupID)} + var rv []*v2.Grant + for _, user := range users { + if !o.connector.shouldIncludeUserAndSetCache(ctx, attrs.Session, user) { + continue + } + rv = append(rv, groupGrant(groupStub, user)) + } + + // The validator rides only the final page; interim pages stamp rows + // with an empty etag (the SDK writes the manifest entry when the + // non-empty etag arrives). + etag := tok.Validator + nextToken := rolesToken + if nextPage != "" { + etag = "" + nextToken, err = (&groupGrantsCursor{ + GroupID: tok.GroupID, + Validator: tok.Validator, + UsersCount: tok.UsersCount, + Phase: groupCursorPhaseMembers, + Page: nextPage, + }).marshal() + if err != nil { + return nil, nil, err + } + } + annos.Update(&v2.SourceCacheScope{ScopeHash: scope, Etag: etag}) + + l.Debug("okta-connectorv2: group members page", + zap.String("mode", replayModeCold), + zap.String("group_id", tok.GroupID), + zap.Int("rows", len(rv)), + zap.Bool("final_page", nextPage == ""), + ) + + return rv, &sdkResource.SyncOpResults{NextPageToken: nextToken, Annotations: annos}, nil +} + +// groupRolesPage enumerates the group's role assignments — always fresh, +// never scope-stamped: role assign/revoke does not bump either group +// timestamp (probed live), so there is nothing to validate replay with. +// Mirrors the role leg of the per-resource Grants path, including the +// access-denied skip. The endpoint is effectively unpaginated (the +// per-resource path also issues a single unparameterized call). +func (o *groupResourceType) groupRolesPage( + ctx context.Context, + tok *groupGrantsCursor, + attrs sdkResource.SyncOpAttrs, +) ([]*v2.Grant, *sdkResource.SyncOpResults, error) { + l := ctxzap.Extract(ctx) + + roles, resp, err := listGroupAssignedRoles(ctx, o.connector.client, tok.GroupID, nil) + if err != nil { + if resp == nil { + return nil, nil, fmt.Errorf("okta-connectorv2: failed to list group roles: %w", err) + } + defer resp.Body.Close() + errOkta, err2 := getError(resp) + if err2 != nil { + return nil, nil, err2 + } + if errOkta.ErrorCode == AccessDeniedErrorCode { + l.Debug("okta-connectorv2: skipping group role grants (access denied)", + zap.String("group_id", tok.GroupID)) + return nil, &sdkResource.SyncOpResults{}, nil + } + return nil, nil, convertNotFoundError(&errOkta, "okta-connectorv2: failed to list group roles") + } + + _, annos, err := parseResp(resp) + if err != nil { + return nil, nil, fmt.Errorf("okta-connectorv2: failed to parse response: %w", err) + } + + shouldExpand := tok.UsersCount == nil || *tok.UsersCount > 0 + var rv []*v2.Grant + for _, role := range roles { + if role.Status == roleStatusInactive || role.AssignmentType != groupRoleAssignmentType { + continue + } + if !o.connector.SyncCustomRoles && role.Type == roleTypeCustom { + continue + } + + var roleResourceVal *v2.Resource + if role.Type == roleTypeCustom { + roleResourceVal, err = roleResource(ctx, &okta.Role{ + Id: role.Role, + Label: role.Label, + }, resourceTypeCustomRole) + } else { + roleResourceVal, err = roleResource(ctx, &okta.Role{ + Id: role.Role, + Label: role.Label, + Type: role.Type, + }, resourceTypeRole) + } + if err != nil { + return nil, nil, err + } + + if !shouldExpand { + l.Debug("okta-connectorv2: skipping expand for role group grant since users_count is 0") + } + rv = append(rv, roleGroupGrant(tok.GroupID, roleResourceVal, shouldExpand)) + } + + return rv, &sdkResource.SyncOpResults{Annotations: annos}, nil +} diff --git a/pkg/connector/request_log.go b/pkg/connector/request_log.go new file mode 100644 index 00000000..935ac549 --- /dev/null +++ b/pkg/connector/request_log.go @@ -0,0 +1,48 @@ +package connector + +import ( + "fmt" + "net/http" + "os" + "sync" +) + +// requestLogEnvVar names a file to append one "METHOD URL" line per Okta +// API request. Demo instrumentation for the source-cache measurement +// harness (warm-vs-cold request counting); unset (the normal case) means +// no wrapping at all. +const requestLogEnvVar = "BATON_OKTA_REQUEST_LOG" + +type countingTransport struct { + base http.RoundTripper + logMu sync.Mutex + logF *os.File +} + +func (t *countingTransport) RoundTrip(req *http.Request) (*http.Response, error) { + t.logMu.Lock() + _, _ = fmt.Fprintf(t.logF, "%s %s\n", req.Method, req.URL.String()) + t.logMu.Unlock() + + base := t.base + if base == nil { + base = http.DefaultTransport + } + return base.RoundTrip(req) +} + +// wrapRequestCounting installs the counting transport when +// BATON_OKTA_REQUEST_LOG is set. Returns the client unchanged otherwise. +func wrapRequestCounting(httpClient *http.Client) (*http.Client, error) { + logPath := os.Getenv(requestLogEnvVar) + if logPath == "" { + return httpClient, nil + } + f, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644) //nolint:gosec // path comes from the operator's own env var, not untrusted input + if err != nil { + return nil, fmt.Errorf("okta-connectorv2: failed to open request log %s: %w", logPath, err) + } + wrapped := *httpClient + wrapped.Transport = &countingTransport{base: httpClient.Transport, logF: f} + return &wrapped, nil +} diff --git a/pkg/connector/sourcecache_fuzz_test.go b/pkg/connector/sourcecache_fuzz_test.go new file mode 100644 index 00000000..b18e7d5d --- /dev/null +++ b/pkg/connector/sourcecache_fuzz_test.go @@ -0,0 +1,384 @@ +package connector + +// Randomized churn equivalence fuzzer for the source-cache warm path. +// +// The scripted scenarios in sourcecache_sync_test.go each pin one known +// hazard. This test covers the space BETWEEN them: every round applies a +// random batch of org mutations (user lifecycle, membership, group +// lifecycle, renames, role assignments, validator-only touches, and +// occasional mass invalidation), runs a warm sync chained off the previous +// round's output, runs a fresh uncached control sync of the same org +// state, and requires the two to be byte-identical at the reader surface. +// Any divergence — a stale replay, a missed dirty group, a leaked grant — +// fails with the round's seed and mutation log, which replays +// deterministically. +// +// Runs are deterministic by default (fixed seed) so CI is stable; set +// BATON_FUZZ_SEED to explore a different trajectory, and BATON_FUZZ_ROUNDS +// to run longer soaks. + +import ( + "fmt" + "math/rand" + "os" + "strconv" + "testing" + + "github.com/conductorone/baton-sdk/pkg/logging" + "github.com/stretchr/testify/require" +) + +// fuzzOrgView is a locked copy of the mock org's mutable state, used to +// pick valid mutation targets without racing the handler. +type fuzzOrgView struct { + activeUsers []string + allUsers []string + groups []string + members map[string][]string + roles map[string][]string // group id → role types +} + +func (m *mockOkta) fuzzView() fuzzOrgView { + m.mu.Lock() + defer m.mu.Unlock() + v := fuzzOrgView{ + members: map[string][]string{}, + roles: map[string][]string{}, + } + for _, id := range m.userOrder { + v.allUsers = append(v.allUsers, id) + if m.users[id].Status == mockStatusActive { + v.activeUsers = append(v.activeUsers, id) + } + } + for _, gid := range m.groupOrder { + g := m.groups[gid] + v.groups = append(v.groups, gid) + v.members[gid] = append([]string{}, g.Members...) + for _, r := range g.Roles { + v.roles[gid] = append(v.roles[gid], r.Type) + } + } + return v +} + +type fuzzOp struct { + name string + // ready reports whether the op has a valid target in this state. + ready func(v fuzzOrgView) bool + apply func(f *fuzzRun, v fuzzOrgView) +} + +type fuzzRun struct { + t *testing.T + m *mockOkta + rng *rand.Rand + nextID int + log []string +} + +func (f *fuzzRun) id(prefix string) string { + f.nextID++ + return fmt.Sprintf("fz-%s-%03d", prefix, f.nextID) +} + +func (f *fuzzRun) pick(items []string) string { + return items[f.rng.Intn(len(items))] +} + +func (f *fuzzRun) notef(format string, args ...any) { + f.log = append(f.log, fmt.Sprintf(format, args...)) +} + +// fuzzableRoleTypes are standard org roles the fuzzer assigns to groups +// (drawn from standardRoleTypes so the role resources exist). +var fuzzableRoleTypes = []string{roleTypeUserAdmin, roleTypeHelpDesk, "APP_ADMIN", "REPORT_ADMIN"} + +func fuzzOps() []fuzzOp { + return []fuzzOp{ + { + name: "add-user", + ready: func(v fuzzOrgView) bool { return true }, + apply: func(f *fuzzRun, v fuzzOrgView) { + id := f.id("user") + f.m.addUser(&mockOktaUser{ + ID: id, FirstName: "Fuzz", LastName: id, Email: id + "@x.test", + }) + f.notef("add-user %s", id) + }, + }, + { + name: "deactivate-user", + ready: func(v fuzzOrgView) bool { return len(v.activeUsers) > 1 }, + apply: func(f *fuzzRun, v fuzzOrgView) { + uid := f.pick(v.activeUsers) + f.m.deactivateUser(uid) + f.notef("deactivate-user %s", uid) + }, + }, + { + name: "delete-user", + ready: func(v fuzzOrgView) bool { return len(v.allUsers) > 2 }, + apply: func(f *fuzzRun, v fuzzOrgView) { + uid := f.pick(v.allUsers) + f.m.deleteUser(uid) + f.notef("delete-user %s", uid) + }, + }, + { + name: "create-group", + ready: func(v fuzzOrgView) bool { return true }, + apply: func(f *fuzzRun, v fuzzOrgView) { + gid := f.id("group") + g := &mockOktaGroup{ID: gid, Name: "Fuzz " + gid} + if len(v.activeUsers) > 0 && f.rng.Intn(2) == 0 { + g.Members = []string{f.pick(v.activeUsers)} + } + f.m.addGroup(g) + f.notef("create-group %s (members=%v)", gid, g.Members) + }, + }, + { + name: "delete-group", + ready: func(v fuzzOrgView) bool { return len(v.groups) > 1 }, + apply: func(f *fuzzRun, v fuzzOrgView) { + gid := f.pick(v.groups) + f.m.deleteGroup(gid) + f.notef("delete-group %s", gid) + }, + }, + { + name: "add-member", + ready: func(v fuzzOrgView) bool { + return len(v.groups) > 0 && len(v.activeUsers) > 0 + }, + apply: func(f *fuzzRun, v fuzzOrgView) { + gid := f.pick(v.groups) + current := map[string]bool{} + for _, mid := range v.members[gid] { + current[mid] = true + } + var cands []string + for _, uid := range v.activeUsers { + if !current[uid] { + cands = append(cands, uid) + } + } + if len(cands) == 0 { + return + } + uid := f.pick(cands) + f.m.addMember(gid, uid) + f.notef("add-member %s -> %s", uid, gid) + }, + }, + { + name: "remove-member", + ready: func(v fuzzOrgView) bool { + for _, gid := range v.groups { + if len(v.members[gid]) > 0 { + return true + } + } + return false + }, + apply: func(f *fuzzRun, v fuzzOrgView) { + var withMembers []string + for _, gid := range v.groups { + if len(v.members[gid]) > 0 { + withMembers = append(withMembers, gid) + } + } + gid := f.pick(withMembers) + uid := f.pick(v.members[gid]) + f.m.removeMember(gid, uid) + f.notef("remove-member %s <- %s", uid, gid) + }, + }, + { + name: "rename-group", + ready: func(v fuzzOrgView) bool { return len(v.groups) > 0 }, + apply: func(f *fuzzRun, v fuzzOrgView) { + gid := f.pick(v.groups) + f.m.renameGroup(gid, "Renamed "+f.id("nm")) + f.notef("rename-group %s", gid) + }, + }, + { + name: "touch-group", + ready: func(v fuzzOrgView) bool { return len(v.groups) > 0 }, + apply: func(f *fuzzRun, v fuzzOrgView) { + gid := f.pick(v.groups) + f.m.touchGroup(gid) + f.notef("touch-group %s", gid) + }, + }, + { + name: "assign-group-role", + ready: func(v fuzzOrgView) bool { + for _, gid := range v.groups { + if len(v.roles[gid]) < len(fuzzableRoleTypes) { + return true + } + } + return false + }, + apply: func(f *fuzzRun, v fuzzOrgView) { + var cands []string + for _, gid := range v.groups { + if len(v.roles[gid]) < len(fuzzableRoleTypes) { + cands = append(cands, gid) + } + } + gid := f.pick(cands) + held := map[string]bool{} + for _, rt := range v.roles[gid] { + held[rt] = true + } + var free []string + for _, rt := range fuzzableRoleTypes { + if !held[rt] { + free = append(free, rt) + } + } + rt := f.pick(free) + f.m.assignGroupRole(gid, mockOktaGroupRole{AssignmentID: f.id("gra"), Type: rt, Label: rt}) + f.notef("assign-group-role %s -> %s", rt, gid) + }, + }, + { + name: "revoke-group-role", + ready: func(v fuzzOrgView) bool { + for _, gid := range v.groups { + if len(v.roles[gid]) > 0 { + return true + } + } + return false + }, + apply: func(f *fuzzRun, v fuzzOrgView) { + var withRoles []string + for _, gid := range v.groups { + if len(v.roles[gid]) > 0 { + withRoles = append(withRoles, gid) + } + } + gid := f.pick(withRoles) + rt := f.pick(v.roles[gid]) + f.m.revokeGroupRole(gid, rt) + f.notef("revoke-group-role %s <- %s", rt, gid) + }, + }, + } +} + +func fuzzEnvInt(name string, def int) int { + if s := os.Getenv(name); s != "" { + if n, err := strconv.Atoi(s); err == nil { + return n + } + } + return def +} + +func TestSourceCacheChurnFuzz(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + + seed := int64(fuzzEnvInt("BATON_FUZZ_SEED", 20260711)) + rounds := fuzzEnvInt("BATON_FUZZ_ROUNDS", 8) + t.Logf("churn fuzz: seed=%d rounds=%d (override with BATON_FUZZ_SEED / BATON_FUZZ_ROUNDS)", seed, rounds) + + mock := newMockOkta(t) + + // Seed org: shape mirrors the scripted test so every leg has data from + // round zero. + for i := 1; i <= 4; i++ { + mock.addUser(&mockOktaUser{ + ID: fmt.Sprintf("u%d", i), FirstName: "Member", LastName: fmt.Sprintf("N%d", i), + Email: fmt.Sprintf("u%d@x.test", i), + }) + } + mock.addGroup(&mockOktaGroup{ID: "g1", Name: "Group One", Members: []string{"u1", "u2"}}) + mock.addGroup(&mockOktaGroup{ID: "g2", Name: "Group Two", Members: []string{"u3"}}) + mock.addGroup(&mockOktaGroup{ID: "g3", Name: "Empty"}) + mock.assignGroupRole("g1", mockOktaGroupRole{AssignmentID: "gra-seed", Type: roleTypeUserAdmin, Label: roleLabelGroupAdmin}) + + h := newSyncHarness(ctx, t, mock) + f := &fuzzRun{t: t, m: mock, rng: rand.New(rand.NewSource(seed))} //nolint:gosec // deterministic replayable fuzzing requires seeded math/rand + ops := fuzzOps() + + prev := h.runSync("fuzz-cold", "") + + for round := 1; round <= rounds; round++ { + nOps := 1 + f.rng.Intn(3) + f.log = f.log[:0] + for i := 0; i < nOps; i++ { + // Re-view after each mutation so ops in the same round compose + // against current state, exactly as real churn would. + v := mock.fuzzView() + var ready []fuzzOp + for _, op := range ops { + if op.ready(v) { + ready = append(ready, op) + } + } + require.NotEmpty(t, ready) + ready[f.rng.Intn(len(ready))].apply(f, v) + } + + // ~1 round in 6 also invalidates every group's validator, so mass + // invalidation is fuzzed IN COMBINATION with churn, not only in + // isolation. + if f.rng.Intn(6) == 0 { + for _, gid := range mock.fuzzView().groups { + mock.touchGroup(gid) + } + f.notef("mass-invalidation") + } + + warm := h.runSync(fmt.Sprintf("fuzz-warm-%02d", round), prev) + control := h.runSync(fmt.Sprintf("fuzz-control-%02d", round), "") + + wSnap := h.snapshot(warm) + cSnap := h.snapshot(control) + if !assertSnapshotsEqual(t, cSnap, wSnap) { + t.Fatalf("round %d diverged (seed=%d); mutations this round:\n %s", + round, seed, joinLines(f.log)) + } + prev = warm + } +} + +func assertSnapshotsEqual(t *testing.T, control, warm map[string]string) bool { + t.Helper() + ok := true + for k, cv := range control { + wv, found := warm[k] + if !found { + t.Errorf("warm sync MISSING %s", k) + ok = false + } else if wv != cv { + t.Errorf("warm sync DIFFERS at %s:\n control: %s\n warm: %s", k, cv, wv) + ok = false + } + } + for k := range warm { + if _, found := control[k]; !found { + t.Errorf("warm sync EXTRA %s: %s", k, warm[k]) + ok = false + } + } + return ok +} + +func joinLines(lines []string) string { + out := "" + for i, l := range lines { + if i > 0 { + out += "\n " + } + out += l + } + return out +} diff --git a/pkg/connector/sourcecache_sync_test.go b/pkg/connector/sourcecache_sync_test.go new file mode 100644 index 00000000..bff7d6dc --- /dev/null +++ b/pkg/connector/sourcecache_sync_test.go @@ -0,0 +1,1003 @@ +package connector + +// End-to-end source-cache replay harness for the lastMembershipUpdated +// dirty-scope model (group_type_scoped.go). +// +// Runs the real connector against a strict mock Okta org (exact query +// verification, unknown requests fail the test) through the real SDK sync +// loop on the Pebble engine, chaining each sync's c1z as the next sync's +// replay source. Timestamps follow the semantics probed live +// (docs/replay-probe-results.md): member add/remove and user deletion bump +// lastMembershipUpdated; profile changes bump only lastUpdated; user +// deactivation bumps neither and the user stays in the member listing. +// +// The paramount assertion is equivalence: after every warm sync a control +// sync (no previous c1z) runs against the same org state and the two must +// be identical at the v2 reader surface. Request-count ceilings assert the +// point of the exercise: clean groups spend ZERO member-listing requests. + +import ( + "context" + "encoding/json" + "fmt" + "net" + "net/http" + "net/http/httptest" + "net/url" + "path/filepath" + "sort" + "strings" + "sync" + "testing" + "time" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/connectorbuilder" + "github.com/conductorone/baton-sdk/pkg/connectorclient" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/logging" + "github.com/conductorone/baton-sdk/pkg/sourcecache" + sdkSync "github.com/conductorone/baton-sdk/pkg/sync" + "github.com/conductorone/baton-sdk/pkg/types" + "github.com/okta/okta-sdk-golang/v2/okta" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" +) + +// --- mock Okta org ----------------------------------------------------------- + +// mockPageSize forces pagination everywhere (groups listing, member +// listings) so multi-page scopes and the planner's cross-page SpawnCursors +// path are always exercised. +const mockPageSize = 2 + +var mockTimeBase = time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + +// Test-local constants for repeated literals (keeps package-wide goconst +// counts at their pre-existing baseline). +const ( + mockStatusActive = "ACTIVE" + mockErrNotFound = "E0000007" + roleTypeUserAdmin = "USER_ADMIN" + roleTypeHelpDesk = "HELP_DESK_ADMIN" + roleLabelGroupAdmin = "Group Administrator" + mockKeyStatus = "status" +) + +type mockOktaUser struct { + ID string + FirstName string + LastName string + Email string + Status string // ACTIVE | DEPROVISIONED | ... +} + +type mockOktaGroupRole struct { + AssignmentID string + Type string // e.g. USER_ADMIN + Label string +} + +type mockOktaGroup struct { + ID string + Name string + Description string + Type string // OKTA_GROUP | APP_GROUP | BUILT_IN + Members []string + Roles []mockOktaGroupRole + + lastMembershipUpdated int64 // logical seconds since mockTimeBase + lastUpdated int64 +} + +type mockOkta struct { + mu sync.Mutex + t *testing.T + base string // server URL, for Link headers + + clock int64 + + users map[string]*mockOktaUser + userOrder []string + + groups map[string]*mockOktaGroup + groupOrder []string + + counts map[string]int +} + +func newMockOkta(t *testing.T) *mockOkta { + return &mockOkta{ + t: t, + users: map[string]*mockOktaUser{}, + groups: map[string]*mockOktaGroup{}, + counts: map[string]int{}, + } +} + +func (m *mockOkta) tick() int64 { + m.clock++ + return m.clock +} + +func mockTS(n int64) string { + return mockTimeBase.Add(time.Duration(n) * time.Second).Format("2006-01-02T15:04:05.000Z") +} + +func (m *mockOkta) addUser(u *mockOktaUser) { + m.mu.Lock() + defer m.mu.Unlock() + if u.Status == "" { + u.Status = mockStatusActive + } + m.users[u.ID] = u + m.userOrder = append(m.userOrder, u.ID) +} + +// deactivateUser flips the user's status. Probed live: the DEPROVISIONED +// user REMAINS in group member listings and no group timestamp bumps. +func (m *mockOkta) deactivateUser(id string) { + m.mu.Lock() + defer m.mu.Unlock() + m.users[id].Status = "DEPROVISIONED" +} + +// deleteUser removes the user from the org and from every group's member +// list, bumping those groups' lastMembershipUpdated (probed live). +func (m *mockOkta) deleteUser(id string) { + m.mu.Lock() + defer m.mu.Unlock() + delete(m.users, id) + for i, uid := range m.userOrder { + if uid == id { + m.userOrder = append(m.userOrder[:i], m.userOrder[i+1:]...) + break + } + } + for _, g := range m.groups { + for i, mid := range g.Members { + if mid == id { + g.Members = append(g.Members[:i], g.Members[i+1:]...) + g.lastMembershipUpdated = m.tick() + break + } + } + } +} + +func (m *mockOkta) addGroup(g *mockOktaGroup) { + m.mu.Lock() + defer m.mu.Unlock() + if g.Type == "" { + g.Type = oktaGroupType + } + now := m.tick() + g.lastMembershipUpdated = now + g.lastUpdated = now + m.groups[g.ID] = g + m.groupOrder = append(m.groupOrder, g.ID) +} + +func (m *mockOkta) deleteGroup(id string) { + m.mu.Lock() + defer m.mu.Unlock() + delete(m.groups, id) + for i, gid := range m.groupOrder { + if gid == id { + m.groupOrder = append(m.groupOrder[:i], m.groupOrder[i+1:]...) + break + } + } +} + +func (m *mockOkta) addMember(groupID, userID string) { + m.mu.Lock() + defer m.mu.Unlock() + g := m.groups[groupID] + g.Members = append(g.Members, userID) + g.lastMembershipUpdated = m.tick() +} + +func (m *mockOkta) removeMember(groupID, userID string) { + m.mu.Lock() + defer m.mu.Unlock() + g := m.groups[groupID] + for i, mid := range g.Members { + if mid == userID { + g.Members = append(g.Members[:i], g.Members[i+1:]...) + g.lastMembershipUpdated = m.tick() + return + } + } + m.t.Fatalf("removeMember: %s not in %s", userID, groupID) +} + +// touchGroup bumps lastMembershipUpdated without changing the member set — +// the validator-regression case (e.g. an add+remove that nets to zero, or +// a rule re-evaluation). The connector must re-enumerate and produce +// identical rows. +func (m *mockOkta) touchGroup(groupID string) { + m.mu.Lock() + defer m.mu.Unlock() + m.groups[groupID].lastMembershipUpdated = m.tick() +} + +// renameGroup bumps only lastUpdated (probed live): the member scope's +// validator must NOT rotate. +func (m *mockOkta) renameGroup(groupID, name string) { + m.mu.Lock() + defer m.mu.Unlock() + g := m.groups[groupID] + g.Name = name + g.lastUpdated = m.tick() +} + +// assignGroupRole attaches an admin role to the group. Probed live: role +// assignment changes NEITHER group timestamp. +func (m *mockOkta) assignGroupRole(groupID string, role mockOktaGroupRole) { + m.mu.Lock() + defer m.mu.Unlock() + g := m.groups[groupID] + g.Roles = append(g.Roles, role) +} + +func (m *mockOkta) revokeGroupRole(groupID string, roleType string) { + m.mu.Lock() + defer m.mu.Unlock() + g := m.groups[groupID] + for i, r := range g.Roles { + if r.Type == roleType { + g.Roles = append(g.Roles[:i], g.Roles[i+1:]...) + return + } + } + m.t.Fatalf("revokeGroupRole: %s has no %s", groupID, roleType) +} + +// snapshotCounts returns a copy of the request counters and resets them. +func (m *mockOkta) snapshotCounts() map[string]int { + m.mu.Lock() + defer m.mu.Unlock() + out := map[string]int{} + for k, v := range m.counts { + out[k] = v + } + m.counts = map[string]int{} + return out +} + +func (m *mockOkta) userJSON(u *mockOktaUser) map[string]any { + return map[string]any{ + "id": u.ID, + mockKeyStatus: u.Status, + "created": mockTS(0), + "lastUpdated": mockTS(0), + "profile": map[string]any{ + "firstName": u.FirstName, + "lastName": u.LastName, + "email": u.Email, + "login": u.Email, + }, + } +} + +func (m *mockOkta) groupJSON(g *mockOktaGroup, withStats bool) map[string]any { + obj := map[string]any{ + "id": g.ID, + groupTypeProfileKey: g.Type, + "created": mockTS(0), + "lastUpdated": mockTS(g.lastUpdated), + "lastMembershipUpdated": mockTS(g.lastMembershipUpdated), + "profile": map[string]any{ + "name": g.Name, + "description": g.Description, + }, + } + if withStats { + obj["_embedded"] = map[string]any{ + "stats": map[string]any{ + "usersCount": float64(len(g.Members)), + "appsCount": float64(0), + "groupPushMappingsCount": float64(0), + }, + } + } + return obj +} + +// pageOf slices order after the given cursor. The cursor is the last id of +// the previous page (Okta's after-cursor is opaque; ids work fine). +func pageOf(order []string, after string, size int) ([]string, string) { + start := 0 + if after != "" { + for i, id := range order { + if id == after { + start = i + 1 + break + } + } + } + end := start + size + if end >= len(order) { + return order[start:], "" + } + return order[start:end], order[end-1] +} + +func mockWriteJSON(w http.ResponseWriter, obj any) { + w.Header().Set("Content-Type", "application/json") + data, err := json.Marshal(obj) + if err != nil { + panic(err) + } + _, _ = w.Write(data) +} + +// requireQueryKeys fails the test when the request carries query params +// outside the allowed set — the strict-mock discipline that catches scope +// signature drift and unexpected request shapes. +func (m *mockOkta) requireQueryKeys(r *http.Request, allowed ...string) { + q := r.URL.Query() + for k := range q { + ok := false + for _, a := range allowed { + if k == a { + ok = true + break + } + } + if !ok { + m.t.Errorf("mock okta: unexpected query param %q on %s (allowed: %v)", k, r.URL.String(), allowed) + } + } +} + +func (m *mockOkta) linkNext(w http.ResponseWriter, path string, next string) { + if next == "" { + return + } + w.Header().Set("Link", fmt.Sprintf("<%s%s?after=%s&limit=%d>; rel=\"next\"", m.base, path, url.QueryEscape(next), mockPageSize)) +} + +func (m *mockOkta) handler() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + m.mu.Lock() + defer m.mu.Unlock() + + if r.Method != http.MethodGet { + m.t.Errorf("mock okta: unexpected method %s %s", r.Method, r.URL.String()) + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + + path := r.URL.Path + q := r.URL.Query() + parts := strings.Split(strings.TrimPrefix(path, "/"), "/") + + switch { + case path == "/api/v1/org": + m.counts["org"]++ + mockWriteJSON(w, map[string]any{"id": "org1", "companyName": "Mock Org", "subdomain": "mock"}) + + case path == "/api/v1/users": + m.counts["users-list"]++ + m.requireQueryKeys(r, "limit", "after", "search") + if q.Get("search") != "status pr" { + m.t.Errorf("mock okta: users listing missing search=\"status pr\": %s", r.URL.String()) + } + ids, next := pageOf(m.userOrder, q.Get("after"), mockPageSize) + out := make([]map[string]any, 0, len(ids)) + for _, id := range ids { + out = append(out, m.userJSON(m.users[id])) + } + m.linkNext(w, "/api/v1/users", next) + mockWriteJSON(w, out) + + case path == "/api/v1/groups": + m.counts["groups-list"]++ + m.requireQueryKeys(r, "limit", "after", "expand") + if q.Get("expand") != "stats" { + m.t.Errorf("mock okta: groups listing missing expand=stats: %s", r.URL.String()) + } + ids, next := pageOf(m.groupOrder, q.Get("after"), mockPageSize) + out := make([]map[string]any, 0, len(ids)) + for _, id := range ids { + out = append(out, m.groupJSON(m.groups[id], true)) + } + m.linkNext(w, "/api/v1/groups", next) + mockWriteJSON(w, out) + + case len(parts) == 5 && parts[2] == "groups" && parts[4] == "users": + gid := parts[3] + m.counts["group-users:"+gid]++ + m.requireQueryKeys(r, "limit", "after") + g, ok := m.groups[gid] + if !ok { + w.WriteHeader(http.StatusNotFound) + mockWriteJSON(w, map[string]any{"errorCode": mockErrNotFound, "errorSummary": "Not found: " + gid}) + return + } + ids, next := pageOf(g.Members, q.Get("after"), mockPageSize) + out := make([]map[string]any, 0, len(ids)) + for _, id := range ids { + out = append(out, m.userJSON(m.users[id])) + } + m.linkNext(w, path, next) + mockWriteJSON(w, out) + + case len(parts) == 5 && parts[2] == "groups" && parts[4] == "roles": + gid := parts[3] + m.counts["group-roles:"+gid]++ + m.requireQueryKeys(r) + g, ok := m.groups[gid] + if !ok { + w.WriteHeader(http.StatusNotFound) + mockWriteJSON(w, map[string]any{"errorCode": mockErrNotFound, "errorSummary": "Not found: " + gid}) + return + } + out := make([]map[string]any, 0, len(g.Roles)) + for _, role := range g.Roles { + out = append(out, map[string]any{ + "id": role.AssignmentID, + groupTypeProfileKey: role.Type, + "label": role.Label, + mockKeyStatus: mockStatusActive, + "assignmentType": "GROUP", + }) + } + mockWriteJSON(w, out) + + case path == "/api/v1/iam/assignees/users": + m.counts["role-assignees"]++ + mockWriteJSON(w, map[string]any{"value": []any{}}) + + default: + m.t.Errorf("mock okta: unexpected request %s %s", r.Method, r.URL.String()) + w.WriteHeader(http.StatusNotFound) + mockWriteJSON(w, map[string]any{"errorCode": mockErrNotFound, "errorSummary": "unhandled: " + path}) + } + } +} + +// --- harness ----------------------------------------------------------------- + +var harnessSyncResourceTypes = []string{resourceTypeUser.Id, resourceTypeGroup.Id, resourceTypeRole.Id} + +type syncHarness struct { + t *testing.T + ctx context.Context + mock *mockOkta + cc types.ConnectorClient + tmpDir string + syncN int +} + +func newSyncHarness(ctx context.Context, t *testing.T, mock *mockOkta) *syncHarness { + return newSyncHarnessTopology(ctx, t, mock, false) +} + +// newSyncHarnessTopology builds the harness in one of two lookup +// topologies: +// +// - direct (deferredLookup=false): the syncer's per-sync lookup is wired +// into the builder via SetSourceCacheSetter, mirroring in-process and +// subprocess runtimes — connector lookups are answered inline. +// - deferred (deferredLookup=true): the setter wiring is OMITTED, so the +// builder has no direct lookup and the ask/answer continuation runs +// for real over the gRPC loopback, mirroring single-shot transports +// (gRPC-over-Lambda): the syncer attaches SourceCacheLookupOffer on +// warm syncs, the connector's phase-1 lookup defers with +// ErrLookupDeferred, the builder answers with a SourceCacheLookupAsk, +// and the syncer re-invokes with SourceCacheLookupAnswers. +// +// Every scenario assertion is topology-independent: a warm round that +// spends zero member-listing requests in deferred mode proves the bounce +// delivered the validator (a broken continuation would miss and go cold). +func newSyncHarnessTopology(ctx context.Context, t *testing.T, mock *mockOkta, deferredLookup bool) *syncHarness { + t.Helper() + + server := httptest.NewServer(mock.handler()) + t.Cleanup(server.Close) + mock.base = server.URL + serverURL, err := url.Parse(server.URL) + require.NoError(t, err) + + _, oktaClient, err := okta.NewClient(ctx, + okta.WithOrgUrl(server.URL), + okta.WithToken("test-token"), + okta.WithTestingDisableHttpsCheck(true), + okta.WithHttpClientPtr(server.Client()), + okta.WithCache(false), + okta.WithRateLimitMaxRetries(0), + ) + require.NoError(t, err) + + c := &Okta{ + client: oktaClient, + domain: serverURL.Host, + apiToken: "test-token", + userFilters: &userFilterConfig{}, + } + + srv, err := connectorbuilder.NewConnector(ctx, c) + require.NoError(t, err) + + // Serve the connector over local gRPC and talk to it through the real + // connector client, mirroring how the CLI runs syncs. + gs := grpc.NewServer() + v2.RegisterConnectorServiceServer(gs, srv) + v2.RegisterGrantsServiceServer(gs, srv) + v2.RegisterEntitlementsServiceServer(gs, srv) + v2.RegisterResourcesServiceServer(gs, srv) + v2.RegisterResourceTypesServiceServer(gs, srv) + v2.RegisterAssetServiceServer(gs, srv) + v2.RegisterEventServiceServer(gs, srv) + v2.RegisterResourceGetterServiceServer(gs, srv) + v2.RegisterTicketsServiceServer(gs, srv) + v2.RegisterActionServiceServer(gs, srv) + v2.RegisterGrantManagerServiceServer(gs, srv) + v2.RegisterResourceManagerServiceServer(gs, srv) + v2.RegisterResourceDeleterServiceServer(gs, srv) + v2.RegisterAccountManagerServiceServer(gs, srv) + v2.RegisterCredentialManagerServiceServer(gs, srv) + + lis, err := net.Listen("tcp", "127.0.0.1:0") //nolint:noctx // test-scoped loopback listener + require.NoError(t, err) + go func() { _ = gs.Serve(lis) }() + t.Cleanup(gs.Stop) + + conn, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(t, err) + t.Cleanup(func() { _ = conn.Close() }) + + cc := connectorclient.NewConnectorClient(ctx, conn) + + // Direct lookup delivery: the syncer installs its per-sync lookup on + // the client, which forwards it to the builder (the CLI wrapper does + // this same wiring in internal/connector). Skipped in the deferred + // topology: with no direct lookup, the builder installs a per-request + // ContinuationLookup and the ask/answer protocol carries validators + // across the loopback instead. + if !deferredLookup { + setter, ok := cc.(interface { + SetSourceCacheSetter(sourcecache.SetLookup) + }) + require.True(t, ok, "connector client must accept a source-cache setter") + lookupSink, ok := srv.(sourcecache.SetLookup) + require.True(t, ok, "connectorbuilder server must implement sourcecache.SetLookup") + setter.SetSourceCacheSetter(lookupSink) + } + + return &syncHarness{t: t, ctx: ctx, mock: mock, cc: cc, tmpDir: t.TempDir()} +} + +// runSync executes one full sync into a fresh Pebble c1z, optionally +// replaying from prevPath. Returns the new file's path. +func (h *syncHarness) runSync(name string, prevPath string) string { + h.t.Helper() + h.syncN++ + path := filepath.Join(h.tmpDir, fmt.Sprintf("%02d-%s.c1z", h.syncN, name)) + + store, err := dotc1z.NewStore(h.ctx, path, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithTmpDir(h.tmpDir), + ) + require.NoError(h.t, err) + + opts := []sdkSync.SyncOpt{ + sdkSync.WithConnectorStore(store), + sdkSync.WithTmpDir(h.tmpDir), + sdkSync.WithSyncResourceTypes(harnessSyncResourceTypes), + } + if prevPath != "" { + opts = append(opts, sdkSync.WithPreviousSyncC1ZPath(prevPath)) + } + + syncer, err := sdkSync.NewSyncer(h.ctx, h.cc, opts...) + require.NoError(h.t, err) + require.NoError(h.t, syncer.Sync(h.ctx)) + require.NoError(h.t, syncer.Close(h.ctx)) + return path +} + +// runControlSync runs an uncached control sync and discards its request +// counts, so the next scenario's counters see only its own warm sync. +func (h *syncHarness) runControlSync(name string) string { + h.t.Helper() + path := h.runSync(name, "") + h.mock.snapshotCounts() + return path +} + +// snapshot reads a finished c1z at the v2 reader surface and returns +// id → canonical JSON for resources, entitlements, and grants. +func (h *syncHarness) snapshot(path string) map[string]string { + h.t.Helper() + store, err := dotc1z.NewStore(h.ctx, path, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithReadOnly(true), + dotc1z.WithTmpDir(h.tmpDir), + ) + require.NoError(h.t, err) + defer func() { _ = store.Close(h.ctx) }() + + latest, err := store.SyncMeta().LatestFullSync(h.ctx) + require.NoError(h.t, err) + require.NotNil(h.t, latest) + require.NoError(h.t, store.SetCurrentSync(h.ctx, latest.ID)) + + out := map[string]string{} + put := func(prefix, id string, msg proto.Message) { + jb, err := protojson.Marshal(msg) + require.NoError(h.t, err) + // protojson output spacing is deliberately unstable; re-marshal + // through encoding/json for canonical (sorted-key) bytes. + var v any + require.NoError(h.t, json.Unmarshal(jb, &v)) + cb, err := json.Marshal(v) + require.NoError(h.t, err) + key := prefix + ":" + id + require.NotContains(h.t, out, key, "duplicate id at reader surface") + out[key] = string(cb) + } + + for _, rt := range harnessSyncResourceTypes { + pageToken := "" + for { + resp, err := store.ListResources(h.ctx, v2.ResourcesServiceListResourcesRequest_builder{ + ResourceTypeId: rt, + PageToken: pageToken, + }.Build()) + require.NoError(h.t, err) + for _, r := range resp.GetList() { + put("resource", r.GetId().GetResourceType()+"/"+r.GetId().GetResource(), r) + } + pageToken = resp.GetNextPageToken() + if pageToken == "" { + break + } + } + } + + pageToken := "" + for { + resp, err := store.ListEntitlements(h.ctx, v2.EntitlementsServiceListEntitlementsRequest_builder{ + PageToken: pageToken, + }.Build()) + require.NoError(h.t, err) + for _, e := range resp.GetList() { + put("entitlement", e.GetId(), e) + } + pageToken = resp.GetNextPageToken() + if pageToken == "" { + break + } + } + + pageToken = "" + for { + resp, err := store.ListGrants(h.ctx, v2.GrantsServiceListGrantsRequest_builder{ + PageToken: pageToken, + }.Build()) + require.NoError(h.t, err) + for _, g := range resp.GetList() { + put("grant", g.GetId(), g) + } + pageToken = resp.GetNextPageToken() + if pageToken == "" { + break + } + } + + return out +} + +// requireEquivalent is the release-blocker check: a warm (replayed) sync +// must be byte-identical to an uncached control sync at the reader surface. +func (h *syncHarness) requireEquivalent(warmPath, controlPath string, scenario string) { + h.t.Helper() + warm := h.snapshot(warmPath) + control := h.snapshot(controlPath) + require.Equal(h.t, control, warm, + "%s: warm sync diverged from uncached control sync — replay equivalence violated", scenario) +} + +// memberListingCalls sums group-users request counts, keyed per group. +func memberListingCalls(counts map[string]int) map[string]int { + out := map[string]int{} + for k, v := range counts { + if gid, ok := strings.CutPrefix(k, "group-users:"); ok { + out[gid] = v + } + } + return out +} + +func sortedKeys(m map[string]int) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} + +func memberGrantID(groupID, userID string) string { + return fmt.Sprintf("group:%s:member:user:%s", groupID, userID) +} + +func roleGroupGrantID(roleType, groupID string) string { + return fmt.Sprintf("role:%s:assigned:group:%s", roleType, groupID) +} + +// --- the scenarios ----------------------------------------------------------- + +func TestSourceCacheReplayEndToEnd(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + + mock := newMockOkta(t) + + // Org: five users; g1 (3 members, paginates at mockPageSize=2, carries + // an admin role for the always-fresh roles leg + grant expansion), + // g2 (1 member), g3 (empty — exercises the users_count==0 skip). + for i := 1; i <= 5; i++ { + mock.addUser(&mockOktaUser{ + ID: fmt.Sprintf("u%d", i), + FirstName: "Member", + LastName: fmt.Sprintf("Number%d", i), + Email: fmt.Sprintf("u%d@x.test", i), + }) + } + mock.addGroup(&mockOktaGroup{ID: "g1", Name: "Engineering", Members: []string{"u1", "u2", "u3"}}) + mock.addGroup(&mockOktaGroup{ID: "g2", Name: "Sales", Members: []string{"u4"}}) + mock.addGroup(&mockOktaGroup{ID: "g3", Name: "Empty"}) + mock.assignGroupRole("g1", mockOktaGroupRole{AssignmentID: "gra1", Type: roleTypeUserAdmin, Label: roleLabelGroupAdmin}) + + h := newSyncHarness(ctx, t, mock) + + // --- Sync 1: cold --------------------------------------------------------- + sync1 := h.runSync("cold", "") + c1 := mock.snapshotCounts() + mc1 := memberListingCalls(c1) + require.Equal(t, 2, mc1["g1"], "3 members at page size 2 = 2 requests") + require.Equal(t, 1, mc1["g2"]) + require.Zero(t, mc1["g3"], "users_count==0 skips the member listing even cold") + require.Equal(t, 1, c1["group-roles:g1"], "roles leg runs once per group") + require.Equal(t, 1, c1["group-roles:g2"]) + require.Equal(t, 1, c1["group-roles:g3"]) + + snap1 := h.snapshot(sync1) + require.Contains(t, snap1, "resource:user/u1") + require.Contains(t, snap1, "resource:group/g1") + require.Contains(t, snap1, "resource:group/g3") + require.Contains(t, snap1, "grant:"+memberGrantID("g1", "u1")) + require.Contains(t, snap1, "grant:"+memberGrantID("g1", "u3")) + require.Contains(t, snap1, "grant:"+memberGrantID("g2", "u4")) + require.Contains(t, snap1, "grant:"+roleGroupGrantID(roleTypeUserAdmin, "g1"), "group role grant from the fresh roles leg") + // Grant expansion: g1's members must hold derived USER_ADMIN grants. + foundDerived := false + for k := range snap1 { + if strings.HasPrefix(k, "grant:role:USER_ADMIN:assigned:user:u1") { + foundDerived = true + } + } + require.True(t, foundDerived, "expansion must derive u1's USER_ADMIN grant via g1") + + // --- Scenario 1: no-op round ---------------------------------------------- + sync2 := h.runSync("noop", sync1) + c2 := mock.snapshotCounts() + mc2 := memberListingCalls(c2) + require.Empty(t, sortedKeys(mc2), "no-op warm sync must spend ZERO member-listing requests, got %v", mc2) + require.Equal(t, 1, c2["group-roles:g1"], "roles leg stays fresh on warm rounds") + control2 := h.runControlSync("noop-control") + h.requireEquivalent(sync2, control2, "no-op round") + + // --- Scenario 2: member ADD ----------------------------------------------- + mock.addMember("g2", "u5") + sync3 := h.runSync("add", sync2) + mc3 := memberListingCalls(mock.snapshotCounts()) + require.Equal(t, []string{"g2"}, sortedKeys(mc3), "only the dirty group re-enumerates, got %v", mc3) + require.Contains(t, h.snapshot(sync3), "grant:"+memberGrantID("g2", "u5")) + control3 := h.runControlSync("add-control") + h.requireEquivalent(sync3, control3, "member add") + + // --- Scenario 3: member REMOVE (the model-critical direction) -------------- + mock.removeMember("g1", "u2") + sync4 := h.runSync("remove", sync3) + mc4 := memberListingCalls(mock.snapshotCounts()) + require.Equal(t, []string{"g1"}, sortedKeys(mc4), "only the dirty group re-enumerates, got %v", mc4) + snap4 := h.snapshot(sync4) + require.NotContains(t, snap4, "grant:"+memberGrantID("g1", "u2"), "revoked membership must disappear from the warm sync") + require.Contains(t, snap4, "grant:"+memberGrantID("g1", "u1")) + control4 := h.runControlSync("remove-control") + h.requireEquivalent(sync4, control4, "member remove") + + // --- Scenario 4: empty group gains its first member ------------------------ + mock.addMember("g3", "u2") + sync5 := h.runSync("empty-fill", sync4) + mc5 := memberListingCalls(mock.snapshotCounts()) + require.Equal(t, []string{"g3"}, sortedKeys(mc5), "formerly-empty dirty group must enumerate, got %v", mc5) + require.Contains(t, h.snapshot(sync5), "grant:"+memberGrantID("g3", "u2")) + control5 := h.runControlSync("empty-fill-control") + h.requireEquivalent(sync5, control5, "empty group fill") + + // --- Scenario 5: group create + delete ------------------------------------- + mock.addGroup(&mockOktaGroup{ID: "g4", Name: "Newcomers", Members: []string{"u5"}}) + sync6 := h.runSync("create", sync5) + mc6 := memberListingCalls(mock.snapshotCounts()) + require.Equal(t, []string{"g4"}, sortedKeys(mc6), "only the new group enumerates, got %v", mc6) + require.Contains(t, h.snapshot(sync6), "grant:"+memberGrantID("g4", "u5")) + control6 := h.runControlSync("create-control") + h.requireEquivalent(sync6, control6, "group create") + + mock.deleteGroup("g4") + sync7 := h.runSync("delete", sync6) + mc7 := memberListingCalls(mock.snapshotCounts()) + require.Empty(t, sortedKeys(mc7), "deleting a group must not dirty the others, got %v", mc7) + snap7 := h.snapshot(sync7) + require.NotContains(t, snap7, "resource:group/g4") + require.NotContains(t, snap7, "grant:"+memberGrantID("g4", "u5")) + control7 := h.runControlSync("delete-control") + h.requireEquivalent(sync7, control7, "group delete") + + // --- Scenario 6: validator regression (bump without member change) --------- + mock.touchGroup("g2") + sync8 := h.runSync("touch", sync7) + mc8 := memberListingCalls(mock.snapshotCounts()) + require.Equal(t, []string{"g2"}, sortedKeys(mc8), "touched group re-enumerates (fails toward cold), got %v", mc8) + control8 := h.runControlSync("touch-control") + h.requireEquivalent(sync8, control8, "validator regression") + + // --- Scenario 7: profile rename must NOT rotate the member validator ------- + mock.renameGroup("g1", "Engineering Platform") + sync9 := h.runSync("rename", sync8) + mc9 := memberListingCalls(mock.snapshotCounts()) + require.Empty(t, sortedKeys(mc9), "profile change must not dirty the member scope, got %v", mc9) + snap9 := h.snapshot(sync9) + require.Contains(t, snap9["resource:group/g1"], "Engineering Platform", "rename lands via the resources phase") + control9 := h.runControlSync("rename-control") + h.requireEquivalent(sync9, control9, "group rename") + + // --- Scenario 8: user deactivation (member stays; probed live) ------------- + mock.deactivateUser("u4") + sync10 := h.runSync("deactivate", sync9) + mc10 := memberListingCalls(mock.snapshotCounts()) + require.Empty(t, sortedKeys(mc10), "deactivation bumps no group timestamp; replay must hold, got %v", mc10) + snap10 := h.snapshot(sync10) + require.Contains(t, snap10, "grant:"+memberGrantID("g2", "u4"), "deprovisioned member remains in the listing (probed) — grant stays") + control10 := h.runControlSync("deactivate-control") + h.requireEquivalent(sync10, control10, "user deactivation") + + // --- Scenario 9: user DELETION (removes membership + bumps; probed live) --- + mock.deleteUser("u4") + sync11 := h.runSync("user-delete", sync10) + mc11 := memberListingCalls(mock.snapshotCounts()) + require.Equal(t, []string{"g2"}, sortedKeys(mc11), "user deletion dirties exactly the groups they belonged to, got %v", mc11) + snap11 := h.snapshot(sync11) + require.NotContains(t, snap11, "resource:user/u4") + require.NotContains(t, snap11, "grant:"+memberGrantID("g2", "u4")) + control11 := h.runControlSync("user-delete-control") + h.requireEquivalent(sync11, control11, "user deletion") + + // --- Scenario 10: role assignment changes ride the fresh leg --------------- + mock.assignGroupRole("g2", mockOktaGroupRole{AssignmentID: "gra2", Type: roleTypeHelpDesk, Label: "Help Desk Administrator"}) + sync12 := h.runSync("role-assign", sync11) + mc12 := memberListingCalls(mock.snapshotCounts()) + require.Empty(t, sortedKeys(mc12), "role assignment must not dirty the member scope, got %v", mc12) + require.Contains(t, h.snapshot(sync12), "grant:"+roleGroupGrantID(roleTypeHelpDesk, "g2"), "new role grant arrives on a fully-warm round via the fresh leg") + control12 := h.runControlSync("role-assign-control") + h.requireEquivalent(sync12, control12, "group role assignment") + + mock.revokeGroupRole("g2", roleTypeHelpDesk) + sync13 := h.runSync("role-revoke", sync12) + require.NotContains(t, h.snapshot(sync13), "grant:"+roleGroupGrantID(roleTypeHelpDesk, "g2")) + control13 := h.runControlSync("role-revoke-control") + h.requireEquivalent(sync13, control13, "group role revocation") + + // --- Scenario 11: mass invalidation + recovery ------------------------------ + for _, gid := range []string{"g1", "g2", "g3"} { + mock.touchGroup(gid) + } + sync14 := h.runSync("mass-invalidation", sync13) + mc14 := memberListingCalls(mock.snapshotCounts()) + // g3 gained u2 in scenario 4, so all three groups are non-empty here. + require.Equal(t, []string{"g1", "g2", "g3"}, sortedKeys(mc14), "every non-empty group re-enumerates, got %v", mc14) + control14 := h.runControlSync("mass-invalidation-control") + h.requireEquivalent(sync14, control14, "mass invalidation") + + sync15 := h.runSync("recovery", sync14) + mc15 := memberListingCalls(mock.snapshotCounts()) + require.Empty(t, sortedKeys(mc15), "the round after mass invalidation must be fully warm again, got %v", mc15) + control15 := h.runControlSync("recovery-control") + h.requireEquivalent(sync15, control15, "post-invalidation recovery") +} + +// TestSourceCacheReplayDeferredLookup runs the replay flow through the +// ask/answer lookup continuation — the topology of single-shot transports +// (gRPC-over-Lambda), where the connector cannot reach the syncer's lookup +// service mid-request. The harness omits the direct-lookup wiring, so on +// warm syncs every group cursor's first call genuinely bounces: phase 1 +// defers at the scope lookup (before any upstream request — the mock's +// strict counters prove no phase-1 double work), the builder answers with +// a SourceCacheLookupAsk, and the syncer re-invokes with answers resolved +// from the previous c1z. +// +// The assertions are deliberately the same ones the direct-topology test +// makes: zero member listings on clean groups is only possible if the +// bounce delivered the stored validator (a broken continuation degrades to +// cold and fails the count ceilings), and reader-surface equivalence +// proves phase-2 re-execution changed nothing. +func TestSourceCacheReplayDeferredLookup(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + + mock := newMockOkta(t) + for i := 1; i <= 4; i++ { + mock.addUser(&mockOktaUser{ + ID: fmt.Sprintf("u%d", i), + FirstName: "Member", + LastName: fmt.Sprintf("Number%d", i), + Email: fmt.Sprintf("u%d@x.test", i), + }) + } + mock.addGroup(&mockOktaGroup{ID: "g1", Name: "Engineering", Members: []string{"u1", "u2", "u3"}}) + mock.addGroup(&mockOktaGroup{ID: "g2", Name: "Sales", Members: []string{"u4"}}) + mock.addGroup(&mockOktaGroup{ID: "g3", Name: "Empty"}) + mock.assignGroupRole("g1", mockOktaGroupRole{AssignmentID: "gra1", Type: roleTypeUserAdmin, Label: roleLabelGroupAdmin}) + + h := newSyncHarnessTopology(ctx, t, mock, true) + + // Cold: no previous sync means no offer, no deferral — plain cold + // enumeration that seeds the scope manifest. + sync1 := h.runSync("deferred-cold", "") + c1 := mock.snapshotCounts() + mc1 := memberListingCalls(c1) + require.Equal(t, 2, mc1["g1"], "3 members at page size 2 = 2 requests") + require.Equal(t, 1, mc1["g2"]) + require.Zero(t, mc1["g3"], "users_count==0 skips the member listing even cold") + + // Warm no-op: every cursor bounces (ask → answers) and replays. + sync2 := h.runSync("deferred-noop", sync1) + c2 := mock.snapshotCounts() + mc2 := memberListingCalls(c2) + require.Empty(t, sortedKeys(mc2), "warm round over the continuation must spend ZERO member-listing requests, got %v", mc2) + require.Equal(t, 1, c2["group-roles:g1"], "phase-2 re-execution must not double the roles leg") + require.Equal(t, 1, c2["group-roles:g2"]) + require.Equal(t, 1, c2["group-roles:g3"]) + control2 := h.runControlSync("deferred-noop-control") + h.requireEquivalent(sync2, control2, "deferred no-op round") + + // Member REMOVE: the bounce answers with a stale validator, the dirty + // group re-enumerates cold in phase 2, clean groups still replay. + mock.removeMember("g1", "u2") + sync3 := h.runSync("deferred-remove", sync2) + mc3 := memberListingCalls(mock.snapshotCounts()) + require.Equal(t, []string{"g1"}, sortedKeys(mc3), "only the dirty group re-enumerates, got %v", mc3) + snap3 := h.snapshot(sync3) + require.NotContains(t, snap3, "grant:"+memberGrantID("g1", "u2")) + require.Contains(t, snap3, "grant:"+memberGrantID("g1", "u1")) + control3 := h.runControlSync("deferred-remove-control") + h.requireEquivalent(sync3, control3, "deferred member remove") + + // Mass invalidation + recovery: every scope misses on the bounce, all + // non-empty groups go cold, and the next round is fully warm again. + for _, gid := range []string{"g1", "g2", "g3"} { + mock.touchGroup(gid) + } + sync4 := h.runSync("deferred-mass-invalidation", sync3) + mc4 := memberListingCalls(mock.snapshotCounts()) + require.Equal(t, []string{"g1", "g2"}, sortedKeys(mc4), "non-empty groups re-enumerate, got %v", mc4) + control4 := h.runControlSync("deferred-mass-invalidation-control") + h.requireEquivalent(sync4, control4, "deferred mass invalidation") + + sync5 := h.runSync("deferred-recovery", sync4) + mc5 := memberListingCalls(mock.snapshotCounts()) + require.Empty(t, sortedKeys(mc5), "post-invalidation round must be fully warm again, got %v", mc5) + control5 := h.runControlSync("deferred-recovery-control") + h.requireEquivalent(sync5, control5, "deferred recovery") +} diff --git a/vendor/github.com/conductorone/baton-sdk/internal/connector/connector.go b/vendor/github.com/conductorone/baton-sdk/internal/connector/connector.go index 04170efd..066fcff7 100644 --- a/vendor/github.com/conductorone/baton-sdk/internal/connector/connector.go +++ b/vendor/github.com/conductorone/baton-sdk/internal/connector/connector.go @@ -23,11 +23,13 @@ import ( connectorV2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" connectorwrapperV1 "github.com/conductorone/baton-sdk/pb/c1/connector_wrapper/v1" + batonV1 "github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1" ratelimitV1 "github.com/conductorone/baton-sdk/pb/c1/ratelimit/v1" tlsV1 "github.com/conductorone/baton-sdk/pb/c1/utls/v1" "github.com/conductorone/baton-sdk/pkg/bid" ratelimit2 "github.com/conductorone/baton-sdk/pkg/ratelimit" "github.com/conductorone/baton-sdk/pkg/session" + "github.com/conductorone/baton-sdk/pkg/sourcecache" "github.com/conductorone/baton-sdk/pkg/types" "github.com/conductorone/baton-sdk/pkg/types/sessions" "github.com/conductorone/baton-sdk/pkg/ugrpc" @@ -55,19 +57,42 @@ type connectorClient struct { connectorV2.ActionServiceClient sessionStoreSetter sessions.SetSessionStore // this is the session store server + sourceCacheSetter sourcecache.SetLookup // this is the source-cache lookup server } var _ sessions.SetSessionStore = (*connectorClient)(nil) +var _ sourcecache.SetLookup = (*connectorClient)(nil) var _ SetSessionStoreSetter = (*connectorClient)(nil) +var _ SetSourceCacheSetter = (*connectorClient)(nil) type SetSessionStoreSetter interface { SetSessionStoreSetter(setsessionStoreSetter sessions.SetSessionStore) } +type SetSourceCacheSetter interface { + SetSourceCacheSetter(sourceCacheSetter sourcecache.SetLookup) +} + func (c *connectorClient) SetSessionStoreSetter(sessionStoreSetter sessions.SetSessionStore) { c.sessionStoreSetter = sessionStoreSetter } +func (c *connectorClient) SetSourceCacheSetter(sourceCacheSetter sourcecache.SetLookup) { + c.sourceCacheSetter = sourceCacheSetter +} + +// SetSourceCache forwards the syncer's per-sync lookup to whatever +// receives it: the subprocess-mode BatonSourceCacheService server, or the +// in-process builder. A nil setter means the connector never opted in; +// the syncer calls this unconditionally, so stay quiet at debug level. +func (c *connectorClient) SetSourceCache(ctx context.Context, lookup sourcecache.Lookup) { + if c.sourceCacheSetter == nil { + ctxzap.Extract(ctx).Debug("connectorClient's source cache setter is nil — connector did not opt into source caching") + return + } + c.sourceCacheSetter.SetSourceCache(ctx, lookup) +} + func (c *connectorClient) SetSessionStore(ctx context.Context, store sessions.SessionStore) { if c.sessionStoreSetter == nil { // Demoted from Warn to Debug: this path is the normal case for @@ -109,7 +134,8 @@ type wrapper struct { now func() time.Time - SessionServer sessions.SetSessionStore + SessionServer sessions.SetSessionStore + SourceCacheServer sourcecache.SetLookup } type Option func(ctx context.Context, w *wrapper) error @@ -197,6 +223,12 @@ func NewWrapper(ctx context.Context, server interface{}, opts ...Option) (*wrapp server: connectorServer, now: time.Now, } + // In-process delivery: a connectorbuilder-based server implements + // sourcecache.SetLookup itself, so the syncer's per-sync lookup can be + // installed without the subprocess gRPC hop. + if sourceCacheServer, ok := connectorServer.(sourcecache.SetLookup); ok { + w.SourceCacheServer = sourceCacheServer + } for _, o := range opts { err := o(ctx, w) @@ -276,18 +308,29 @@ func (cw *wrapper) runServer(ctx context.Context, serverCred *tlsV1.Credential) return 0, fmt.Errorf("failed to create session listener config: %w", err) } - // TODO(kans): block until we send a request or something/error handling in general. + // One listener serves BatonSessionService (connector session data) + // and BatonSourceCacheService (source-cache scope lookups). Keeping + // them as separate RPCs keeps validator lookups out of the + // connector's local MemorySessionCache and its TTL/eviction rules. l.Info("starting session store server") - server := session.NewGRPCSessionServer() - cw.SessionServer = server + sessionServer := session.NewGRPCSessionServer() + sourceCacheServer := sourcecache.NewGRPCServer() + cw.SessionServer = sessionServer + cw.SourceCacheServer = sourceCacheServer go func() { defer sessionListener.Close() - serverErr := session.StartGRPCSessionServerWithOptions(ctx, sessionListener, server, + grpcServer := grpc.NewServer( grpc.Creds(credentials.NewTLS(tlsConfig)), grpc.ChainUnaryInterceptor(ugrpc.UnaryServerInterceptor(ctx)...), ) - if serverErr != nil { - l.Error("failed to create session store server", zap.Error(serverErr)) + batonV1.RegisterBatonSessionServiceServer(grpcServer, sessionServer) + batonV1.RegisterBatonSourceCacheServiceServer(grpcServer, sourceCacheServer) + go func() { + <-ctx.Done() + grpcServer.GracefulStop() + }() + if serveErr := grpcServer.Serve(sessionListener); serveErr != nil { + l.Error("session/source-cache server stopped", zap.Error(serveErr)) return } }() @@ -440,6 +483,7 @@ func (cw *wrapper) C(ctx context.Context) (types.ConnectorClient, error) { cw.conn = conn client := NewConnectorClient(ctx, cw.conn) client.SetSessionStoreSetter(cw.SessionServer) + client.SetSourceCacheSetter(cw.SourceCacheServer) cw.client = client return client, nil diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_source_cache.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_source_cache.pb.go new file mode 100644 index 00000000..437dfd89 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_source_cache.pb.go @@ -0,0 +1,914 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.10 +// protoc (unknown) +// source: c1/connector/v2/annotation_source_cache.proto + +//go:build !protoopaque + +package v2 + +import ( + _ "github.com/envoyproxy/protoc-gen-validate/validate" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type SourceCacheCapability_Mode int32 + +const ( + SourceCacheCapability_MODE_UNSPECIFIED SourceCacheCapability_Mode = 0 + SourceCacheCapability_MODE_DISABLED SourceCacheCapability_Mode = 1 + SourceCacheCapability_MODE_READ_WRITE SourceCacheCapability_Mode = 2 +) + +// Enum value maps for SourceCacheCapability_Mode. +var ( + SourceCacheCapability_Mode_name = map[int32]string{ + 0: "MODE_UNSPECIFIED", + 1: "MODE_DISABLED", + 2: "MODE_READ_WRITE", + } + SourceCacheCapability_Mode_value = map[string]int32{ + "MODE_UNSPECIFIED": 0, + "MODE_DISABLED": 1, + "MODE_READ_WRITE": 2, + } +) + +func (x SourceCacheCapability_Mode) Enum() *SourceCacheCapability_Mode { + p := new(SourceCacheCapability_Mode) + *p = x + return p +} + +func (x SourceCacheCapability_Mode) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (SourceCacheCapability_Mode) Descriptor() protoreflect.EnumDescriptor { + return file_c1_connector_v2_annotation_source_cache_proto_enumTypes[0].Descriptor() +} + +func (SourceCacheCapability_Mode) Type() protoreflect.EnumType { + return &file_c1_connector_v2_annotation_source_cache_proto_enumTypes[0] +} + +func (x SourceCacheCapability_Mode) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// SourceCacheCapability is attached to ConnectorServiceValidateResponse +// annotations to opt in to source-cache replay. Absent or any mode other +// than MODE_READ_WRITE means all source-cache annotations are ignored. +type SourceCacheCapability struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + Mode SourceCacheCapability_Mode `protobuf:"varint,1,opt,name=mode,proto3,enum=c1.connector.v2.SourceCacheCapability_Mode" json:"mode,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheCapability) Reset() { + *x = SourceCacheCapability{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheCapability) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheCapability) ProtoMessage() {} + +func (x *SourceCacheCapability) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheCapability) GetMode() SourceCacheCapability_Mode { + if x != nil { + return x.Mode + } + return SourceCacheCapability_MODE_UNSPECIFIED +} + +func (x *SourceCacheCapability) SetMode(v SourceCacheCapability_Mode) { + x.Mode = v +} + +type SourceCacheCapability_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + Mode SourceCacheCapability_Mode +} + +func (b0 SourceCacheCapability_builder) Build() *SourceCacheCapability { + m0 := &SourceCacheCapability{} + b, x := &b0, m0 + _, _ = b, x + x.Mode = b.Mode + return m0 +} + +// SourceCacheScope is attached to a list-response page whose rows were +// freshly fetched from upstream. The SDK stamps the page's rows with +// scope_hash so a future sync can replay them as a unit. +type SourceCacheScope struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + // Connector-computed stable identifier for the canonical scope. Must be + // byte-stable across syncs for the same logical scope. Prefer an ORDERED + // natural identifier (e.g. the request URL, or "groups/{id}/members") + // over a random hash: scope-index writes lead with this value, so + // identifiers that correlate with fetch order keep index writes nearly + // append-ordered at scale. sourcecache.HashScope is the fallback when no + // compact natural form exists. + ScopeHash string `protobuf:"bytes,1,opt,name=scope_hash,json=scopeHash,proto3" json:"scope_hash,omitempty"` + // Opaque validator to persist for this scope. May be empty on interim + // pages of a multi-page scope (e.g. Graph @odata.nextLink pages); the + // SDK writes the scope's manifest entry when a non-empty etag arrives. + // A 200 response with zero rows still persists the entry. + Etag string `protobuf:"bytes,2,opt,name=etag,proto3" json:"etag,omitempty"` + // Tombstones applied after this page's rows commit. Lets every page of + // a multi-page delta round carry its own deletions as the provider + // delivers them, instead of buffering a whole round onto the first + // (replay-annotated) page. Same formats as SourceCacheReplay. + // + // PRECONDITION for tombstones anywhere in a round: the provider's delta + // must be coalesced — at most one add-or-tombstone per object per round + // (Microsoft Graph guarantees this by returning final object state). + // With interleaved add/remove events for one object, per-page ordering + // is deterministic (a page's rows upsert before its deletions apply) + // but cross-page re-adds after a tombstone are the connector's + // responsibility to order. + DeletedIds []string `protobuf:"bytes,3,rep,name=deleted_ids,json=deletedIds,proto3" json:"deleted_ids,omitempty"` + // Principal-scoped grant tombstones: for RowKindGrants pages, each + // entry deletes EVERY grant row stamped with this scope whose principal + // id equals the entry — no principal resource type and no canonical + // grant-id reconstruction required (delta tombstones usually carry only + // a bare object id, and the object may no longer exist to look up). + // + // PRECONDITION: the scope must be partitioned so that "principal + // removed from scope" means every grant they have in the scope is gone + // — one scope per navigation with independent removal semantics (e.g. + // members and owners of a group are separate scopes, or membership + // removal would take the owner grant with it). + // + // For RowKindResources pages, each entry deletes the resource row(s) + // stamped with this scope whose resource id equals the entry, any + // resource type. + DeletedPrincipalIds []string `protobuf:"bytes,4,rep,name=deleted_principal_ids,json=deletedPrincipalIds,proto3" json:"deleted_principal_ids,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheScope) Reset() { + *x = SourceCacheScope{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheScope) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheScope) ProtoMessage() {} + +func (x *SourceCacheScope) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheScope) GetScopeHash() string { + if x != nil { + return x.ScopeHash + } + return "" +} + +func (x *SourceCacheScope) GetEtag() string { + if x != nil { + return x.Etag + } + return "" +} + +func (x *SourceCacheScope) GetDeletedIds() []string { + if x != nil { + return x.DeletedIds + } + return nil +} + +func (x *SourceCacheScope) GetDeletedPrincipalIds() []string { + if x != nil { + return x.DeletedPrincipalIds + } + return nil +} + +func (x *SourceCacheScope) SetScopeHash(v string) { + x.ScopeHash = v +} + +func (x *SourceCacheScope) SetEtag(v string) { + x.Etag = v +} + +func (x *SourceCacheScope) SetDeletedIds(v []string) { + x.DeletedIds = v +} + +func (x *SourceCacheScope) SetDeletedPrincipalIds(v []string) { + x.DeletedPrincipalIds = v +} + +type SourceCacheScope_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Connector-computed stable identifier for the canonical scope. Must be + // byte-stable across syncs for the same logical scope. Prefer an ORDERED + // natural identifier (e.g. the request URL, or "groups/{id}/members") + // over a random hash: scope-index writes lead with this value, so + // identifiers that correlate with fetch order keep index writes nearly + // append-ordered at scale. sourcecache.HashScope is the fallback when no + // compact natural form exists. + ScopeHash string + // Opaque validator to persist for this scope. May be empty on interim + // pages of a multi-page scope (e.g. Graph @odata.nextLink pages); the + // SDK writes the scope's manifest entry when a non-empty etag arrives. + // A 200 response with zero rows still persists the entry. + Etag string + // Tombstones applied after this page's rows commit. Lets every page of + // a multi-page delta round carry its own deletions as the provider + // delivers them, instead of buffering a whole round onto the first + // (replay-annotated) page. Same formats as SourceCacheReplay. + // + // PRECONDITION for tombstones anywhere in a round: the provider's delta + // must be coalesced — at most one add-or-tombstone per object per round + // (Microsoft Graph guarantees this by returning final object state). + // With interleaved add/remove events for one object, per-page ordering + // is deterministic (a page's rows upsert before its deletions apply) + // but cross-page re-adds after a tombstone are the connector's + // responsibility to order. + DeletedIds []string + // Principal-scoped grant tombstones: for RowKindGrants pages, each + // entry deletes EVERY grant row stamped with this scope whose principal + // id equals the entry — no principal resource type and no canonical + // grant-id reconstruction required (delta tombstones usually carry only + // a bare object id, and the object may no longer exist to look up). + // + // PRECONDITION: the scope must be partitioned so that "principal + // removed from scope" means every grant they have in the scope is gone + // — one scope per navigation with independent removal semantics (e.g. + // members and owners of a group are separate scopes, or membership + // removal would take the owner grant with it). + // + // For RowKindResources pages, each entry deletes the resource row(s) + // stamped with this scope whose resource id equals the entry, any + // resource type. + DeletedPrincipalIds []string +} + +func (b0 SourceCacheScope_builder) Build() *SourceCacheScope { + m0 := &SourceCacheScope{} + b, x := &b0, m0 + _, _ = b, x + x.ScopeHash = b.ScopeHash + x.Etag = b.Etag + x.DeletedIds = b.DeletedIds + x.DeletedPrincipalIds = b.DeletedPrincipalIds + return m0 +} + +// SourceCacheReplay is attached to a list-response page to tell the SDK to +// copy the previous sync's rows for scope_hash into the current sync. +// +// The row kind (resources, entitlements, grants) is determined by which +// RPC the annotation arrived on, never by the annotation itself. +// +// The connector must only emit this for a scope whose etag it received +// from the SDK's source-cache lookup during this same sync. A replay for +// an unknown scope fails the sync: the connector has already skipped row +// generation, so there is nothing to fall back to. +type SourceCacheReplay struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + ScopeHash string `protobuf:"bytes,1,opt,name=scope_hash,json=scopeHash,proto3" json:"scope_hash,omitempty"` + // Validator to persist for this scope in the current sync. For an HTTP + // 304 this is the unchanged etag. For a delta query this is the NEW + // token; it may instead be supplied by the final overlay page's + // SourceCacheScope.etag, in which case this field may be left empty. + Etag string `protobuf:"bytes,2,opt,name=etag,proto3" json:"etag,omitempty"` + // When true, the response (and subsequent pages carrying + // SourceCacheScope with the same scope_hash) contains changed rows to + // upsert on top of the replayed base. When false the response must + // contain no rows for this scope. + Overlay bool `protobuf:"varint,3,opt,name=overlay,proto3" json:"overlay,omitempty"` + // Public canonical IDs (grant/entitlement IDs, or resource BIDs for + // RowKindResources) to delete from the current sync after the replay + // copy and this page's upserts. Used for delta-query tombstones (e.g. + // Microsoft Graph @removed entries). Subsequent pages of the round + // carry their tombstones on SourceCacheScope.deleted_ids. + DeletedIds []string `protobuf:"bytes,4,rep,name=deleted_ids,json=deletedIds,proto3" json:"deleted_ids,omitempty"` + // Principal-scoped tombstones for this page; see + // SourceCacheScope.deleted_principal_ids for semantics and + // preconditions. + DeletedPrincipalIds []string `protobuf:"bytes,5,rep,name=deleted_principal_ids,json=deletedPrincipalIds,proto3" json:"deleted_principal_ids,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheReplay) Reset() { + *x = SourceCacheReplay{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheReplay) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheReplay) ProtoMessage() {} + +func (x *SourceCacheReplay) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheReplay) GetScopeHash() string { + if x != nil { + return x.ScopeHash + } + return "" +} + +func (x *SourceCacheReplay) GetEtag() string { + if x != nil { + return x.Etag + } + return "" +} + +func (x *SourceCacheReplay) GetOverlay() bool { + if x != nil { + return x.Overlay + } + return false +} + +func (x *SourceCacheReplay) GetDeletedIds() []string { + if x != nil { + return x.DeletedIds + } + return nil +} + +func (x *SourceCacheReplay) GetDeletedPrincipalIds() []string { + if x != nil { + return x.DeletedPrincipalIds + } + return nil +} + +func (x *SourceCacheReplay) SetScopeHash(v string) { + x.ScopeHash = v +} + +func (x *SourceCacheReplay) SetEtag(v string) { + x.Etag = v +} + +func (x *SourceCacheReplay) SetOverlay(v bool) { + x.Overlay = v +} + +func (x *SourceCacheReplay) SetDeletedIds(v []string) { + x.DeletedIds = v +} + +func (x *SourceCacheReplay) SetDeletedPrincipalIds(v []string) { + x.DeletedPrincipalIds = v +} + +type SourceCacheReplay_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + ScopeHash string + // Validator to persist for this scope in the current sync. For an HTTP + // 304 this is the unchanged etag. For a delta query this is the NEW + // token; it may instead be supplied by the final overlay page's + // SourceCacheScope.etag, in which case this field may be left empty. + Etag string + // When true, the response (and subsequent pages carrying + // SourceCacheScope with the same scope_hash) contains changed rows to + // upsert on top of the replayed base. When false the response must + // contain no rows for this scope. + Overlay bool + // Public canonical IDs (grant/entitlement IDs, or resource BIDs for + // RowKindResources) to delete from the current sync after the replay + // copy and this page's upserts. Used for delta-query tombstones (e.g. + // Microsoft Graph @removed entries). Subsequent pages of the round + // carry their tombstones on SourceCacheScope.deleted_ids. + DeletedIds []string + // Principal-scoped tombstones for this page; see + // SourceCacheScope.deleted_principal_ids for semantics and + // preconditions. + DeletedPrincipalIds []string +} + +func (b0 SourceCacheReplay_builder) Build() *SourceCacheReplay { + m0 := &SourceCacheReplay{} + b, x := &b0, m0 + _, _ = b, x + x.ScopeHash = b.ScopeHash + x.Etag = b.Etag + x.Overlay = b.Overlay + x.DeletedIds = b.DeletedIds + x.DeletedPrincipalIds = b.DeletedPrincipalIds + return m0 +} + +// SourceCacheLookupOffer is attached by the SDK to list REQUESTS when the +// syncer can answer source-cache lookup asks: the connector declared +// SourceCacheCapability and a warm previous-sync lookup is installed. +// Its presence is the connector's permission to answer with +// SourceCacheLookupAsk instead of rows. Connectors with a direct lookup +// (in-process, subprocess) never need to defer and may ignore it. +type SourceCacheLookupOffer struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupOffer) Reset() { + *x = SourceCacheLookupOffer{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupOffer) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupOffer) ProtoMessage() {} + +func (x *SourceCacheLookupOffer) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +type SourceCacheLookupOffer_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + +} + +func (b0 SourceCacheLookupOffer_builder) Build() *SourceCacheLookupOffer { + m0 := &SourceCacheLookupOffer{} + b, x := &b0, m0 + _, _ = b, x + return m0 +} + +// SourceCacheLookupAsk is attached to a list RESPONSE in place of rows: +// the connector needs previous-sync validators before it can serve the +// page. An ask response must carry NO rows, NO next page token, and no +// other source-cache annotations; the syncer consumes it (it never +// reaches page handling), resolves every query, and re-invokes the same +// request with SourceCacheLookupAnswers attached. +// +// Only legal on responses to requests that carried +// SourceCacheLookupOffer. Bounces are capped per action; a connector +// that keeps asking past the cap fails the sync loudly. +type SourceCacheLookupAsk struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + Queries []*SourceCacheLookupAsk_Query `protobuf:"bytes,1,rep,name=queries,proto3" json:"queries,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupAsk) Reset() { + *x = SourceCacheLookupAsk{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupAsk) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupAsk) ProtoMessage() {} + +func (x *SourceCacheLookupAsk) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheLookupAsk) GetQueries() []*SourceCacheLookupAsk_Query { + if x != nil { + return x.Queries + } + return nil +} + +func (x *SourceCacheLookupAsk) SetQueries(v []*SourceCacheLookupAsk_Query) { + x.Queries = v +} + +type SourceCacheLookupAsk_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + Queries []*SourceCacheLookupAsk_Query +} + +func (b0 SourceCacheLookupAsk_builder) Build() *SourceCacheLookupAsk { + m0 := &SourceCacheLookupAsk{} + b, x := &b0, m0 + _, _ = b, x + x.Queries = b.Queries + return m0 +} + +// SourceCacheLookupAnswers is attached by the SDK to the re-invoked +// REQUEST, carrying the resolution of a prior ask's queries. +// +// An ABSENT query (asked but not answered — e.g. dropped to the answer +// size budget) is distinct from found=false: not-found means the previous +// sync has no entry (fetch fresh); absent means unresolved (the connector +// may ask again, subject to the bounce cap). Not-found answers are always +// complete for the queried set — only found answers with large etags are +// ever dropped to budget. +type SourceCacheLookupAnswers struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + Answers []*SourceCacheLookupAnswers_Answer `protobuf:"bytes,1,rep,name=answers,proto3" json:"answers,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupAnswers) Reset() { + *x = SourceCacheLookupAnswers{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupAnswers) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupAnswers) ProtoMessage() {} + +func (x *SourceCacheLookupAnswers) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheLookupAnswers) GetAnswers() []*SourceCacheLookupAnswers_Answer { + if x != nil { + return x.Answers + } + return nil +} + +func (x *SourceCacheLookupAnswers) SetAnswers(v []*SourceCacheLookupAnswers_Answer) { + x.Answers = v +} + +type SourceCacheLookupAnswers_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + Answers []*SourceCacheLookupAnswers_Answer +} + +func (b0 SourceCacheLookupAnswers_builder) Build() *SourceCacheLookupAnswers { + m0 := &SourceCacheLookupAnswers{} + b, x := &b0, m0 + _, _ = b, x + x.Answers = b.Answers + return m0 +} + +type SourceCacheLookupAsk_Query struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + // One of the sourcecache.RowKind values: "resources", + // "entitlements", "grants". + RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3" json:"row_kind,omitempty"` + ScopeHash string `protobuf:"bytes,2,opt,name=scope_hash,json=scopeHash,proto3" json:"scope_hash,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupAsk_Query) Reset() { + *x = SourceCacheLookupAsk_Query{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupAsk_Query) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupAsk_Query) ProtoMessage() {} + +func (x *SourceCacheLookupAsk_Query) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheLookupAsk_Query) GetRowKind() string { + if x != nil { + return x.RowKind + } + return "" +} + +func (x *SourceCacheLookupAsk_Query) GetScopeHash() string { + if x != nil { + return x.ScopeHash + } + return "" +} + +func (x *SourceCacheLookupAsk_Query) SetRowKind(v string) { + x.RowKind = v +} + +func (x *SourceCacheLookupAsk_Query) SetScopeHash(v string) { + x.ScopeHash = v +} + +type SourceCacheLookupAsk_Query_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // One of the sourcecache.RowKind values: "resources", + // "entitlements", "grants". + RowKind string + ScopeHash string +} + +func (b0 SourceCacheLookupAsk_Query_builder) Build() *SourceCacheLookupAsk_Query { + m0 := &SourceCacheLookupAsk_Query{} + b, x := &b0, m0 + _, _ = b, x + x.RowKind = b.RowKind + x.ScopeHash = b.ScopeHash + return m0 +} + +type SourceCacheLookupAnswers_Answer struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3" json:"row_kind,omitempty"` + ScopeHash string `protobuf:"bytes,2,opt,name=scope_hash,json=scopeHash,proto3" json:"scope_hash,omitempty"` + Found bool `protobuf:"varint,3,opt,name=found,proto3" json:"found,omitempty"` + // The previous sync's validator; empty when found is false. Cap + // matches the lookup RPC (Graph delta tokens run long). + Etag string `protobuf:"bytes,4,opt,name=etag,proto3" json:"etag,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupAnswers_Answer) Reset() { + *x = SourceCacheLookupAnswers_Answer{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupAnswers_Answer) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupAnswers_Answer) ProtoMessage() {} + +func (x *SourceCacheLookupAnswers_Answer) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheLookupAnswers_Answer) GetRowKind() string { + if x != nil { + return x.RowKind + } + return "" +} + +func (x *SourceCacheLookupAnswers_Answer) GetScopeHash() string { + if x != nil { + return x.ScopeHash + } + return "" +} + +func (x *SourceCacheLookupAnswers_Answer) GetFound() bool { + if x != nil { + return x.Found + } + return false +} + +func (x *SourceCacheLookupAnswers_Answer) GetEtag() string { + if x != nil { + return x.Etag + } + return "" +} + +func (x *SourceCacheLookupAnswers_Answer) SetRowKind(v string) { + x.RowKind = v +} + +func (x *SourceCacheLookupAnswers_Answer) SetScopeHash(v string) { + x.ScopeHash = v +} + +func (x *SourceCacheLookupAnswers_Answer) SetFound(v bool) { + x.Found = v +} + +func (x *SourceCacheLookupAnswers_Answer) SetEtag(v string) { + x.Etag = v +} + +type SourceCacheLookupAnswers_Answer_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + RowKind string + ScopeHash string + Found bool + // The previous sync's validator; empty when found is false. Cap + // matches the lookup RPC (Graph delta tokens run long). + Etag string +} + +func (b0 SourceCacheLookupAnswers_Answer_builder) Build() *SourceCacheLookupAnswers_Answer { + m0 := &SourceCacheLookupAnswers_Answer{} + b, x := &b0, m0 + _, _ = b, x + x.RowKind = b.RowKind + x.ScopeHash = b.ScopeHash + x.Found = b.Found + x.Etag = b.Etag + return m0 +} + +var File_c1_connector_v2_annotation_source_cache_proto protoreflect.FileDescriptor + +const file_c1_connector_v2_annotation_source_cache_proto_rawDesc = "" + + "\n" + + "-c1/connector/v2/annotation_source_cache.proto\x12\x0fc1.connector.v2\x1a\x17validate/validate.proto\"\x9e\x01\n" + + "\x15SourceCacheCapability\x12?\n" + + "\x04mode\x18\x01 \x01(\x0e2+.c1.connector.v2.SourceCacheCapability.ModeR\x04mode\"D\n" + + "\x04Mode\x12\x14\n" + + "\x10MODE_UNSPECIFIED\x10\x00\x12\x11\n" + + "\rMODE_DISABLED\x10\x01\x12\x13\n" + + "\x0fMODE_READ_WRITE\x10\x02\"\x9a\x01\n" + + "\x10SourceCacheScope\x12\x1d\n" + + "\n" + + "scope_hash\x18\x01 \x01(\tR\tscopeHash\x12\x12\n" + + "\x04etag\x18\x02 \x01(\tR\x04etag\x12\x1f\n" + + "\vdeleted_ids\x18\x03 \x03(\tR\n" + + "deletedIds\x122\n" + + "\x15deleted_principal_ids\x18\x04 \x03(\tR\x13deletedPrincipalIds\"\xb5\x01\n" + + "\x11SourceCacheReplay\x12\x1d\n" + + "\n" + + "scope_hash\x18\x01 \x01(\tR\tscopeHash\x12\x12\n" + + "\x04etag\x18\x02 \x01(\tR\x04etag\x12\x18\n" + + "\aoverlay\x18\x03 \x01(\bR\aoverlay\x12\x1f\n" + + "\vdeleted_ids\x18\x04 \x03(\tR\n" + + "deletedIds\x122\n" + + "\x15deleted_principal_ids\x18\x05 \x03(\tR\x13deletedPrincipalIds\"\x18\n" + + "\x16SourceCacheLookupOffer\"\xc4\x01\n" + + "\x14SourceCacheLookupAsk\x12R\n" + + "\aqueries\x18\x01 \x03(\v2+.c1.connector.v2.SourceCacheLookupAsk.QueryB\v\xfaB\b\x92\x01\x05\b\x01\x10\x80 R\aqueries\x1aX\n" + + "\x05Query\x12$\n" + + "\brow_kind\x18\x01 \x01(\tB\t\xfaB\x06r\x04 \x01(@R\arowKind\x12)\n" + + "\n" + + "scope_hash\x18\x02 \x01(\tB\n" + + "\xfaB\ar\x05 \x01(\x80\x02R\tscopeHash\"\xfa\x01\n" + + "\x18SourceCacheLookupAnswers\x12J\n" + + "\aanswers\x18\x01 \x03(\v20.c1.connector.v2.SourceCacheLookupAnswers.AnswerR\aanswers\x1a\x91\x01\n" + + "\x06Answer\x12$\n" + + "\brow_kind\x18\x01 \x01(\tB\t\xfaB\x06r\x04 \x01(@R\arowKind\x12)\n" + + "\n" + + "scope_hash\x18\x02 \x01(\tB\n" + + "\xfaB\ar\x05 \x01(\x80\x02R\tscopeHash\x12\x14\n" + + "\x05found\x18\x03 \x01(\bR\x05found\x12 \n" + + "\x04etag\x18\x04 \x01(\tB\f\xfaB\tr\a(\x80\x80\x04\xd0\x01\x01R\x04etagB6Z4github.com/conductorone/baton-sdk/pb/c1/connector/v2b\x06proto3" + +var file_c1_connector_v2_annotation_source_cache_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_c1_connector_v2_annotation_source_cache_proto_msgTypes = make([]protoimpl.MessageInfo, 8) +var file_c1_connector_v2_annotation_source_cache_proto_goTypes = []any{ + (SourceCacheCapability_Mode)(0), // 0: c1.connector.v2.SourceCacheCapability.Mode + (*SourceCacheCapability)(nil), // 1: c1.connector.v2.SourceCacheCapability + (*SourceCacheScope)(nil), // 2: c1.connector.v2.SourceCacheScope + (*SourceCacheReplay)(nil), // 3: c1.connector.v2.SourceCacheReplay + (*SourceCacheLookupOffer)(nil), // 4: c1.connector.v2.SourceCacheLookupOffer + (*SourceCacheLookupAsk)(nil), // 5: c1.connector.v2.SourceCacheLookupAsk + (*SourceCacheLookupAnswers)(nil), // 6: c1.connector.v2.SourceCacheLookupAnswers + (*SourceCacheLookupAsk_Query)(nil), // 7: c1.connector.v2.SourceCacheLookupAsk.Query + (*SourceCacheLookupAnswers_Answer)(nil), // 8: c1.connector.v2.SourceCacheLookupAnswers.Answer +} +var file_c1_connector_v2_annotation_source_cache_proto_depIdxs = []int32{ + 0, // 0: c1.connector.v2.SourceCacheCapability.mode:type_name -> c1.connector.v2.SourceCacheCapability.Mode + 7, // 1: c1.connector.v2.SourceCacheLookupAsk.queries:type_name -> c1.connector.v2.SourceCacheLookupAsk.Query + 8, // 2: c1.connector.v2.SourceCacheLookupAnswers.answers:type_name -> c1.connector.v2.SourceCacheLookupAnswers.Answer + 3, // [3:3] is the sub-list for method output_type + 3, // [3:3] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name +} + +func init() { file_c1_connector_v2_annotation_source_cache_proto_init() } +func file_c1_connector_v2_annotation_source_cache_proto_init() { + if File_c1_connector_v2_annotation_source_cache_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_connector_v2_annotation_source_cache_proto_rawDesc), len(file_c1_connector_v2_annotation_source_cache_proto_rawDesc)), + NumEnums: 1, + NumMessages: 8, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_c1_connector_v2_annotation_source_cache_proto_goTypes, + DependencyIndexes: file_c1_connector_v2_annotation_source_cache_proto_depIdxs, + EnumInfos: file_c1_connector_v2_annotation_source_cache_proto_enumTypes, + MessageInfos: file_c1_connector_v2_annotation_source_cache_proto_msgTypes, + }.Build() + File_c1_connector_v2_annotation_source_cache_proto = out.File + file_c1_connector_v2_annotation_source_cache_proto_goTypes = nil + file_c1_connector_v2_annotation_source_cache_proto_depIdxs = nil +} diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_source_cache.pb.validate.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_source_cache.pb.validate.go new file mode 100644 index 00000000..a301fbcb --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_source_cache.pb.validate.go @@ -0,0 +1,1003 @@ +// Code generated by protoc-gen-validate. DO NOT EDIT. +// source: c1/connector/v2/annotation_source_cache.proto + +package v2 + +import ( + "bytes" + "errors" + "fmt" + "net" + "net/mail" + "net/url" + "regexp" + "sort" + "strings" + "time" + "unicode/utf8" + + "google.golang.org/protobuf/types/known/anypb" +) + +// ensure the imports are used +var ( + _ = bytes.MinRead + _ = errors.New("") + _ = fmt.Print + _ = utf8.UTFMax + _ = (*regexp.Regexp)(nil) + _ = (*strings.Reader)(nil) + _ = net.IPv4len + _ = time.Duration(0) + _ = (*url.URL)(nil) + _ = (*mail.Address)(nil) + _ = anypb.Any{} + _ = sort.Sort +) + +// Validate checks the field values on SourceCacheCapability with the rules +// defined in the proto definition for this message. If any rules are +// violated, the first error encountered is returned, or nil if there are no violations. +func (m *SourceCacheCapability) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SourceCacheCapability with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// SourceCacheCapabilityMultiError, or nil if none found. +func (m *SourceCacheCapability) ValidateAll() error { + return m.validate(true) +} + +func (m *SourceCacheCapability) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + // no validation rules for Mode + + if len(errors) > 0 { + return SourceCacheCapabilityMultiError(errors) + } + + return nil +} + +// SourceCacheCapabilityMultiError is an error wrapping multiple validation +// errors returned by SourceCacheCapability.ValidateAll() if the designated +// constraints aren't met. +type SourceCacheCapabilityMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SourceCacheCapabilityMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SourceCacheCapabilityMultiError) AllErrors() []error { return m } + +// SourceCacheCapabilityValidationError is the validation error returned by +// SourceCacheCapability.Validate if the designated constraints aren't met. +type SourceCacheCapabilityValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SourceCacheCapabilityValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SourceCacheCapabilityValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SourceCacheCapabilityValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SourceCacheCapabilityValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SourceCacheCapabilityValidationError) ErrorName() string { + return "SourceCacheCapabilityValidationError" +} + +// Error satisfies the builtin error interface +func (e SourceCacheCapabilityValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSourceCacheCapability.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SourceCacheCapabilityValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SourceCacheCapabilityValidationError{} + +// Validate checks the field values on SourceCacheScope with the rules defined +// in the proto definition for this message. If any rules are violated, the +// first error encountered is returned, or nil if there are no violations. +func (m *SourceCacheScope) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SourceCacheScope with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// SourceCacheScopeMultiError, or nil if none found. +func (m *SourceCacheScope) ValidateAll() error { + return m.validate(true) +} + +func (m *SourceCacheScope) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + // no validation rules for ScopeHash + + // no validation rules for Etag + + if len(errors) > 0 { + return SourceCacheScopeMultiError(errors) + } + + return nil +} + +// SourceCacheScopeMultiError is an error wrapping multiple validation errors +// returned by SourceCacheScope.ValidateAll() if the designated constraints +// aren't met. +type SourceCacheScopeMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SourceCacheScopeMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SourceCacheScopeMultiError) AllErrors() []error { return m } + +// SourceCacheScopeValidationError is the validation error returned by +// SourceCacheScope.Validate if the designated constraints aren't met. +type SourceCacheScopeValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SourceCacheScopeValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SourceCacheScopeValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SourceCacheScopeValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SourceCacheScopeValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SourceCacheScopeValidationError) ErrorName() string { return "SourceCacheScopeValidationError" } + +// Error satisfies the builtin error interface +func (e SourceCacheScopeValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSourceCacheScope.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SourceCacheScopeValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SourceCacheScopeValidationError{} + +// Validate checks the field values on SourceCacheReplay with the rules defined +// in the proto definition for this message. If any rules are violated, the +// first error encountered is returned, or nil if there are no violations. +func (m *SourceCacheReplay) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SourceCacheReplay with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// SourceCacheReplayMultiError, or nil if none found. +func (m *SourceCacheReplay) ValidateAll() error { + return m.validate(true) +} + +func (m *SourceCacheReplay) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + // no validation rules for ScopeHash + + // no validation rules for Etag + + // no validation rules for Overlay + + if len(errors) > 0 { + return SourceCacheReplayMultiError(errors) + } + + return nil +} + +// SourceCacheReplayMultiError is an error wrapping multiple validation errors +// returned by SourceCacheReplay.ValidateAll() if the designated constraints +// aren't met. +type SourceCacheReplayMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SourceCacheReplayMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SourceCacheReplayMultiError) AllErrors() []error { return m } + +// SourceCacheReplayValidationError is the validation error returned by +// SourceCacheReplay.Validate if the designated constraints aren't met. +type SourceCacheReplayValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SourceCacheReplayValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SourceCacheReplayValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SourceCacheReplayValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SourceCacheReplayValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SourceCacheReplayValidationError) ErrorName() string { + return "SourceCacheReplayValidationError" +} + +// Error satisfies the builtin error interface +func (e SourceCacheReplayValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSourceCacheReplay.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SourceCacheReplayValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SourceCacheReplayValidationError{} + +// Validate checks the field values on SourceCacheLookupOffer with the rules +// defined in the proto definition for this message. If any rules are +// violated, the first error encountered is returned, or nil if there are no violations. +func (m *SourceCacheLookupOffer) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SourceCacheLookupOffer with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// SourceCacheLookupOfferMultiError, or nil if none found. +func (m *SourceCacheLookupOffer) ValidateAll() error { + return m.validate(true) +} + +func (m *SourceCacheLookupOffer) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + if len(errors) > 0 { + return SourceCacheLookupOfferMultiError(errors) + } + + return nil +} + +// SourceCacheLookupOfferMultiError is an error wrapping multiple validation +// errors returned by SourceCacheLookupOffer.ValidateAll() if the designated +// constraints aren't met. +type SourceCacheLookupOfferMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SourceCacheLookupOfferMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SourceCacheLookupOfferMultiError) AllErrors() []error { return m } + +// SourceCacheLookupOfferValidationError is the validation error returned by +// SourceCacheLookupOffer.Validate if the designated constraints aren't met. +type SourceCacheLookupOfferValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SourceCacheLookupOfferValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SourceCacheLookupOfferValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SourceCacheLookupOfferValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SourceCacheLookupOfferValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SourceCacheLookupOfferValidationError) ErrorName() string { + return "SourceCacheLookupOfferValidationError" +} + +// Error satisfies the builtin error interface +func (e SourceCacheLookupOfferValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSourceCacheLookupOffer.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SourceCacheLookupOfferValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SourceCacheLookupOfferValidationError{} + +// Validate checks the field values on SourceCacheLookupAsk with the rules +// defined in the proto definition for this message. If any rules are +// violated, the first error encountered is returned, or nil if there are no violations. +func (m *SourceCacheLookupAsk) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SourceCacheLookupAsk with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// SourceCacheLookupAskMultiError, or nil if none found. +func (m *SourceCacheLookupAsk) ValidateAll() error { + return m.validate(true) +} + +func (m *SourceCacheLookupAsk) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + if l := len(m.GetQueries()); l < 1 || l > 4096 { + err := SourceCacheLookupAskValidationError{ + field: "Queries", + reason: "value must contain between 1 and 4096 items, inclusive", + } + if !all { + return err + } + errors = append(errors, err) + } + + for idx, item := range m.GetQueries() { + _, _ = idx, item + + if all { + switch v := interface{}(item).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, SourceCacheLookupAskValidationError{ + field: fmt.Sprintf("Queries[%v]", idx), + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, SourceCacheLookupAskValidationError{ + field: fmt.Sprintf("Queries[%v]", idx), + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(item).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return SourceCacheLookupAskValidationError{ + field: fmt.Sprintf("Queries[%v]", idx), + reason: "embedded message failed validation", + cause: err, + } + } + } + + } + + if len(errors) > 0 { + return SourceCacheLookupAskMultiError(errors) + } + + return nil +} + +// SourceCacheLookupAskMultiError is an error wrapping multiple validation +// errors returned by SourceCacheLookupAsk.ValidateAll() if the designated +// constraints aren't met. +type SourceCacheLookupAskMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SourceCacheLookupAskMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SourceCacheLookupAskMultiError) AllErrors() []error { return m } + +// SourceCacheLookupAskValidationError is the validation error returned by +// SourceCacheLookupAsk.Validate if the designated constraints aren't met. +type SourceCacheLookupAskValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SourceCacheLookupAskValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SourceCacheLookupAskValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SourceCacheLookupAskValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SourceCacheLookupAskValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SourceCacheLookupAskValidationError) ErrorName() string { + return "SourceCacheLookupAskValidationError" +} + +// Error satisfies the builtin error interface +func (e SourceCacheLookupAskValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSourceCacheLookupAsk.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SourceCacheLookupAskValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SourceCacheLookupAskValidationError{} + +// Validate checks the field values on SourceCacheLookupAnswers with the rules +// defined in the proto definition for this message. If any rules are +// violated, the first error encountered is returned, or nil if there are no violations. +func (m *SourceCacheLookupAnswers) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SourceCacheLookupAnswers with the +// rules defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// SourceCacheLookupAnswersMultiError, or nil if none found. +func (m *SourceCacheLookupAnswers) ValidateAll() error { + return m.validate(true) +} + +func (m *SourceCacheLookupAnswers) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + for idx, item := range m.GetAnswers() { + _, _ = idx, item + + if all { + switch v := interface{}(item).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, SourceCacheLookupAnswersValidationError{ + field: fmt.Sprintf("Answers[%v]", idx), + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, SourceCacheLookupAnswersValidationError{ + field: fmt.Sprintf("Answers[%v]", idx), + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(item).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return SourceCacheLookupAnswersValidationError{ + field: fmt.Sprintf("Answers[%v]", idx), + reason: "embedded message failed validation", + cause: err, + } + } + } + + } + + if len(errors) > 0 { + return SourceCacheLookupAnswersMultiError(errors) + } + + return nil +} + +// SourceCacheLookupAnswersMultiError is an error wrapping multiple validation +// errors returned by SourceCacheLookupAnswers.ValidateAll() if the designated +// constraints aren't met. +type SourceCacheLookupAnswersMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SourceCacheLookupAnswersMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SourceCacheLookupAnswersMultiError) AllErrors() []error { return m } + +// SourceCacheLookupAnswersValidationError is the validation error returned by +// SourceCacheLookupAnswers.Validate if the designated constraints aren't met. +type SourceCacheLookupAnswersValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SourceCacheLookupAnswersValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SourceCacheLookupAnswersValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SourceCacheLookupAnswersValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SourceCacheLookupAnswersValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SourceCacheLookupAnswersValidationError) ErrorName() string { + return "SourceCacheLookupAnswersValidationError" +} + +// Error satisfies the builtin error interface +func (e SourceCacheLookupAnswersValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSourceCacheLookupAnswers.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SourceCacheLookupAnswersValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SourceCacheLookupAnswersValidationError{} + +// Validate checks the field values on SourceCacheLookupAsk_Query with the +// rules defined in the proto definition for this message. If any rules are +// violated, the first error encountered is returned, or nil if there are no violations. +func (m *SourceCacheLookupAsk_Query) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SourceCacheLookupAsk_Query with the +// rules defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// SourceCacheLookupAsk_QueryMultiError, or nil if none found. +func (m *SourceCacheLookupAsk_Query) ValidateAll() error { + return m.validate(true) +} + +func (m *SourceCacheLookupAsk_Query) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + if l := len(m.GetRowKind()); l < 1 || l > 64 { + err := SourceCacheLookupAsk_QueryValidationError{ + field: "RowKind", + reason: "value length must be between 1 and 64 bytes, inclusive", + } + if !all { + return err + } + errors = append(errors, err) + } + + if l := len(m.GetScopeHash()); l < 1 || l > 256 { + err := SourceCacheLookupAsk_QueryValidationError{ + field: "ScopeHash", + reason: "value length must be between 1 and 256 bytes, inclusive", + } + if !all { + return err + } + errors = append(errors, err) + } + + if len(errors) > 0 { + return SourceCacheLookupAsk_QueryMultiError(errors) + } + + return nil +} + +// SourceCacheLookupAsk_QueryMultiError is an error wrapping multiple +// validation errors returned by SourceCacheLookupAsk_Query.ValidateAll() if +// the designated constraints aren't met. +type SourceCacheLookupAsk_QueryMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SourceCacheLookupAsk_QueryMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SourceCacheLookupAsk_QueryMultiError) AllErrors() []error { return m } + +// SourceCacheLookupAsk_QueryValidationError is the validation error returned +// by SourceCacheLookupAsk_Query.Validate if the designated constraints aren't met. +type SourceCacheLookupAsk_QueryValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SourceCacheLookupAsk_QueryValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SourceCacheLookupAsk_QueryValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SourceCacheLookupAsk_QueryValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SourceCacheLookupAsk_QueryValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SourceCacheLookupAsk_QueryValidationError) ErrorName() string { + return "SourceCacheLookupAsk_QueryValidationError" +} + +// Error satisfies the builtin error interface +func (e SourceCacheLookupAsk_QueryValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSourceCacheLookupAsk_Query.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SourceCacheLookupAsk_QueryValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SourceCacheLookupAsk_QueryValidationError{} + +// Validate checks the field values on SourceCacheLookupAnswers_Answer with the +// rules defined in the proto definition for this message. If any rules are +// violated, the first error encountered is returned, or nil if there are no violations. +func (m *SourceCacheLookupAnswers_Answer) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SourceCacheLookupAnswers_Answer with +// the rules defined in the proto definition for this message. If any rules +// are violated, the result is a list of violation errors wrapped in +// SourceCacheLookupAnswers_AnswerMultiError, or nil if none found. +func (m *SourceCacheLookupAnswers_Answer) ValidateAll() error { + return m.validate(true) +} + +func (m *SourceCacheLookupAnswers_Answer) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + if l := len(m.GetRowKind()); l < 1 || l > 64 { + err := SourceCacheLookupAnswers_AnswerValidationError{ + field: "RowKind", + reason: "value length must be between 1 and 64 bytes, inclusive", + } + if !all { + return err + } + errors = append(errors, err) + } + + if l := len(m.GetScopeHash()); l < 1 || l > 256 { + err := SourceCacheLookupAnswers_AnswerValidationError{ + field: "ScopeHash", + reason: "value length must be between 1 and 256 bytes, inclusive", + } + if !all { + return err + } + errors = append(errors, err) + } + + // no validation rules for Found + + if m.GetEtag() != "" { + + if len(m.GetEtag()) > 65536 { + err := SourceCacheLookupAnswers_AnswerValidationError{ + field: "Etag", + reason: "value length must be at most 65536 bytes", + } + if !all { + return err + } + errors = append(errors, err) + } + + } + + if len(errors) > 0 { + return SourceCacheLookupAnswers_AnswerMultiError(errors) + } + + return nil +} + +// SourceCacheLookupAnswers_AnswerMultiError is an error wrapping multiple +// validation errors returned by SourceCacheLookupAnswers_Answer.ValidateAll() +// if the designated constraints aren't met. +type SourceCacheLookupAnswers_AnswerMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SourceCacheLookupAnswers_AnswerMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SourceCacheLookupAnswers_AnswerMultiError) AllErrors() []error { return m } + +// SourceCacheLookupAnswers_AnswerValidationError is the validation error +// returned by SourceCacheLookupAnswers_Answer.Validate if the designated +// constraints aren't met. +type SourceCacheLookupAnswers_AnswerValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SourceCacheLookupAnswers_AnswerValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SourceCacheLookupAnswers_AnswerValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SourceCacheLookupAnswers_AnswerValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SourceCacheLookupAnswers_AnswerValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SourceCacheLookupAnswers_AnswerValidationError) ErrorName() string { + return "SourceCacheLookupAnswers_AnswerValidationError" +} + +// Error satisfies the builtin error interface +func (e SourceCacheLookupAnswers_AnswerValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSourceCacheLookupAnswers_Answer.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SourceCacheLookupAnswers_AnswerValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SourceCacheLookupAnswers_AnswerValidationError{} diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_source_cache_protoopaque.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_source_cache_protoopaque.pb.go new file mode 100644 index 00000000..23ad8c5b --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_source_cache_protoopaque.pb.go @@ -0,0 +1,860 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.10 +// protoc (unknown) +// source: c1/connector/v2/annotation_source_cache.proto + +//go:build protoopaque + +package v2 + +import ( + _ "github.com/envoyproxy/protoc-gen-validate/validate" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type SourceCacheCapability_Mode int32 + +const ( + SourceCacheCapability_MODE_UNSPECIFIED SourceCacheCapability_Mode = 0 + SourceCacheCapability_MODE_DISABLED SourceCacheCapability_Mode = 1 + SourceCacheCapability_MODE_READ_WRITE SourceCacheCapability_Mode = 2 +) + +// Enum value maps for SourceCacheCapability_Mode. +var ( + SourceCacheCapability_Mode_name = map[int32]string{ + 0: "MODE_UNSPECIFIED", + 1: "MODE_DISABLED", + 2: "MODE_READ_WRITE", + } + SourceCacheCapability_Mode_value = map[string]int32{ + "MODE_UNSPECIFIED": 0, + "MODE_DISABLED": 1, + "MODE_READ_WRITE": 2, + } +) + +func (x SourceCacheCapability_Mode) Enum() *SourceCacheCapability_Mode { + p := new(SourceCacheCapability_Mode) + *p = x + return p +} + +func (x SourceCacheCapability_Mode) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (SourceCacheCapability_Mode) Descriptor() protoreflect.EnumDescriptor { + return file_c1_connector_v2_annotation_source_cache_proto_enumTypes[0].Descriptor() +} + +func (SourceCacheCapability_Mode) Type() protoreflect.EnumType { + return &file_c1_connector_v2_annotation_source_cache_proto_enumTypes[0] +} + +func (x SourceCacheCapability_Mode) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// SourceCacheCapability is attached to ConnectorServiceValidateResponse +// annotations to opt in to source-cache replay. Absent or any mode other +// than MODE_READ_WRITE means all source-cache annotations are ignored. +type SourceCacheCapability struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Mode SourceCacheCapability_Mode `protobuf:"varint,1,opt,name=mode,proto3,enum=c1.connector.v2.SourceCacheCapability_Mode"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheCapability) Reset() { + *x = SourceCacheCapability{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheCapability) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheCapability) ProtoMessage() {} + +func (x *SourceCacheCapability) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheCapability) GetMode() SourceCacheCapability_Mode { + if x != nil { + return x.xxx_hidden_Mode + } + return SourceCacheCapability_MODE_UNSPECIFIED +} + +func (x *SourceCacheCapability) SetMode(v SourceCacheCapability_Mode) { + x.xxx_hidden_Mode = v +} + +type SourceCacheCapability_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + Mode SourceCacheCapability_Mode +} + +func (b0 SourceCacheCapability_builder) Build() *SourceCacheCapability { + m0 := &SourceCacheCapability{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_Mode = b.Mode + return m0 +} + +// SourceCacheScope is attached to a list-response page whose rows were +// freshly fetched from upstream. The SDK stamps the page's rows with +// scope_hash so a future sync can replay them as a unit. +type SourceCacheScope struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_ScopeHash string `protobuf:"bytes,1,opt,name=scope_hash,json=scopeHash,proto3"` + xxx_hidden_Etag string `protobuf:"bytes,2,opt,name=etag,proto3"` + xxx_hidden_DeletedIds []string `protobuf:"bytes,3,rep,name=deleted_ids,json=deletedIds,proto3"` + xxx_hidden_DeletedPrincipalIds []string `protobuf:"bytes,4,rep,name=deleted_principal_ids,json=deletedPrincipalIds,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheScope) Reset() { + *x = SourceCacheScope{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheScope) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheScope) ProtoMessage() {} + +func (x *SourceCacheScope) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheScope) GetScopeHash() string { + if x != nil { + return x.xxx_hidden_ScopeHash + } + return "" +} + +func (x *SourceCacheScope) GetEtag() string { + if x != nil { + return x.xxx_hidden_Etag + } + return "" +} + +func (x *SourceCacheScope) GetDeletedIds() []string { + if x != nil { + return x.xxx_hidden_DeletedIds + } + return nil +} + +func (x *SourceCacheScope) GetDeletedPrincipalIds() []string { + if x != nil { + return x.xxx_hidden_DeletedPrincipalIds + } + return nil +} + +func (x *SourceCacheScope) SetScopeHash(v string) { + x.xxx_hidden_ScopeHash = v +} + +func (x *SourceCacheScope) SetEtag(v string) { + x.xxx_hidden_Etag = v +} + +func (x *SourceCacheScope) SetDeletedIds(v []string) { + x.xxx_hidden_DeletedIds = v +} + +func (x *SourceCacheScope) SetDeletedPrincipalIds(v []string) { + x.xxx_hidden_DeletedPrincipalIds = v +} + +type SourceCacheScope_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Connector-computed stable identifier for the canonical scope. Must be + // byte-stable across syncs for the same logical scope. Prefer an ORDERED + // natural identifier (e.g. the request URL, or "groups/{id}/members") + // over a random hash: scope-index writes lead with this value, so + // identifiers that correlate with fetch order keep index writes nearly + // append-ordered at scale. sourcecache.HashScope is the fallback when no + // compact natural form exists. + ScopeHash string + // Opaque validator to persist for this scope. May be empty on interim + // pages of a multi-page scope (e.g. Graph @odata.nextLink pages); the + // SDK writes the scope's manifest entry when a non-empty etag arrives. + // A 200 response with zero rows still persists the entry. + Etag string + // Tombstones applied after this page's rows commit. Lets every page of + // a multi-page delta round carry its own deletions as the provider + // delivers them, instead of buffering a whole round onto the first + // (replay-annotated) page. Same formats as SourceCacheReplay. + // + // PRECONDITION for tombstones anywhere in a round: the provider's delta + // must be coalesced — at most one add-or-tombstone per object per round + // (Microsoft Graph guarantees this by returning final object state). + // With interleaved add/remove events for one object, per-page ordering + // is deterministic (a page's rows upsert before its deletions apply) + // but cross-page re-adds after a tombstone are the connector's + // responsibility to order. + DeletedIds []string + // Principal-scoped grant tombstones: for RowKindGrants pages, each + // entry deletes EVERY grant row stamped with this scope whose principal + // id equals the entry — no principal resource type and no canonical + // grant-id reconstruction required (delta tombstones usually carry only + // a bare object id, and the object may no longer exist to look up). + // + // PRECONDITION: the scope must be partitioned so that "principal + // removed from scope" means every grant they have in the scope is gone + // — one scope per navigation with independent removal semantics (e.g. + // members and owners of a group are separate scopes, or membership + // removal would take the owner grant with it). + // + // For RowKindResources pages, each entry deletes the resource row(s) + // stamped with this scope whose resource id equals the entry, any + // resource type. + DeletedPrincipalIds []string +} + +func (b0 SourceCacheScope_builder) Build() *SourceCacheScope { + m0 := &SourceCacheScope{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_ScopeHash = b.ScopeHash + x.xxx_hidden_Etag = b.Etag + x.xxx_hidden_DeletedIds = b.DeletedIds + x.xxx_hidden_DeletedPrincipalIds = b.DeletedPrincipalIds + return m0 +} + +// SourceCacheReplay is attached to a list-response page to tell the SDK to +// copy the previous sync's rows for scope_hash into the current sync. +// +// The row kind (resources, entitlements, grants) is determined by which +// RPC the annotation arrived on, never by the annotation itself. +// +// The connector must only emit this for a scope whose etag it received +// from the SDK's source-cache lookup during this same sync. A replay for +// an unknown scope fails the sync: the connector has already skipped row +// generation, so there is nothing to fall back to. +type SourceCacheReplay struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_ScopeHash string `protobuf:"bytes,1,opt,name=scope_hash,json=scopeHash,proto3"` + xxx_hidden_Etag string `protobuf:"bytes,2,opt,name=etag,proto3"` + xxx_hidden_Overlay bool `protobuf:"varint,3,opt,name=overlay,proto3"` + xxx_hidden_DeletedIds []string `protobuf:"bytes,4,rep,name=deleted_ids,json=deletedIds,proto3"` + xxx_hidden_DeletedPrincipalIds []string `protobuf:"bytes,5,rep,name=deleted_principal_ids,json=deletedPrincipalIds,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheReplay) Reset() { + *x = SourceCacheReplay{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheReplay) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheReplay) ProtoMessage() {} + +func (x *SourceCacheReplay) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheReplay) GetScopeHash() string { + if x != nil { + return x.xxx_hidden_ScopeHash + } + return "" +} + +func (x *SourceCacheReplay) GetEtag() string { + if x != nil { + return x.xxx_hidden_Etag + } + return "" +} + +func (x *SourceCacheReplay) GetOverlay() bool { + if x != nil { + return x.xxx_hidden_Overlay + } + return false +} + +func (x *SourceCacheReplay) GetDeletedIds() []string { + if x != nil { + return x.xxx_hidden_DeletedIds + } + return nil +} + +func (x *SourceCacheReplay) GetDeletedPrincipalIds() []string { + if x != nil { + return x.xxx_hidden_DeletedPrincipalIds + } + return nil +} + +func (x *SourceCacheReplay) SetScopeHash(v string) { + x.xxx_hidden_ScopeHash = v +} + +func (x *SourceCacheReplay) SetEtag(v string) { + x.xxx_hidden_Etag = v +} + +func (x *SourceCacheReplay) SetOverlay(v bool) { + x.xxx_hidden_Overlay = v +} + +func (x *SourceCacheReplay) SetDeletedIds(v []string) { + x.xxx_hidden_DeletedIds = v +} + +func (x *SourceCacheReplay) SetDeletedPrincipalIds(v []string) { + x.xxx_hidden_DeletedPrincipalIds = v +} + +type SourceCacheReplay_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + ScopeHash string + // Validator to persist for this scope in the current sync. For an HTTP + // 304 this is the unchanged etag. For a delta query this is the NEW + // token; it may instead be supplied by the final overlay page's + // SourceCacheScope.etag, in which case this field may be left empty. + Etag string + // When true, the response (and subsequent pages carrying + // SourceCacheScope with the same scope_hash) contains changed rows to + // upsert on top of the replayed base. When false the response must + // contain no rows for this scope. + Overlay bool + // Public canonical IDs (grant/entitlement IDs, or resource BIDs for + // RowKindResources) to delete from the current sync after the replay + // copy and this page's upserts. Used for delta-query tombstones (e.g. + // Microsoft Graph @removed entries). Subsequent pages of the round + // carry their tombstones on SourceCacheScope.deleted_ids. + DeletedIds []string + // Principal-scoped tombstones for this page; see + // SourceCacheScope.deleted_principal_ids for semantics and + // preconditions. + DeletedPrincipalIds []string +} + +func (b0 SourceCacheReplay_builder) Build() *SourceCacheReplay { + m0 := &SourceCacheReplay{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_ScopeHash = b.ScopeHash + x.xxx_hidden_Etag = b.Etag + x.xxx_hidden_Overlay = b.Overlay + x.xxx_hidden_DeletedIds = b.DeletedIds + x.xxx_hidden_DeletedPrincipalIds = b.DeletedPrincipalIds + return m0 +} + +// SourceCacheLookupOffer is attached by the SDK to list REQUESTS when the +// syncer can answer source-cache lookup asks: the connector declared +// SourceCacheCapability and a warm previous-sync lookup is installed. +// Its presence is the connector's permission to answer with +// SourceCacheLookupAsk instead of rows. Connectors with a direct lookup +// (in-process, subprocess) never need to defer and may ignore it. +type SourceCacheLookupOffer struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupOffer) Reset() { + *x = SourceCacheLookupOffer{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupOffer) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupOffer) ProtoMessage() {} + +func (x *SourceCacheLookupOffer) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +type SourceCacheLookupOffer_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + +} + +func (b0 SourceCacheLookupOffer_builder) Build() *SourceCacheLookupOffer { + m0 := &SourceCacheLookupOffer{} + b, x := &b0, m0 + _, _ = b, x + return m0 +} + +// SourceCacheLookupAsk is attached to a list RESPONSE in place of rows: +// the connector needs previous-sync validators before it can serve the +// page. An ask response must carry NO rows, NO next page token, and no +// other source-cache annotations; the syncer consumes it (it never +// reaches page handling), resolves every query, and re-invokes the same +// request with SourceCacheLookupAnswers attached. +// +// Only legal on responses to requests that carried +// SourceCacheLookupOffer. Bounces are capped per action; a connector +// that keeps asking past the cap fails the sync loudly. +type SourceCacheLookupAsk struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Queries *[]*SourceCacheLookupAsk_Query `protobuf:"bytes,1,rep,name=queries,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupAsk) Reset() { + *x = SourceCacheLookupAsk{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupAsk) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupAsk) ProtoMessage() {} + +func (x *SourceCacheLookupAsk) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheLookupAsk) GetQueries() []*SourceCacheLookupAsk_Query { + if x != nil { + if x.xxx_hidden_Queries != nil { + return *x.xxx_hidden_Queries + } + } + return nil +} + +func (x *SourceCacheLookupAsk) SetQueries(v []*SourceCacheLookupAsk_Query) { + x.xxx_hidden_Queries = &v +} + +type SourceCacheLookupAsk_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + Queries []*SourceCacheLookupAsk_Query +} + +func (b0 SourceCacheLookupAsk_builder) Build() *SourceCacheLookupAsk { + m0 := &SourceCacheLookupAsk{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_Queries = &b.Queries + return m0 +} + +// SourceCacheLookupAnswers is attached by the SDK to the re-invoked +// REQUEST, carrying the resolution of a prior ask's queries. +// +// An ABSENT query (asked but not answered — e.g. dropped to the answer +// size budget) is distinct from found=false: not-found means the previous +// sync has no entry (fetch fresh); absent means unresolved (the connector +// may ask again, subject to the bounce cap). Not-found answers are always +// complete for the queried set — only found answers with large etags are +// ever dropped to budget. +type SourceCacheLookupAnswers struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Answers *[]*SourceCacheLookupAnswers_Answer `protobuf:"bytes,1,rep,name=answers,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupAnswers) Reset() { + *x = SourceCacheLookupAnswers{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupAnswers) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupAnswers) ProtoMessage() {} + +func (x *SourceCacheLookupAnswers) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheLookupAnswers) GetAnswers() []*SourceCacheLookupAnswers_Answer { + if x != nil { + if x.xxx_hidden_Answers != nil { + return *x.xxx_hidden_Answers + } + } + return nil +} + +func (x *SourceCacheLookupAnswers) SetAnswers(v []*SourceCacheLookupAnswers_Answer) { + x.xxx_hidden_Answers = &v +} + +type SourceCacheLookupAnswers_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + Answers []*SourceCacheLookupAnswers_Answer +} + +func (b0 SourceCacheLookupAnswers_builder) Build() *SourceCacheLookupAnswers { + m0 := &SourceCacheLookupAnswers{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_Answers = &b.Answers + return m0 +} + +type SourceCacheLookupAsk_Query struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3"` + xxx_hidden_ScopeHash string `protobuf:"bytes,2,opt,name=scope_hash,json=scopeHash,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupAsk_Query) Reset() { + *x = SourceCacheLookupAsk_Query{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupAsk_Query) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupAsk_Query) ProtoMessage() {} + +func (x *SourceCacheLookupAsk_Query) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheLookupAsk_Query) GetRowKind() string { + if x != nil { + return x.xxx_hidden_RowKind + } + return "" +} + +func (x *SourceCacheLookupAsk_Query) GetScopeHash() string { + if x != nil { + return x.xxx_hidden_ScopeHash + } + return "" +} + +func (x *SourceCacheLookupAsk_Query) SetRowKind(v string) { + x.xxx_hidden_RowKind = v +} + +func (x *SourceCacheLookupAsk_Query) SetScopeHash(v string) { + x.xxx_hidden_ScopeHash = v +} + +type SourceCacheLookupAsk_Query_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // One of the sourcecache.RowKind values: "resources", + // "entitlements", "grants". + RowKind string + ScopeHash string +} + +func (b0 SourceCacheLookupAsk_Query_builder) Build() *SourceCacheLookupAsk_Query { + m0 := &SourceCacheLookupAsk_Query{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_RowKind = b.RowKind + x.xxx_hidden_ScopeHash = b.ScopeHash + return m0 +} + +type SourceCacheLookupAnswers_Answer struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3"` + xxx_hidden_ScopeHash string `protobuf:"bytes,2,opt,name=scope_hash,json=scopeHash,proto3"` + xxx_hidden_Found bool `protobuf:"varint,3,opt,name=found,proto3"` + xxx_hidden_Etag string `protobuf:"bytes,4,opt,name=etag,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupAnswers_Answer) Reset() { + *x = SourceCacheLookupAnswers_Answer{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupAnswers_Answer) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupAnswers_Answer) ProtoMessage() {} + +func (x *SourceCacheLookupAnswers_Answer) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheLookupAnswers_Answer) GetRowKind() string { + if x != nil { + return x.xxx_hidden_RowKind + } + return "" +} + +func (x *SourceCacheLookupAnswers_Answer) GetScopeHash() string { + if x != nil { + return x.xxx_hidden_ScopeHash + } + return "" +} + +func (x *SourceCacheLookupAnswers_Answer) GetFound() bool { + if x != nil { + return x.xxx_hidden_Found + } + return false +} + +func (x *SourceCacheLookupAnswers_Answer) GetEtag() string { + if x != nil { + return x.xxx_hidden_Etag + } + return "" +} + +func (x *SourceCacheLookupAnswers_Answer) SetRowKind(v string) { + x.xxx_hidden_RowKind = v +} + +func (x *SourceCacheLookupAnswers_Answer) SetScopeHash(v string) { + x.xxx_hidden_ScopeHash = v +} + +func (x *SourceCacheLookupAnswers_Answer) SetFound(v bool) { + x.xxx_hidden_Found = v +} + +func (x *SourceCacheLookupAnswers_Answer) SetEtag(v string) { + x.xxx_hidden_Etag = v +} + +type SourceCacheLookupAnswers_Answer_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + RowKind string + ScopeHash string + Found bool + // The previous sync's validator; empty when found is false. Cap + // matches the lookup RPC (Graph delta tokens run long). + Etag string +} + +func (b0 SourceCacheLookupAnswers_Answer_builder) Build() *SourceCacheLookupAnswers_Answer { + m0 := &SourceCacheLookupAnswers_Answer{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_RowKind = b.RowKind + x.xxx_hidden_ScopeHash = b.ScopeHash + x.xxx_hidden_Found = b.Found + x.xxx_hidden_Etag = b.Etag + return m0 +} + +var File_c1_connector_v2_annotation_source_cache_proto protoreflect.FileDescriptor + +const file_c1_connector_v2_annotation_source_cache_proto_rawDesc = "" + + "\n" + + "-c1/connector/v2/annotation_source_cache.proto\x12\x0fc1.connector.v2\x1a\x17validate/validate.proto\"\x9e\x01\n" + + "\x15SourceCacheCapability\x12?\n" + + "\x04mode\x18\x01 \x01(\x0e2+.c1.connector.v2.SourceCacheCapability.ModeR\x04mode\"D\n" + + "\x04Mode\x12\x14\n" + + "\x10MODE_UNSPECIFIED\x10\x00\x12\x11\n" + + "\rMODE_DISABLED\x10\x01\x12\x13\n" + + "\x0fMODE_READ_WRITE\x10\x02\"\x9a\x01\n" + + "\x10SourceCacheScope\x12\x1d\n" + + "\n" + + "scope_hash\x18\x01 \x01(\tR\tscopeHash\x12\x12\n" + + "\x04etag\x18\x02 \x01(\tR\x04etag\x12\x1f\n" + + "\vdeleted_ids\x18\x03 \x03(\tR\n" + + "deletedIds\x122\n" + + "\x15deleted_principal_ids\x18\x04 \x03(\tR\x13deletedPrincipalIds\"\xb5\x01\n" + + "\x11SourceCacheReplay\x12\x1d\n" + + "\n" + + "scope_hash\x18\x01 \x01(\tR\tscopeHash\x12\x12\n" + + "\x04etag\x18\x02 \x01(\tR\x04etag\x12\x18\n" + + "\aoverlay\x18\x03 \x01(\bR\aoverlay\x12\x1f\n" + + "\vdeleted_ids\x18\x04 \x03(\tR\n" + + "deletedIds\x122\n" + + "\x15deleted_principal_ids\x18\x05 \x03(\tR\x13deletedPrincipalIds\"\x18\n" + + "\x16SourceCacheLookupOffer\"\xc4\x01\n" + + "\x14SourceCacheLookupAsk\x12R\n" + + "\aqueries\x18\x01 \x03(\v2+.c1.connector.v2.SourceCacheLookupAsk.QueryB\v\xfaB\b\x92\x01\x05\b\x01\x10\x80 R\aqueries\x1aX\n" + + "\x05Query\x12$\n" + + "\brow_kind\x18\x01 \x01(\tB\t\xfaB\x06r\x04 \x01(@R\arowKind\x12)\n" + + "\n" + + "scope_hash\x18\x02 \x01(\tB\n" + + "\xfaB\ar\x05 \x01(\x80\x02R\tscopeHash\"\xfa\x01\n" + + "\x18SourceCacheLookupAnswers\x12J\n" + + "\aanswers\x18\x01 \x03(\v20.c1.connector.v2.SourceCacheLookupAnswers.AnswerR\aanswers\x1a\x91\x01\n" + + "\x06Answer\x12$\n" + + "\brow_kind\x18\x01 \x01(\tB\t\xfaB\x06r\x04 \x01(@R\arowKind\x12)\n" + + "\n" + + "scope_hash\x18\x02 \x01(\tB\n" + + "\xfaB\ar\x05 \x01(\x80\x02R\tscopeHash\x12\x14\n" + + "\x05found\x18\x03 \x01(\bR\x05found\x12 \n" + + "\x04etag\x18\x04 \x01(\tB\f\xfaB\tr\a(\x80\x80\x04\xd0\x01\x01R\x04etagB6Z4github.com/conductorone/baton-sdk/pb/c1/connector/v2b\x06proto3" + +var file_c1_connector_v2_annotation_source_cache_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_c1_connector_v2_annotation_source_cache_proto_msgTypes = make([]protoimpl.MessageInfo, 8) +var file_c1_connector_v2_annotation_source_cache_proto_goTypes = []any{ + (SourceCacheCapability_Mode)(0), // 0: c1.connector.v2.SourceCacheCapability.Mode + (*SourceCacheCapability)(nil), // 1: c1.connector.v2.SourceCacheCapability + (*SourceCacheScope)(nil), // 2: c1.connector.v2.SourceCacheScope + (*SourceCacheReplay)(nil), // 3: c1.connector.v2.SourceCacheReplay + (*SourceCacheLookupOffer)(nil), // 4: c1.connector.v2.SourceCacheLookupOffer + (*SourceCacheLookupAsk)(nil), // 5: c1.connector.v2.SourceCacheLookupAsk + (*SourceCacheLookupAnswers)(nil), // 6: c1.connector.v2.SourceCacheLookupAnswers + (*SourceCacheLookupAsk_Query)(nil), // 7: c1.connector.v2.SourceCacheLookupAsk.Query + (*SourceCacheLookupAnswers_Answer)(nil), // 8: c1.connector.v2.SourceCacheLookupAnswers.Answer +} +var file_c1_connector_v2_annotation_source_cache_proto_depIdxs = []int32{ + 0, // 0: c1.connector.v2.SourceCacheCapability.mode:type_name -> c1.connector.v2.SourceCacheCapability.Mode + 7, // 1: c1.connector.v2.SourceCacheLookupAsk.queries:type_name -> c1.connector.v2.SourceCacheLookupAsk.Query + 8, // 2: c1.connector.v2.SourceCacheLookupAnswers.answers:type_name -> c1.connector.v2.SourceCacheLookupAnswers.Answer + 3, // [3:3] is the sub-list for method output_type + 3, // [3:3] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name +} + +func init() { file_c1_connector_v2_annotation_source_cache_proto_init() } +func file_c1_connector_v2_annotation_source_cache_proto_init() { + if File_c1_connector_v2_annotation_source_cache_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_connector_v2_annotation_source_cache_proto_rawDesc), len(file_c1_connector_v2_annotation_source_cache_proto_rawDesc)), + NumEnums: 1, + NumMessages: 8, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_c1_connector_v2_annotation_source_cache_proto_goTypes, + DependencyIndexes: file_c1_connector_v2_annotation_source_cache_proto_depIdxs, + EnumInfos: file_c1_connector_v2_annotation_source_cache_proto_enumTypes, + MessageInfos: file_c1_connector_v2_annotation_source_cache_proto_msgTypes, + }.Build() + File_c1_connector_v2_annotation_source_cache_proto = out.File + file_c1_connector_v2_annotation_source_cache_proto_goTypes = nil + file_c1_connector_v2_annotation_source_cache_proto_depIdxs = nil +} diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_type_scoped_grants.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_type_scoped_grants.pb.go new file mode 100644 index 00000000..52a9383b --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_type_scoped_grants.pb.go @@ -0,0 +1,250 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.10 +// protoc (unknown) +// source: c1/connector/v2/annotation_type_scoped_grants.proto + +//go:build !protoopaque + +package v2 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Type-scoped grant ingestion. +// +// Some providers expose change/enumeration APIs whose natural unit is a +// whole collection (or a connector-defined shard of one), not a single +// resource — e.g. Microsoft Graph delta queries, where one stream yields +// membership changes for up to 50 groups. Forcing that shape through the +// per-resource ListGrants fan-out costs one request per resource even when +// nothing changed. +// +// A resource type annotated with TypeScopedGrants is EXCLUDED from the +// per-resource grants fan-out. Instead the syncer issues ListGrants calls +// whose resource carries ONLY the resource type (empty resource id); the +// connector answers with grants for the whole type, across as many +// paginated cursors as it chooses to spawn (see SpawnCursors). +// +// This is the grants-phase analogue of StaticEntitlements: the connector +// takes over enumeration for the type, and every downstream behavior +// (storage, source-cache scopes/replay/tombstones, grant expansion, rate +// limiting, page-token checkpointing) is unchanged because grants are +// self-describing rows. +// +// COMPLETENESS CONTRACT: a full sync of an annotated type must emit (or +// replay, via source-cache annotations) EVERY grant of that type. The +// syncer no longer visits each resource, so completeness rests entirely on +// the connector's enumeration. +type TypeScopedGrants struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TypeScopedGrants) Reset() { + *x = TypeScopedGrants{} + mi := &file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TypeScopedGrants) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TypeScopedGrants) ProtoMessage() {} + +func (x *TypeScopedGrants) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +type TypeScopedGrants_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + +} + +func (b0 TypeScopedGrants_builder) Build() *TypeScopedGrants { + m0 := &TypeScopedGrants{} + b, x := &b0, m0 + _, _ = b, x + return m0 +} + +// SpawnCursors is attached to a ListGrants response to enqueue additional +// independent sibling cursors. Each token is delivered back to the +// connector as the page token of its own action — scheduled by the +// syncer's worker pool, rate-limited, and checkpointed like any other +// pagination. Honored on BOTH type-scoped and per-resource ListGrants +// responses; the spawned actions inherit the response's resource identity +// (type only for type-scoped calls, type+resource for per-resource calls). +// +// Typical uses: +// +// - Type-scoped planning: the first call computes shard assignments — +// e.g. one 50-id delta filter chunk per cursor — returns no rows, and +// spawns one cursor per shard. Cold enumeration then parallelizes +// across shards instead of serializing through one stream. +// - Per-resource parallel warm revalidation (page-numbered APIs): on the +// first page of a collection the connector already knows every other +// page's URL and stored validator, so it answers page one and spawns +// pages 2..N; the worker pool revalidates them concurrently instead +// of paying one round trip per page serially. +// +// Spawned cursors are ORDINARY pages that happen to be enqueued eagerly. +// The SDK assumes nothing about how they resolve: a spawned page may hit +// its source-cache lookup and replay, miss and fetch cold (e.g. a page +// boundary shifted since the last sync), and may continue a chain via its +// response's next page token. Any page of a fan-out may itself spawn. +// +// Progress accounting counts a resource as covered when its ORIGIN +// action's chain ends; spawned siblings never double-count it. +// +// Tokens are opaque to the SDK. They must be self-contained: a cursor may +// execute after a suspend/resume, on a different worker, so anything the +// connector needs to serve the cursor must be inside the token (or +// re-derivable from it) — for per-resource spawns the resource identity +// rides the action, so tokens only need the page coordinate. +type SpawnCursors struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + // Page tokens for the sibling cursors to enqueue, one action each. + PageTokens []string `protobuf:"bytes,1,rep,name=page_tokens,json=pageTokens,proto3" json:"page_tokens,omitempty"` + // Optional connector estimate of total grants across all cursors of this + // type, for progress reporting. Zero means unknown. + EstimatedTotal int64 `protobuf:"varint,2,opt,name=estimated_total,json=estimatedTotal,proto3" json:"estimated_total,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SpawnCursors) Reset() { + *x = SpawnCursors{} + mi := &file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SpawnCursors) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SpawnCursors) ProtoMessage() {} + +func (x *SpawnCursors) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SpawnCursors) GetPageTokens() []string { + if x != nil { + return x.PageTokens + } + return nil +} + +func (x *SpawnCursors) GetEstimatedTotal() int64 { + if x != nil { + return x.EstimatedTotal + } + return 0 +} + +func (x *SpawnCursors) SetPageTokens(v []string) { + x.PageTokens = v +} + +func (x *SpawnCursors) SetEstimatedTotal(v int64) { + x.EstimatedTotal = v +} + +type SpawnCursors_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Page tokens for the sibling cursors to enqueue, one action each. + PageTokens []string + // Optional connector estimate of total grants across all cursors of this + // type, for progress reporting. Zero means unknown. + EstimatedTotal int64 +} + +func (b0 SpawnCursors_builder) Build() *SpawnCursors { + m0 := &SpawnCursors{} + b, x := &b0, m0 + _, _ = b, x + x.PageTokens = b.PageTokens + x.EstimatedTotal = b.EstimatedTotal + return m0 +} + +var File_c1_connector_v2_annotation_type_scoped_grants_proto protoreflect.FileDescriptor + +const file_c1_connector_v2_annotation_type_scoped_grants_proto_rawDesc = "" + + "\n" + + "3c1/connector/v2/annotation_type_scoped_grants.proto\x12\x0fc1.connector.v2\"\x12\n" + + "\x10TypeScopedGrants\"X\n" + + "\fSpawnCursors\x12\x1f\n" + + "\vpage_tokens\x18\x01 \x03(\tR\n" + + "pageTokens\x12'\n" + + "\x0festimated_total\x18\x02 \x01(\x03R\x0eestimatedTotalB6Z4github.com/conductorone/baton-sdk/pb/c1/connector/v2b\x06proto3" + +var file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_c1_connector_v2_annotation_type_scoped_grants_proto_goTypes = []any{ + (*TypeScopedGrants)(nil), // 0: c1.connector.v2.TypeScopedGrants + (*SpawnCursors)(nil), // 1: c1.connector.v2.SpawnCursors +} +var file_c1_connector_v2_annotation_type_scoped_grants_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_c1_connector_v2_annotation_type_scoped_grants_proto_init() } +func file_c1_connector_v2_annotation_type_scoped_grants_proto_init() { + if File_c1_connector_v2_annotation_type_scoped_grants_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_connector_v2_annotation_type_scoped_grants_proto_rawDesc), len(file_c1_connector_v2_annotation_type_scoped_grants_proto_rawDesc)), + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_c1_connector_v2_annotation_type_scoped_grants_proto_goTypes, + DependencyIndexes: file_c1_connector_v2_annotation_type_scoped_grants_proto_depIdxs, + MessageInfos: file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes, + }.Build() + File_c1_connector_v2_annotation_type_scoped_grants_proto = out.File + file_c1_connector_v2_annotation_type_scoped_grants_proto_goTypes = nil + file_c1_connector_v2_annotation_type_scoped_grants_proto_depIdxs = nil +} diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_type_scoped_grants.pb.validate.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_type_scoped_grants.pb.validate.go new file mode 100644 index 00000000..5679e6e9 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_type_scoped_grants.pb.validate.go @@ -0,0 +1,237 @@ +// Code generated by protoc-gen-validate. DO NOT EDIT. +// source: c1/connector/v2/annotation_type_scoped_grants.proto + +package v2 + +import ( + "bytes" + "errors" + "fmt" + "net" + "net/mail" + "net/url" + "regexp" + "sort" + "strings" + "time" + "unicode/utf8" + + "google.golang.org/protobuf/types/known/anypb" +) + +// ensure the imports are used +var ( + _ = bytes.MinRead + _ = errors.New("") + _ = fmt.Print + _ = utf8.UTFMax + _ = (*regexp.Regexp)(nil) + _ = (*strings.Reader)(nil) + _ = net.IPv4len + _ = time.Duration(0) + _ = (*url.URL)(nil) + _ = (*mail.Address)(nil) + _ = anypb.Any{} + _ = sort.Sort +) + +// Validate checks the field values on TypeScopedGrants with the rules defined +// in the proto definition for this message. If any rules are violated, the +// first error encountered is returned, or nil if there are no violations. +func (m *TypeScopedGrants) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on TypeScopedGrants with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// TypeScopedGrantsMultiError, or nil if none found. +func (m *TypeScopedGrants) ValidateAll() error { + return m.validate(true) +} + +func (m *TypeScopedGrants) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + if len(errors) > 0 { + return TypeScopedGrantsMultiError(errors) + } + + return nil +} + +// TypeScopedGrantsMultiError is an error wrapping multiple validation errors +// returned by TypeScopedGrants.ValidateAll() if the designated constraints +// aren't met. +type TypeScopedGrantsMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m TypeScopedGrantsMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m TypeScopedGrantsMultiError) AllErrors() []error { return m } + +// TypeScopedGrantsValidationError is the validation error returned by +// TypeScopedGrants.Validate if the designated constraints aren't met. +type TypeScopedGrantsValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e TypeScopedGrantsValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e TypeScopedGrantsValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e TypeScopedGrantsValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e TypeScopedGrantsValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e TypeScopedGrantsValidationError) ErrorName() string { return "TypeScopedGrantsValidationError" } + +// Error satisfies the builtin error interface +func (e TypeScopedGrantsValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sTypeScopedGrants.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = TypeScopedGrantsValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = TypeScopedGrantsValidationError{} + +// Validate checks the field values on SpawnCursors with the rules defined in +// the proto definition for this message. If any rules are violated, the first +// error encountered is returned, or nil if there are no violations. +func (m *SpawnCursors) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SpawnCursors with the rules defined +// in the proto definition for this message. If any rules are violated, the +// result is a list of violation errors wrapped in SpawnCursorsMultiError, or +// nil if none found. +func (m *SpawnCursors) ValidateAll() error { + return m.validate(true) +} + +func (m *SpawnCursors) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + // no validation rules for EstimatedTotal + + if len(errors) > 0 { + return SpawnCursorsMultiError(errors) + } + + return nil +} + +// SpawnCursorsMultiError is an error wrapping multiple validation errors +// returned by SpawnCursors.ValidateAll() if the designated constraints aren't met. +type SpawnCursorsMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SpawnCursorsMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SpawnCursorsMultiError) AllErrors() []error { return m } + +// SpawnCursorsValidationError is the validation error returned by +// SpawnCursors.Validate if the designated constraints aren't met. +type SpawnCursorsValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SpawnCursorsValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SpawnCursorsValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SpawnCursorsValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SpawnCursorsValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SpawnCursorsValidationError) ErrorName() string { return "SpawnCursorsValidationError" } + +// Error satisfies the builtin error interface +func (e SpawnCursorsValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSpawnCursors.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SpawnCursorsValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SpawnCursorsValidationError{} diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_type_scoped_grants_protoopaque.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_type_scoped_grants_protoopaque.pb.go new file mode 100644 index 00000000..e9b3571b --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_type_scoped_grants_protoopaque.pb.go @@ -0,0 +1,247 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.10 +// protoc (unknown) +// source: c1/connector/v2/annotation_type_scoped_grants.proto + +//go:build protoopaque + +package v2 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Type-scoped grant ingestion. +// +// Some providers expose change/enumeration APIs whose natural unit is a +// whole collection (or a connector-defined shard of one), not a single +// resource — e.g. Microsoft Graph delta queries, where one stream yields +// membership changes for up to 50 groups. Forcing that shape through the +// per-resource ListGrants fan-out costs one request per resource even when +// nothing changed. +// +// A resource type annotated with TypeScopedGrants is EXCLUDED from the +// per-resource grants fan-out. Instead the syncer issues ListGrants calls +// whose resource carries ONLY the resource type (empty resource id); the +// connector answers with grants for the whole type, across as many +// paginated cursors as it chooses to spawn (see SpawnCursors). +// +// This is the grants-phase analogue of StaticEntitlements: the connector +// takes over enumeration for the type, and every downstream behavior +// (storage, source-cache scopes/replay/tombstones, grant expansion, rate +// limiting, page-token checkpointing) is unchanged because grants are +// self-describing rows. +// +// COMPLETENESS CONTRACT: a full sync of an annotated type must emit (or +// replay, via source-cache annotations) EVERY grant of that type. The +// syncer no longer visits each resource, so completeness rests entirely on +// the connector's enumeration. +type TypeScopedGrants struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TypeScopedGrants) Reset() { + *x = TypeScopedGrants{} + mi := &file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TypeScopedGrants) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TypeScopedGrants) ProtoMessage() {} + +func (x *TypeScopedGrants) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +type TypeScopedGrants_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + +} + +func (b0 TypeScopedGrants_builder) Build() *TypeScopedGrants { + m0 := &TypeScopedGrants{} + b, x := &b0, m0 + _, _ = b, x + return m0 +} + +// SpawnCursors is attached to a ListGrants response to enqueue additional +// independent sibling cursors. Each token is delivered back to the +// connector as the page token of its own action — scheduled by the +// syncer's worker pool, rate-limited, and checkpointed like any other +// pagination. Honored on BOTH type-scoped and per-resource ListGrants +// responses; the spawned actions inherit the response's resource identity +// (type only for type-scoped calls, type+resource for per-resource calls). +// +// Typical uses: +// +// - Type-scoped planning: the first call computes shard assignments — +// e.g. one 50-id delta filter chunk per cursor — returns no rows, and +// spawns one cursor per shard. Cold enumeration then parallelizes +// across shards instead of serializing through one stream. +// - Per-resource parallel warm revalidation (page-numbered APIs): on the +// first page of a collection the connector already knows every other +// page's URL and stored validator, so it answers page one and spawns +// pages 2..N; the worker pool revalidates them concurrently instead +// of paying one round trip per page serially. +// +// Spawned cursors are ORDINARY pages that happen to be enqueued eagerly. +// The SDK assumes nothing about how they resolve: a spawned page may hit +// its source-cache lookup and replay, miss and fetch cold (e.g. a page +// boundary shifted since the last sync), and may continue a chain via its +// response's next page token. Any page of a fan-out may itself spawn. +// +// Progress accounting counts a resource as covered when its ORIGIN +// action's chain ends; spawned siblings never double-count it. +// +// Tokens are opaque to the SDK. They must be self-contained: a cursor may +// execute after a suspend/resume, on a different worker, so anything the +// connector needs to serve the cursor must be inside the token (or +// re-derivable from it) — for per-resource spawns the resource identity +// rides the action, so tokens only need the page coordinate. +type SpawnCursors struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_PageTokens []string `protobuf:"bytes,1,rep,name=page_tokens,json=pageTokens,proto3"` + xxx_hidden_EstimatedTotal int64 `protobuf:"varint,2,opt,name=estimated_total,json=estimatedTotal,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SpawnCursors) Reset() { + *x = SpawnCursors{} + mi := &file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SpawnCursors) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SpawnCursors) ProtoMessage() {} + +func (x *SpawnCursors) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SpawnCursors) GetPageTokens() []string { + if x != nil { + return x.xxx_hidden_PageTokens + } + return nil +} + +func (x *SpawnCursors) GetEstimatedTotal() int64 { + if x != nil { + return x.xxx_hidden_EstimatedTotal + } + return 0 +} + +func (x *SpawnCursors) SetPageTokens(v []string) { + x.xxx_hidden_PageTokens = v +} + +func (x *SpawnCursors) SetEstimatedTotal(v int64) { + x.xxx_hidden_EstimatedTotal = v +} + +type SpawnCursors_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Page tokens for the sibling cursors to enqueue, one action each. + PageTokens []string + // Optional connector estimate of total grants across all cursors of this + // type, for progress reporting. Zero means unknown. + EstimatedTotal int64 +} + +func (b0 SpawnCursors_builder) Build() *SpawnCursors { + m0 := &SpawnCursors{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_PageTokens = b.PageTokens + x.xxx_hidden_EstimatedTotal = b.EstimatedTotal + return m0 +} + +var File_c1_connector_v2_annotation_type_scoped_grants_proto protoreflect.FileDescriptor + +const file_c1_connector_v2_annotation_type_scoped_grants_proto_rawDesc = "" + + "\n" + + "3c1/connector/v2/annotation_type_scoped_grants.proto\x12\x0fc1.connector.v2\"\x12\n" + + "\x10TypeScopedGrants\"X\n" + + "\fSpawnCursors\x12\x1f\n" + + "\vpage_tokens\x18\x01 \x03(\tR\n" + + "pageTokens\x12'\n" + + "\x0festimated_total\x18\x02 \x01(\x03R\x0eestimatedTotalB6Z4github.com/conductorone/baton-sdk/pb/c1/connector/v2b\x06proto3" + +var file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_c1_connector_v2_annotation_type_scoped_grants_proto_goTypes = []any{ + (*TypeScopedGrants)(nil), // 0: c1.connector.v2.TypeScopedGrants + (*SpawnCursors)(nil), // 1: c1.connector.v2.SpawnCursors +} +var file_c1_connector_v2_annotation_type_scoped_grants_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_c1_connector_v2_annotation_type_scoped_grants_proto_init() } +func file_c1_connector_v2_annotation_type_scoped_grants_proto_init() { + if File_c1_connector_v2_annotation_type_scoped_grants_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_connector_v2_annotation_type_scoped_grants_proto_rawDesc), len(file_c1_connector_v2_annotation_type_scoped_grants_proto_rawDesc)), + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_c1_connector_v2_annotation_type_scoped_grants_proto_goTypes, + DependencyIndexes: file_c1_connector_v2_annotation_type_scoped_grants_proto_depIdxs, + MessageInfos: file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes, + }.Build() + File_c1_connector_v2_annotation_type_scoped_grants_proto = out.File + file_c1_connector_v2_annotation_type_scoped_grants_proto_goTypes = nil + file_c1_connector_v2_annotation_type_scoped_grants_proto_depIdxs = nil +} diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/source_cache.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/source_cache.pb.go new file mode 100644 index 00000000..5d0c2090 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/source_cache.pb.go @@ -0,0 +1,249 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.10 +// protoc (unknown) +// source: c1/connectorapi/baton/v1/source_cache.proto + +//go:build !protoopaque + +package v1 + +import ( + _ "github.com/envoyproxy/protoc-gen-validate/validate" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type LookupRequest struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + // Row kind: resources / entitlements / grants + // (pkg/sourcecache.RowKind values). Entries are partitioned by row + // kind, so one scope hash can carry a different validator per kind. + RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3" json:"row_kind,omitempty"` + // Connector-defined stable scope identifier (conventionally a hex hash + // of the canonical scope; see pkg/sourcecache.HashScope). Opaque to the + // parent; matched verbatim against the previous sync's source-cache + // entries. + ScopeHash string `protobuf:"bytes,2,opt,name=scope_hash,json=scopeHash,proto3" json:"scope_hash,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LookupRequest) Reset() { + *x = LookupRequest{} + mi := &file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LookupRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LookupRequest) ProtoMessage() {} + +func (x *LookupRequest) ProtoReflect() protoreflect.Message { + mi := &file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *LookupRequest) GetRowKind() string { + if x != nil { + return x.RowKind + } + return "" +} + +func (x *LookupRequest) GetScopeHash() string { + if x != nil { + return x.ScopeHash + } + return "" +} + +func (x *LookupRequest) SetRowKind(v string) { + x.RowKind = v +} + +func (x *LookupRequest) SetScopeHash(v string) { + x.ScopeHash = v +} + +type LookupRequest_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Row kind: resources / entitlements / grants + // (pkg/sourcecache.RowKind values). Entries are partitioned by row + // kind, so one scope hash can carry a different validator per kind. + RowKind string + // Connector-defined stable scope identifier (conventionally a hex hash + // of the canonical scope; see pkg/sourcecache.HashScope). Opaque to the + // parent; matched verbatim against the previous sync's source-cache + // entries. + ScopeHash string +} + +func (b0 LookupRequest_builder) Build() *LookupRequest { + m0 := &LookupRequest{} + b, x := &b0, m0 + _, _ = b, x + x.RowKind = b.RowKind + x.ScopeHash = b.ScopeHash + return m0 +} + +type LookupResponse struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + // False means no prior entry exists for (row_kind, scope_hash): the + // connector must fetch fresh and must not emit SourceCacheReplay for + // this scope. + Found bool `protobuf:"varint,1,opt,name=found,proto3" json:"found,omitempty"` + // The opaque validator the previous sync recorded for this scope (HTTP + // ETag, delta token, ...). Empty when found is false. The cap is a + // sanity bound sized for Microsoft Graph delta tokens, which are known + // to run to thousands of characters; storage imposes no limit. + Etag string `protobuf:"bytes,2,opt,name=etag,proto3" json:"etag,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LookupResponse) Reset() { + *x = LookupResponse{} + mi := &file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LookupResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LookupResponse) ProtoMessage() {} + +func (x *LookupResponse) ProtoReflect() protoreflect.Message { + mi := &file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *LookupResponse) GetFound() bool { + if x != nil { + return x.Found + } + return false +} + +func (x *LookupResponse) GetEtag() string { + if x != nil { + return x.Etag + } + return "" +} + +func (x *LookupResponse) SetFound(v bool) { + x.Found = v +} + +func (x *LookupResponse) SetEtag(v string) { + x.Etag = v +} + +type LookupResponse_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // False means no prior entry exists for (row_kind, scope_hash): the + // connector must fetch fresh and must not emit SourceCacheReplay for + // this scope. + Found bool + // The opaque validator the previous sync recorded for this scope (HTTP + // ETag, delta token, ...). Empty when found is false. The cap is a + // sanity bound sized for Microsoft Graph delta tokens, which are known + // to run to thousands of characters; storage imposes no limit. + Etag string +} + +func (b0 LookupResponse_builder) Build() *LookupResponse { + m0 := &LookupResponse{} + b, x := &b0, m0 + _, _ = b, x + x.Found = b.Found + x.Etag = b.Etag + return m0 +} + +var File_c1_connectorapi_baton_v1_source_cache_proto protoreflect.FileDescriptor + +const file_c1_connectorapi_baton_v1_source_cache_proto_rawDesc = "" + + "\n" + + "+c1/connectorapi/baton/v1/source_cache.proto\x12\x18c1.connectorapi.baton.v1\x1a\x17validate/validate.proto\"`\n" + + "\rLookupRequest\x12$\n" + + "\brow_kind\x18\x01 \x01(\tB\t\xfaB\x06r\x04\x10\x01\x18@R\arowKind\x12)\n" + + "\n" + + "scope_hash\x18\x02 \x01(\tB\n" + + "\xfaB\ar\x05\x10\x01\x18\x80\x02R\tscopeHash\"E\n" + + "\x0eLookupResponse\x12\x14\n" + + "\x05found\x18\x01 \x01(\bR\x05found\x12\x1d\n" + + "\x04etag\x18\x02 \x01(\tB\t\xfaB\x06r\x04\x18\x80\x80\x04R\x04etag2x\n" + + "\x17BatonSourceCacheService\x12]\n" + + "\x06Lookup\x12'.c1.connectorapi.baton.v1.LookupRequest\x1a(.c1.connectorapi.baton.v1.LookupResponse\"\x00B7Z5gitlab.com/ductone/c1/pkg/pb/c1/connectorapi/baton/v1b\x06proto3" + +var file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_c1_connectorapi_baton_v1_source_cache_proto_goTypes = []any{ + (*LookupRequest)(nil), // 0: c1.connectorapi.baton.v1.LookupRequest + (*LookupResponse)(nil), // 1: c1.connectorapi.baton.v1.LookupResponse +} +var file_c1_connectorapi_baton_v1_source_cache_proto_depIdxs = []int32{ + 0, // 0: c1.connectorapi.baton.v1.BatonSourceCacheService.Lookup:input_type -> c1.connectorapi.baton.v1.LookupRequest + 1, // 1: c1.connectorapi.baton.v1.BatonSourceCacheService.Lookup:output_type -> c1.connectorapi.baton.v1.LookupResponse + 1, // [1:2] is the sub-list for method output_type + 0, // [0:1] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_c1_connectorapi_baton_v1_source_cache_proto_init() } +func file_c1_connectorapi_baton_v1_source_cache_proto_init() { + if File_c1_connectorapi_baton_v1_source_cache_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_connectorapi_baton_v1_source_cache_proto_rawDesc), len(file_c1_connectorapi_baton_v1_source_cache_proto_rawDesc)), + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_c1_connectorapi_baton_v1_source_cache_proto_goTypes, + DependencyIndexes: file_c1_connectorapi_baton_v1_source_cache_proto_depIdxs, + MessageInfos: file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes, + }.Build() + File_c1_connectorapi_baton_v1_source_cache_proto = out.File + file_c1_connectorapi_baton_v1_source_cache_proto_goTypes = nil + file_c1_connectorapi_baton_v1_source_cache_proto_depIdxs = nil +} diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/source_cache.pb.validate.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/source_cache.pb.validate.go new file mode 100644 index 00000000..dfc7cca7 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/source_cache.pb.validate.go @@ -0,0 +1,271 @@ +// Code generated by protoc-gen-validate. DO NOT EDIT. +// source: c1/connectorapi/baton/v1/source_cache.proto + +package v1 + +import ( + "bytes" + "errors" + "fmt" + "net" + "net/mail" + "net/url" + "regexp" + "sort" + "strings" + "time" + "unicode/utf8" + + "google.golang.org/protobuf/types/known/anypb" +) + +// ensure the imports are used +var ( + _ = bytes.MinRead + _ = errors.New("") + _ = fmt.Print + _ = utf8.UTFMax + _ = (*regexp.Regexp)(nil) + _ = (*strings.Reader)(nil) + _ = net.IPv4len + _ = time.Duration(0) + _ = (*url.URL)(nil) + _ = (*mail.Address)(nil) + _ = anypb.Any{} + _ = sort.Sort +) + +// Validate checks the field values on LookupRequest with the rules defined in +// the proto definition for this message. If any rules are violated, the first +// error encountered is returned, or nil if there are no violations. +func (m *LookupRequest) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on LookupRequest with the rules defined +// in the proto definition for this message. If any rules are violated, the +// result is a list of violation errors wrapped in LookupRequestMultiError, or +// nil if none found. +func (m *LookupRequest) ValidateAll() error { + return m.validate(true) +} + +func (m *LookupRequest) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + if l := utf8.RuneCountInString(m.GetRowKind()); l < 1 || l > 64 { + err := LookupRequestValidationError{ + field: "RowKind", + reason: "value length must be between 1 and 64 runes, inclusive", + } + if !all { + return err + } + errors = append(errors, err) + } + + if l := utf8.RuneCountInString(m.GetScopeHash()); l < 1 || l > 256 { + err := LookupRequestValidationError{ + field: "ScopeHash", + reason: "value length must be between 1 and 256 runes, inclusive", + } + if !all { + return err + } + errors = append(errors, err) + } + + if len(errors) > 0 { + return LookupRequestMultiError(errors) + } + + return nil +} + +// LookupRequestMultiError is an error wrapping multiple validation errors +// returned by LookupRequest.ValidateAll() if the designated constraints +// aren't met. +type LookupRequestMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m LookupRequestMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m LookupRequestMultiError) AllErrors() []error { return m } + +// LookupRequestValidationError is the validation error returned by +// LookupRequest.Validate if the designated constraints aren't met. +type LookupRequestValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e LookupRequestValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e LookupRequestValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e LookupRequestValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e LookupRequestValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e LookupRequestValidationError) ErrorName() string { return "LookupRequestValidationError" } + +// Error satisfies the builtin error interface +func (e LookupRequestValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sLookupRequest.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = LookupRequestValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = LookupRequestValidationError{} + +// Validate checks the field values on LookupResponse with the rules defined in +// the proto definition for this message. If any rules are violated, the first +// error encountered is returned, or nil if there are no violations. +func (m *LookupResponse) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on LookupResponse with the rules defined +// in the proto definition for this message. If any rules are violated, the +// result is a list of violation errors wrapped in LookupResponseMultiError, +// or nil if none found. +func (m *LookupResponse) ValidateAll() error { + return m.validate(true) +} + +func (m *LookupResponse) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + // no validation rules for Found + + if utf8.RuneCountInString(m.GetEtag()) > 65536 { + err := LookupResponseValidationError{ + field: "Etag", + reason: "value length must be at most 65536 runes", + } + if !all { + return err + } + errors = append(errors, err) + } + + if len(errors) > 0 { + return LookupResponseMultiError(errors) + } + + return nil +} + +// LookupResponseMultiError is an error wrapping multiple validation errors +// returned by LookupResponse.ValidateAll() if the designated constraints +// aren't met. +type LookupResponseMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m LookupResponseMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m LookupResponseMultiError) AllErrors() []error { return m } + +// LookupResponseValidationError is the validation error returned by +// LookupResponse.Validate if the designated constraints aren't met. +type LookupResponseValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e LookupResponseValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e LookupResponseValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e LookupResponseValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e LookupResponseValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e LookupResponseValidationError) ErrorName() string { return "LookupResponseValidationError" } + +// Error satisfies the builtin error interface +func (e LookupResponseValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sLookupResponse.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = LookupResponseValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = LookupResponseValidationError{} diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/source_cache_grpc.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/source_cache_grpc.pb.go new file mode 100644 index 00000000..8db25cd3 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/source_cache_grpc.pb.go @@ -0,0 +1,145 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.5.1 +// - protoc (unknown) +// source: c1/connectorapi/baton/v1/source_cache.proto + +package v1 + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + BatonSourceCacheService_Lookup_FullMethodName = "/c1.connectorapi.baton.v1.BatonSourceCacheService/Lookup" +) + +// BatonSourceCacheServiceClient is the client API for BatonSourceCacheService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// BatonSourceCacheService is the dedicated parent-side RPC the connector +// subprocess uses to ask "do I have a previous validator (etag / delta +// token) for this scope?" before revalidating upstream. +// +// This is intentionally NOT routed through the session-store gRPC service: +// session data goes through the connector's local MemorySessionCache +// (otter), which would subject sync-scoped validator state to generic +// TTL/eviction policies and burn bounded cache weight on it. A dedicated +// service keeps the message shape explicit and the path uncached. +// +// The parent has exactly one active lookup registered at a time (set +// per-sync, cleared at sync end), so no sync_id travels on the wire. +type BatonSourceCacheServiceClient interface { + Lookup(ctx context.Context, in *LookupRequest, opts ...grpc.CallOption) (*LookupResponse, error) +} + +type batonSourceCacheServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewBatonSourceCacheServiceClient(cc grpc.ClientConnInterface) BatonSourceCacheServiceClient { + return &batonSourceCacheServiceClient{cc} +} + +func (c *batonSourceCacheServiceClient) Lookup(ctx context.Context, in *LookupRequest, opts ...grpc.CallOption) (*LookupResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(LookupResponse) + err := c.cc.Invoke(ctx, BatonSourceCacheService_Lookup_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// BatonSourceCacheServiceServer is the server API for BatonSourceCacheService service. +// All implementations should embed UnimplementedBatonSourceCacheServiceServer +// for forward compatibility. +// +// BatonSourceCacheService is the dedicated parent-side RPC the connector +// subprocess uses to ask "do I have a previous validator (etag / delta +// token) for this scope?" before revalidating upstream. +// +// This is intentionally NOT routed through the session-store gRPC service: +// session data goes through the connector's local MemorySessionCache +// (otter), which would subject sync-scoped validator state to generic +// TTL/eviction policies and burn bounded cache weight on it. A dedicated +// service keeps the message shape explicit and the path uncached. +// +// The parent has exactly one active lookup registered at a time (set +// per-sync, cleared at sync end), so no sync_id travels on the wire. +type BatonSourceCacheServiceServer interface { + Lookup(context.Context, *LookupRequest) (*LookupResponse, error) +} + +// UnimplementedBatonSourceCacheServiceServer should be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedBatonSourceCacheServiceServer struct{} + +func (UnimplementedBatonSourceCacheServiceServer) Lookup(context.Context, *LookupRequest) (*LookupResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method Lookup not implemented") +} +func (UnimplementedBatonSourceCacheServiceServer) testEmbeddedByValue() {} + +// UnsafeBatonSourceCacheServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to BatonSourceCacheServiceServer will +// result in compilation errors. +type UnsafeBatonSourceCacheServiceServer interface { + mustEmbedUnimplementedBatonSourceCacheServiceServer() +} + +func RegisterBatonSourceCacheServiceServer(s grpc.ServiceRegistrar, srv BatonSourceCacheServiceServer) { + // If the following call pancis, it indicates UnimplementedBatonSourceCacheServiceServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&BatonSourceCacheService_ServiceDesc, srv) +} + +func _BatonSourceCacheService_Lookup_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(LookupRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(BatonSourceCacheServiceServer).Lookup(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: BatonSourceCacheService_Lookup_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(BatonSourceCacheServiceServer).Lookup(ctx, req.(*LookupRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// BatonSourceCacheService_ServiceDesc is the grpc.ServiceDesc for BatonSourceCacheService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var BatonSourceCacheService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "c1.connectorapi.baton.v1.BatonSourceCacheService", + HandlerType: (*BatonSourceCacheServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Lookup", + Handler: _BatonSourceCacheService_Lookup_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "c1/connectorapi/baton/v1/source_cache.proto", +} diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/source_cache_protoopaque.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/source_cache_protoopaque.pb.go new file mode 100644 index 00000000..9bfc8953 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/source_cache_protoopaque.pb.go @@ -0,0 +1,235 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.10 +// protoc (unknown) +// source: c1/connectorapi/baton/v1/source_cache.proto + +//go:build protoopaque + +package v1 + +import ( + _ "github.com/envoyproxy/protoc-gen-validate/validate" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type LookupRequest struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3"` + xxx_hidden_ScopeHash string `protobuf:"bytes,2,opt,name=scope_hash,json=scopeHash,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LookupRequest) Reset() { + *x = LookupRequest{} + mi := &file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LookupRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LookupRequest) ProtoMessage() {} + +func (x *LookupRequest) ProtoReflect() protoreflect.Message { + mi := &file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *LookupRequest) GetRowKind() string { + if x != nil { + return x.xxx_hidden_RowKind + } + return "" +} + +func (x *LookupRequest) GetScopeHash() string { + if x != nil { + return x.xxx_hidden_ScopeHash + } + return "" +} + +func (x *LookupRequest) SetRowKind(v string) { + x.xxx_hidden_RowKind = v +} + +func (x *LookupRequest) SetScopeHash(v string) { + x.xxx_hidden_ScopeHash = v +} + +type LookupRequest_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Row kind: resources / entitlements / grants + // (pkg/sourcecache.RowKind values). Entries are partitioned by row + // kind, so one scope hash can carry a different validator per kind. + RowKind string + // Connector-defined stable scope identifier (conventionally a hex hash + // of the canonical scope; see pkg/sourcecache.HashScope). Opaque to the + // parent; matched verbatim against the previous sync's source-cache + // entries. + ScopeHash string +} + +func (b0 LookupRequest_builder) Build() *LookupRequest { + m0 := &LookupRequest{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_RowKind = b.RowKind + x.xxx_hidden_ScopeHash = b.ScopeHash + return m0 +} + +type LookupResponse struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Found bool `protobuf:"varint,1,opt,name=found,proto3"` + xxx_hidden_Etag string `protobuf:"bytes,2,opt,name=etag,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LookupResponse) Reset() { + *x = LookupResponse{} + mi := &file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LookupResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LookupResponse) ProtoMessage() {} + +func (x *LookupResponse) ProtoReflect() protoreflect.Message { + mi := &file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *LookupResponse) GetFound() bool { + if x != nil { + return x.xxx_hidden_Found + } + return false +} + +func (x *LookupResponse) GetEtag() string { + if x != nil { + return x.xxx_hidden_Etag + } + return "" +} + +func (x *LookupResponse) SetFound(v bool) { + x.xxx_hidden_Found = v +} + +func (x *LookupResponse) SetEtag(v string) { + x.xxx_hidden_Etag = v +} + +type LookupResponse_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // False means no prior entry exists for (row_kind, scope_hash): the + // connector must fetch fresh and must not emit SourceCacheReplay for + // this scope. + Found bool + // The opaque validator the previous sync recorded for this scope (HTTP + // ETag, delta token, ...). Empty when found is false. The cap is a + // sanity bound sized for Microsoft Graph delta tokens, which are known + // to run to thousands of characters; storage imposes no limit. + Etag string +} + +func (b0 LookupResponse_builder) Build() *LookupResponse { + m0 := &LookupResponse{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_Found = b.Found + x.xxx_hidden_Etag = b.Etag + return m0 +} + +var File_c1_connectorapi_baton_v1_source_cache_proto protoreflect.FileDescriptor + +const file_c1_connectorapi_baton_v1_source_cache_proto_rawDesc = "" + + "\n" + + "+c1/connectorapi/baton/v1/source_cache.proto\x12\x18c1.connectorapi.baton.v1\x1a\x17validate/validate.proto\"`\n" + + "\rLookupRequest\x12$\n" + + "\brow_kind\x18\x01 \x01(\tB\t\xfaB\x06r\x04\x10\x01\x18@R\arowKind\x12)\n" + + "\n" + + "scope_hash\x18\x02 \x01(\tB\n" + + "\xfaB\ar\x05\x10\x01\x18\x80\x02R\tscopeHash\"E\n" + + "\x0eLookupResponse\x12\x14\n" + + "\x05found\x18\x01 \x01(\bR\x05found\x12\x1d\n" + + "\x04etag\x18\x02 \x01(\tB\t\xfaB\x06r\x04\x18\x80\x80\x04R\x04etag2x\n" + + "\x17BatonSourceCacheService\x12]\n" + + "\x06Lookup\x12'.c1.connectorapi.baton.v1.LookupRequest\x1a(.c1.connectorapi.baton.v1.LookupResponse\"\x00B7Z5gitlab.com/ductone/c1/pkg/pb/c1/connectorapi/baton/v1b\x06proto3" + +var file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_c1_connectorapi_baton_v1_source_cache_proto_goTypes = []any{ + (*LookupRequest)(nil), // 0: c1.connectorapi.baton.v1.LookupRequest + (*LookupResponse)(nil), // 1: c1.connectorapi.baton.v1.LookupResponse +} +var file_c1_connectorapi_baton_v1_source_cache_proto_depIdxs = []int32{ + 0, // 0: c1.connectorapi.baton.v1.BatonSourceCacheService.Lookup:input_type -> c1.connectorapi.baton.v1.LookupRequest + 1, // 1: c1.connectorapi.baton.v1.BatonSourceCacheService.Lookup:output_type -> c1.connectorapi.baton.v1.LookupResponse + 1, // [1:2] is the sub-list for method output_type + 0, // [0:1] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_c1_connectorapi_baton_v1_source_cache_proto_init() } +func file_c1_connectorapi_baton_v1_source_cache_proto_init() { + if File_c1_connectorapi_baton_v1_source_cache_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_connectorapi_baton_v1_source_cache_proto_rawDesc), len(file_c1_connectorapi_baton_v1_source_cache_proto_rawDesc)), + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_c1_connectorapi_baton_v1_source_cache_proto_goTypes, + DependencyIndexes: file_c1_connectorapi_baton_v1_source_cache_proto_depIdxs, + MessageInfos: file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes, + }.Build() + File_c1_connectorapi_baton_v1_source_cache_proto = out.File + file_c1_connectorapi_baton_v1_source_cache_proto_goTypes = nil + file_c1_connectorapi_baton_v1_source_cache_proto_depIdxs = nil +} diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records.pb.go index 312f32a0..29552e74 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records.pb.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records.pb.go @@ -455,8 +455,11 @@ type ResourceRecord struct { Parent *ResourceRef `protobuf:"bytes,6,opt,name=parent,proto3" json:"parent,omitempty"` Annotations []*anypb.Any `protobuf:"bytes,7,rep,name=annotations,proto3" json:"annotations,omitempty"` DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,8,opt,name=discovered_at,json=discoveredAt,proto3" json:"discovered_at,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope. + SourceScopeHash string `protobuf:"bytes,9,opt,name=source_scope_hash,json=sourceScopeHash,proto3" json:"source_scope_hash,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ResourceRecord) Reset() { @@ -533,6 +536,13 @@ func (x *ResourceRecord) GetDiscoveredAt() *timestamppb.Timestamp { return nil } +func (x *ResourceRecord) GetSourceScopeHash() string { + if x != nil { + return x.SourceScopeHash + } + return "" +} + func (x *ResourceRecord) SetResourceTypeId(v string) { x.ResourceTypeId = v } @@ -561,6 +571,10 @@ func (x *ResourceRecord) SetDiscoveredAt(v *timestamppb.Timestamp) { x.DiscoveredAt = v } +func (x *ResourceRecord) SetSourceScopeHash(v string) { + x.SourceScopeHash = v +} + func (x *ResourceRecord) HasParent() bool { if x == nil { return false @@ -593,6 +607,9 @@ type ResourceRecord_builder struct { Parent *ResourceRef Annotations []*anypb.Any DiscoveredAt *timestamppb.Timestamp + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope. + SourceScopeHash string } func (b0 ResourceRecord_builder) Build() *ResourceRecord { @@ -606,6 +623,7 @@ func (b0 ResourceRecord_builder) Build() *ResourceRecord { x.Parent = b.Parent x.Annotations = b.Annotations x.DiscoveredAt = b.DiscoveredAt + x.SourceScopeHash = b.SourceScopeHash return m0 } @@ -632,8 +650,11 @@ type EntitlementRecord struct { // resource_types table when needed. Consumed by // pkg/sync/syncer.go's principal-type narrowing. GrantableToResourceTypeIds []string `protobuf:"bytes,10,rep,name=grantable_to_resource_type_ids,json=grantableToResourceTypeIds,proto3" json:"grantable_to_resource_type_ids,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope. + SourceScopeHash string `protobuf:"bytes,11,opt,name=source_scope_hash,json=sourceScopeHash,proto3" json:"source_scope_hash,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *EntitlementRecord) Reset() { @@ -724,6 +745,13 @@ func (x *EntitlementRecord) GetGrantableToResourceTypeIds() []string { return nil } +func (x *EntitlementRecord) GetSourceScopeHash() string { + if x != nil { + return x.SourceScopeHash + } + return "" +} + func (x *EntitlementRecord) SetExternalId(v string) { x.ExternalId = v } @@ -760,6 +788,10 @@ func (x *EntitlementRecord) SetGrantableToResourceTypeIds(v []string) { x.GrantableToResourceTypeIds = v } +func (x *EntitlementRecord) SetSourceScopeHash(v string) { + x.SourceScopeHash = v +} + func (x *EntitlementRecord) HasResource() bool { if x == nil { return false @@ -806,6 +838,9 @@ type EntitlementRecord_builder struct { // resource_types table when needed. Consumed by // pkg/sync/syncer.go's principal-type narrowing. GrantableToResourceTypeIds []string + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope. + SourceScopeHash string } func (b0 EntitlementRecord_builder) Build() *EntitlementRecord { @@ -821,6 +856,7 @@ func (b0 EntitlementRecord_builder) Build() *EntitlementRecord { x.DiscoveredAt = b.DiscoveredAt x.Slug = b.Slug x.GrantableToResourceTypeIds = b.GrantableToResourceTypeIds + x.SourceScopeHash = b.SourceScopeHash return m0 } @@ -846,9 +882,15 @@ type GrantRecord struct { Annotations []*anypb.Any `protobuf:"bytes,8,rep,name=annotations,proto3" json:"annotations,omitempty"` // map — same wire shape as // c1.connector.v2.GrantSources.sources. - Sources map[string]*GrantSourceRecord `protobuf:"bytes,9,rep,name=sources,proto3" json:"sources,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + Sources map[string]*GrantSourceRecord `protobuf:"bytes,9,rep,name=sources,proto3" json:"sources,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope — notably + // expander-derived grants, which are recreated by expansion each sync + // and never replayed. StoreExpandedGrants preserves this field on + // rewrites of existing rows, exactly like expansion/needs_expansion. + SourceScopeHash string `protobuf:"bytes,10,opt,name=source_scope_hash,json=sourceScopeHash,proto3" json:"source_scope_hash,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *GrantRecord) Reset() { @@ -932,6 +974,13 @@ func (x *GrantRecord) GetSources() map[string]*GrantSourceRecord { return nil } +func (x *GrantRecord) GetSourceScopeHash() string { + if x != nil { + return x.SourceScopeHash + } + return "" +} + func (x *GrantRecord) SetExternalId(v string) { x.ExternalId = v } @@ -964,6 +1013,10 @@ func (x *GrantRecord) SetSources(v map[string]*GrantSourceRecord) { x.Sources = v } +func (x *GrantRecord) SetSourceScopeHash(v string) { + x.SourceScopeHash = v +} + func (x *GrantRecord) HasEntitlement() bool { if x == nil { return false @@ -1032,6 +1085,12 @@ type GrantRecord_builder struct { // map — same wire shape as // c1.connector.v2.GrantSources.sources. Sources map[string]*GrantSourceRecord + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope — notably + // expander-derived grants, which are recreated by expansion each sync + // and never replayed. StoreExpandedGrants preserves this field on + // rewrites of existing rows, exactly like expansion/needs_expansion. + SourceScopeHash string } func (b0 GrantRecord_builder) Build() *GrantRecord { @@ -1046,6 +1105,7 @@ func (b0 GrantRecord_builder) Build() *GrantRecord { x.NeedsExpansion = b.NeedsExpansion x.Annotations = b.Annotations x.Sources = b.Sources + x.SourceScopeHash = b.SourceScopeHash return m0 } @@ -1653,6 +1713,130 @@ func (b0 SessionRecord_builder) Build() *SessionRecord { return m0 } +// SourceCacheEntryRecord is the per-scope manifest entry for source-cache +// replay (see c1/connector/v2/annotation_source_cache.proto). One entry +// per (row_kind, scope_hash), written for every freshly fetched scope — +// including zero-row responses — and rewritten on replay with the scope's +// current validator. The previous sync's entries (read from the previous +// c1z) are the lookup surface connectors revalidate against. +type SourceCacheEntryRecord struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + // Row kind partition: "resources", "entitlements", or "grants" + // (pkg/sourcecache.RowKind values). + RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3" json:"row_kind,omitempty"` + // Connector-computed scope hash (lowercase hex). + ScopeHash string `protobuf:"bytes,2,opt,name=scope_hash,json=scopeHash,proto3" json:"scope_hash,omitempty"` + // Opaque upstream validator: HTTP ETag, delta token, etc. + Etag string `protobuf:"bytes,3,opt,name=etag,proto3" json:"etag,omitempty"` + DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=discovered_at,json=discoveredAt,proto3" json:"discovered_at,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheEntryRecord) Reset() { + *x = SourceCacheEntryRecord{} + mi := &file_c1_storage_v3_records_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheEntryRecord) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheEntryRecord) ProtoMessage() {} + +func (x *SourceCacheEntryRecord) ProtoReflect() protoreflect.Message { + mi := &file_c1_storage_v3_records_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheEntryRecord) GetRowKind() string { + if x != nil { + return x.RowKind + } + return "" +} + +func (x *SourceCacheEntryRecord) GetScopeHash() string { + if x != nil { + return x.ScopeHash + } + return "" +} + +func (x *SourceCacheEntryRecord) GetEtag() string { + if x != nil { + return x.Etag + } + return "" +} + +func (x *SourceCacheEntryRecord) GetDiscoveredAt() *timestamppb.Timestamp { + if x != nil { + return x.DiscoveredAt + } + return nil +} + +func (x *SourceCacheEntryRecord) SetRowKind(v string) { + x.RowKind = v +} + +func (x *SourceCacheEntryRecord) SetScopeHash(v string) { + x.ScopeHash = v +} + +func (x *SourceCacheEntryRecord) SetEtag(v string) { + x.Etag = v +} + +func (x *SourceCacheEntryRecord) SetDiscoveredAt(v *timestamppb.Timestamp) { + x.DiscoveredAt = v +} + +func (x *SourceCacheEntryRecord) HasDiscoveredAt() bool { + if x == nil { + return false + } + return x.DiscoveredAt != nil +} + +func (x *SourceCacheEntryRecord) ClearDiscoveredAt() { + x.DiscoveredAt = nil +} + +type SourceCacheEntryRecord_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Row kind partition: "resources", "entitlements", or "grants" + // (pkg/sourcecache.RowKind values). + RowKind string + // Connector-computed scope hash (lowercase hex). + ScopeHash string + // Opaque upstream validator: HTTP ETag, delta token, etc. + Etag string + DiscoveredAt *timestamppb.Timestamp +} + +func (b0 SourceCacheEntryRecord_builder) Build() *SourceCacheEntryRecord { + m0 := &SourceCacheEntryRecord{} + b, x := &b0, m0 + _, _ = b, x + x.RowKind = b.RowKind + x.ScopeHash = b.ScopeHash + x.Etag = b.Etag + x.DiscoveredAt = b.DiscoveredAt + return m0 +} + var File_c1_storage_v3_records_proto protoreflect.FileDescriptor const file_c1_storage_v3_records_proto_rawDesc = "" + @@ -1677,7 +1861,7 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\rdiscovered_at\x18\x06 \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt\x12 \n" + "\vdescription\x18\a \x01(\tR\vdescription\x12-\n" + "\x12sourced_externally\x18\b \x01(\bR\x11sourcedExternally:!\x82\xf9+\x1d\n" + - "\x0eresource_types\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\xbc\x03\n" + + "\x0eresource_types\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\x98\x04\n" + "\x0eResourceRecord\x12(\n" + "\x10resource_type_id\x18\x02 \x01(\tR\x0eresourceTypeId\x12\x1f\n" + "\vresource_id\x18\x03 \x01(\tR\n" + @@ -1687,8 +1871,10 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\x06parent\x18\x06 \x01(\v2\x1a.c1.storage.v3.ResourceRefB.\x8a\xf9+*\n" + "\tby_parent\x1a\x10resource_type_id\x1a\vresource_idR\x06parent\x126\n" + "\vannotations\x18\a \x03(\v2\x14.google.protobuf.AnyR\vannotations\x12?\n" + - "\rdiscovered_at\x18\b \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt:.\x82\xf9+*\n" + - "\tresources\x12\x10resource_type_id\x12\vresource_idJ\x04\b\x01\x10\x02R\async_id\"\xfe\x03\n" + + "\rdiscovered_at\x18\b \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt\x12Z\n" + + "\x11source_scope_hash\x18\t \x01(\tB.\x8a\xf9+*\n" + + "\x0fby_source_scope\"\x17source_scope_hash != ''R\x0fsourceScopeHash:.\x82\xf9+*\n" + + "\tresources\x12\x10resource_type_id\x12\vresource_idJ\x04\b\x01\x10\x02R\async_id\"\xda\x04\n" + "\x11EntitlementRecord\x12\x1f\n" + "\vexternal_id\x18\x02 \x01(\tR\n" + "externalId\x12h\n" + @@ -1701,8 +1887,10 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\rdiscovered_at\x18\b \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt\x12\x12\n" + "\x04slug\x18\t \x01(\tR\x04slug\x12B\n" + "\x1egrantable_to_resource_type_ids\x18\n" + - " \x03(\tR\x1agrantableToResourceTypeIds:\x1f\x82\xf9+\x1b\n" + - "\fentitlements\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\x9a\x06\n" + + " \x03(\tR\x1agrantableToResourceTypeIds\x12Z\n" + + "\x11source_scope_hash\x18\v \x01(\tB.\x8a\xf9+*\n" + + "\x0fby_source_scope\"\x17source_scope_hash != ''R\x0fsourceScopeHash:\x1f\x82\xf9+\x1b\n" + + "\fentitlements\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\xf6\x06\n" + "\vGrantRecord\x12\x1f\n" + "\vexternal_id\x18\x02 \x01(\tR\n" + "externalId\x12\x98\x01\n" + @@ -1715,7 +1903,10 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\x0fneeds_expansion\x18\a \x01(\bB0\x8a\xf9+,\n" + "\x12by_needs_expansion\"\x16needs_expansion = trueR\x0eneedsExpansion\x126\n" + "\vannotations\x18\b \x03(\v2\x14.google.protobuf.AnyR\vannotations\x12A\n" + - "\asources\x18\t \x03(\v2'.c1.storage.v3.GrantRecord.SourcesEntryR\asources\x1a\\\n" + + "\asources\x18\t \x03(\v2'.c1.storage.v3.GrantRecord.SourcesEntryR\asources\x12Z\n" + + "\x11source_scope_hash\x18\n" + + " \x01(\tB.\x8a\xf9+*\n" + + "\x0fby_source_scope\"\x17source_scope_hash != ''R\x0fsourceScopeHash\x1a\\\n" + "\fSourcesEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x126\n" + "\x05value\x18\x02 \x01(\v2 .c1.storage.v3.GrantSourceRecordR\x05value:\x028\x01:\x19\x82\xf9+\x15\n" + @@ -1765,7 +1956,15 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\async_id\x18\x01 \x01(\tR\x06syncId\x12\x10\n" + "\x03key\x18\x02 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x03 \x01(\fR\x05value:\x1c\x82\xf9+\x18\n" + - "\bsessions\x12\async_id\x12\x03key*\xae\x01\n" + + "\bsessions\x12\async_id\x12\x03key\"\xd9\x01\n" + + "\x16SourceCacheEntryRecord\x12\x19\n" + + "\brow_kind\x18\x01 \x01(\tR\arowKind\x12\x1d\n" + + "\n" + + "scope_hash\x18\x02 \x01(\tR\tscopeHash\x12\x12\n" + + "\x04etag\x18\x03 \x01(\tR\x04etag\x12?\n" + + "\rdiscovered_at\x18\x04 \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt:0\x82\xf9+,\n" + + "\x14source_cache_entries\x12\brow_kind\x12\n" + + "scope_hash*\xae\x01\n" + "\bSyncType\x12\x19\n" + "\x15SYNC_TYPE_UNSPECIFIED\x10\x00\x12\x12\n" + "\x0eSYNC_TYPE_FULL\x10\x01\x12\x15\n" + @@ -1775,58 +1974,60 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\x1bSYNC_TYPE_PARTIAL_DELETIONS\x10\x05B4Z2github.com/conductorone/baton-sdk/pb/c1/storage/v3b\x06proto3" var file_c1_storage_v3_records_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_c1_storage_v3_records_proto_msgTypes = make([]protoimpl.MessageInfo, 14) +var file_c1_storage_v3_records_proto_msgTypes = make([]protoimpl.MessageInfo, 15) var file_c1_storage_v3_records_proto_goTypes = []any{ - (SyncType)(0), // 0: c1.storage.v3.SyncType - (*GrantExpandableRecord)(nil), // 1: c1.storage.v3.GrantExpandableRecord - (*GrantSourceRecord)(nil), // 2: c1.storage.v3.GrantSourceRecord - (*ResourceTypeRecord)(nil), // 3: c1.storage.v3.ResourceTypeRecord - (*ResourceRecord)(nil), // 4: c1.storage.v3.ResourceRecord - (*EntitlementRecord)(nil), // 5: c1.storage.v3.EntitlementRecord - (*GrantRecord)(nil), // 6: c1.storage.v3.GrantRecord - (*AssetRecord)(nil), // 7: c1.storage.v3.AssetRecord - (*SyncRunRecord)(nil), // 8: c1.storage.v3.SyncRunRecord - (*SyncStatsRecord)(nil), // 9: c1.storage.v3.SyncStatsRecord - (*SessionRecord)(nil), // 10: c1.storage.v3.SessionRecord - nil, // 11: c1.storage.v3.GrantRecord.SourcesEntry - nil, // 12: c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry - nil, // 13: c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry - nil, // 14: c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry - (*anypb.Any)(nil), // 15: google.protobuf.Any - (*timestamppb.Timestamp)(nil), // 16: google.protobuf.Timestamp - (*ResourceRef)(nil), // 17: c1.storage.v3.ResourceRef - (*EntitlementRef)(nil), // 18: c1.storage.v3.EntitlementRef - (*PrincipalRef)(nil), // 19: c1.storage.v3.PrincipalRef + (SyncType)(0), // 0: c1.storage.v3.SyncType + (*GrantExpandableRecord)(nil), // 1: c1.storage.v3.GrantExpandableRecord + (*GrantSourceRecord)(nil), // 2: c1.storage.v3.GrantSourceRecord + (*ResourceTypeRecord)(nil), // 3: c1.storage.v3.ResourceTypeRecord + (*ResourceRecord)(nil), // 4: c1.storage.v3.ResourceRecord + (*EntitlementRecord)(nil), // 5: c1.storage.v3.EntitlementRecord + (*GrantRecord)(nil), // 6: c1.storage.v3.GrantRecord + (*AssetRecord)(nil), // 7: c1.storage.v3.AssetRecord + (*SyncRunRecord)(nil), // 8: c1.storage.v3.SyncRunRecord + (*SyncStatsRecord)(nil), // 9: c1.storage.v3.SyncStatsRecord + (*SessionRecord)(nil), // 10: c1.storage.v3.SessionRecord + (*SourceCacheEntryRecord)(nil), // 11: c1.storage.v3.SourceCacheEntryRecord + nil, // 12: c1.storage.v3.GrantRecord.SourcesEntry + nil, // 13: c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry + nil, // 14: c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry + nil, // 15: c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry + (*anypb.Any)(nil), // 16: google.protobuf.Any + (*timestamppb.Timestamp)(nil), // 17: google.protobuf.Timestamp + (*ResourceRef)(nil), // 18: c1.storage.v3.ResourceRef + (*EntitlementRef)(nil), // 19: c1.storage.v3.EntitlementRef + (*PrincipalRef)(nil), // 20: c1.storage.v3.PrincipalRef } var file_c1_storage_v3_records_proto_depIdxs = []int32{ - 15, // 0: c1.storage.v3.ResourceTypeRecord.annotations:type_name -> google.protobuf.Any - 16, // 1: c1.storage.v3.ResourceTypeRecord.discovered_at:type_name -> google.protobuf.Timestamp - 17, // 2: c1.storage.v3.ResourceRecord.parent:type_name -> c1.storage.v3.ResourceRef - 15, // 3: c1.storage.v3.ResourceRecord.annotations:type_name -> google.protobuf.Any - 16, // 4: c1.storage.v3.ResourceRecord.discovered_at:type_name -> google.protobuf.Timestamp - 17, // 5: c1.storage.v3.EntitlementRecord.resource:type_name -> c1.storage.v3.ResourceRef - 15, // 6: c1.storage.v3.EntitlementRecord.annotations:type_name -> google.protobuf.Any - 16, // 7: c1.storage.v3.EntitlementRecord.discovered_at:type_name -> google.protobuf.Timestamp - 18, // 8: c1.storage.v3.GrantRecord.entitlement:type_name -> c1.storage.v3.EntitlementRef - 19, // 9: c1.storage.v3.GrantRecord.principal:type_name -> c1.storage.v3.PrincipalRef - 16, // 10: c1.storage.v3.GrantRecord.discovered_at:type_name -> google.protobuf.Timestamp + 16, // 0: c1.storage.v3.ResourceTypeRecord.annotations:type_name -> google.protobuf.Any + 17, // 1: c1.storage.v3.ResourceTypeRecord.discovered_at:type_name -> google.protobuf.Timestamp + 18, // 2: c1.storage.v3.ResourceRecord.parent:type_name -> c1.storage.v3.ResourceRef + 16, // 3: c1.storage.v3.ResourceRecord.annotations:type_name -> google.protobuf.Any + 17, // 4: c1.storage.v3.ResourceRecord.discovered_at:type_name -> google.protobuf.Timestamp + 18, // 5: c1.storage.v3.EntitlementRecord.resource:type_name -> c1.storage.v3.ResourceRef + 16, // 6: c1.storage.v3.EntitlementRecord.annotations:type_name -> google.protobuf.Any + 17, // 7: c1.storage.v3.EntitlementRecord.discovered_at:type_name -> google.protobuf.Timestamp + 19, // 8: c1.storage.v3.GrantRecord.entitlement:type_name -> c1.storage.v3.EntitlementRef + 20, // 9: c1.storage.v3.GrantRecord.principal:type_name -> c1.storage.v3.PrincipalRef + 17, // 10: c1.storage.v3.GrantRecord.discovered_at:type_name -> google.protobuf.Timestamp 1, // 11: c1.storage.v3.GrantRecord.expansion:type_name -> c1.storage.v3.GrantExpandableRecord - 15, // 12: c1.storage.v3.GrantRecord.annotations:type_name -> google.protobuf.Any - 11, // 13: c1.storage.v3.GrantRecord.sources:type_name -> c1.storage.v3.GrantRecord.SourcesEntry - 16, // 14: c1.storage.v3.AssetRecord.discovered_at:type_name -> google.protobuf.Timestamp + 16, // 12: c1.storage.v3.GrantRecord.annotations:type_name -> google.protobuf.Any + 12, // 13: c1.storage.v3.GrantRecord.sources:type_name -> c1.storage.v3.GrantRecord.SourcesEntry + 17, // 14: c1.storage.v3.AssetRecord.discovered_at:type_name -> google.protobuf.Timestamp 0, // 15: c1.storage.v3.SyncRunRecord.type:type_name -> c1.storage.v3.SyncType - 16, // 16: c1.storage.v3.SyncRunRecord.started_at:type_name -> google.protobuf.Timestamp - 16, // 17: c1.storage.v3.SyncRunRecord.ended_at:type_name -> google.protobuf.Timestamp - 12, // 18: c1.storage.v3.SyncStatsRecord.resources_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry - 13, // 19: c1.storage.v3.SyncStatsRecord.grants_by_entitlement_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry - 14, // 20: c1.storage.v3.SyncStatsRecord.entitlements_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry - 16, // 21: c1.storage.v3.SyncStatsRecord.written_at:type_name -> google.protobuf.Timestamp - 2, // 22: c1.storage.v3.GrantRecord.SourcesEntry.value:type_name -> c1.storage.v3.GrantSourceRecord - 23, // [23:23] is the sub-list for method output_type - 23, // [23:23] is the sub-list for method input_type - 23, // [23:23] is the sub-list for extension type_name - 23, // [23:23] is the sub-list for extension extendee - 0, // [0:23] is the sub-list for field type_name + 17, // 16: c1.storage.v3.SyncRunRecord.started_at:type_name -> google.protobuf.Timestamp + 17, // 17: c1.storage.v3.SyncRunRecord.ended_at:type_name -> google.protobuf.Timestamp + 13, // 18: c1.storage.v3.SyncStatsRecord.resources_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry + 14, // 19: c1.storage.v3.SyncStatsRecord.grants_by_entitlement_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry + 15, // 20: c1.storage.v3.SyncStatsRecord.entitlements_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry + 17, // 21: c1.storage.v3.SyncStatsRecord.written_at:type_name -> google.protobuf.Timestamp + 17, // 22: c1.storage.v3.SourceCacheEntryRecord.discovered_at:type_name -> google.protobuf.Timestamp + 2, // 23: c1.storage.v3.GrantRecord.SourcesEntry.value:type_name -> c1.storage.v3.GrantSourceRecord + 24, // [24:24] is the sub-list for method output_type + 24, // [24:24] is the sub-list for method input_type + 24, // [24:24] is the sub-list for extension type_name + 24, // [24:24] is the sub-list for extension extendee + 0, // [0:24] is the sub-list for field type_name } func init() { file_c1_storage_v3_records_proto_init() } @@ -1842,7 +2043,7 @@ func file_c1_storage_v3_records_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_storage_v3_records_proto_rawDesc), len(file_c1_storage_v3_records_proto_rawDesc)), NumEnums: 1, - NumMessages: 14, + NumMessages: 15, NumExtensions: 0, NumServices: 0, }, diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records.pb.validate.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records.pb.validate.go index 29287724..0c0b299e 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records.pb.validate.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records.pb.validate.go @@ -544,6 +544,8 @@ func (m *ResourceRecord) validate(all bool) error { } } + // no validation rules for SourceScopeHash + if len(errors) > 0 { return ResourceRecordMultiError(errors) } @@ -746,6 +748,8 @@ func (m *EntitlementRecord) validate(all bool) error { // no validation rules for Slug + // no validation rules for SourceScopeHash + if len(errors) > 0 { return EntitlementRecordMultiError(errors) } @@ -1048,6 +1052,8 @@ func (m *GrantRecord) validate(all bool) error { } } + // no validation rules for SourceScopeHash + if len(errors) > 0 { return GrantRecordMultiError(errors) } @@ -1683,3 +1689,140 @@ var _ interface { Cause() error ErrorName() string } = SessionRecordValidationError{} + +// Validate checks the field values on SourceCacheEntryRecord with the rules +// defined in the proto definition for this message. If any rules are +// violated, the first error encountered is returned, or nil if there are no violations. +func (m *SourceCacheEntryRecord) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SourceCacheEntryRecord with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// SourceCacheEntryRecordMultiError, or nil if none found. +func (m *SourceCacheEntryRecord) ValidateAll() error { + return m.validate(true) +} + +func (m *SourceCacheEntryRecord) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + // no validation rules for RowKind + + // no validation rules for ScopeHash + + // no validation rules for Etag + + if all { + switch v := interface{}(m.GetDiscoveredAt()).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, SourceCacheEntryRecordValidationError{ + field: "DiscoveredAt", + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, SourceCacheEntryRecordValidationError{ + field: "DiscoveredAt", + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(m.GetDiscoveredAt()).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return SourceCacheEntryRecordValidationError{ + field: "DiscoveredAt", + reason: "embedded message failed validation", + cause: err, + } + } + } + + if len(errors) > 0 { + return SourceCacheEntryRecordMultiError(errors) + } + + return nil +} + +// SourceCacheEntryRecordMultiError is an error wrapping multiple validation +// errors returned by SourceCacheEntryRecord.ValidateAll() if the designated +// constraints aren't met. +type SourceCacheEntryRecordMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SourceCacheEntryRecordMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SourceCacheEntryRecordMultiError) AllErrors() []error { return m } + +// SourceCacheEntryRecordValidationError is the validation error returned by +// SourceCacheEntryRecord.Validate if the designated constraints aren't met. +type SourceCacheEntryRecordValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SourceCacheEntryRecordValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SourceCacheEntryRecordValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SourceCacheEntryRecordValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SourceCacheEntryRecordValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SourceCacheEntryRecordValidationError) ErrorName() string { + return "SourceCacheEntryRecordValidationError" +} + +// Error satisfies the builtin error interface +func (e SourceCacheEntryRecordValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSourceCacheEntryRecord.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SourceCacheEntryRecordValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SourceCacheEntryRecordValidationError{} diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records_protoopaque.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records_protoopaque.pb.go index bf2acc8c..797ad824 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records_protoopaque.pb.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records_protoopaque.pb.go @@ -441,16 +441,17 @@ func (b0 ResourceTypeRecord_builder) Build() *ResourceTypeRecord { } type ResourceRecord struct { - state protoimpl.MessageState `protogen:"opaque.v1"` - xxx_hidden_ResourceTypeId string `protobuf:"bytes,2,opt,name=resource_type_id,json=resourceTypeId,proto3"` - xxx_hidden_ResourceId string `protobuf:"bytes,3,opt,name=resource_id,json=resourceId,proto3"` - xxx_hidden_DisplayName string `protobuf:"bytes,4,opt,name=display_name,json=displayName,proto3"` - xxx_hidden_Description string `protobuf:"bytes,5,opt,name=description,proto3"` - xxx_hidden_Parent *ResourceRef `protobuf:"bytes,6,opt,name=parent,proto3"` - xxx_hidden_Annotations *[]*anypb.Any `protobuf:"bytes,7,rep,name=annotations,proto3"` - xxx_hidden_DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,8,opt,name=discovered_at,json=discoveredAt,proto3"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_ResourceTypeId string `protobuf:"bytes,2,opt,name=resource_type_id,json=resourceTypeId,proto3"` + xxx_hidden_ResourceId string `protobuf:"bytes,3,opt,name=resource_id,json=resourceId,proto3"` + xxx_hidden_DisplayName string `protobuf:"bytes,4,opt,name=display_name,json=displayName,proto3"` + xxx_hidden_Description string `protobuf:"bytes,5,opt,name=description,proto3"` + xxx_hidden_Parent *ResourceRef `protobuf:"bytes,6,opt,name=parent,proto3"` + xxx_hidden_Annotations *[]*anypb.Any `protobuf:"bytes,7,rep,name=annotations,proto3"` + xxx_hidden_DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,8,opt,name=discovered_at,json=discoveredAt,proto3"` + xxx_hidden_SourceScopeHash string `protobuf:"bytes,9,opt,name=source_scope_hash,json=sourceScopeHash,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ResourceRecord) Reset() { @@ -529,6 +530,13 @@ func (x *ResourceRecord) GetDiscoveredAt() *timestamppb.Timestamp { return nil } +func (x *ResourceRecord) GetSourceScopeHash() string { + if x != nil { + return x.xxx_hidden_SourceScopeHash + } + return "" +} + func (x *ResourceRecord) SetResourceTypeId(v string) { x.xxx_hidden_ResourceTypeId = v } @@ -557,6 +565,10 @@ func (x *ResourceRecord) SetDiscoveredAt(v *timestamppb.Timestamp) { x.xxx_hidden_DiscoveredAt = v } +func (x *ResourceRecord) SetSourceScopeHash(v string) { + x.xxx_hidden_SourceScopeHash = v +} + func (x *ResourceRecord) HasParent() bool { if x == nil { return false @@ -589,6 +601,9 @@ type ResourceRecord_builder struct { Parent *ResourceRef Annotations []*anypb.Any DiscoveredAt *timestamppb.Timestamp + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope. + SourceScopeHash string } func (b0 ResourceRecord_builder) Build() *ResourceRecord { @@ -602,6 +617,7 @@ func (b0 ResourceRecord_builder) Build() *ResourceRecord { x.xxx_hidden_Parent = b.Parent x.xxx_hidden_Annotations = &b.Annotations x.xxx_hidden_DiscoveredAt = b.DiscoveredAt + x.xxx_hidden_SourceScopeHash = b.SourceScopeHash return m0 } @@ -616,6 +632,7 @@ type EntitlementRecord struct { xxx_hidden_DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,8,opt,name=discovered_at,json=discoveredAt,proto3"` xxx_hidden_Slug string `protobuf:"bytes,9,opt,name=slug,proto3"` xxx_hidden_GrantableToResourceTypeIds []string `protobuf:"bytes,10,rep,name=grantable_to_resource_type_ids,json=grantableToResourceTypeIds,proto3"` + xxx_hidden_SourceScopeHash string `protobuf:"bytes,11,opt,name=source_scope_hash,json=sourceScopeHash,proto3"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -710,6 +727,13 @@ func (x *EntitlementRecord) GetGrantableToResourceTypeIds() []string { return nil } +func (x *EntitlementRecord) GetSourceScopeHash() string { + if x != nil { + return x.xxx_hidden_SourceScopeHash + } + return "" +} + func (x *EntitlementRecord) SetExternalId(v string) { x.xxx_hidden_ExternalId = v } @@ -746,6 +770,10 @@ func (x *EntitlementRecord) SetGrantableToResourceTypeIds(v []string) { x.xxx_hidden_GrantableToResourceTypeIds = v } +func (x *EntitlementRecord) SetSourceScopeHash(v string) { + x.xxx_hidden_SourceScopeHash = v +} + func (x *EntitlementRecord) HasResource() bool { if x == nil { return false @@ -792,6 +820,9 @@ type EntitlementRecord_builder struct { // resource_types table when needed. Consumed by // pkg/sync/syncer.go's principal-type narrowing. GrantableToResourceTypeIds []string + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope. + SourceScopeHash string } func (b0 EntitlementRecord_builder) Build() *EntitlementRecord { @@ -807,21 +838,23 @@ func (b0 EntitlementRecord_builder) Build() *EntitlementRecord { x.xxx_hidden_DiscoveredAt = b.DiscoveredAt x.xxx_hidden_Slug = b.Slug x.xxx_hidden_GrantableToResourceTypeIds = b.GrantableToResourceTypeIds + x.xxx_hidden_SourceScopeHash = b.SourceScopeHash return m0 } type GrantRecord struct { - state protoimpl.MessageState `protogen:"opaque.v1"` - xxx_hidden_ExternalId string `protobuf:"bytes,2,opt,name=external_id,json=externalId,proto3"` - xxx_hidden_Entitlement *EntitlementRef `protobuf:"bytes,3,opt,name=entitlement,proto3"` - xxx_hidden_Principal *PrincipalRef `protobuf:"bytes,4,opt,name=principal,proto3"` - xxx_hidden_DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=discovered_at,json=discoveredAt,proto3"` - xxx_hidden_Expansion *GrantExpandableRecord `protobuf:"bytes,6,opt,name=expansion,proto3"` - xxx_hidden_NeedsExpansion bool `protobuf:"varint,7,opt,name=needs_expansion,json=needsExpansion,proto3"` - xxx_hidden_Annotations *[]*anypb.Any `protobuf:"bytes,8,rep,name=annotations,proto3"` - xxx_hidden_Sources map[string]*GrantSourceRecord `protobuf:"bytes,9,rep,name=sources,proto3" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_ExternalId string `protobuf:"bytes,2,opt,name=external_id,json=externalId,proto3"` + xxx_hidden_Entitlement *EntitlementRef `protobuf:"bytes,3,opt,name=entitlement,proto3"` + xxx_hidden_Principal *PrincipalRef `protobuf:"bytes,4,opt,name=principal,proto3"` + xxx_hidden_DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=discovered_at,json=discoveredAt,proto3"` + xxx_hidden_Expansion *GrantExpandableRecord `protobuf:"bytes,6,opt,name=expansion,proto3"` + xxx_hidden_NeedsExpansion bool `protobuf:"varint,7,opt,name=needs_expansion,json=needsExpansion,proto3"` + xxx_hidden_Annotations *[]*anypb.Any `protobuf:"bytes,8,rep,name=annotations,proto3"` + xxx_hidden_Sources map[string]*GrantSourceRecord `protobuf:"bytes,9,rep,name=sources,proto3" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + xxx_hidden_SourceScopeHash string `protobuf:"bytes,10,opt,name=source_scope_hash,json=sourceScopeHash,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *GrantRecord) Reset() { @@ -907,6 +940,13 @@ func (x *GrantRecord) GetSources() map[string]*GrantSourceRecord { return nil } +func (x *GrantRecord) GetSourceScopeHash() string { + if x != nil { + return x.xxx_hidden_SourceScopeHash + } + return "" +} + func (x *GrantRecord) SetExternalId(v string) { x.xxx_hidden_ExternalId = v } @@ -939,6 +979,10 @@ func (x *GrantRecord) SetSources(v map[string]*GrantSourceRecord) { x.xxx_hidden_Sources = v } +func (x *GrantRecord) SetSourceScopeHash(v string) { + x.xxx_hidden_SourceScopeHash = v +} + func (x *GrantRecord) HasEntitlement() bool { if x == nil { return false @@ -1007,6 +1051,12 @@ type GrantRecord_builder struct { // map — same wire shape as // c1.connector.v2.GrantSources.sources. Sources map[string]*GrantSourceRecord + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope — notably + // expander-derived grants, which are recreated by expansion each sync + // and never replayed. StoreExpandedGrants preserves this field on + // rewrites of existing rows, exactly like expansion/needs_expansion. + SourceScopeHash string } func (b0 GrantRecord_builder) Build() *GrantRecord { @@ -1021,6 +1071,7 @@ func (b0 GrantRecord_builder) Build() *GrantRecord { x.xxx_hidden_NeedsExpansion = b.NeedsExpansion x.xxx_hidden_Annotations = &b.Annotations x.xxx_hidden_Sources = b.Sources + x.xxx_hidden_SourceScopeHash = b.SourceScopeHash return m0 } @@ -1622,6 +1673,126 @@ func (b0 SessionRecord_builder) Build() *SessionRecord { return m0 } +// SourceCacheEntryRecord is the per-scope manifest entry for source-cache +// replay (see c1/connector/v2/annotation_source_cache.proto). One entry +// per (row_kind, scope_hash), written for every freshly fetched scope — +// including zero-row responses — and rewritten on replay with the scope's +// current validator. The previous sync's entries (read from the previous +// c1z) are the lookup surface connectors revalidate against. +type SourceCacheEntryRecord struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3"` + xxx_hidden_ScopeHash string `protobuf:"bytes,2,opt,name=scope_hash,json=scopeHash,proto3"` + xxx_hidden_Etag string `protobuf:"bytes,3,opt,name=etag,proto3"` + xxx_hidden_DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=discovered_at,json=discoveredAt,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheEntryRecord) Reset() { + *x = SourceCacheEntryRecord{} + mi := &file_c1_storage_v3_records_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheEntryRecord) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheEntryRecord) ProtoMessage() {} + +func (x *SourceCacheEntryRecord) ProtoReflect() protoreflect.Message { + mi := &file_c1_storage_v3_records_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheEntryRecord) GetRowKind() string { + if x != nil { + return x.xxx_hidden_RowKind + } + return "" +} + +func (x *SourceCacheEntryRecord) GetScopeHash() string { + if x != nil { + return x.xxx_hidden_ScopeHash + } + return "" +} + +func (x *SourceCacheEntryRecord) GetEtag() string { + if x != nil { + return x.xxx_hidden_Etag + } + return "" +} + +func (x *SourceCacheEntryRecord) GetDiscoveredAt() *timestamppb.Timestamp { + if x != nil { + return x.xxx_hidden_DiscoveredAt + } + return nil +} + +func (x *SourceCacheEntryRecord) SetRowKind(v string) { + x.xxx_hidden_RowKind = v +} + +func (x *SourceCacheEntryRecord) SetScopeHash(v string) { + x.xxx_hidden_ScopeHash = v +} + +func (x *SourceCacheEntryRecord) SetEtag(v string) { + x.xxx_hidden_Etag = v +} + +func (x *SourceCacheEntryRecord) SetDiscoveredAt(v *timestamppb.Timestamp) { + x.xxx_hidden_DiscoveredAt = v +} + +func (x *SourceCacheEntryRecord) HasDiscoveredAt() bool { + if x == nil { + return false + } + return x.xxx_hidden_DiscoveredAt != nil +} + +func (x *SourceCacheEntryRecord) ClearDiscoveredAt() { + x.xxx_hidden_DiscoveredAt = nil +} + +type SourceCacheEntryRecord_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Row kind partition: "resources", "entitlements", or "grants" + // (pkg/sourcecache.RowKind values). + RowKind string + // Connector-computed scope hash (lowercase hex). + ScopeHash string + // Opaque upstream validator: HTTP ETag, delta token, etc. + Etag string + DiscoveredAt *timestamppb.Timestamp +} + +func (b0 SourceCacheEntryRecord_builder) Build() *SourceCacheEntryRecord { + m0 := &SourceCacheEntryRecord{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_RowKind = b.RowKind + x.xxx_hidden_ScopeHash = b.ScopeHash + x.xxx_hidden_Etag = b.Etag + x.xxx_hidden_DiscoveredAt = b.DiscoveredAt + return m0 +} + var File_c1_storage_v3_records_proto protoreflect.FileDescriptor const file_c1_storage_v3_records_proto_rawDesc = "" + @@ -1646,7 +1817,7 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\rdiscovered_at\x18\x06 \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt\x12 \n" + "\vdescription\x18\a \x01(\tR\vdescription\x12-\n" + "\x12sourced_externally\x18\b \x01(\bR\x11sourcedExternally:!\x82\xf9+\x1d\n" + - "\x0eresource_types\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\xbc\x03\n" + + "\x0eresource_types\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\x98\x04\n" + "\x0eResourceRecord\x12(\n" + "\x10resource_type_id\x18\x02 \x01(\tR\x0eresourceTypeId\x12\x1f\n" + "\vresource_id\x18\x03 \x01(\tR\n" + @@ -1656,8 +1827,10 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\x06parent\x18\x06 \x01(\v2\x1a.c1.storage.v3.ResourceRefB.\x8a\xf9+*\n" + "\tby_parent\x1a\x10resource_type_id\x1a\vresource_idR\x06parent\x126\n" + "\vannotations\x18\a \x03(\v2\x14.google.protobuf.AnyR\vannotations\x12?\n" + - "\rdiscovered_at\x18\b \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt:.\x82\xf9+*\n" + - "\tresources\x12\x10resource_type_id\x12\vresource_idJ\x04\b\x01\x10\x02R\async_id\"\xfe\x03\n" + + "\rdiscovered_at\x18\b \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt\x12Z\n" + + "\x11source_scope_hash\x18\t \x01(\tB.\x8a\xf9+*\n" + + "\x0fby_source_scope\"\x17source_scope_hash != ''R\x0fsourceScopeHash:.\x82\xf9+*\n" + + "\tresources\x12\x10resource_type_id\x12\vresource_idJ\x04\b\x01\x10\x02R\async_id\"\xda\x04\n" + "\x11EntitlementRecord\x12\x1f\n" + "\vexternal_id\x18\x02 \x01(\tR\n" + "externalId\x12h\n" + @@ -1670,8 +1843,10 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\rdiscovered_at\x18\b \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt\x12\x12\n" + "\x04slug\x18\t \x01(\tR\x04slug\x12B\n" + "\x1egrantable_to_resource_type_ids\x18\n" + - " \x03(\tR\x1agrantableToResourceTypeIds:\x1f\x82\xf9+\x1b\n" + - "\fentitlements\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\x9a\x06\n" + + " \x03(\tR\x1agrantableToResourceTypeIds\x12Z\n" + + "\x11source_scope_hash\x18\v \x01(\tB.\x8a\xf9+*\n" + + "\x0fby_source_scope\"\x17source_scope_hash != ''R\x0fsourceScopeHash:\x1f\x82\xf9+\x1b\n" + + "\fentitlements\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\xf6\x06\n" + "\vGrantRecord\x12\x1f\n" + "\vexternal_id\x18\x02 \x01(\tR\n" + "externalId\x12\x98\x01\n" + @@ -1684,7 +1859,10 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\x0fneeds_expansion\x18\a \x01(\bB0\x8a\xf9+,\n" + "\x12by_needs_expansion\"\x16needs_expansion = trueR\x0eneedsExpansion\x126\n" + "\vannotations\x18\b \x03(\v2\x14.google.protobuf.AnyR\vannotations\x12A\n" + - "\asources\x18\t \x03(\v2'.c1.storage.v3.GrantRecord.SourcesEntryR\asources\x1a\\\n" + + "\asources\x18\t \x03(\v2'.c1.storage.v3.GrantRecord.SourcesEntryR\asources\x12Z\n" + + "\x11source_scope_hash\x18\n" + + " \x01(\tB.\x8a\xf9+*\n" + + "\x0fby_source_scope\"\x17source_scope_hash != ''R\x0fsourceScopeHash\x1a\\\n" + "\fSourcesEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x126\n" + "\x05value\x18\x02 \x01(\v2 .c1.storage.v3.GrantSourceRecordR\x05value:\x028\x01:\x19\x82\xf9+\x15\n" + @@ -1734,7 +1912,15 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\async_id\x18\x01 \x01(\tR\x06syncId\x12\x10\n" + "\x03key\x18\x02 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x03 \x01(\fR\x05value:\x1c\x82\xf9+\x18\n" + - "\bsessions\x12\async_id\x12\x03key*\xae\x01\n" + + "\bsessions\x12\async_id\x12\x03key\"\xd9\x01\n" + + "\x16SourceCacheEntryRecord\x12\x19\n" + + "\brow_kind\x18\x01 \x01(\tR\arowKind\x12\x1d\n" + + "\n" + + "scope_hash\x18\x02 \x01(\tR\tscopeHash\x12\x12\n" + + "\x04etag\x18\x03 \x01(\tR\x04etag\x12?\n" + + "\rdiscovered_at\x18\x04 \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt:0\x82\xf9+,\n" + + "\x14source_cache_entries\x12\brow_kind\x12\n" + + "scope_hash*\xae\x01\n" + "\bSyncType\x12\x19\n" + "\x15SYNC_TYPE_UNSPECIFIED\x10\x00\x12\x12\n" + "\x0eSYNC_TYPE_FULL\x10\x01\x12\x15\n" + @@ -1744,58 +1930,60 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\x1bSYNC_TYPE_PARTIAL_DELETIONS\x10\x05B4Z2github.com/conductorone/baton-sdk/pb/c1/storage/v3b\x06proto3" var file_c1_storage_v3_records_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_c1_storage_v3_records_proto_msgTypes = make([]protoimpl.MessageInfo, 14) +var file_c1_storage_v3_records_proto_msgTypes = make([]protoimpl.MessageInfo, 15) var file_c1_storage_v3_records_proto_goTypes = []any{ - (SyncType)(0), // 0: c1.storage.v3.SyncType - (*GrantExpandableRecord)(nil), // 1: c1.storage.v3.GrantExpandableRecord - (*GrantSourceRecord)(nil), // 2: c1.storage.v3.GrantSourceRecord - (*ResourceTypeRecord)(nil), // 3: c1.storage.v3.ResourceTypeRecord - (*ResourceRecord)(nil), // 4: c1.storage.v3.ResourceRecord - (*EntitlementRecord)(nil), // 5: c1.storage.v3.EntitlementRecord - (*GrantRecord)(nil), // 6: c1.storage.v3.GrantRecord - (*AssetRecord)(nil), // 7: c1.storage.v3.AssetRecord - (*SyncRunRecord)(nil), // 8: c1.storage.v3.SyncRunRecord - (*SyncStatsRecord)(nil), // 9: c1.storage.v3.SyncStatsRecord - (*SessionRecord)(nil), // 10: c1.storage.v3.SessionRecord - nil, // 11: c1.storage.v3.GrantRecord.SourcesEntry - nil, // 12: c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry - nil, // 13: c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry - nil, // 14: c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry - (*anypb.Any)(nil), // 15: google.protobuf.Any - (*timestamppb.Timestamp)(nil), // 16: google.protobuf.Timestamp - (*ResourceRef)(nil), // 17: c1.storage.v3.ResourceRef - (*EntitlementRef)(nil), // 18: c1.storage.v3.EntitlementRef - (*PrincipalRef)(nil), // 19: c1.storage.v3.PrincipalRef + (SyncType)(0), // 0: c1.storage.v3.SyncType + (*GrantExpandableRecord)(nil), // 1: c1.storage.v3.GrantExpandableRecord + (*GrantSourceRecord)(nil), // 2: c1.storage.v3.GrantSourceRecord + (*ResourceTypeRecord)(nil), // 3: c1.storage.v3.ResourceTypeRecord + (*ResourceRecord)(nil), // 4: c1.storage.v3.ResourceRecord + (*EntitlementRecord)(nil), // 5: c1.storage.v3.EntitlementRecord + (*GrantRecord)(nil), // 6: c1.storage.v3.GrantRecord + (*AssetRecord)(nil), // 7: c1.storage.v3.AssetRecord + (*SyncRunRecord)(nil), // 8: c1.storage.v3.SyncRunRecord + (*SyncStatsRecord)(nil), // 9: c1.storage.v3.SyncStatsRecord + (*SessionRecord)(nil), // 10: c1.storage.v3.SessionRecord + (*SourceCacheEntryRecord)(nil), // 11: c1.storage.v3.SourceCacheEntryRecord + nil, // 12: c1.storage.v3.GrantRecord.SourcesEntry + nil, // 13: c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry + nil, // 14: c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry + nil, // 15: c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry + (*anypb.Any)(nil), // 16: google.protobuf.Any + (*timestamppb.Timestamp)(nil), // 17: google.protobuf.Timestamp + (*ResourceRef)(nil), // 18: c1.storage.v3.ResourceRef + (*EntitlementRef)(nil), // 19: c1.storage.v3.EntitlementRef + (*PrincipalRef)(nil), // 20: c1.storage.v3.PrincipalRef } var file_c1_storage_v3_records_proto_depIdxs = []int32{ - 15, // 0: c1.storage.v3.ResourceTypeRecord.annotations:type_name -> google.protobuf.Any - 16, // 1: c1.storage.v3.ResourceTypeRecord.discovered_at:type_name -> google.protobuf.Timestamp - 17, // 2: c1.storage.v3.ResourceRecord.parent:type_name -> c1.storage.v3.ResourceRef - 15, // 3: c1.storage.v3.ResourceRecord.annotations:type_name -> google.protobuf.Any - 16, // 4: c1.storage.v3.ResourceRecord.discovered_at:type_name -> google.protobuf.Timestamp - 17, // 5: c1.storage.v3.EntitlementRecord.resource:type_name -> c1.storage.v3.ResourceRef - 15, // 6: c1.storage.v3.EntitlementRecord.annotations:type_name -> google.protobuf.Any - 16, // 7: c1.storage.v3.EntitlementRecord.discovered_at:type_name -> google.protobuf.Timestamp - 18, // 8: c1.storage.v3.GrantRecord.entitlement:type_name -> c1.storage.v3.EntitlementRef - 19, // 9: c1.storage.v3.GrantRecord.principal:type_name -> c1.storage.v3.PrincipalRef - 16, // 10: c1.storage.v3.GrantRecord.discovered_at:type_name -> google.protobuf.Timestamp + 16, // 0: c1.storage.v3.ResourceTypeRecord.annotations:type_name -> google.protobuf.Any + 17, // 1: c1.storage.v3.ResourceTypeRecord.discovered_at:type_name -> google.protobuf.Timestamp + 18, // 2: c1.storage.v3.ResourceRecord.parent:type_name -> c1.storage.v3.ResourceRef + 16, // 3: c1.storage.v3.ResourceRecord.annotations:type_name -> google.protobuf.Any + 17, // 4: c1.storage.v3.ResourceRecord.discovered_at:type_name -> google.protobuf.Timestamp + 18, // 5: c1.storage.v3.EntitlementRecord.resource:type_name -> c1.storage.v3.ResourceRef + 16, // 6: c1.storage.v3.EntitlementRecord.annotations:type_name -> google.protobuf.Any + 17, // 7: c1.storage.v3.EntitlementRecord.discovered_at:type_name -> google.protobuf.Timestamp + 19, // 8: c1.storage.v3.GrantRecord.entitlement:type_name -> c1.storage.v3.EntitlementRef + 20, // 9: c1.storage.v3.GrantRecord.principal:type_name -> c1.storage.v3.PrincipalRef + 17, // 10: c1.storage.v3.GrantRecord.discovered_at:type_name -> google.protobuf.Timestamp 1, // 11: c1.storage.v3.GrantRecord.expansion:type_name -> c1.storage.v3.GrantExpandableRecord - 15, // 12: c1.storage.v3.GrantRecord.annotations:type_name -> google.protobuf.Any - 11, // 13: c1.storage.v3.GrantRecord.sources:type_name -> c1.storage.v3.GrantRecord.SourcesEntry - 16, // 14: c1.storage.v3.AssetRecord.discovered_at:type_name -> google.protobuf.Timestamp + 16, // 12: c1.storage.v3.GrantRecord.annotations:type_name -> google.protobuf.Any + 12, // 13: c1.storage.v3.GrantRecord.sources:type_name -> c1.storage.v3.GrantRecord.SourcesEntry + 17, // 14: c1.storage.v3.AssetRecord.discovered_at:type_name -> google.protobuf.Timestamp 0, // 15: c1.storage.v3.SyncRunRecord.type:type_name -> c1.storage.v3.SyncType - 16, // 16: c1.storage.v3.SyncRunRecord.started_at:type_name -> google.protobuf.Timestamp - 16, // 17: c1.storage.v3.SyncRunRecord.ended_at:type_name -> google.protobuf.Timestamp - 12, // 18: c1.storage.v3.SyncStatsRecord.resources_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry - 13, // 19: c1.storage.v3.SyncStatsRecord.grants_by_entitlement_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry - 14, // 20: c1.storage.v3.SyncStatsRecord.entitlements_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry - 16, // 21: c1.storage.v3.SyncStatsRecord.written_at:type_name -> google.protobuf.Timestamp - 2, // 22: c1.storage.v3.GrantRecord.SourcesEntry.value:type_name -> c1.storage.v3.GrantSourceRecord - 23, // [23:23] is the sub-list for method output_type - 23, // [23:23] is the sub-list for method input_type - 23, // [23:23] is the sub-list for extension type_name - 23, // [23:23] is the sub-list for extension extendee - 0, // [0:23] is the sub-list for field type_name + 17, // 16: c1.storage.v3.SyncRunRecord.started_at:type_name -> google.protobuf.Timestamp + 17, // 17: c1.storage.v3.SyncRunRecord.ended_at:type_name -> google.protobuf.Timestamp + 13, // 18: c1.storage.v3.SyncStatsRecord.resources_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry + 14, // 19: c1.storage.v3.SyncStatsRecord.grants_by_entitlement_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry + 15, // 20: c1.storage.v3.SyncStatsRecord.entitlements_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry + 17, // 21: c1.storage.v3.SyncStatsRecord.written_at:type_name -> google.protobuf.Timestamp + 17, // 22: c1.storage.v3.SourceCacheEntryRecord.discovered_at:type_name -> google.protobuf.Timestamp + 2, // 23: c1.storage.v3.GrantRecord.SourcesEntry.value:type_name -> c1.storage.v3.GrantSourceRecord + 24, // [24:24] is the sub-list for method output_type + 24, // [24:24] is the sub-list for method input_type + 24, // [24:24] is the sub-list for extension type_name + 24, // [24:24] is the sub-list for extension extendee + 0, // [0:24] is the sub-list for field type_name } func init() { file_c1_storage_v3_records_proto_init() } @@ -1811,7 +1999,7 @@ func file_c1_storage_v3_records_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_storage_v3_records_proto_rawDesc), len(file_c1_storage_v3_records_proto_rawDesc)), NumEnums: 1, - NumMessages: 14, + NumMessages: 15, NumExtensions: 0, NumServices: 0, }, diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/cli/cli.go b/vendor/github.com/conductorone/baton-sdk/pkg/cli/cli.go index 036e9ee9..59d87248 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/cli/cli.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/cli/cli.go @@ -11,6 +11,7 @@ import ( "github.com/conductorone/baton-sdk/pkg/connectorbuilder" "github.com/conductorone/baton-sdk/pkg/field" + "github.com/conductorone/baton-sdk/pkg/sourcecache" "github.com/conductorone/baton-sdk/pkg/types" "github.com/conductorone/baton-sdk/pkg/types/sessions" "github.com/spf13/cobra" @@ -20,7 +21,12 @@ import ( ) type RunTimeOpts struct { - SessionStore sessions.SessionStore + SessionStore sessions.SessionStore + // SourceCacheLookup resolves a scope's previous-sync validator for + // source-cache replay (see pkg/sourcecache). In subprocess mode this + // is a gRPC client to the parent SDK's BatonSourceCacheService; when + // unset the framework falls back to NoopLookup. + SourceCacheLookup sourcecache.Lookup TokenSource oauth2.TokenSource SelectedAuthMethod string SyncResourceTypeIDs []string diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/cli/commands.go b/vendor/github.com/conductorone/baton-sdk/pkg/cli/commands.go index 19dd598f..7bde72bd 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/cli/commands.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/cli/commands.go @@ -9,6 +9,7 @@ import ( "fmt" "io" "os" + "sync" "time" "github.com/conductorone/baton-sdk/pkg/connectorbuilder" @@ -35,6 +36,7 @@ import ( "github.com/conductorone/baton-sdk/pkg/field" "github.com/conductorone/baton-sdk/pkg/logging" "github.com/conductorone/baton-sdk/pkg/session" + "github.com/conductorone/baton-sdk/pkg/sourcecache" "github.com/conductorone/baton-sdk/pkg/tempdir" "github.com/conductorone/baton-sdk/pkg/types/sessions" "github.com/conductorone/baton-sdk/pkg/uhttp" @@ -53,29 +55,55 @@ type eventLogEnabledKey struct{} type ContrainstSetter func(*cobra.Command, field.Configuration) error -// In one shot & service mode, the child process uses this client to connect to the session store server... +// parentControlPlaneDialer opens (lazily, at most once) a single gRPC +// connection from the connector subprocess back to the parent SDK's +// control-plane listener. The same connection multiplexes +// BatonSessionService (connector session data) and BatonSourceCacheService +// (source-cache scope lookups) — the two services share a listener on the +// parent (internal/connector.runServer), so they share a client conn here. // -// which uses the C1Z for storage. Unfortunately the C1Z is instantiated well after we fork the child process, -// so there is quite a bit of pass through. -func getGRPCSessionStoreClient(ctx context.Context, serverCfg *v1.ServerConfig) func(ctx context.Context, opt ...sessions.SessionStoreConstructorOption) (sessions.SessionStore, error) { - return func(_ context.Context, opt ...sessions.SessionStoreConstructorOption) (sessions.SessionStore, error) { - l := ctxzap.Extract(ctx) - clientTLSConfig, err := utls2.ClientConfig(ctx, serverCfg.GetCredential()) - if err != nil { - return nil, err +// dial returns (nil, nil) when the parent did not start a control-plane +// listener (session store disabled); callers substitute no-op +// implementations in that case. +type parentControlPlaneDialer struct { + once sync.Once + conn *grpc.ClientConn + dialErr error + ctx context.Context + cfg *v1.ServerConfig +} + +func newParentControlPlaneDialer(ctx context.Context, serverCfg *v1.ServerConfig) *parentControlPlaneDialer { + return &parentControlPlaneDialer{ctx: ctx, cfg: serverCfg} +} + +// Close releases the underlying gRPC connection if dial ever succeeded. +// Safe when dial never ran or failed, and safe to call multiple times. +func (d *parentControlPlaneDialer) Close() { + if d.conn == nil { + return + } + _ = d.conn.Close() +} + +func (d *parentControlPlaneDialer) dial() (*grpc.ClientConn, error) { + d.once.Do(func() { + if d.cfg.GetSessionStoreListenPort() == 0 { + return } - if serverCfg.GetSessionStoreListenPort() == 0 { - return &session.NoOpSessionStore{}, nil + clientTLSConfig, err := utls2.ClientConfig(d.ctx, d.cfg.GetCredential()) + if err != nil { + d.dialErr = err + return } - // connected, grpc will handle retries for us. - dialCtx, canc := context.WithTimeout(ctx, 5*time.Second) + dialCtx, canc := context.WithTimeout(d.ctx, 5*time.Second) defer canc() var dialErr error var conn *grpc.ClientConn for { conn, err = grpc.DialContext( //nolint:staticcheck // grpc.DialContext is deprecated but we are using it still. - ctx, - fmt.Sprintf("127.0.0.1:%d", serverCfg.GetSessionStoreListenPort()), + d.ctx, + fmt.Sprintf("127.0.0.1:%d", d.cfg.GetSessionStoreListenPort()), grpc.WithTransportCredentials(credentials.NewTLS(clientTLSConfig)), grpc.WithBlock(), //nolint:staticcheck // grpc.WithBlock is deprecated but we are using it still. ) @@ -84,26 +112,59 @@ func getGRPCSessionStoreClient(ctx context.Context, serverCfg *v1.ServerConfig) select { case <-time.After(time.Millisecond * 500): case <-dialCtx.Done(): - return nil, dialErr + d.dialErr = dialErr + return } continue } break } + d.conn = conn + }) + return d.conn, d.dialErr +} +// In one shot & service mode, the child process uses this client to connect to the session store server... +// +// which uses the C1Z for storage. Unfortunately the C1Z is instantiated well after we fork the child process, +// so there is quite a bit of pass through. +func getGRPCSessionStoreClient(ctx context.Context, dialer *parentControlPlaneDialer) func(ctx context.Context, opt ...sessions.SessionStoreConstructorOption) (sessions.SessionStore, error) { + return func(_ context.Context, _ ...sessions.SessionStoreConstructorOption) (sessions.SessionStore, error) { + l := ctxzap.Extract(ctx) + conn, err := dialer.dial() + if err != nil { + return nil, err + } + if conn == nil { + return &session.NoOpSessionStore{}, nil + } client := baton_v1.NewBatonSessionServiceClient(conn) - ss, err := session.NewGRPCSessionStore(ctx, client, opt...) + ss, err := session.NewGRPCSessionStore(ctx, client) if err != nil { - err2 := conn.Close() - if err2 != nil { - l.Error("error closing connection", zap.Error(err2)) - } + l.Error("error creating session store client", zap.Error(err)) return nil, err } return ss, nil } } +// buildGRPCSourceCacheLookup returns the connector-side source-cache Lookup +// that talks to the parent's BatonSourceCacheService over the same +// loopback-TLS connection the session client uses. Returns NoopLookup when +// no parent control-plane listener is configured (matches the "no previous +// sync" behavior). +func buildGRPCSourceCacheLookup(dialer *parentControlPlaneDialer) (sourcecache.Lookup, error) { + conn, err := dialer.dial() + if err != nil { + return nil, err + } + if conn == nil { + return sourcecache.NoopLookup{}, nil + } + client := baton_v1.NewBatonSourceCacheServiceClient(conn) + return sourcecache.NewGRPCLookup(client), nil +} + func MakeMainCommand[T field.Configurable]( ctx context.Context, name string, @@ -377,13 +438,20 @@ func MakeMainCommand[T field.Configurable]( } } - if v.GetBool(field.ParallelSyncField.GetName()) { - opts = append(opts, connectorrunner.WithWorkerCount(-1)) - } - + // Worker count resolves through viper with no zero sentinel in the + // default chain: explicit flag > env (BATON_WORKERS) > config file > + // connector default (field.WithConnectorDefault on WorkerCountField) + // > shared default (0). A resolved 0 — explicit or defaulted — means + // sequential, which is the runner's zero-value behavior (normalized + // to one worker downstream), so no option is appended; -1 means + // auto-detect. The deprecated --parallel-sync only applies when + // workers resolved to 0. workers := v.GetInt(field.WorkerCountField.GetName()) - if workers != 0 { + switch { + case workers != 0: opts = append(opts, connectorrunner.WithWorkerCount(workers)) + case v.GetBool(field.ParallelSyncField.GetName()): + opts = append(opts, connectorrunner.WithWorkerCount(-1)) } c1zTmpDir := tempdir.Resolve(v.GetString("c1z-temp-dir")) @@ -403,6 +471,15 @@ func MakeMainCommand[T field.Configurable]( opts = append(opts, connectorrunner.WithExternalResourceC1Z(externalResourceC1ZPath)) } + if v.GetString(field.PreviousSyncC1ZField.GetName()) != "" { + previousSyncC1ZPath := v.GetString(field.PreviousSyncC1ZField.GetName()) + _, err := os.Open(previousSyncC1ZPath) + if err != nil { + return fmt.Errorf("the specified previous sync c1z file does not exist: %s", previousSyncC1ZPath) + } + opts = append(opts, connectorrunner.WithPreviousSyncC1Z(previousSyncC1ZPath)) + } + if v.GetString("external-resource-entitlement-id-filter") != "" { externalResourceEntitlementIdFilter := v.GetString("external-resource-entitlement-id-filter") opts = append(opts, connectorrunner.WithExternalResourceEntitlementFilter(externalResourceEntitlementIdFilter)) @@ -626,7 +703,17 @@ func MakeGRPCServerCommand[T field.Configurable]( runCtx = context.WithValue(runCtx, uhttp.ContextHTTPTimeoutKey, time.Duration(httpTimeout)*time.Second) sessionStoreMaximumSize := v.GetInt(field.ServerSessionStoreMaximumSizeField.GetName()) - sessionConstructor := getGRPCSessionStoreClient(runCtx, serverCfg) + // One dialer per subprocess; the session store client and the + // source-cache lookup client share the underlying gRPC connection + // (both services are registered on the same parent listener in + // internal/connector.runServer). + controlPlaneDialer := newParentControlPlaneDialer(runCtx, serverCfg) + defer controlPlaneDialer.Close() + sessionConstructor := getGRPCSessionStoreClient(runCtx, controlPlaneDialer) + sourceCacheLookup, err := buildGRPCSourceCacheLookup(controlPlaneDialer) + if err != nil { + return fmt.Errorf("failed to build source cache lookup: %w", err) + } c, err := getconnector(runCtx, t, RunTimeOpts{ SessionStore: NewLazyCachingSessionStore(sessionConstructor, func(otterOptions *otter.Options[string, []byte]) { if sessionStoreMaximumSize <= 0 { @@ -635,6 +722,7 @@ func MakeGRPCServerCommand[T field.Configurable]( otterOptions.MaximumWeight = uint64(sessionStoreMaximumSize) } }), + SourceCacheLookup: sourceCacheLookup, SelectedAuthMethod: v.GetString("auth-method"), SyncResourceTypeIDs: v.GetStringSlice("sync-resource-types"), }) diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/config/config.go b/vendor/github.com/conductorone/baton-sdk/pkg/config/config.go index 3586605f..f250a20a 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/config/config.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/config/config.go @@ -40,6 +40,7 @@ func RunConnector[T field.Configurable]( } builderOpts = append(builderOpts, connectorbuilder.WithSessionStore(runTimeOpts.SessionStore)) + builderOpts = append(builderOpts, connectorbuilder.WithSourceCache(runTimeOpts.SourceCacheLookup)) c, err := connectorbuilder.NewConnector(ctx, connector, builderOpts...) if err != nil { @@ -160,6 +161,20 @@ func DefineConfigurationV2[T field.Configurable]( } confschema.Fields = fields + // The replay flags are hidden by default (source-cache replay is + // author-opt-in functionality); surface them in help only for + // connectors whose author baked the capability into their runner + // options. Hidden flags still parse, so this is a visibility decision, + // not a behavioral one. + if connectorrunner.DeclaresPreviousSyncCapability(ctx, options...) { + for i, f := range confschema.Fields { + switch f.FieldName { + case field.PreviousSyncC1ZField.FieldName, field.KeepPreviousSyncC1ZField.FieldName: + confschema.Fields[i].SyncerConfig.Hidden = false + } + } + } + // setup CLI with cobra mainCMD := &cobra.Command{ Use: connectorName, @@ -292,6 +307,12 @@ func verifyStructFields[T field.Configurable](schema field.Configuration) error return fmt.Errorf("T must be a struct type, got %v", configType.Kind()) //nolint:staticcheck // we want to capital letter here } for _, field := range schema.Fields { + if field.WasReExported { + // Re-exported shared SDK fields (field.WithConnectorDefault) + // are parsed by the SDK's own flag handling, not the + // connector's configuration struct. + continue + } fieldFound := false for i := 0; i < configType.NumField(); i++ { structField := configType.Field(i) diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/connectorbuilder.go b/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/connectorbuilder.go index bfc1a39e..68fb27d8 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/connectorbuilder.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/connectorbuilder.go @@ -22,6 +22,7 @@ import ( "github.com/conductorone/baton-sdk/pkg/metrics" "github.com/conductorone/baton-sdk/pkg/retry" "github.com/conductorone/baton-sdk/pkg/sdk" + "github.com/conductorone/baton-sdk/pkg/sourcecache" "github.com/conductorone/baton-sdk/pkg/types" "github.com/conductorone/baton-sdk/pkg/types/sessions" "github.com/conductorone/baton-sdk/pkg/types/tasks" @@ -76,6 +77,7 @@ type builder struct { nowFunc func() time.Time clientSecret *jose.JSONWebKey sessionStore sessions.SessionStore + sourceCache sourcecache.Lookup metadataProvider MetadataProvider validateProvider ValidateProvider ticketManager TicketManagerLimited @@ -241,6 +243,30 @@ func WithSessionStore(ss sessions.SessionStore) Opt { } } +// WithSourceCache supplies the connector-facing source-cache Lookup exposed +// on SyncOpAttrs.SourceCache (see pkg/sourcecache). A nil lookup is +// normalized to NoopLookup so connectors never nil-check. +func WithSourceCache(lookup sourcecache.Lookup) Opt { + return func(b *builder) error { + if lookup == nil { + lookup = sourcecache.NoopLookup{} + } + b.sourceCache = lookup + return nil + } +} + +// SetSourceCache implements sourcecache.SetLookup so an in-process runner +// (the syncer, via its connector client) can install/clear the active +// lookup per sync. Subprocess mode instead routes lookups over +// BatonSourceCacheService and never calls this. +func (b *builder) SetSourceCache(_ context.Context, lookup sourcecache.Lookup) { + if lookup == nil { + lookup = sourcecache.NoopLookup{} + } + b.sourceCache = lookup +} + func (b *builder) options(opts ...Opt) error { for _, opt := range opts { if err := opt(b); err != nil { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/resource_syncer.go b/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/resource_syncer.go index a7d99394..3193ed84 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/resource_syncer.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/resource_syncer.go @@ -2,11 +2,14 @@ package connectorbuilder import ( "context" + "errors" "fmt" v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" "github.com/conductorone/baton-sdk/pkg/annotations" "github.com/conductorone/baton-sdk/pkg/pagination" + "github.com/conductorone/baton-sdk/pkg/session" + "github.com/conductorone/baton-sdk/pkg/sourcecache" "github.com/conductorone/baton-sdk/pkg/types/resource" "github.com/conductorone/baton-sdk/pkg/types/tasks" "github.com/conductorone/baton-sdk/pkg/uotel" @@ -64,6 +67,111 @@ type StaticEntitlementSyncerV2 interface { StaticEntitlements(ctx context.Context, opts resource.SyncOpAttrs) ([]*v2.Entitlement, *resource.SyncOpResults, error) } +// TypeScopedGrantsSyncer is the grants-phase analogue of +// StaticEntitlementSyncerV2: the connector enumerates grants for a WHOLE +// resource type instead of being called once per resource. Implement it on +// a resource syncer whose ResourceType carries the v2.TypeScopedGrants +// annotation; the syncer then issues ListGrants calls with an empty +// resource id, which the builder routes here. +// +// The first call of a sync arrives with an empty page token (the planning +// call); the connector may answer with rows directly and/or spawn +// additional independent cursors by attaching a v2.SpawnCursors annotation +// whose page tokens are delivered back through opts.PageToken, one action +// each. Source-cache scope/replay/tombstone annotations work exactly as on +// per-resource grants pages. +type TypeScopedGrantsSyncer interface { + GrantsForResourceType(ctx context.Context, resourceTypeID string, opts resource.SyncOpAttrs) ([]*v2.Grant, *resource.SyncOpResults, error) +} + +// syncOpAttrs assembles the per-call SyncOpAttrs handed to V2 resource +// syncers. SourceCache is never nil: when the runner supplied no lookup +// (source cache disabled/degraded) connectors see NoopLookup and every +// lookup misses. Session is likewise never a nil-backed wrapper: without a +// configured store, connectors see the erroring no-op store instead of a +// panic on first use. +func (b *builder) syncOpAttrs(activeSyncID string, token pagination.Token) resource.SyncOpAttrs { + sc := b.sourceCache + if sc == nil { + sc = sourcecache.NoopLookup{} + } + ss := b.sessionStore + if ss == nil { + ss = &session.NoOpSessionStore{} + } + return resource.SyncOpAttrs{ + SyncID: activeSyncID, + PageToken: token, + Session: WithSyncId(ss, activeSyncID), + SourceCache: sc, + } +} + +// continuationOpAttrs builds SyncOpAttrs for a list RPC, selecting the +// source-cache lookup by topology: +// +// - a runner-supplied lookup (in-process interface, subprocess loopback +// gRPC) always wins: those topologies answer lookups directly and +// never bounce; +// - otherwise, when the request carries SourceCacheLookupOffer (and/or +// SourceCacheLookupAnswers from a previous bounce), a per-request +// ContinuationLookup is installed — it serves answered scopes and +// defers the rest via ErrLookupDeferred; +// - otherwise NoopLookup (today's behavior). +// +// The returned ContinuationLookup is nil when the continuation is not in +// play for this request. +func (b *builder) continuationOpAttrs(activeSyncID string, token pagination.Token, reqAnnos annotations.Annotations) (resource.SyncOpAttrs, *sourcecache.ContinuationLookup, error) { + attrs := b.syncOpAttrs(activeSyncID, token) + if b.sourceCache != nil { + return attrs, nil, nil + } + answersMsg := &v2.SourceCacheLookupAnswers{} + hasAnswers, err := reqAnnos.Pick(answersMsg) + if err != nil { + return attrs, nil, fmt.Errorf("error parsing source-cache lookup answers annotation: %w", err) + } + if !hasAnswers && !reqAnnos.Contains(&v2.SourceCacheLookupOffer{}) { + return attrs, nil, nil + } + var answers []sourcecache.Answer + if hasAnswers { + answers = sourcecache.AnswersFromProto(answersMsg) + } + cl := sourcecache.NewContinuationLookup(answers) + attrs.SourceCache = cl + return attrs, cl, nil +} + +// continuationOutcome inspects a list handler's result when a +// ContinuationLookup was installed. +// +// Returns (askAnnos, deferred, misuseErr): +// - deferred=true: the handler deferred on recorded scopes. Respond with +// askAnnos (the SourceCacheLookupAsk), NO rows, NO next page token, +// and no failure metrics — this is a protocol turn, not a failure. +// - misuseErr != nil: the handler swallowed a deferred lookup (scopes +// were recorded but no ErrLookupDeferred propagated). Loud failure: +// silently continuing past a deferral re-asks forever and dies at the +// bounce cap in production, so it must fail here, visibly. +// - all-zero: the handler resolved normally; use its result as-is. +func continuationOutcome(cl *sourcecache.ContinuationLookup, handlerErr error) (annotations.Annotations, bool, error) { + if cl == nil { + return nil, false, nil + } + asked := cl.Asked() + if errors.Is(handlerErr, sourcecache.ErrLookupDeferred) && len(asked) > 0 { + annos := annotations.Annotations{} + annos.Update(sourcecache.AskProto(asked)) + return annos, true, nil + } + if handlerErr == nil && len(asked) > 0 { + return nil, false, status.Errorf(codes.Internal, + "connector swallowed a deferred source-cache lookup (%d scope(s) asked, no error propagated): ErrLookupDeferred must be returned — wrap with %%w, never swallow", len(asked)) + } + return nil, false, nil +} + // ResourceTargetedSyncer extends ResourceSyncer to add capabilities for directly syncing an individual resource // // Implementing this interface indicates the connector supports calling "get" on a resource @@ -129,12 +237,26 @@ func (b *builder) ListResources(ctx context.Context, request *v2.ResourcesServic Size: int(request.GetPageSize()), Token: request.GetPageToken(), } - opts := resource.SyncOpAttrs{SyncID: request.GetActiveSyncId(), PageToken: token, Session: WithSyncId(b.sessionStore, request.GetActiveSyncId())} + opts, contLookup, err := b.continuationOpAttrs(request.GetActiveSyncId(), token, annotations.Annotations(request.GetAnnotations())) + if err != nil { + b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start), err) + return nil, err + } out, retOptions, err := rb.List(ctx, request.GetParentResourceId(), opts) if retOptions == nil { retOptions = &resource.SyncOpResults{} } + askAnnos, deferred, misuseErr := continuationOutcome(contLookup, err) + if misuseErr != nil { + b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start), misuseErr) + return nil, misuseErr + } + if deferred { + b.m.RecordTaskSuccess(ctx, tt, b.nowFunc().Sub(start)) + return v2.ResourcesServiceListResourcesResponse_builder{Annotations: askAnnos}.Build(), nil + } + resp := v2.ResourcesServiceListResourcesResponse_builder{ List: out, NextPageToken: retOptions.NextPageToken, @@ -217,7 +339,7 @@ func (b *builder) ListStaticEntitlements(ctx context.Context, request *v2.Entitl Size: int(request.GetPageSize()), Token: request.GetPageToken(), } - opts := resource.SyncOpAttrs{SyncID: request.GetActiveSyncId(), PageToken: token, Session: WithSyncId(b.sessionStore, request.GetActiveSyncId())} + opts := b.syncOpAttrs(request.GetActiveSyncId(), token) out, retOptions, err := rbse.StaticEntitlements(ctx, opts) if retOptions == nil { retOptions = &resource.SyncOpResults{} @@ -265,12 +387,26 @@ func (b *builder) ListEntitlements(ctx context.Context, request *v2.Entitlements Size: int(request.GetPageSize()), Token: request.GetPageToken(), } - opts := resource.SyncOpAttrs{SyncID: request.GetActiveSyncId(), PageToken: token, Session: WithSyncId(b.sessionStore, request.GetActiveSyncId())} + opts, contLookup, err := b.continuationOpAttrs(request.GetActiveSyncId(), token, annotations.Annotations(request.GetAnnotations())) + if err != nil { + b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start), err) + return nil, err + } out, retOptions, err := rb.Entitlements(ctx, request.GetResource(), opts) if retOptions == nil { retOptions = &resource.SyncOpResults{} } + askAnnos, deferred, misuseErr := continuationOutcome(contLookup, err) + if misuseErr != nil { + b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start), misuseErr) + return nil, misuseErr + } + if deferred { + b.m.RecordTaskSuccess(ctx, tt, b.nowFunc().Sub(start)) + return v2.EntitlementsServiceListEntitlementsResponse_builder{Annotations: askAnnos}.Build(), nil + } + resp := v2.EntitlementsServiceListEntitlementsResponse_builder{ List: out, NextPageToken: retOptions.NextPageToken, @@ -322,12 +458,47 @@ func (b *builder) ListGrants(ctx context.Context, request *v2.GrantsServiceListG Size: int(request.GetPageSize()), Token: request.GetPageToken(), } - opts := resource.SyncOpAttrs{SyncID: request.GetActiveSyncId(), PageToken: token, Session: WithSyncId(b.sessionStore, request.GetActiveSyncId())} - out, retOptions, err := rb.Grants(ctx, request.GetResource(), opts) + reqAnnos := annotations.Annotations(request.GetAnnotations()) + opts, contLookup, err := b.continuationOpAttrs(request.GetActiveSyncId(), token, reqAnnos) + if err != nil { + b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start), err) + return nil, err + } + + typeScoped := reqAnnos.Contains(&v2.TypeScopedGrants{}) + + var out []*v2.Grant + var retOptions *resource.SyncOpResults + if typeScoped { + // Type-scoped grants call (see v2.TypeScopedGrants): the request + // annotation is the routing marker (the resource is a + // self-referential {type, type} stub to satisfy wire validation). + // Only legal against a syncer that opted in. + tsgs, tsOk := rb.(TypeScopedGrantsSyncer) + if !tsOk { + err = status.Errorf(codes.InvalidArgument, + "error: type-scoped list grants for resource type %s, but its syncer does not implement TypeScopedGrantsSyncer", rid.GetResourceType()) + b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start), err) + return nil, err + } + out, retOptions, err = tsgs.GrantsForResourceType(ctx, rid.GetResourceType(), opts) + } else { + out, retOptions, err = rb.Grants(ctx, request.GetResource(), opts) + } if retOptions == nil { retOptions = &resource.SyncOpResults{} } + askAnnos, deferred, misuseErr := continuationOutcome(contLookup, err) + if misuseErr != nil { + b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start), misuseErr) + return nil, misuseErr + } + if deferred { + b.m.RecordTaskSuccess(ctx, tt, b.nowFunc().Sub(start)) + return v2.GrantsServiceListGrantsResponse_builder{Annotations: askAnnos}.Build(), nil + } + resp := v2.GrantsServiceListGrantsResponse_builder{ List: out, Annotations: retOptions.Annotations, diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/connectorclient/connectorclient.go b/vendor/github.com/conductorone/baton-sdk/pkg/connectorclient/connectorclient.go new file mode 100644 index 00000000..bd085ac0 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/connectorclient/connectorclient.go @@ -0,0 +1,15 @@ +package connectorclient + +import ( + "context" + + "github.com/conductorone/baton-sdk/internal/connector" + "github.com/conductorone/baton-sdk/pkg/types" + "google.golang.org/grpc" +) + +// NewConnectorClient takes a grpc.ClientConnInterface and returns an implementation of the ConnectorClient interface. +// Note: lambda functions directly instantiate the connector client, so this function is not used in this package. +func NewConnectorClient(ctx context.Context, cc grpc.ClientConnInterface) types.ConnectorClient { + return connector.NewConnectorClient(ctx, cc) +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/connectorrunner/runner.go b/vendor/github.com/conductorone/baton-sdk/pkg/connectorrunner/runner.go index c404401b..4db9cf18 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/connectorrunner/runner.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/connectorrunner/runner.go @@ -426,6 +426,7 @@ type runnerConfig struct { workerCount int targetedSyncResourceIDs []string externalResourceC1Z string + previousSyncC1Z string externalResourceEntitlementIdFilter string keepPreviousSyncC1ZCapable bool keepPreviousSyncC1ZEnabled bool @@ -760,6 +761,13 @@ func WithExternalResourceC1Z(externalResourceC1Z string) Option { } } +func WithPreviousSyncC1Z(previousSyncC1Z string) Option { + return func(ctx context.Context, cfg *runnerConfig) error { + cfg.previousSyncC1Z = previousSyncC1Z + return nil + } +} + func WithExternalResourceEntitlementFilter(entitlementId string) Option { return func(ctx context.Context, cfg *runnerConfig) error { cfg.externalResourceEntitlementIdFilter = entitlementId @@ -802,6 +810,25 @@ func WithKeepPreviousSyncC1ZRuntimeOptIn() Option { } } +// DeclaresPreviousSyncCapability reports whether opts include the connector +// author's replay declaration (WithKeepPreviousSyncC1Z). Used at +// command-definition time to decide whether the replay CLI flags +// (--previous-sync-c1z / --keep-previous-sync-c1z) appear in help: they are +// hidden for the overwhelming majority of connectors that don't support +// replay, and surfaced only for the ones whose author baked the capability +// into their RunConnector options. Options are applied to a scratch config; +// they are plain setters, so this is side-effect free. +func DeclaresPreviousSyncCapability(ctx context.Context, opts ...Option) bool { + cfg := &runnerConfig{} + for _, o := range opts { + if o == nil { + continue + } + _ = o(ctx, cfg) + } + return cfg.keepPreviousSyncC1ZCapable +} + func WithDiffSyncs(c1zPath string, baseSyncID string, newSyncID string) Option { return func(ctx context.Context, cfg *runnerConfig) error { cfg.onDemand = true @@ -973,7 +1000,15 @@ func NewConnectorRunner(ctx context.Context, c types.ConnectorServer, opts ...Op wrapperOpts = append(wrapperOpts, connector.WithTargetedSyncResources(cfg.targetedSyncResourceIDs)) } - if cfg.sessionStoreEnabled { + // The parent control-plane listener serves BOTH BatonSessionService and + // BatonSourceCacheService (internal/connector.runServer). Source-cache + // replay therefore needs the listener even when the connector never uses + // sessions: without it the subprocess connector's lookup degrades to + // NoopLookup, every scope misses, and every sync runs a full cold + // enumeration despite a valid --previous-sync-c1z. Start it whenever + // replay could be in play: the author declared the capability + // (WithKeepPreviousSyncC1Z) or a previous-sync c1z was configured. + if cfg.sessionStoreEnabled || cfg.keepPreviousSyncC1ZCapable || cfg.previousSyncC1Z != "" { wrapperOpts = append(wrapperOpts, connector.WithSessionStoreEnabled()) } @@ -1081,6 +1116,7 @@ func NewConnectorRunner(ctx context.Context, c types.ConnectorServer, opts ...Op tm, err = local.NewSyncer(ctx, cfg.c1zPath, local.WithTmpDir(cfg.tempDir), local.WithExternalResourceC1Z(cfg.externalResourceC1Z), + local.WithPreviousSyncC1Z(cfg.previousSyncC1Z), local.WithExternalResourceEntitlementIdFilter(cfg.externalResourceEntitlementIdFilter), local.WithTargetedSyncResources(resources), local.WithSkipEntitlementsAndGrants(cfg.skipEntitlementsAndGrants), diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter.go index c18ef1b5..b2c38511 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter.go @@ -21,6 +21,7 @@ import ( v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" "github.com/conductorone/baton-sdk/pkg/connectorstore" "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/sourcecache" ) // Adapter wraps an *Engine and implements connectorstore.Writer @@ -400,12 +401,27 @@ func (a *Adapter) PutGrants(ctx context.Context, grants ...*v2.Grant) error { return ErrNoCurrentSync } records := translateGrants(syncID, grants) + stampSourceScope(ctx, records, func(r *v3.GrantRecord, s string) { r.SetSourceScopeHash(s) }) if err := a.engine.PutGrantRecords(ctx, records...); err != nil { return fmt.Errorf("PutGrants: %w", err) } return nil } +// stampSourceScope stamps the source-cache scope hash carried by ctx +// (sourcecache.WithScope) onto freshly translated records. The syncer +// sets the scope around a page's store writes when the page carried a +// SourceCacheScope annotation; everything else writes unstamped rows. +func stampSourceScope[T any](ctx context.Context, records []T, set func(T, string)) { + scope := sourcecache.ScopeFromContext(ctx) + if scope == "" { + return + } + for _, r := range records { + set(r, scope) + } +} + // UnsafePutUniqueGrants writes grants on the trusted-import path: records // are encoded in parallel and written unconditionally, with no read-before-write // and no dedup pass. Do not use it for live connector output. The destination @@ -563,6 +579,7 @@ func (a *Adapter) PutResources(ctx context.Context, resources ...*v2.Resource) e } records = append(records, rec) } + stampSourceScope(ctx, records, func(r *v3.ResourceRecord, s string) { r.SetSourceScopeHash(s) }) if err := a.engine.PutResourceRecords(ctx, records...); err != nil { return fmt.Errorf("PutResources: %w", err) } @@ -590,6 +607,7 @@ func (a *Adapter) PutEntitlements(ctx context.Context, entitlements ...*v2.Entit } records = append(records, rec) } + stampSourceScope(ctx, records, func(r *v3.EntitlementRecord, s string) { r.SetSourceScopeHash(s) }) if err := a.engine.PutEntitlementRecords(ctx, records...); err != nil { return fmt.Errorf("PutEntitlements: %w", err) } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter_grants_store.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter_grants_store.go index 30736c59..fe19c1f2 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter_grants_store.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter_grants_store.go @@ -220,6 +220,10 @@ func (g pebbleGrantStore) translateExpanded(syncID string, grants []*v2.Grant) [ // because the caller left a residual GrantExpandable annotation. newRec.SetExpansion(nil) newRec.SetNeedsExpansion(false) + // Same shape for the source-cache scope stamp: existing records get + // their prior stamp restored in PutExpandedGrantRecords; brand-new + // expander-derived rows are never part of a source scope. + newRec.SetSourceScopeHash("") merged = append(merged, newRec) } return merged diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/bulk_import.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/bulk_import.go index 621779a3..fb27441f 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/bulk_import.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/bulk_import.go @@ -54,6 +54,7 @@ const bulkSpillBufferSize = 1 << 20 var grantIndexFamilies = []byte{ idxGrantByPrincipal, idxGrantByNeedsExpansion, + idxGrantBySourceScope, } // bulkSSTWriter builds one SST file for a single disjoint key bucket. diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/cleanup.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/cleanup.go index a132572d..f412b874 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/cleanup.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/cleanup.go @@ -35,6 +35,10 @@ func scopedRanges() [][2][]byte { {GrantByPrincipalLowerBound(), GrantByPrincipalUpperBound()}, {GrantByPrincipalResourceTypeLowerBound(), GrantByPrincipalResourceTypeUpperBound()}, {GrantByNeedsExpansionLowerBound(), GrantByNeedsExpansionUpperBound()}, + {GrantBySourceScopeLowerBound(), GrantBySourceScopeUpperBound()}, + {EntitlementBySourceScopeLowerBound(), EntitlementBySourceScopeUpperBound()}, + {ResourceBySourceScopeLowerBound(), ResourceBySourceScopeUpperBound()}, + {SourceCacheEntryLowerBound(), SourceCacheEntryUpperBound()}, {encodeAssetPrefix(), upperBoundOf(encodeAssetPrefix())}, // Stats sidecar — single key; the half-open range shape // contains exactly that one key. diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/entitlements.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/entitlements.go index 2182c3a2..7037b4db 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/entitlements.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/entitlements.go @@ -75,6 +75,18 @@ func (e *Engine) PutEntitlementRecords(ctx context.Context, records ...*v3.Entit if err := priBatch.Set(key, val, nil); err != nil { return err } + // by_source_scope is the only entitlement secondary index. + // No read-before-write cleanup here (matching this method's + // no-Get philosophy): a same-identity rewrite under a + // different scope within one sync can leave a stale index + // entry, which replay tolerates — the copied record carries + // its true scope and the file holds a single sync, so the + // staleness cannot outlive it. + if sh := r.GetSourceScopeHash(); sh != "" { + if err := priBatch.Set(encodeEntitlementBySourceScopeIndexKey(sh, id), nil, nil); err != nil { + return err + } + } } opts := writeOpts(e.opts.durability) if fresh { @@ -122,6 +134,22 @@ func (e *Engine) DeleteEntitlementRecord(ctx context.Context, externalID string) key := encodeEntitlementIdentityKey(id) batch := e.db.NewBatch() defer batch.Close() + // Clean up the source-scope index entry (the only entitlement + // secondary index) so a replayed scope can't resurrect the row. + if oldVal, closer, getErr := e.db.Get(key); getErr == nil { + oldScope, scanErr := scanEntitlementSourceScopeRaw(oldVal) + closer.Close() + if scanErr != nil { + return scanErr + } + if oldScope != "" { + if err := batch.Delete(encodeEntitlementBySourceScopeIndexKey(oldScope, id), nil); err != nil { + return err + } + } + } else if !errors.Is(getErr, pebble.ErrNotFound) { + return getErr + } if err := batch.Delete(key, nil); err != nil { return err } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grants.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grants.go index e004584f..a3758116 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grants.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grants.go @@ -263,6 +263,12 @@ func (e *Engine) PutExpandedGrantRecords(ctx context.Context, records []*v3.Gran r.SetExpansion(prior.GetExpansion()) r.SetNeedsExpansion(prior.GetNeedsExpansion()) r.SetDiscoveredAt(prior.GetDiscoveredAt()) + // Preserve the source-cache scope stamp exactly like the + // expansion side-state: the expander rewrites existing + // direct grants to bake in Sources, and clobbering the + // stamp here would silently drop every expander-touched + // grant from the next sync's replay of its scope. + r.SetSourceScopeHash(prior.GetSourceScopeHash()) var err error idxScratch, err = e.deleteGrantIndexesScratch(idxBatch, ext, oldVal, idxScratch) if err != nil { @@ -746,14 +752,17 @@ func (e *Engine) putSynthesizedGrantContributionsBatch(ctx context.Context, reco if err != nil { return err } - // sources (field 9) is the record's highest field, so appending - // it after the base marshal matches the deterministic byte order. + // sources (field 9) is the highest field this record carries, so + // appending it after the base marshal matches the deterministic + // byte order. (source_scope_hash is field 10, but synthesized + // grants never carry a scope stamp — fillSynthGrantRecord leaves + // it unset — so field 9 stays last on the wire.) val, srcScratch = appendGrantSourcesWire(val, srcScratch, rec.sources) valScratch = val if err := priBatch.Set(keyScratch, val, nil); err != nil { return err } - idxScratch, err = e.writeGrantIndexesForIdentityScratch(idxBatch, rec.id, false, idxScratch) + idxScratch, err = e.writeGrantIndexesForIdentityScratch(idxBatch, rec.id, false, "", idxScratch) if err != nil { return err } @@ -986,11 +995,14 @@ func grantIndexKeys(r *v3.GrantRecord) [][]byte { if err != nil { return nil } - keys := make([][]byte, 0, 2) + keys := make([][]byte, 0, 3) keys = append(keys, encodeGrantByPrincipalIdentityIndexKey(id)) if r.GetNeedsExpansion() { keys = append(keys, encodeGrantByNeedsExpansionIdentityIndexKey(id)) } + if sh := r.GetSourceScopeHash(); sh != "" { + keys = append(keys, encodeGrantBySourceScopeIndexKey(sh, id)) + } return keys } @@ -1015,7 +1027,7 @@ func (e *Engine) writeGrantIndexesScratch(batch *pebble.Batch, r *v3.GrantRecord if err != nil { return scratch, err } - return e.writeGrantIndexesForIdentityScratch(batch, id, r.GetNeedsExpansion(), scratch) + return e.writeGrantIndexesForIdentityScratch(batch, id, r.GetNeedsExpansion(), r.GetSourceScopeHash(), scratch) } // markDeferredIdxPending arms the deferred by_principal rebuild, durably. @@ -1058,7 +1070,10 @@ func (e *Engine) clearDeferredIdxPending() error { // for one grant. by_principal is never written inline: it is scattered // relative to the entitlement-first write order, so it is always rebuilt as // one sorted SST at EndSync (deferredIdxPending → BuildDeferredGrantIndexes). -func (e *Engine) writeGrantIndexesForIdentityScratch(batch *pebble.Batch, id grantIdentity, needsExpansion bool, scratch []byte) ([]byte, error) { +// by_needs_expansion and by_source_scope are written inline: both share the +// entitlement-first ordering of the primary keyspace, so their writes stay +// sorted. +func (e *Engine) writeGrantIndexesForIdentityScratch(batch *pebble.Batch, id grantIdentity, needsExpansion bool, sourceScopeHash string, scratch []byte) ([]byte, error) { if err := e.markDeferredIdxPending(); err != nil { return scratch, err } @@ -1068,6 +1083,12 @@ func (e *Engine) writeGrantIndexesForIdentityScratch(batch *pebble.Batch, id gra return scratch, err } } + if sourceScopeHash != "" { + scratch = appendGrantBySourceScopeIndexKey(scratch[:0], sourceScopeHash, id) + if err := batch.Set(scratch, nil, nil); err != nil { + return scratch, err + } + } return scratch, nil } @@ -1080,7 +1101,7 @@ func (e *Engine) writeGrantIndexesForIdentityScratch(batch *pebble.Batch, id gra // which also clears any stale entries an overwrite would have left. Returns // the (possibly grown) scratch buffer. func (e *Engine) deleteGrantIndexesScratch(batch *pebble.Batch, externalID string, value, scratch []byte) ([]byte, error) { - entRT, entRID, entID, principalRT, principalID, _, err := scanGrantIndexFieldsRaw(value) + entRT, entRID, entID, principalRT, principalID, _, sourceScopeHash, err := scanGrantIndexFieldsRaw(value) if err != nil { return scratch, err } @@ -1096,6 +1117,12 @@ func (e *Engine) deleteGrantIndexesScratch(batch *pebble.Batch, externalID strin if err := batch.Delete(scratch, nil); err != nil { return scratch, err } + if sourceScopeHash != "" { + scratch = appendGrantBySourceScopeIndexKey(scratch[:0], sourceScopeHash, id) + if err := batch.Delete(scratch, nil); err != nil { + return scratch, err + } + } return scratch, nil } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grants_synth_encode.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grants_synth_encode.go index 8fa0a780..e63e72a3 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grants_synth_encode.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grants_synth_encode.go @@ -30,11 +30,19 @@ var expandedGrantImmutableAnnotations = []*anypb.Any{expandedGrantImmutableAnnot // appendSynthGrantExternalIDWire) and sources (field 9, appended by // appendGrantSourcesWire). Expansion/NeedsExpansion stay zero for // synthesized grants. +// +// SourceScopeHash is cleared unconditionally, for two reasons: synthesized +// (expander-derived) grants are never part of a source-cache scope, and — +// load-bearing for appendGrantSourcesWire — an empty proto3 scalar emits no +// bytes, which keeps sources (field 9) the highest field on the wire so the +// hand encoder's append-after-base-marshal stays canonical. See +// TestGrantSourcesWireSchemaPin. func fillSynthGrantRecord(r *v3.GrantRecord, rec *synthesizedGrantRecord, now *timestamppb.Timestamp) { r.SetEntitlement(rec.entitlement) r.SetPrincipal(rec.principal) r.SetAnnotations(expandedGrantImmutableAnnotations) r.SetDiscoveredAt(now) + r.SetSourceScopeHash("") } // grantExternalIDFieldTag is the wire tag for GrantRecord.external_id (2). diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/id_index_migration.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/id_index_migration.go index 019aa715..c592f89f 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/id_index_migration.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/id_index_migration.go @@ -213,7 +213,7 @@ func (e *Engine) emitStructuredGrantMigration(ctx context.Context, primary *spil lastLog = now } } - entRT, entRID, entID, principalRT, principalID, _, err := scanGrantIndexFieldsRaw(iter.Value()) + entRT, entRID, entID, principalRT, principalID, _, _, err := scanGrantIndexFieldsRaw(iter.Value()) if err != nil { return rows, fmt.Errorf("id-index migration: scan grant: %w", err) } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/if_newer.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/if_newer.go index 347091a0..e166f8dc 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/if_newer.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/if_newer.go @@ -182,7 +182,16 @@ func (e *Engine) PutEntitlementRecordsIfNewer(ctx context.Context, records ...*v closer.Close() continue } + oldScope, scanErr := scanEntitlementSourceScopeRaw(oldVal) closer.Close() + if scanErr != nil { + return scanErr + } + if oldScope != "" && oldScope != r.GetSourceScopeHash() { + if err := batch.Delete(encodeEntitlementBySourceScopeIndexKey(oldScope, id), nil); err != nil { + return err + } + } case errors.Is(getErr, pebble.ErrNotFound): default: return fmt.Errorf("PutEntitlementRecordsIfNewer: get: %w", getErr) @@ -194,6 +203,11 @@ func (e *Engine) PutEntitlementRecordsIfNewer(ctx context.Context, records ...*v if err := batch.Set(key, val, nil); err != nil { return err } + if sh := r.GetSourceScopeHash(); sh != "" { + if err := batch.Set(encodeEntitlementBySourceScopeIndexKey(sh, id), nil, nil); err != nil { + return err + } + } written++ } if written == 0 { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/keys.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/keys.go index f1484d08..a9ca13c6 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/keys.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/keys.go @@ -63,7 +63,16 @@ const ( typeIndex byte = 0x07 typeCounter byte = 0x08 typeSession byte = 0x09 - typeEngineMeta byte = 0xFF + // typeSourceCache holds one SourceCacheEntryRecord per + // (row_kind, scope_hash) — the source-cache replay manifest (see + // proto/c1/connector/v2/annotation_source_cache.proto). Placed + // after typeSession so the clone path's counter/session excise + // span ([typeCounter, upperBoundOf(typeSession))) leaves it in the + // clone: a cloned sync artifact stays usable as a replay source. + // ResetForNewSync's [typeResourceType, typeEngineMeta) span wipes + // it with the rest of the sync-scoped data. + typeSourceCache byte = 0x0A + typeEngineMeta byte = 0xFF ) // Index-discriminator bytes (second byte after typeIndex). One byte @@ -78,6 +87,13 @@ const ( idxGrantByNeedsExpansion byte = 0x05 idxGrantByPrincipalResourceType byte = 0x06 // retired: served by idxGrantByPrincipal prefix scans. idxGrantByEntitlementResource byte = 0x07 // retired: served by grant primary entitlement-resource prefix scans. + // by_source_scope families: partial indexes over records whose + // source_scope_hash is non-empty (source-cache replay). Tails are + // identity tuples per the keys.go convention, so replay derives + // each primary key from the index key without touching the record. + idxGrantBySourceScope byte = 0x08 + idxEntitlementBySourceScope byte = 0x09 + idxResourceBySourceScope byte = 0x0A ) // --- Grant --- @@ -270,6 +286,120 @@ func encodeGrantByPrincipalResourceTypeIdentityPrefix(principalRT string) []byte return codec.AppendTupleSeparator(buf) } +// --- Source-cache (by_source_scope indexes + entry keyspace) --- + +// encodeGrantBySourceScopeIndexKey is the partial by_source_scope index +// over grants whose SourceScopeHash is non-empty: +// +// v3 | typeIndex | idxGrantBySourceScope | 0x00 | +// scope_hash | 0x00 | +// ent_rt | 0x00 | ent_rid | 0x00 | ent_flag | 0x00 | ent_tail | 0x00 | +// principal_rt | 0x00 | principal_id +// +// The tail after scope_hash is the grant identity tuple — byte-identical +// to the grant primary key's tail — so replay derives the primary key from +// the index key alone. Paired with encodeGrantBySourceScopePrefix +// (by-value prefix, with trailing sep). +func encodeGrantBySourceScopeIndexKey(scopeHash string, id grantIdentity) []byte { + return appendGrantBySourceScopeIndexKey(make([]byte, 0, 128), scopeHash, id) +} + +func appendGrantBySourceScopeIndexKey(dst []byte, scopeHash string, id grantIdentity) []byte { + dst = append(dst, versionV3, typeIndex, idxGrantBySourceScope) + dst = codec.AppendTupleSeparator(dst) + return codec.AppendTupleStrings( + dst, + scopeHash, + id.entitlement.resourceTypeID, + id.entitlement.resourceID, + id.entitlement.flagComponent(), + id.entitlement.tail, + id.principalTypeID, + id.principalID, + ) +} + +// encodeGrantBySourceScopePrefix is the by-value prefix for "all grants +// stamped with this scope hash". Trailing sep is load-bearing — see the +// keys.go convention doc. +func encodeGrantBySourceScopePrefix(scopeHash string) []byte { + buf := make([]byte, 0, 32+len(scopeHash)) + buf = append(buf, versionV3, typeIndex, idxGrantBySourceScope) + buf = codec.AppendTupleSeparator(buf) + buf = codec.AppendTupleStrings(buf, scopeHash) + return codec.AppendTupleSeparator(buf) +} + +// encodeEntitlementBySourceScopeIndexKey: +// +// v3 | typeIndex | idxEntitlementBySourceScope | 0x00 | +// scope_hash | 0x00 | rt | 0x00 | rid | 0x00 | flag | 0x00 | tail +// +// Tail is the entitlement identity tuple, byte-identical to the +// entitlement primary key's tail. Paired with +// encodeEntitlementBySourceScopePrefix. +func encodeEntitlementBySourceScopeIndexKey(scopeHash string, id entitlementIdentity) []byte { + return appendEntitlementBySourceScopeIndexKey(make([]byte, 0, 128), scopeHash, id) +} + +func appendEntitlementBySourceScopeIndexKey(dst []byte, scopeHash string, id entitlementIdentity) []byte { + dst = append(dst, versionV3, typeIndex, idxEntitlementBySourceScope) + dst = codec.AppendTupleSeparator(dst) + return codec.AppendTupleStrings(dst, scopeHash, id.resourceTypeID, id.resourceID, id.flagComponent(), id.tail) +} + +func encodeEntitlementBySourceScopePrefix(scopeHash string) []byte { + buf := make([]byte, 0, 32+len(scopeHash)) + buf = append(buf, versionV3, typeIndex, idxEntitlementBySourceScope) + buf = codec.AppendTupleSeparator(buf) + buf = codec.AppendTupleStrings(buf, scopeHash) + return codec.AppendTupleSeparator(buf) +} + +// encodeResourceBySourceScopeIndexKey: +// +// v3 | typeIndex | idxResourceBySourceScope | 0x00 | +// scope_hash | 0x00 | resource_type_id | 0x00 | resource_id +// +// Tail is the resource primary tuple. Paired with +// encodeResourceBySourceScopePrefix. +func encodeResourceBySourceScopeIndexKey(scopeHash, resourceTypeID, resourceID string) []byte { + return appendResourceBySourceScopeIndexKey(make([]byte, 0, 96), scopeHash, resourceTypeID, resourceID) +} + +func appendResourceBySourceScopeIndexKey(dst []byte, scopeHash, resourceTypeID, resourceID string) []byte { + dst = append(dst, versionV3, typeIndex, idxResourceBySourceScope) + dst = codec.AppendTupleSeparator(dst) + return codec.AppendTupleStrings(dst, scopeHash, resourceTypeID, resourceID) +} + +func encodeResourceBySourceScopePrefix(scopeHash string) []byte { + buf := make([]byte, 0, 32+len(scopeHash)) + buf = append(buf, versionV3, typeIndex, idxResourceBySourceScope) + buf = codec.AppendTupleSeparator(buf) + buf = codec.AppendTupleStrings(buf, scopeHash) + return codec.AppendTupleSeparator(buf) +} + +// encodeSourceCacheEntryKey is the primary key for one source-cache +// manifest entry: +// +// v3 | typeSourceCache | 0x00 | row_kind | 0x00 | scope_hash +// +// Paired with encodeSourceCachePrefix (by-type prefix). +func encodeSourceCacheEntryKey(rowKind, scopeHash string) []byte { + buf := make([]byte, 0, 32+len(rowKind)+len(scopeHash)) + buf = append(buf, versionV3, typeSourceCache) + buf = codec.AppendTupleSeparator(buf) + return codec.AppendTupleStrings(buf, rowKind, scopeHash) +} + +// encodeSourceCachePrefix is the by-type prefix for all source-cache +// entries. +func encodeSourceCachePrefix() []byte { + return []byte{versionV3, typeSourceCache} +} + // --- ResourceType --- // encodeResourceTypeKey returns the primary key for a resource_type: @@ -498,6 +628,28 @@ func GrantByEntitlementResourceUpperBound() []byte { return upperBoundOf(GrantByEntitlementResourceLowerBound()) } +func GrantBySourceScopeLowerBound() []byte { + return []byte{versionV3, typeIndex, idxGrantBySourceScope} +} +func GrantBySourceScopeUpperBound() []byte { return upperBoundOf(GrantBySourceScopeLowerBound()) } + +func EntitlementBySourceScopeLowerBound() []byte { + return []byte{versionV3, typeIndex, idxEntitlementBySourceScope} +} +func EntitlementBySourceScopeUpperBound() []byte { + return upperBoundOf(EntitlementBySourceScopeLowerBound()) +} + +func ResourceBySourceScopeLowerBound() []byte { + return []byte{versionV3, typeIndex, idxResourceBySourceScope} +} +func ResourceBySourceScopeUpperBound() []byte { + return upperBoundOf(ResourceBySourceScopeLowerBound()) +} + +func SourceCacheEntryLowerBound() []byte { return encodeSourceCachePrefix() } +func SourceCacheEntryUpperBound() []byte { return upperBoundOf(encodeSourceCachePrefix()) } + func AssetLowerBound() []byte { return encodeAssetPrefix() } func AssetUpperBound() []byte { return upperBoundOf(encodeAssetPrefix()) } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/lookup.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/lookup.go index 2b70e59f..282c3a9c 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/lookup.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/lookup.go @@ -265,6 +265,28 @@ func (e *Engine) grantPrimaryPrefixNonEmpty(prefix []byte) (bool, error) { // the row instead of the concat). Exactly one hit wins; zero is // pebble.ErrNotFound; several is ErrAmbiguousExternalID. func (e *Engine) resolveGrantIdentityByExternalID(ctx context.Context, grantID string) (grantIdentity, error) { + id, err := e.resolveGrantIdentityByCandidates(ctx, grantID) + if err == nil || !errors.Is(err, errNoGrantCandidateHits) { + return id, err + } + // Candidate probing proved nothing either way: the id may still be a + // connector-custom STORED external id (with or without colons), which + // only the O(all grants) scan can find. Interactive edges pay it; + // bounded callers (DeleteGrantRecordBounded) stop before this line. + return e.scanGrantIdentityByStoredExternalID(ctx, grantID) +} + +// errNoGrantCandidateHits reports that combinatorial candidate probing +// completed without a hit — distinct from pebble.ErrNotFound, which the +// full resolution reserves for "provably absent after the scan of last +// resort". Internal to the resolution layer. +var errNoGrantCandidateHits = errors.New("pebble: no grant candidate parse hit") + +// resolveGrantIdentityByCandidates is the bounded stage of grant-id +// resolution: combinatorial concat splits probed with point Gets, no +// keyspace scan. Returns errNoGrantCandidateHits when the shape yields no +// candidates (fewer than two colons) or every candidate missed. +func (e *Engine) resolveGrantIdentityByCandidates(ctx context.Context, grantID string) (grantIdentity, error) { var colons []int for i := 0; i < len(grantID); i++ { if grantID[i] == ':' { @@ -272,10 +294,8 @@ func (e *Engine) resolveGrantIdentityByExternalID(ctx context.Context, grantID s } } if len(colons) < 2 { - // No concat shape to split: connector-custom ids (SQLite keyed rows - // by these, and provisioner revokes address grants with them) are - // findable only by their STORED external id. - return e.scanGrantIdentityByStoredExternalID(ctx, grantID) + // No concat shape to split: findable only by STORED external id. + return grantIdentity{}, errNoGrantCandidateHits } if len(colons) > maxBareIDColons { return grantIdentity{}, fmt.Errorf("%w: grant id has %d colons; too complex to resolve safely by string", ErrAmbiguousExternalID, len(colons)) @@ -378,9 +398,7 @@ func (e *Engine) resolveGrantIdentityByExternalID(ctx context.Context, grantID s } switch len(hits) { case 0: - // Every concat split missed: the id may still be a connector-custom - // STORED external id that merely contains colons. - return e.scanGrantIdentityByStoredExternalID(ctx, grantID) + return grantIdentity{}, errNoGrantCandidateHits case 1: return hits[0], nil default: diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/raw_records.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/raw_records.go index 89ddfd37..40640954 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/raw_records.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/raw_records.go @@ -110,18 +110,25 @@ func rawTimestampNanos(value []byte) (int64, error) { } func (e *Engine) deleteResourceIndexesRaw(batch *pebble.Batch, resourceTypeID string, resourceID string, value []byte) error { - parentRT, parentID, err := scanResourceParentRaw(value) + parentRT, parentID, sourceScopeHash, err := scanResourceIndexFieldsRaw(value) if err != nil { return err } - if parentID == "" { - return nil + if parentID != "" { + if err := batch.Delete(encodeResourceByParentIndexKey(parentRT, parentID, resourceTypeID, resourceID), nil); err != nil { + return err + } + } + if sourceScopeHash != "" { + if err := batch.Delete(encodeResourceBySourceScopeIndexKey(sourceScopeHash, resourceTypeID, resourceID), nil); err != nil { + return err + } } - return batch.Delete(encodeResourceByParentIndexKey(parentRT, parentID, resourceTypeID, resourceID), nil) + return nil } func (e *Engine) deleteGrantIndexesRaw(batch *pebble.Batch, externalID string, value []byte) error { - entRT, entRID, entID, principalRT, principalID, _, err := scanGrantIndexFieldsRaw(value) + entRT, entRID, entID, principalRT, principalID, _, sourceScopeHash, err := scanGrantIndexFieldsRaw(value) if err != nil { return err } @@ -136,6 +143,11 @@ func (e *Engine) deleteGrantIndexesRaw(batch *pebble.Batch, externalID string, v if err := batch.Delete(encodeGrantByPrincipalIdentityIndexKey(id), nil); err != nil { return err } + if sourceScopeHash != "" { + if err := batch.Delete(encodeGrantBySourceScopeIndexKey(sourceScopeHash, id), nil); err != nil { + return err + } + } return batch.Delete(encodeGrantByNeedsExpansionIdentityIndexKey(id), nil) } @@ -257,37 +269,83 @@ func scanEntitlementResourceTypeRaw(value []byte) ([]byte, error) { // occurrence of the target field, matching scanGrantIndexFieldsRaw and // approximating proto merge semantics. Values written by this SDK carry // at most one occurrence, so this only matters for foreign writers. -func scanResourceParentRaw(value []byte) (string, string, error) { - var rt, id string +// scanResourceIndexFieldsRaw extracts the parent ref (field 6) and +// source_scope_hash (field 9) from a marshaled ResourceRecord — the +// fields that key resource secondary indexes. +func scanResourceIndexFieldsRaw(value []byte) (string, string, string, error) { + var rt, id, sourceScopeHash string for len(value) > 0 { num, typ, n := protowire.ConsumeTag(value) if n < 0 { - return "", "", protowire.ParseError(n) + return "", "", "", protowire.ParseError(n) } value = value[n:] - if num != 6 { + switch num { + case 6: + if typ != protowire.BytesType { + return "", "", "", fmt.Errorf("raw record: resource parent has wire type %v", typ) + } + msg, n := protowire.ConsumeBytes(value) + if n < 0 { + return "", "", "", protowire.ParseError(n) + } + var err error + rt, id, err = scanResourceRefRaw(msg) + if err != nil { + return "", "", "", err + } + value = value[n:] + case 9: + if typ != protowire.BytesType { + return "", "", "", fmt.Errorf("raw record: resource source_scope_hash has wire type %v", typ) + } + s, n := protowire.ConsumeBytes(value) + if n < 0 { + return "", "", "", protowire.ParseError(n) + } + sourceScopeHash = string(s) + value = value[n:] + default: n = protowire.ConsumeFieldValue(num, typ, value) if n < 0 { - return "", "", protowire.ParseError(n) + return "", "", "", protowire.ParseError(n) + } + value = value[n:] + } + } + return rt, id, sourceScopeHash, nil +} + +// scanEntitlementSourceScopeRaw extracts source_scope_hash (field 11) +// from a marshaled EntitlementRecord. Keys the entitlement +// by_source_scope index — the only entitlement secondary index. +func scanEntitlementSourceScopeRaw(value []byte) (string, error) { + var sourceScopeHash string + for len(value) > 0 { + num, typ, n := protowire.ConsumeTag(value) + if n < 0 { + return "", protowire.ParseError(n) + } + value = value[n:] + if num != 11 { + n = protowire.ConsumeFieldValue(num, typ, value) + if n < 0 { + return "", protowire.ParseError(n) } value = value[n:] continue } if typ != protowire.BytesType { - return "", "", fmt.Errorf("raw record: resource parent has wire type %v", typ) + return "", fmt.Errorf("raw record: entitlement source_scope_hash has wire type %v", typ) } - msg, n := protowire.ConsumeBytes(value) + s, n := protowire.ConsumeBytes(value) if n < 0 { - return "", "", protowire.ParseError(n) - } - var err error - rt, id, err = scanResourceRefRaw(msg) - if err != nil { - return "", "", err + return "", protowire.ParseError(n) } + sourceScopeHash = string(s) value = value[n:] } - return rt, id, nil + return sourceScopeHash, nil } func scanEntitlementResourceRaw(value []byte) (string, string, error) { @@ -367,63 +425,73 @@ func scanEntitlementIdentityFieldsRaw(value []byte) (string, string, string, err return rt, id, externalID, nil } -func scanGrantIndexFieldsRaw(value []byte) (string, string, string, string, string, bool, error) { - var entRT, entRID, entID, principalRT, principalID string +func scanGrantIndexFieldsRaw(value []byte) (string, string, string, string, string, bool, string, error) { + var entRT, entRID, entID, principalRT, principalID, sourceScopeHash string var needsExpansion bool for len(value) > 0 { num, typ, n := protowire.ConsumeTag(value) if n < 0 { - return "", "", "", "", "", false, protowire.ParseError(n) + return "", "", "", "", "", false, "", protowire.ParseError(n) } value = value[n:] switch num { case 3: if typ != protowire.BytesType { - return "", "", "", "", "", false, fmt.Errorf("raw record: grant entitlement has wire type %v", typ) + return "", "", "", "", "", false, "", fmt.Errorf("raw record: grant entitlement has wire type %v", typ) } msg, n := protowire.ConsumeBytes(value) if n < 0 { - return "", "", "", "", "", false, protowire.ParseError(n) + return "", "", "", "", "", false, "", protowire.ParseError(n) } var err error entRT, entRID, entID, err = scanEntitlementRefRaw(msg) if err != nil { - return "", "", "", "", "", false, err + return "", "", "", "", "", false, "", err } value = value[n:] case 4: if typ != protowire.BytesType { - return "", "", "", "", "", false, fmt.Errorf("raw record: grant principal has wire type %v", typ) + return "", "", "", "", "", false, "", fmt.Errorf("raw record: grant principal has wire type %v", typ) } msg, n := protowire.ConsumeBytes(value) if n < 0 { - return "", "", "", "", "", false, protowire.ParseError(n) + return "", "", "", "", "", false, "", protowire.ParseError(n) } var err error principalRT, principalID, err = scanPrincipalRefRaw(msg) if err != nil { - return "", "", "", "", "", false, err + return "", "", "", "", "", false, "", err } value = value[n:] case 7: if typ != protowire.VarintType { - return "", "", "", "", "", false, fmt.Errorf("raw record: grant needs_expansion has wire type %v", typ) + return "", "", "", "", "", false, "", fmt.Errorf("raw record: grant needs_expansion has wire type %v", typ) } v, n := protowire.ConsumeVarint(value) if n < 0 { - return "", "", "", "", "", false, protowire.ParseError(n) + return "", "", "", "", "", false, "", protowire.ParseError(n) } needsExpansion = v != 0 value = value[n:] + case 10: + if typ != protowire.BytesType { + return "", "", "", "", "", false, "", fmt.Errorf("raw record: grant source_scope_hash has wire type %v", typ) + } + s, n := protowire.ConsumeBytes(value) + if n < 0 { + return "", "", "", "", "", false, "", protowire.ParseError(n) + } + sourceScopeHash = string(s) + value = value[n:] default: n = protowire.ConsumeFieldValue(num, typ, value) if n < 0 { - return "", "", "", "", "", false, protowire.ParseError(n) + return "", "", "", "", "", false, "", protowire.ParseError(n) } value = value[n:] } } - return entRT, entRID, entID, principalRT, principalID, needsExpansion, nil + return entRT, entRID, entID, principalRT, principalID, needsExpansion, sourceScopeHash, nil } // scanGrantNeedsExpansionRaw extracts only the needs_expansion flag diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/resources.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/resources.go index 564a0430..0f0d820c 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/resources.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/resources.go @@ -151,15 +151,22 @@ func (e *Engine) DeleteResourceRecord(ctx context.Context, resourceTypeID, resou } func (e *Engine) writeResourceIndexes(batch *pebble.Batch, r *v3.ResourceRecord) error { - parent := r.GetParent() - if parent == nil || parent.GetResourceId() == "" { - return nil + if parent := r.GetParent(); parent != nil && parent.GetResourceId() != "" { + k := encodeResourceByParentIndexKey( + parent.GetResourceTypeId(), parent.GetResourceId(), + r.GetResourceTypeId(), r.GetResourceId(), + ) + if err := batch.Set(k, nil, nil); err != nil { + return err + } + } + if sh := r.GetSourceScopeHash(); sh != "" { + k := encodeResourceBySourceScopeIndexKey(sh, r.GetResourceTypeId(), r.GetResourceId()) + if err := batch.Set(k, nil, nil); err != nil { + return err + } } - k := encodeResourceByParentIndexKey( - parent.GetResourceTypeId(), parent.GetResourceId(), - r.GetResourceTypeId(), r.GetResourceId(), - ) - return batch.Set(k, nil, nil) + return nil } func (e *Engine) IterateResources(ctx context.Context, yield func(*v3.ResourceRecord) bool) error { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/source_cache.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/source_cache.go new file mode 100644 index 00000000..dfb238c4 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/source_cache.go @@ -0,0 +1,835 @@ +package pebble + +import ( + "context" + "errors" + "fmt" + + "github.com/cockroachdb/pebble/v2" + "google.golang.org/protobuf/encoding/protowire" + "google.golang.org/protobuf/types/known/timestamppb" + + v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" + "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/codec" +) + +// Source-cache replay, engine side. +// +// The typeSourceCache keyspace holds one SourceCacheEntryRecord per +// (row_kind, scope_hash): the opaque upstream validator (etag / delta +// token) the sync recorded for that scope. Rows produced under a scope +// are stamped with source_scope_hash and indexed under the +// by_source_scope families, whose tails are identity tuples — so a +// replay derives every primary key from the index key alone and copies +// raw values across files without a proto unmarshal. +// +// The previous sync lives in a separate read-only engine (a Pebble c1z +// holds exactly one sync); replay copies from prev into the receiver. + +// replayBatchRows bounds how many rows accumulate in one pebble.Batch +// before an intermediate commit. Replay of a delta-query collection can +// be the whole previous row set, so the batch must not grow unbounded. +const replayBatchRows = 10_000 + +// SourceCacheReplayResult reports what one scope's replay copied. +type SourceCacheReplayResult struct { + Rows int64 + // NeedsExpansion is true when at least one copied grant row carried + // needs_expansion. The syncer must arm grant expansion in this case: + // replayed pages never pass GrantExpandable-annotated rows through + // the syncer's connector-response path, which is otherwise the only + // thing that enables the expansion phase. + NeedsExpansion bool +} + +// PutSourceCacheEntry writes the manifest entry for (rowKind, scopeHash). +// Zero-row scopes still get entries — the validator must survive to the +// next sync even when the scope produced no rows. +func (e *Engine) PutSourceCacheEntry(ctx context.Context, rowKind, scopeHash, etag string) error { + return e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + rec := &v3.SourceCacheEntryRecord{} + rec.SetRowKind(rowKind) + rec.SetScopeHash(scopeHash) + rec.SetEtag(etag) + rec.SetDiscoveredAt(timestamppb.Now()) + val, err := marshalRecord(rec) + if err != nil { + return err + } + opts := writeOpts(e.opts.durability) + if e.IsFreshSync() { + opts = pebble.NoSync + } + return e.db.Set(encodeSourceCacheEntryKey(rowKind, scopeHash), val, opts) + }) +} + +// GetSourceCacheEntry returns the manifest entry for (rowKind, scopeHash), +// or pebble.ErrNotFound. +func (e *Engine) GetSourceCacheEntry(ctx context.Context, rowKind, scopeHash string) (*v3.SourceCacheEntryRecord, error) { + val, closer, err := e.db.Get(encodeSourceCacheEntryKey(rowKind, scopeHash)) + if err != nil { + return nil, err + } + defer closer.Close() + rec := &v3.SourceCacheEntryRecord{} + if err := unmarshalRecord(val, rec); err != nil { + return nil, fmt.Errorf("GetSourceCacheEntry: unmarshal: %w", err) + } + return rec, nil +} + +// DeleteGrantRecordBounded deletes a grant by canonical public id WITHOUT +// the O(all grants) stored-external-id scan fallback that the interactive +// DeleteGrantRecord path is allowed to take. Used by the source-cache +// tombstone path, where a mass-removal round would otherwise pay a full +// keyspace scan PER already-absent id. +// +// Consequence, by design: a grant stored under a connector-CUSTOM id (one +// that isn't the SDK concat shape) is unreachable here and the delete +// no-ops. Connectors with custom grant ids must use principal-scoped +// tombstones (SourceCacheScope.deleted_principal_ids) instead — documented +// in the annotation proto. +func (e *Engine) DeleteGrantRecordBounded(ctx context.Context, externalID string) error { + return e.withWrite(func() error { + id, err := e.resolveGrantIdentityByCandidates(ctx, externalID) + if err != nil { + if errors.Is(err, errNoGrantCandidateHits) || errors.Is(err, pebble.ErrNotFound) { + return nil // absent (or custom-id) — tombstone no-op + } + return err + } + return e.deleteGrantByIdentityLocked(id) + }) +} + +// DeleteGrantsByPrincipalsInScope deletes every grant row in the CURRENT +// store stamped with scopeHash whose principal id is in principalIDs — +// the engine side of principal-scoped delta tombstones +// (SourceCacheScope.deleted_principal_ids). +// +// One prefix scan of the scope's by_source_scope index resolves +// everything: the index tail IS the grant identity, so the primary key +// and every secondary index key for a match are constructible from the +// index key alone — no value reads, no string resolution, no guessing. +// A principal with no rows in the scope is a no-op (providers tombstone +// objects the client never synced). Deleting a missing secondary index +// entry is a pebble no-op, which covers the mixed inline/deferred +// by_principal state mid-sync. +// +// Complexity: O(scope size) tuple-walks per call regardless of tombstone +// count — callers batch a page's tombstones into one call. +func (e *Engine) DeleteGrantsByPrincipalsInScope(ctx context.Context, scopeHash string, principalIDs map[string]struct{}) (int64, error) { + if len(principalIDs) == 0 { + return 0, nil + } + prefix := encodeGrantBySourceScopePrefix(scopeHash) + var deleted int64 + + err := e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return err + } + defer iter.Close() + + opts := writeOpts(e.opts.durability) + if e.IsFreshSync() { + opts = pebble.NoSync + } + batch := e.db.NewBatch() + defer func() { _ = batch.Close() }() + + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return err + } + key := iter.Key() + tail := key[len(prefix):] + // Tail layout: ent_rt | ent_rid | flag | ent_tail | prin_rt | prin_id + // (identical to the grant primary key tail; decoder shared with + // the primary-prefix scan paths in grants.go). + id, ok := decodeGrantIdentityTail(key, prefix) + if !ok { + continue // malformed index key — defensive skip + } + if _, hit := principalIDs[id.principalID]; !hit { + continue + } + // Primary key = grant header + the identity tail verbatim. + priKey := make([]byte, 0, 3+len(tail)) + priKey = append(priKey, versionV3, typeGrant) + priKey = codec.AppendTupleSeparator(priKey) + priKey = append(priKey, tail...) + if err := batch.Delete(priKey, nil); err != nil { + return err + } + if err := batch.Delete(encodeGrantByPrincipalIdentityIndexKey(id), nil); err != nil { + return err + } + if err := batch.Delete(encodeGrantByNeedsExpansionIdentityIndexKey(id), nil); err != nil { + return err + } + // The scope index entry itself (the key under the iterator — + // safe: the iterator reads a snapshot). + if err := batch.Delete(key, nil); err != nil { + return err + } + deleted++ + } + if err := iter.Error(); err != nil { + return err + } + return batch.Commit(opts) + }) + if err != nil { + return 0, err + } + return deleted, nil +} + +// DeleteGrantsByExternalIDsInScope deletes every grant row in the CURRENT +// store stamped with scopeHash whose STORED grant id (external id, which +// may be a connector-custom shape) is in ids. One scan of the scope's +// index, loading each candidate's primary row to compare the stored id — +// bounded by the scope's row count, never the whole keyspace. This is the +// tombstone path for connectors with custom grant ids whose scopes span +// multiple resources (so principal-scoped deletes would over-delete). +func (e *Engine) DeleteGrantsByExternalIDsInScope(ctx context.Context, scopeHash string, ids map[string]struct{}) (int64, error) { + if len(ids) == 0 { + return 0, nil + } + prefix := encodeGrantBySourceScopePrefix(scopeHash) + var deleted int64 + + err := e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return err + } + defer iter.Close() + + opts := writeOpts(e.opts.durability) + if e.IsFreshSync() { + opts = pebble.NoSync + } + batch := e.db.NewBatch() + defer func() { _ = batch.Close() }() + + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return err + } + key := iter.Key() + tail := key[len(prefix):] + id, ok := decodeGrantIdentityTail(key, prefix) + if !ok { + continue // malformed index key — defensive skip + } + // Primary key = grant header + the identity tail verbatim. + priKey := make([]byte, 0, 3+len(tail)) + priKey = append(priKey, versionV3, typeGrant) + priKey = codec.AppendTupleSeparator(priKey) + priKey = append(priKey, tail...) + + val, closer, err := e.db.Get(priKey) + if err != nil { + if errors.Is(err, pebble.ErrNotFound) { + continue // index ahead of primary — defensive skip + } + return err + } + rec := &v3.GrantRecord{} + uerr := unmarshalRecord(val, rec) + _ = closer.Close() + if uerr != nil { + return fmt.Errorf("DeleteGrantsByExternalIDsInScope: unmarshal: %w", uerr) + } + if _, hit := ids[rec.GetExternalId()]; !hit { + continue + } + if err := batch.Delete(priKey, nil); err != nil { + return err + } + if err := batch.Delete(encodeGrantByPrincipalIdentityIndexKey(id), nil); err != nil { + return err + } + if err := batch.Delete(encodeGrantByNeedsExpansionIdentityIndexKey(id), nil); err != nil { + return err + } + if err := batch.Delete(key, nil); err != nil { + return err + } + deleted++ + } + if err := iter.Error(); err != nil { + return err + } + return batch.Commit(opts) + }) + if err != nil { + return 0, err + } + return deleted, nil +} + +// DeleteResourcesByIDsInScope deletes every resource row in the CURRENT +// store stamped with scopeHash whose resource id is in resourceIDs (any +// resource type) — principal-scoped tombstones for RowKindResources. +func (e *Engine) DeleteResourcesByIDsInScope(ctx context.Context, scopeHash string, resourceIDs map[string]struct{}) (int64, error) { + if len(resourceIDs) == 0 { + return 0, nil + } + prefix := encodeResourceBySourceScopePrefix(scopeHash) + var deleted int64 + + err := e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return err + } + defer iter.Close() + + opts := writeOpts(e.opts.durability) + if e.IsFreshSync() { + opts = pebble.NoSync + } + batch := e.db.NewBatch() + defer func() { _ = batch.Close() }() + + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return err + } + key := iter.Key() + tail := key[len(prefix):] + // Tail layout: resource_type_id | resource_id. + rtBytes, next, ok := codec.DecodeTupleStringAlias(tail, 0) + if !ok || next >= len(tail) { + continue + } + ridBytes, _, ok := codec.DecodeTupleStringAlias(tail, next+1) + if !ok { + continue + } + if _, hit := resourceIDs[string(ridBytes)]; !hit { + continue + } + rt, rid := string(rtBytes), string(ridBytes) + priKey := make([]byte, 0, 3+len(tail)) + priKey = append(priKey, versionV3, typeResource) + priKey = codec.AppendTupleSeparator(priKey) + priKey = append(priKey, tail...) + // by_parent cleanup needs the parent ref from the value. + if val, closer, getErr := e.db.Get(priKey); getErr == nil { + err := e.deleteResourceIndexesRaw(batch, rt, rid, val) + closer.Close() + if err != nil { + return err + } + } else if !errors.Is(getErr, pebble.ErrNotFound) { + return getErr + } + if err := batch.Delete(priKey, nil); err != nil { + return err + } + // deleteResourceIndexesRaw already covered the scope entry + // (source_scope_hash is in the value), but delete the iterated + // key too in case the value read missed (orphan entry). + if err := batch.Delete(key, nil); err != nil { + return err + } + deleted++ + } + if err := iter.Error(); err != nil { + return err + } + return batch.Commit(opts) + }) + if err != nil { + return 0, err + } + return deleted, nil +} + +// grantValueHasSourcesRaw reports whether a marshaled GrantRecord carries +// at least one sources entry (field 9), without unmarshaling. +func grantValueHasSourcesRaw(value []byte) (bool, error) { + for len(value) > 0 { + num, typ, n := protowire.ConsumeTag(value) + if n < 0 { + return false, protowire.ParseError(n) + } + value = value[n:] + if num == 9 { + return true, nil + } + n = protowire.ConsumeFieldValue(num, typ, value) + if n < 0 { + return false, protowire.ParseError(n) + } + value = value[n:] + } + return false, nil +} + +// stripExpanderSourcesRaw clears a replayed grant's Sources map when it is +// expander-written, so the current sync's expansion recomputes it from +// true state instead of inheriting contributions that may have been +// removed upstream. Classification mirrors RollbackExpansion: a Sources +// map containing a self-source entry (keyed by the grant's own entitlement +// id) was written by the expander; one without a self-source is +// connector-set public data and is preserved. Returns (newValue, true) when +// the record was rewritten, (nil, false) when the original bytes should be +// copied verbatim. +func stripExpanderSourcesRaw(value []byte, ownEntitlementID string) ([]byte, bool, error) { + r := &v3.GrantRecord{} + if err := unmarshalRecord(value, r); err != nil { + return nil, false, fmt.Errorf("source cache replay: unmarshal grant for sources strip: %w", err) + } + sources := r.GetSources() + if len(sources) == 0 { + return nil, false, nil + } + if _, hasSelf := sources[ownEntitlementID]; !hasSelf { + // No self-source: connector-set Sources. Preserve verbatim. + return nil, false, nil + } + r.SetSources(nil) + stripped, err := marshalRecord(r) + if err != nil { + return nil, false, fmt.Errorf("source cache replay: re-marshal grant after sources strip: %w", err) + } + return stripped, true, nil +} + +// decodeResourcePrimaryTail decodes (resource_type_id, resource_id) +// from a resource primary key (v3 | typeResource | 0x00 | rt | 0x00 | rid). +func decodeResourcePrimaryTail(priKey []byte) (string, string, error) { + const headerLen = 3 // versionV3, typeResource, separator + if len(priKey) <= headerLen { + return "", "", fmt.Errorf("source cache replay: malformed resource primary key %x", priKey) + } + tail := priKey[headerLen:] + rtBytes, next, err := codec.DecodeTupleStringTo(nil, tail, 0) + if err != nil { + return "", "", err + } + if next >= len(tail) { + return "", "", fmt.Errorf("source cache replay: resource primary key missing resource_id: %x", priKey) + } + ridBytes, _, err := codec.DecodeTupleStringTo(nil, tail, next+1) + if err != nil { + return "", "", err + } + return string(rtBytes), string(ridBytes), nil +} + +// replayPrimaryFromIndexKey derives a record's primary key from its +// by_source_scope index key. The index prefix (header|0x00|scope|0x00) +// is followed by exactly the identity tuple that forms the primary +// key's tail, so the primary is header' + 0x00-separated remainder. +func replayPrimaryFromIndexKey(indexKey, indexPrefix []byte, primaryHeader [2]byte) ([]byte, error) { + if len(indexKey) <= len(indexPrefix) { + return nil, fmt.Errorf("source cache replay: malformed index key %x", indexKey) + } + tail := indexKey[len(indexPrefix):] + key := make([]byte, 0, 3+len(tail)) + key = append(key, primaryHeader[0], primaryHeader[1], 0x00) + return append(key, tail...), nil +} + +// ReplaySourceCacheGrants copies every grant stamped with scopeHash from +// prev into the receiver: raw primary copy plus index synthesis from the +// raw value (principal, needs_expansion, source-scope families). Mirrors +// PutGrantRecords' read-before-write index cleanup when the receiver +// already holds a record at the same identity. +func (e *Engine) ReplaySourceCacheGrants(ctx context.Context, prev *Engine, scopeHash string) (SourceCacheReplayResult, error) { + var res SourceCacheReplayResult + prefix := encodeGrantBySourceScopePrefix(scopeHash) + primaryHeader := [2]byte{versionV3, typeGrant} + + err := e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + iter, err := prev.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return err + } + defer iter.Close() + + opts := writeOpts(e.opts.durability) + if e.IsFreshSync() { + opts = pebble.NoSync + } + batch := e.db.NewBatch() + defer func() { _ = batch.Close() }() + rowsInBatch := 0 + + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return err + } + priKey, err := replayPrimaryFromIndexKey(iter.Key(), prefix, primaryHeader) + if err != nil { + return err + } + val, closer, getErr := prev.db.Get(priKey) + if getErr != nil { + if errors.Is(getErr, pebble.ErrNotFound) { + // Orphan index entry in the previous file — skip, + // matching the defensive-skip semantic of the other + // index read paths. + continue + } + return fmt.Errorf("source cache replay: get prev grant: %w", getErr) + } + + entRT, entRID, entID, principalRT, principalID, needsExpansion, srcScope, scanErr := scanGrantIndexFieldsRaw(val) + if scanErr != nil { + closer.Close() + return scanErr + } + // Stale-index defense: only copy rows whose VALUE stamp matches + // the queried scope. An index entry pointing at a row stamped + // differently (or not at all) is left over from a path that + // replaced the row without cleaning the index — e.g. a fold + // compaction predating the source-cache bucket plans, or an + // in-sync same-identity rewrite under a different scope. Copying + // it would inject rows upstream never returned for this scope. + if srcScope != scopeHash { + closer.Close() + continue + } + + // Clean up index entries for any record the current sync + // already wrote at this identity (same discipline as + // PutGrantRecords' read-before-write). + if oldVal, oldCloser, oldErr := e.db.Get(priKey); oldErr == nil { + if err := e.deleteGrantIndexesRaw(batch, "", oldVal); err != nil { + oldCloser.Close() + closer.Close() + return err + } + oldCloser.Close() + } else if !errors.Is(oldErr, pebble.ErrNotFound) { + closer.Close() + return fmt.Errorf("source cache replay: get current grant: %w", oldErr) + } + + // Replay-equivalence: a cached sync must reproduce what a full + // resync would produce. The one field where a verbatim copy + // diverges is expander-written Sources — the previous sync's + // expansion baked contributions into direct grants, and + // re-expansion only ADDS, so a contribution removed this sync + // (via a delta tombstone or a refetched page) would survive + // forever. Strip expander-written Sources so the current sync's + // expansion recomputes them from true state; connector-set + // Sources (no self-source entry — same classification as + // RollbackExpansion) are connector data and are preserved + // verbatim. The probe is a cheap protowire scan; the vast + // majority of rows carry no Sources and stay on the raw-copy + // path. + writeVal := val + if hasSources, probeErr := grantValueHasSourcesRaw(val); probeErr != nil { + closer.Close() + return probeErr + } else if hasSources { + stripped, strippedOK, stripErr := stripExpanderSourcesRaw(val, entID) + if stripErr != nil { + closer.Close() + return stripErr + } + if strippedOK { + writeVal = stripped + } + } + if err := batch.Set(priKey, writeVal, nil); err != nil { + closer.Close() + return err + } + closer.Close() + + if entID != "" && entRT != "" && entRID != "" && principalRT != "" && principalID != "" { + id := grantIdentity{ + entitlement: entitlementIdentityFromParts(entRT, entRID, entID), + principalTypeID: principalRT, + principalID: principalID, + } + if _, err := e.writeGrantIndexesForIdentityScratch(batch, id, needsExpansion, srcScope, nil); err != nil { + return err + } + } + if needsExpansion { + res.NeedsExpansion = true + } + res.Rows++ + rowsInBatch++ + if rowsInBatch >= replayBatchRows { + if err := batch.Commit(opts); err != nil { + return err + } + _ = batch.Close() + batch = e.db.NewBatch() + rowsInBatch = 0 + } + } + if err := iter.Error(); err != nil { + return err + } + return batch.Commit(opts) + }) + if err != nil { + return SourceCacheReplayResult{}, err + } + return res, nil +} + +// ReplaySourceCacheEntitlements copies every entitlement stamped with +// scopeHash from prev into the receiver. +func (e *Engine) ReplaySourceCacheEntitlements(ctx context.Context, prev *Engine, scopeHash string) (SourceCacheReplayResult, error) { + var res SourceCacheReplayResult + prefix := encodeEntitlementBySourceScopePrefix(scopeHash) + primaryHeader := [2]byte{versionV3, typeEntitlement} + + err := e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + iter, err := prev.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return err + } + defer iter.Close() + + opts := writeOpts(e.opts.durability) + if e.IsFreshSync() { + opts = pebble.NoSync + } + batch := e.db.NewBatch() + defer func() { _ = batch.Close() }() + rowsInBatch := 0 + + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return err + } + priKey, err := replayPrimaryFromIndexKey(iter.Key(), prefix, primaryHeader) + if err != nil { + return err + } + val, closer, getErr := prev.db.Get(priKey) + if getErr != nil { + if errors.Is(getErr, pebble.ErrNotFound) { + continue + } + return fmt.Errorf("source cache replay: get prev entitlement: %w", getErr) + } + // Stale-index defense: only copy rows whose VALUE stamp matches + // the queried scope (see the grants replay for rationale). + prevScope, prevScanErr := scanEntitlementSourceScopeRaw(val) + if prevScanErr != nil { + closer.Close() + return prevScanErr + } + if prevScope != scopeHash { + closer.Close() + continue + } + // The replayed row's source-scope index key is byte-identical + // to prev's — the only entitlement secondary index. Clean up a + // differing stamp on any record the current sync already wrote + // at this identity. + if oldVal, oldCloser, oldErr := e.db.Get(priKey); oldErr == nil { + oldScope, scanErr := scanEntitlementSourceScopeRaw(oldVal) + oldCloser.Close() + if scanErr != nil { + closer.Close() + return scanErr + } + if oldScope != "" && oldScope != scopeHash { + // The old index key's tail equals the primary key's + // tail (identity tuple), so rebuild it byte-wise. + oldIdxKey := make([]byte, 0, 8+len(oldScope)+len(priKey)) + oldIdxKey = append(oldIdxKey, versionV3, typeIndex, idxEntitlementBySourceScope) + oldIdxKey = codec.AppendTupleSeparator(oldIdxKey) + oldIdxKey = codec.AppendTupleStrings(oldIdxKey, oldScope) + oldIdxKey = codec.AppendTupleSeparator(oldIdxKey) + oldIdxKey = append(oldIdxKey, priKey[3:]...) + if err := batch.Delete(oldIdxKey, nil); err != nil { + closer.Close() + return err + } + } + } else if !errors.Is(oldErr, pebble.ErrNotFound) { + closer.Close() + return fmt.Errorf("source cache replay: get current entitlement: %w", oldErr) + } + if err := batch.Set(priKey, val, nil); err != nil { + closer.Close() + return err + } + closer.Close() + if err := batch.Set(iter.Key(), nil, nil); err != nil { + return err + } + res.Rows++ + rowsInBatch++ + if rowsInBatch >= replayBatchRows { + if err := batch.Commit(opts); err != nil { + return err + } + _ = batch.Close() + batch = e.db.NewBatch() + rowsInBatch = 0 + } + } + if err := iter.Error(); err != nil { + return err + } + return batch.Commit(opts) + }) + if err != nil { + return SourceCacheReplayResult{}, err + } + return res, nil +} + +// ReplaySourceCacheResources copies every resource stamped with scopeHash +// from prev into the receiver, synthesizing by_parent and by_source_scope +// index entries from the raw value. +func (e *Engine) ReplaySourceCacheResources(ctx context.Context, prev *Engine, scopeHash string) (SourceCacheReplayResult, error) { + var res SourceCacheReplayResult + prefix := encodeResourceBySourceScopePrefix(scopeHash) + primaryHeader := [2]byte{versionV3, typeResource} + + err := e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + iter, err := prev.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return err + } + defer iter.Close() + + opts := writeOpts(e.opts.durability) + if e.IsFreshSync() { + opts = pebble.NoSync + } + batch := e.db.NewBatch() + defer func() { _ = batch.Close() }() + rowsInBatch := 0 + + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return err + } + priKey, err := replayPrimaryFromIndexKey(iter.Key(), prefix, primaryHeader) + if err != nil { + return err + } + val, closer, getErr := prev.db.Get(priKey) + if getErr != nil { + if errors.Is(getErr, pebble.ErrNotFound) { + continue + } + return fmt.Errorf("source cache replay: get prev resource: %w", getErr) + } + + rt, rid, decodeErr := decodeResourcePrimaryTail(priKey) + if decodeErr != nil { + closer.Close() + return decodeErr + } + parentRT, parentID, srcScope, scanErr := scanResourceIndexFieldsRaw(val) + if scanErr != nil { + closer.Close() + return scanErr + } + // Stale-index defense: only copy rows whose VALUE stamp matches + // the queried scope (see the grants replay for rationale). + if srcScope != scopeHash { + closer.Close() + continue + } + if oldVal, oldCloser, oldErr := e.db.Get(priKey); oldErr == nil { + if err := e.deleteResourceIndexesRaw(batch, rt, rid, oldVal); err != nil { + oldCloser.Close() + closer.Close() + return err + } + oldCloser.Close() + } else if !errors.Is(oldErr, pebble.ErrNotFound) { + closer.Close() + return fmt.Errorf("source cache replay: get current resource: %w", oldErr) + } + + if err := batch.Set(priKey, val, nil); err != nil { + closer.Close() + return err + } + closer.Close() + + if parentID != "" { + if err := batch.Set(encodeResourceByParentIndexKey(parentRT, parentID, rt, rid), nil, nil); err != nil { + return err + } + } + if srcScope != "" { + if err := batch.Set(encodeResourceBySourceScopeIndexKey(srcScope, rt, rid), nil, nil); err != nil { + return err + } + } + res.Rows++ + rowsInBatch++ + if rowsInBatch >= replayBatchRows { + if err := batch.Commit(opts); err != nil { + return err + } + _ = batch.Close() + batch = e.db.NewBatch() + rowsInBatch = 0 + } + } + if err := iter.Error(); err != nil { + return err + } + return batch.Commit(opts) + }) + if err != nil { + return SourceCacheReplayResult{}, err + } + return res, nil +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/source_cache.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/source_cache.go new file mode 100644 index 00000000..80171981 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/source_cache.go @@ -0,0 +1,213 @@ +package dotc1z + +import ( + "context" + "errors" + "fmt" + + cdbpebble "github.com/cockroachdb/pebble/v2" + + "github.com/conductorone/baton-sdk/pkg/bid" + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble" + "github.com/conductorone/baton-sdk/pkg/sourcecache" +) + +// SourceCacheReplayResult reports what one scope's replay copied. +type SourceCacheReplayResult = pebble.SourceCacheReplayResult + +// SourceCacheStore is the optional store capability backing source-cache +// replay (see proto/c1/connector/v2/annotation_source_cache.proto). It is +// implemented ONLY by the Pebble engine; the syncer type-asserts for it and +// treats a store without it as "source cache unsupported" (no-op lookup, +// no replay). It is deliberately NOT part of c1zstore.Store. +type SourceCacheStore interface { + // LookupSourceCacheEntry returns this store's manifest entry for + // (kind, scopeHash). Backs the connector-facing lookup when this + // store is the previous sync. + LookupSourceCacheEntry(ctx context.Context, kind sourcecache.RowKind, scopeHash string) (sourcecache.Entry, bool, error) + + // PutSourceCacheEntry writes the current sync's manifest entry for + // (kind, scopeHash). Zero-row scopes still get entries. + PutSourceCacheEntry(ctx context.Context, kind sourcecache.RowKind, scopeHash string, etag string) error + + // ReplaySourceCache copies every row stamped with scopeHash from prev + // (the previous sync's store, opened read-only) into this store. prev + // must be a Pebble store. Does NOT write the manifest entry — the + // caller writes it after the scope's overlay/deletes complete, so a + // failed replay can't leave a phantom hit for the next sync. + ReplaySourceCache(ctx context.Context, prev connectorstore.Reader, kind sourcecache.RowKind, scopeHash string) (SourceCacheReplayResult, error) + + // DeleteSourceCacheRows removes rows by public canonical ID from the + // current sync, after replay + overlay (delta-query tombstones). + // ID formats per kind: grants and entitlements use their canonical + // IDs; resources use Baton resource BIDs ("bid:r:..."). + // + // Grant resolution is BOUNDED: candidate probing only, never the + // O(all grants) stored-external-id scan. Grants stored under + // connector-custom ids are unreachable here (delete no-ops); such + // connectors use DeleteSourceCacheRowsInScope instead. + DeleteSourceCacheRows(ctx context.Context, kind sourcecache.RowKind, ids []string) error + + // DeleteSourceCacheRowsInScope removes rows stamped with scopeHash by + // bare object id — grants by principal id (no principal type, no + // canonical-id reconstruction), resources by resource id (any type). + // One index scan of the scope per call; a page's tombstones are + // batched into one call. Ids with no matching rows are no-ops. + // Not supported for entitlements. + DeleteSourceCacheRowsInScope(ctx context.Context, kind sourcecache.RowKind, scopeHash string, ids []string) (int64, error) + + // DeleteSourceCacheGrantsByIDInScope removes grant rows stamped with + // scopeHash whose STORED grant id is in ids — works for + // connector-custom grant-id shapes that the global bounded delete + // cannot resolve, and stays bounded by the scope's row count. Ids with + // no matching rows are no-ops. + DeleteSourceCacheGrantsByIDInScope(ctx context.Context, scopeHash string, ids []string) (int64, error) +} + +var _ SourceCacheStore = (*pebbleStore)(nil) + +// sourceCacheEngine recovers the Pebble engine from an arbitrary store, +// nil-safe. Mirrors pebble.AsEngine but accepts any value so the syncer +// can probe its previous-sync reader without caring about its static type. +func sourceCacheEngine(store any) (*pebble.Engine, bool) { + a, ok := store.(interface{ PebbleEngine() *pebble.Engine }) + if !ok { + return nil, false + } + e := a.PebbleEngine() + return e, e != nil +} + +func (s *pebbleStore) LookupSourceCacheEntry(ctx context.Context, kind sourcecache.RowKind, scopeHash string) (sourcecache.Entry, bool, error) { + if err := sourcecache.ValidateRowKind(kind); err != nil { + return sourcecache.Entry{}, false, err + } + rec, err := s.engine.GetSourceCacheEntry(ctx, string(kind), scopeHash) + if err != nil { + if errors.Is(err, cdbpebble.ErrNotFound) { + return sourcecache.Entry{}, false, nil + } + return sourcecache.Entry{}, false, err + } + return sourcecache.Entry{ + ETag: rec.GetEtag(), + DiscoveredAt: rec.GetDiscoveredAt().AsTime(), + }, true, nil +} + +func (s *pebbleStore) PutSourceCacheEntry(ctx context.Context, kind sourcecache.RowKind, scopeHash string, etag string) error { + if err := sourcecache.ValidateRowKind(kind); err != nil { + return err + } + return s.markDirty(s.engine.PutSourceCacheEntry(ctx, string(kind), scopeHash, etag)) +} + +func (s *pebbleStore) ReplaySourceCache(ctx context.Context, prev connectorstore.Reader, kind sourcecache.RowKind, scopeHash string) (SourceCacheReplayResult, error) { + prevEngine, ok := sourceCacheEngine(prev) + if !ok { + return SourceCacheReplayResult{}, errors.New("source cache replay: previous sync store is not a pebble store") + } + var res SourceCacheReplayResult + var err error + switch kind { + case sourcecache.RowKindResources: + res, err = s.engine.ReplaySourceCacheResources(ctx, prevEngine, scopeHash) + case sourcecache.RowKindEntitlements: + res, err = s.engine.ReplaySourceCacheEntitlements(ctx, prevEngine, scopeHash) + case sourcecache.RowKindGrants: + res, err = s.engine.ReplaySourceCacheGrants(ctx, prevEngine, scopeHash) + default: + return SourceCacheReplayResult{}, fmt.Errorf("source cache replay: invalid row kind %q", kind) + } + if err != nil { + return SourceCacheReplayResult{}, err + } + if res.Rows > 0 { + s.MarkDirty() + } + return res, nil +} + +// DeleteSourceCacheRows deletes delta tombstones by public id string. +// +// NOTE on the bare-id lookup safety contract (engine/pebble/lookup.go): +// sync paths normally must not resolve grants by string. This path is a +// deliberate, narrow exception: tombstone ids are strings the connector +// itself emitted for these rows, volumes are delta-sized (not O(rows)), +// and resolution keeps the exactly-one rule — an ambiguous id fails the +// sync loudly rather than guessing a delete, which matches the +// source-cache replay-phase error policy. +func (s *pebbleStore) DeleteSourceCacheRows(ctx context.Context, kind sourcecache.RowKind, ids []string) error { + for _, id := range ids { + switch kind { + case sourcecache.RowKindGrants: + if err := s.markDirty(s.engine.DeleteGrantRecordBounded(ctx, id)); err != nil { + return fmt.Errorf("source cache delete grant %q: %w", id, err) + } + case sourcecache.RowKindEntitlements: + if err := s.markDirty(s.engine.DeleteEntitlementRecord(ctx, id)); err != nil { + return fmt.Errorf("source cache delete entitlement %q: %w", id, err) + } + case sourcecache.RowKindResources: + r, err := bid.ParseResourceBid(id) + if err != nil { + return fmt.Errorf("source cache delete resource: invalid resource bid %q: %w", id, err) + } + rid := r.GetId() + if err := s.markDirty(s.engine.DeleteResourceRecord(ctx, rid.GetResourceType(), rid.GetResource())); err != nil { + return fmt.Errorf("source cache delete resource %q: %w", id, err) + } + default: + return fmt.Errorf("source cache delete: invalid row kind %q", kind) + } + } + return nil +} + +func (s *pebbleStore) DeleteSourceCacheGrantsByIDInScope(ctx context.Context, scopeHash string, ids []string) (int64, error) { + if len(ids) == 0 { + return 0, nil + } + idSet := make(map[string]struct{}, len(ids)) + for _, id := range ids { + idSet[id] = struct{}{} + } + deleted, err := s.engine.DeleteGrantsByExternalIDsInScope(ctx, scopeHash, idSet) + if err != nil { + return 0, fmt.Errorf("source cache grant-id delete for scope %q: %w", scopeHash, err) + } + if deleted > 0 { + s.MarkDirty() + } + return deleted, nil +} + +func (s *pebbleStore) DeleteSourceCacheRowsInScope(ctx context.Context, kind sourcecache.RowKind, scopeHash string, ids []string) (int64, error) { + if len(ids) == 0 { + return 0, nil + } + idSet := make(map[string]struct{}, len(ids)) + for _, id := range ids { + idSet[id] = struct{}{} + } + var deleted int64 + var err error + switch kind { + case sourcecache.RowKindGrants: + deleted, err = s.engine.DeleteGrantsByPrincipalsInScope(ctx, scopeHash, idSet) + case sourcecache.RowKindResources: + deleted, err = s.engine.DeleteResourcesByIDsInScope(ctx, scopeHash, idSet) + case sourcecache.RowKindEntitlements: + return 0, fmt.Errorf("source cache scoped delete: not supported for entitlements") + default: + return 0, fmt.Errorf("source cache scoped delete: invalid row kind %q", kind) + } + if err != nil { + return 0, fmt.Errorf("source cache scoped delete for scope %q: %w", scopeHash, err) + } + if deleted > 0 { + s.MarkDirty() + } + return deleted, nil +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/field/defaults.go b/vendor/github.com/conductorone/baton-sdk/pkg/field/defaults.go index e4572664..40c4e551 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/field/defaults.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/field/defaults.go @@ -264,6 +264,17 @@ var ( WithDescription("The path to the c1z file to sync external baton resources with"), WithPersistent(true), WithExportTarget(ExportTargetNone)) + // PreviousSyncC1ZField is hidden by default: source-cache replay is + // author-opt-in functionality (connectorrunner.WithKeepPreviousSyncC1Z), + // and non-replaying connectors shouldn't advertise it to operators. + // DefineConfiguration unhides it for connectors that declare the + // capability; the flag still parses everywhere (hidden ≠ disabled) so + // expert/debug use keeps working. + PreviousSyncC1ZField = StringField("previous-sync-c1z", + WithDescription("The path to the previous sync c1z file to use as a source-cache replay input"), + WithPersistent(true), + WithHidden(true), + WithExportTarget(ExportTargetNone)) externalResourceEntitlementIdFilter = StringField("external-resource-entitlement-id-filter", WithDescription("The entitlement that external users, groups must have access to sync external baton resources"), WithPersistent(true), @@ -276,11 +287,14 @@ var ( // connectors whose author also declared ETag-replay support at build // time (connectorrunner.WithKeepPreviousSyncC1Z) — both are // required. Costs one c1z of local disk. + // Hidden by default for the same reason as PreviousSyncC1ZField; unhidden + // when the connector author declares the replay capability. KeepPreviousSyncC1ZField = BoolField("keep-previous-sync-c1z", WithDescription("Keep the previously synced c1z on disk to enable ETag replay across service-mode syncs "+ "(requires a connector that supports ETag replay; costs one c1z of local disk)"), WithDefaultValue(false), WithPersistent(true), + WithHidden(true), WithExportTarget(ExportTargetNone)) LambdaServerClientIDField = StringField("lambda-client-id", WithRequired(true), WithDescription("The oauth client id to use with the configuration endpoint"), @@ -431,6 +445,7 @@ var DefaultFields = append([]SchemaField{ skipEntitlementsAndGrants, skipGrants, externalResourceC1ZField, + PreviousSyncC1ZField, externalResourceEntitlementIdFilter, KeepPreviousSyncC1ZField, diffSyncsField, diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/field/fields.go b/vendor/github.com/conductorone/baton-sdk/pkg/field/fields.go index 1107861c..738d1ca6 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/field/fields.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/field/fields.go @@ -126,6 +126,34 @@ func (s SchemaField) ExportAs(et ExportTarget) SchemaField { return c } +// WithConnectorDefault returns a copy of a shared/default SDK field carrying +// a connector-specific default value. Include the copy in the connector's +// field.Configuration Fields — DefineConfiguration replaces the SDK's copy +// with it (same re-export mechanism as ExportAs), so --help, flag parsing, +// and exported config schemas all reflect the connector's default. +// +// Value-resolution precedence is unchanged and sentinel-free: an explicit +// flag beats the environment beats the config file beats this default (the +// default lives on the flag itself, so a user-supplied zero value remains +// distinguishable from "unset"). +// +// Example — a connector whose sync fans out well declares its own worker +// default without hiding the shared flag's semantics: +// +// field.NewConfiguration([]field.SchemaField{ +// field.WithConnectorDefault(field.WorkerCountField, 16), +// ... +// }) +// +// The value's type must match the field's variant (int for IntField, etc.); +// a mismatch fails loudly at startup when the flag is registered. +func WithConnectorDefault[T SchemaTypes](s SchemaField, defaultValue T) SchemaField { + c := s + c.DefaultValue = defaultValue + c.WasReExported = true + return c +} + // Go doesn't allow generic methods on a non-generic struct. func ValidateField[T SchemaTypes](s *SchemaField, value T) (bool, error) { return s.validate(value) diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sdk/version.go b/vendor/github.com/conductorone/baton-sdk/pkg/sdk/version.go index e9dc1322..ff96f32a 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sdk/version.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sdk/version.go @@ -1,3 +1,3 @@ package sdk -const Version = "v0.18.1" +const Version = "v0.18.2" diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/context.go b/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/context.go new file mode 100644 index 00000000..266dd5c0 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/context.go @@ -0,0 +1,20 @@ +package sourcecache + +import "context" + +type scopeContextKey struct{} + +// WithScope returns a context carrying the source-cache scope hash for +// rows written under it. The syncer wraps a page's store writes in this +// context when the page carried a SourceCacheScope annotation; the Pebble +// write path stamps the record's source_scope_hash from it. +func WithScope(ctx context.Context, scopeHash string) context.Context { + return context.WithValue(ctx, scopeContextKey{}, scopeHash) +} + +// ScopeFromContext returns the scope hash set by WithScope, or "" when +// the context carries none (the common, unstamped case). +func ScopeFromContext(ctx context.Context) string { + s, _ := ctx.Value(scopeContextKey{}).(string) + return s +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/continuation.go b/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/continuation.go new file mode 100644 index 00000000..78058fe2 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/continuation.go @@ -0,0 +1,232 @@ +package sourcecache + +// Lookup continuation (ask/answer): the lookup transport for connector +// runtimes that cannot call back to the syncer mid-request (single-shot +// request/response tunnels, e.g. gRPC-over-Lambda). The connector's first +// execution of a page ("phase 1") records the scopes it needs and fails +// with ErrLookupDeferred; the SDK converts that into a +// SourceCacheLookupAsk response; the syncer resolves the queries against +// its local previous-sync store and re-invokes the same request with +// SourceCacheLookupAnswers attached ("phase 2"), where the same connector +// code gets real answers. See docs/tasks/source-cache-lambda-lookup.md. + +import ( + "context" + "errors" + "fmt" + "sync" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" +) + +// ErrLookupDeferred is returned by a deferring Lookup (phase 1 of the +// ask/answer continuation) when the answer is not yet available. The SDK +// intercepts it and answers the RPC with a SourceCacheLookupAsk. +// +// Propagation contract for connectors: wrap with %w if you must, NEVER +// swallow. The SDK matches with errors.Is, so idiomatic wrapping +// (fmt.Errorf("listing members: %w", err)) is fine. A connector that +// logs-and-continues past this error re-asks for scopes it already +// "handled" and turns every warm sync into a hard failure at the bounce +// cap. +var ErrLookupDeferred = errors.New("source cache lookup deferred: answer arrives on re-invoke") + +// Query identifies one scope to resolve against the previous sync. +type Query struct { + RowKind RowKind + ScopeHash string +} + +// Answer resolves one Query. Found=false means the previous sync has no +// entry for the scope: fetch fresh. (Distinct from a query that got no +// Answer at all, which means unresolved: ask again.) +type Answer struct { + Query + Found bool + ETag string +} + +// BatchLookup is optionally implemented by Lookup implementations that can +// resolve many scopes in one round trip. Connectors should not type-assert +// for it directly; call LookupMany, which falls back to per-query lookups. +type BatchLookup interface { + LookupPreviousSourceCacheMany(ctx context.Context, queries []Query) ([]Answer, error) +} + +// LookupMany resolves a batch of queries through lookup, using one round +// trip when the implementation supports it (BatchLookup) and a loop of +// single lookups otherwise. This is the topology-uniform batch API: +// in-process and subprocess lookups loop over local/loopback point-reads; +// a deferring lookup collects the whole batch into ONE ask. +// +// The returned answers are exact and complete for the queried set — one +// Answer per Query, order preserved, explicit Found per entry. (A +// deferring lookup returns ErrLookupDeferred instead of answers; phase 2 +// then answers the same calls. Answers dropped to the transport size +// budget surface as ErrLookupDeferred again on the affected queries, never +// as silent omissions or false not-founds.) +func LookupMany(ctx context.Context, lookup Lookup, queries []Query) ([]Answer, error) { + if bl, ok := lookup.(BatchLookup); ok { + return bl.LookupPreviousSourceCacheMany(ctx, queries) + } + answers := make([]Answer, 0, len(queries)) + for _, q := range queries { + entry, found, err := lookup.LookupPreviousSourceCache(ctx, q.RowKind, q.ScopeHash) + if err != nil { + return nil, err + } + a := Answer{Query: q, Found: found} + if found { + a.ETag = entry.ETag + } + answers = append(answers, a) + } + return answers, nil +} + +// ContinuationLookup is the per-request Lookup installed for the +// ask/answer continuation. It serves lookups from the answers delivered on +// the request (phase 2) and defers everything else by recording the query +// and returning ErrLookupDeferred (phase 1, or an under-answered phase 2 — +// e.g. answers dropped to the size budget). +// +// It is constructed per RPC by the SDK; connectors only ever see it as +// SyncOpAttrs.SourceCache. +type ContinuationLookup struct { + mu sync.Mutex + answers map[Query]Answer + asked []Query + askedSet map[Query]struct{} +} + +var ( + _ Lookup = (*ContinuationLookup)(nil) + _ BatchLookup = (*ContinuationLookup)(nil) +) + +// NewContinuationLookup builds a ContinuationLookup pre-loaded with the +// request's answers (nil/empty on phase 1). +func NewContinuationLookup(answers []Answer) *ContinuationLookup { + m := make(map[Query]Answer, len(answers)) + for _, a := range answers { + m[a.Query] = a + } + return &ContinuationLookup{ + answers: m, + askedSet: map[Query]struct{}{}, + } +} + +func (c *ContinuationLookup) LookupPreviousSourceCache(_ context.Context, rowKind RowKind, scopeHash string) (Entry, bool, error) { + q := Query{RowKind: rowKind, ScopeHash: scopeHash} + c.mu.Lock() + defer c.mu.Unlock() + if a, ok := c.answers[q]; ok { + if !a.Found { + return Entry{}, false, nil + } + return Entry{ETag: a.ETag}, true, nil + } + c.recordLocked(q) + return Entry{}, false, fmt.Errorf("lookup %s/%s: %w", rowKind, scopeHash, ErrLookupDeferred) +} + +func (c *ContinuationLookup) LookupPreviousSourceCacheMany(_ context.Context, queries []Query) ([]Answer, error) { + c.mu.Lock() + defer c.mu.Unlock() + answers := make([]Answer, 0, len(queries)) + missing := 0 + for _, q := range queries { + a, ok := c.answers[q] + if !ok { + c.recordLocked(q) + missing++ + continue + } + answers = append(answers, a) + } + if missing > 0 { + // Defer the whole batch: phase 2 re-runs the same call with the + // full answer set (already-answered queries remain answered on the + // re-invoked request). + return nil, fmt.Errorf("batch lookup: %d of %d queries unresolved: %w", missing, len(queries), ErrLookupDeferred) + } + return answers, nil +} + +func (c *ContinuationLookup) recordLocked(q Query) { + if _, seen := c.askedSet[q]; seen { + return + } + c.askedSet[q] = struct{}{} + c.asked = append(c.asked, q) +} + +// Asked returns the queries recorded by deferred lookups, deduplicated, in +// first-ask order. Empty means no lookup deferred and the handler's result +// stands. +func (c *ContinuationLookup) Asked() []Query { + c.mu.Lock() + defer c.mu.Unlock() + out := make([]Query, len(c.asked)) + copy(out, c.asked) + return out +} + +// --- proto conversions (shared by connectorbuilder and the syncer) ------ + +// AskProto builds the SourceCacheLookupAsk annotation for a set of +// deferred queries. +func AskProto(queries []Query) *v2.SourceCacheLookupAsk { + qs := make([]*v2.SourceCacheLookupAsk_Query, 0, len(queries)) + for _, q := range queries { + qs = append(qs, v2.SourceCacheLookupAsk_Query_builder{ + RowKind: string(q.RowKind), + ScopeHash: q.ScopeHash, + }.Build()) + } + return v2.SourceCacheLookupAsk_builder{Queries: qs}.Build() +} + +// QueriesFromProto extracts and validates the queries of an ask. +func QueriesFromProto(ask *v2.SourceCacheLookupAsk) ([]Query, error) { + out := make([]Query, 0, len(ask.GetQueries())) + for _, q := range ask.GetQueries() { + kind := RowKind(q.GetRowKind()) + if err := ValidateRowKind(kind); err != nil { + return nil, err + } + if err := ValidateScopeHash(q.GetScopeHash()); err != nil { + return nil, err + } + out = append(out, Query{RowKind: kind, ScopeHash: q.GetScopeHash()}) + } + return out, nil +} + +// AnswersProto builds the SourceCacheLookupAnswers annotation. +func AnswersProto(answers []Answer) *v2.SourceCacheLookupAnswers { + as := make([]*v2.SourceCacheLookupAnswers_Answer, 0, len(answers)) + for _, a := range answers { + as = append(as, v2.SourceCacheLookupAnswers_Answer_builder{ + RowKind: string(a.RowKind), + ScopeHash: a.ScopeHash, + Found: a.Found, + Etag: a.ETag, + }.Build()) + } + return v2.SourceCacheLookupAnswers_builder{Answers: as}.Build() +} + +// AnswersFromProto extracts the answers delivered on a request. +func AnswersFromProto(msg *v2.SourceCacheLookupAnswers) []Answer { + out := make([]Answer, 0, len(msg.GetAnswers())) + for _, a := range msg.GetAnswers() { + out = append(out, Answer{ + Query: Query{RowKind: RowKind(a.GetRowKind()), ScopeHash: a.GetScopeHash()}, + Found: a.GetFound(), + ETag: a.GetEtag(), + }) + } + return out +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/grpc_lookup.go b/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/grpc_lookup.go new file mode 100644 index 00000000..199ecda9 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/grpc_lookup.go @@ -0,0 +1,54 @@ +package sourcecache + +import ( + "context" + "fmt" + + v1 "github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1" +) + +// GRPCLookup is the connector-side Lookup implementation that talks to +// BatonSourceCacheService on the parent SDK. +// +// This is deliberately not routed through the session store: session data +// passes through the connector's local MemorySessionCache (otter), which +// would apply generic TTL/eviction policies to sync-scoped validator state. +// The dedicated service keeps the path uncached and the message shape +// explicit. +// +// The parent has exactly one active Lookup registered at a time (set per +// sync via SetSourceCache on the server), so the wire format carries no +// sync_id; routing is implicit. +type GRPCLookup struct { + client v1.BatonSourceCacheServiceClient +} + +// NewGRPCLookup returns a Lookup backed by the given client. A nil client +// yields NoopLookup so callers can configure the client optionally without +// nil checks at every call site. +func NewGRPCLookup(client v1.BatonSourceCacheServiceClient) Lookup { + if client == nil { + return NoopLookup{} + } + return &GRPCLookup{client: client} +} + +func (g *GRPCLookup) LookupPreviousSourceCache(ctx context.Context, rowKind RowKind, scopeHash string) (Entry, bool, error) { + if err := ValidateRowKind(rowKind); err != nil { + return Entry{}, false, err + } + if err := ValidateScopeHash(scopeHash); err != nil { + return Entry{}, false, err + } + resp, err := g.client.Lookup(ctx, v1.LookupRequest_builder{ + RowKind: string(rowKind), + ScopeHash: scopeHash, + }.Build()) + if err != nil { + return Entry{}, false, fmt.Errorf("source cache rpc lookup: %w", err) + } + if !resp.GetFound() { + return Entry{}, false, nil + } + return Entry{ETag: resp.GetEtag()}, true, nil +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/grpc_server.go b/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/grpc_server.go new file mode 100644 index 00000000..950aaf7c --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/grpc_server.go @@ -0,0 +1,71 @@ +package sourcecache + +import ( + "context" + "fmt" + "sync/atomic" + + v1 "github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1" +) + +// GRPCServer is the parent-side BatonSourceCacheService implementation. +// +// The parent SDK holds a single GRPCServer for the lifetime of the connector +// subprocess and swaps the active Lookup via SetSourceCache as syncs come +// and go. The syncer installs a real lookup once it has resolved a usable +// previous sync, and clears it when the sync ends so a late RPC can't serve +// from a store the syncer no longer owns. +// +// Until the first SetSourceCache call the server answers every lookup with +// found=false, which the connector treats as "no previous sync" and falls +// back to an unconditional fetch. +type GRPCServer struct { + v1.UnimplementedBatonSourceCacheServiceServer + lookup atomic.Pointer[Lookup] +} + +var _ v1.BatonSourceCacheServiceServer = (*GRPCServer)(nil) +var _ SetLookup = (*GRPCServer)(nil) + +// NewGRPCServer returns a GRPCServer with no active Lookup registered. +func NewGRPCServer() *GRPCServer { + return &GRPCServer{} +} + +// SetSourceCache replaces the active lookup. Safe to call concurrently with +// in-flight RPCs: existing RPCs continue against the value they read at +// entry; new RPCs see the swapped value. +func (s *GRPCServer) SetSourceCache(ctx context.Context, lookup Lookup) { + if lookup == nil { + s.lookup.Store(nil) + return + } + s.lookup.Store(&lookup) +} + +func (s *GRPCServer) Lookup(ctx context.Context, req *v1.LookupRequest) (*v1.LookupResponse, error) { + rowKind := RowKind(req.GetRowKind()) + if err := ValidateRowKind(rowKind); err != nil { + return nil, err + } + scopeHash := req.GetScopeHash() + if err := ValidateScopeHash(scopeHash); err != nil { + return nil, err + } + + lookupPtr := s.lookup.Load() + if lookupPtr == nil { + return v1.LookupResponse_builder{Found: false}.Build(), nil + } + entry, found, err := (*lookupPtr).LookupPreviousSourceCache(ctx, rowKind, scopeHash) + if err != nil { + return nil, fmt.Errorf("source cache lookup: %w", err) + } + if !found { + return v1.LookupResponse_builder{Found: false}.Build(), nil + } + return v1.LookupResponse_builder{ + Found: true, + Etag: entry.ETag, + }.Build(), nil +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/sourcecache.go b/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/sourcecache.go new file mode 100644 index 00000000..1892cda3 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/sourcecache.go @@ -0,0 +1,140 @@ +// Package sourcecache defines the connector-facing surface of source-cache +// replay (see proto/c1/connector/v2/annotation_source_cache.proto). +// +// A connector that can cheaply revalidate upstream data — HTTP conditional +// requests (GitHub), delta queries (Microsoft Graph) — opts in by attaching +// SourceCacheCapability MODE_READ_WRITE to its Validate response. During a +// sync it looks up the previous validator for a scope via the Lookup the SDK +// provides on SyncOpAttrs, revalidates upstream, and either emits fresh rows +// tagged with SourceCacheScope or asks the SDK to replay the previous rows +// with SourceCacheReplay. +// +// The connector owns scope computation; the SDK only keys storage by the +// connector-supplied scope hash. The validator (etag, delta token) is opaque +// to the SDK. +// +// Invariant that keeps replay safe: a connector must only emit +// SourceCacheReplay for a scope whose validator it received from THIS sync's +// Lookup. The lookup need not happen in the same call that emits the +// replay: a planning call may batch-resolve many scopes and pass the +// verdicts to sibling cursors through SpawnCursors page tokens — that +// satisfies the invariant, because the validator still originates from the +// consuming sync. What's forbidden is a validator that outlives a sync +// (connector-side caches, config, upstream echoes). When source cache is +// disabled or degraded (no capability, no usable previous sync, unsupported +// storage engine) the SDK installs NoopLookup, every lookup misses, and a +// well-behaved connector naturally falls back to full fetch. +// +// Replay equivalence: a cached sync must reproduce what a full resync +// would produce. Replayed rows are verbatim copies of the previous sync's +// rows with one deliberate exception — expander-written Sources on direct +// grants (classified by a self-source entry, mirroring RollbackExpansion) +// are stripped at copy time so the current sync's expansion recomputes +// them from true state; re-expansion only adds contributions, so carrying +// them verbatim would immortalize contributions removed upstream. +// Connector-set Sources (no self-source) are public connector data and +// survive replay byte-for-byte. +package sourcecache + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "time" +) + +// RowKind partitions source-cache scopes by the row type they produce. +// It doubles as the row_kind value stored in SourceCacheEntryRecord. +type RowKind string + +const ( + RowKindResources RowKind = "resources" + RowKindEntitlements RowKind = "entitlements" + RowKindGrants RowKind = "grants" +) + +// Valid reports whether k is one of the defined row kinds. +func (k RowKind) Valid() bool { + switch k { + case RowKindResources, RowKindEntitlements, RowKindGrants: + return true + } + return false +} + +// ValidateRowKind returns an error if rowKind is not one of the known +// RowKind* constants. +func ValidateRowKind(rowKind RowKind) error { + if !rowKind.Valid() { + return fmt.Errorf("invalid source cache row kind: %q", rowKind) + } + return nil +} + +// maxScopeHashLen bounds scope identifiers on the wire and in storage +// keys. Deliberately generous: the shape is a connector convention +// (HashScope produces 64 hex chars) and is not enforced beyond +// non-emptiness and this cap while the model is being proven out against +// real providers. +const maxScopeHashLen = 256 + +// ValidateScopeHash returns an error when scopeHash is empty or +// unreasonably long. Connectors conventionally use HashScope, but any +// stable identifier is accepted. +func ValidateScopeHash(scopeHash string) error { + if scopeHash == "" { + return fmt.Errorf("source cache scope hash is required") + } + if len(scopeHash) > maxScopeHashLen { + return fmt.Errorf("source cache scope hash too long: %d bytes (max %d)", len(scopeHash), maxScopeHashLen) + } + return nil +} + +// Entry is a previous sync's persisted validator for one scope. +type Entry struct { + // ETag is the opaque upstream validator: a literal HTTP ETag, a delta + // token, etc. Never interpreted by the SDK. + ETag string + + // DiscoveredAt is when the entry was written. + DiscoveredAt time.Time +} + +// Lookup resolves a scope's previous-sync validator. The SDK provides an +// implementation on SyncOpAttrs; connectors call it before revalidating +// upstream. +type Lookup interface { + // LookupPreviousSourceCache returns the previous sync's entry for + // (rowKind, scopeHash). found=false means no entry: fetch fresh. + // Implementations must treat internal read errors that leave fresh + // fetch available as misses rather than failing the connector call. + LookupPreviousSourceCache(ctx context.Context, rowKind RowKind, scopeHash string) (entry Entry, found bool, err error) +} + +// NoopLookup is the Lookup installed when source cache is disabled or +// degraded. Every lookup misses. +type NoopLookup struct{} + +var _ Lookup = NoopLookup{} + +func (NoopLookup) LookupPreviousSourceCache(context.Context, RowKind, string) (Entry, bool, error) { + return Entry{}, false, nil +} + +// SetLookup is implemented by connector clients/servers that can receive a +// source-cache lookup implementation from the sync runner. The SDK calls +// SetSourceCache(lookup) at the start of each sync and SetSourceCache(nil) +// when the sync ends so a late RPC can't read stale state. +type SetLookup interface { + SetSourceCache(ctx context.Context, lookup Lookup) +} + +// HashScope returns the lowercase-hex sha256 of a canonical scope string. +// Convenience for connectors; any stable identifier is acceptable as a +// scope hash (only non-emptiness and a length cap are enforced). +func HashScope(canonicalScope string) string { + sum := sha256.Sum256([]byte(canonicalScope)) + return hex.EncodeToString(sum[:]) +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/progresslog/progresslog.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/progresslog/progresslog.go index 734ccd54..51be5a36 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sync/progresslog/progresslog.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/progresslog/progresslog.go @@ -38,6 +38,7 @@ type ProgressLog struct { lastEntitlementLog map[string]time.Time grantsProgress map[string]int lastGrantLog map[string]time.Time + grantsCountOnly map[string]bool mu sync.RWMutex l *zap.Logger maxLogFrequency time.Duration @@ -150,6 +151,7 @@ func NewProgressCounts(ctx context.Context, opts ...Option) *ProgressLog { lastEntitlementLog: make(map[string]time.Time), grantsProgress: make(map[string]int), lastGrantLog: make(map[string]time.Time), + grantsCountOnly: make(map[string]bool), l: ctxzap.Extract(ctx), maxLogFrequency: defaultMaxLogFrequency, mu: sync.RWMutex{}, @@ -256,18 +258,46 @@ func (p *ProgressLog) LogEntitlementsProgress(ctx context.Context, resourceType } } +// SetGrantsCountOnly marks a resource type's grant progress as a plain +// count with no resources-covered denominator. Used for type-scoped grant +// enumeration (v2.TypeScopedGrants), where cursors don't map 1:1 to +// resources: the per-cursor accounting would exceed the type's resource +// total and trip the "more grant resources than resources" warning for a +// perfectly healthy sync. Count-only types log synced row counts +// periodically and never compute the ratio. +func (p *ProgressLog) SetGrantsCountOnly(resourceType string) { + p.mu.Lock() + defer p.mu.Unlock() + p.grantsCountOnly[resourceType] = true +} + +// GrantsProgress returns the current grant-coverage counter for a resource +// type. For per-resource types this is "resources covered" and must never +// exceed the type's resource total — spawned sibling cursors +// (v2.SpawnCursors) don't increment it, only the origin action's chain end +// does. Exposed for tests pinning that accounting. +func (p *ProgressLog) GrantsProgress(resourceType string) int { + p.mu.RLock() + defer p.mu.RUnlock() + return p.grantsProgress[resourceType] +} + func (p *ProgressLog) LogGrantsProgress(ctx context.Context, resourceType string) { var grantsProgress, resources int var lastLogTime time.Time + var countOnly bool p.mu.RLock() grantsProgress = p.grantsProgress[resourceType] resources = p.resources[resourceType] lastLogTime = p.lastGrantLog[resourceType] + countOnly = p.grantsCountOnly[resourceType] p.mu.RUnlock() - if resources == 0 { - // if resuming sync, resource counts will be zero, so don't calculate percentage. just log every 10 seconds. + if resources == 0 || countOnly { + // Count-only: either a resumed sync (resource counts are zero) or a + // type-scoped grants type (no meaningful denominator). Log the raw + // synced count every log window; never compute a percentage. if time.Since(lastLogTime) > p.maxLogFrequency { p.l.Info("Syncing grants", zap.String("resource_type_id", resourceType), diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/source_cache.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/source_cache.go new file mode 100644 index 00000000..7ae42348 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/source_cache.go @@ -0,0 +1,354 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +import ( + "context" + "fmt" + stdsync "sync" + + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "go.uber.org/zap" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/sourcecache" +) + +// Source-cache replay, syncer side. See +// proto/c1/connector/v2/annotation_source_cache.proto for the contract. +// +// Setup degrades, replay fails loudly. Any setup problem (capability +// absent, store engine unsupported, no usable previous sync) installs the +// no-op lookup: the connector never sees a previous validator, never gets +// a conditional-request hit, and therefore never emits SourceCacheReplay +// — which is what makes it safe to treat a replay annotation arriving +// while degraded as a hard error (the connector already skipped row +// generation; there is nothing to fall back to). + +// syncerSourceCache is the per-sync source-cache state resolved by +// configureSourceCache. +// +// Write side and read side enable independently: the FIRST sync of a chain +// has no previous sync but must still stamp rows and write manifest +// entries, or the second sync would have nothing to replay. enabled covers +// the write side (capability declared + current store supports it); prev +// is non-nil only when a usable previous sync exists (read side — lookup +// hits and replay). +type syncerSourceCache struct { + enabled bool + // current is the writable output store's source-cache capability. + current dotc1z.SourceCacheStore + // prev is the previous sync's lookup/replay source (read-only). Nil + // when no usable previous sync exists; lookups then miss and replay + // annotations are hard errors. + prev dotc1z.SourceCacheStore + // prevReader is the same store as prev, typed for ReplaySourceCache. + prevReader connectorstore.Reader + // lookup is the connector-facing lookup built from prev (NoopLookup + // when prev is nil). The syncer also uses it to answer lookup asks + // from connectors on single-shot transports (the ask/answer + // continuation); both paths see identical results by construction. + lookup sourcecache.Lookup + // contStats accumulates ask/answer continuation counters for the + // sync-complete log line. + contStats *continuationStats +} + +// prevStoreLookup adapts the previous store's manifest to the +// connector-facing Lookup. Mid-sync read errors are logged once and +// treated as misses: at lookup time the connector can still fetch fresh, +// so degrading beats failing the sync. +type prevStoreLookup struct { + prev dotc1z.SourceCacheStore + logOnce *stdsync.Once +} + +var _ sourcecache.Lookup = prevStoreLookup{} + +func (p prevStoreLookup) LookupPreviousSourceCache(ctx context.Context, kind sourcecache.RowKind, scopeHash string) (sourcecache.Entry, bool, error) { + entry, found, err := p.prev.LookupSourceCacheEntry(ctx, kind, scopeHash) + if err != nil { + p.logOnce.Do(func() { + ctxzap.Extract(ctx).Warn("source cache lookup failed; treating as miss", zap.Error(err)) + }) + return sourcecache.Entry{}, false, nil //nolint:nilerr // intentional: a failed lookup degrades to a miss (connector fetches fresh) rather than failing the connector call + } + return entry, found, nil +} + +// configureSourceCache resolves per-sync source-cache state from the +// connector's Validate response and installs the connector-facing lookup. +// Called once per Sync, after Validate. +func (s *syncer) configureSourceCache(ctx context.Context, resp *v2.ConnectorServiceValidateResponse) error { + l := ctxzap.Extract(ctx) + s.sourceCache = syncerSourceCache{} + + setLookup, canSetLookup := s.connector.(sourcecache.SetLookup) + degrade := func(reason string) error { + if canSetLookup { + setLookup.SetSourceCache(ctx, sourcecache.NoopLookup{}) + } + if reason != "" { + l.Info("source cache disabled", zap.String("reason", reason)) + } + return nil + } + + capability := &v2.SourceCacheCapability{} + annos := annotations.Annotations(resp.GetAnnotations()) + ok, err := annos.Pick(capability) + if err != nil { + return fmt.Errorf("error parsing source cache capability annotation: %w", err) + } + if !ok || capability.GetMode() != v2.SourceCacheCapability_MODE_READ_WRITE { + // The common case; stay quiet. + return degrade("") + } + current, ok := s.store.(dotc1z.SourceCacheStore) + if !ok { + return degrade("storage engine does not support source cache") + } + + // Write side enabled: rows produced under a scope get stamped and + // manifest entries get written, so this sync is usable as the NEXT + // sync's replay source even when this one has nothing to replay from. + s.sourceCache = syncerSourceCache{enabled: true, current: current} + + // Read side: a usable previous sync makes lookups hit and replay legal. + var readReason string + if s.previousSyncReader == nil { + readReason = "no previous-sync c1z configured" + } else if prev, ok := s.previousSyncReader.(dotc1z.SourceCacheStore); !ok { + readReason = "previous-sync store engine does not support source cache" + } else { + s.sourceCache.prev = prev + s.sourceCache.prevReader = s.previousSyncReader + } + + lookup := sourcecache.Lookup(sourcecache.NoopLookup{}) + if s.sourceCache.prev != nil { + lookup = prevStoreLookup{prev: s.sourceCache.prev, logOnce: &stdsync.Once{}} + } + s.sourceCache.lookup = lookup + s.sourceCache.contStats = &continuationStats{} + if canSetLookup { + setLookup.SetSourceCache(ctx, lookup) + } else { + // The connector declared the capability but the client offers no + // way to deliver lookups. Its own lookup stays no-op, so every + // scope misses and no replay annotations can legally arrive. + l.Warn("source cache capability declared but connector client cannot receive lookups") + } + l.Info("source cache enabled", + zap.Bool("replay_available", s.sourceCache.prev != nil), + zap.String("replay_unavailable_reason", readReason), + ) + return nil +} + +// clearSourceCacheLookup detaches the per-sync lookup so a late RPC from +// the connector cannot read a store the syncer no longer owns. +func (s *syncer) clearSourceCacheLookup(ctx context.Context) { + if setLookup, ok := s.connector.(sourcecache.SetLookup); ok { + setLookup.SetSourceCache(ctx, nil) + } +} + +// sourceCachePage carries one list response's source-cache instructions +// from beginSourceCachePage (before rows are written) to +// finishSourceCachePage (after rows are written). +type sourceCachePage struct { + kind sourcecache.RowKind + scopeHash string + etag string + // replayed reports that beginSourceCachePage copied the previous + // sync's rows for this scope into the current sync BEFORE the page's + // own rows commit. Consumers that dedupe against "already synced this + // sync" state (the resources path) must not skip this page's rows: + // they are the overlay, and the already-present row is the stale + // replayed base they exist to overwrite. + replayed bool + // deletedIDs are canonical-id tombstones (grant/entitlement ids, + // resource BIDs); deletedPrincipalIDs are bare-object-id tombstones + // applied scope-relatively. Both may arrive on any page of a scope + // (replay annotation on the first page, scope annotation on every + // page) and apply after the page's rows commit. + deletedIDs []string + deletedPrincipalIDs []string +} + +// beginSourceCachePage inspects a list response's annotations, performs +// any requested replay, and returns the context to write the page's rows +// under (stamped with the scope when one is present). A nil page means the +// response carried no source-cache instructions. +// +// rowCount is the number of rows in the response; a non-overlay replay +// that also returned rows gets a warning (the rows are upserted anyway). +func (s *syncer) beginSourceCachePage( + ctx context.Context, + kind sourcecache.RowKind, + respAnnos annotations.Annotations, + rowCount int, +) (context.Context, *sourceCachePage, error) { + replay := &v2.SourceCacheReplay{} + hasReplay, err := respAnnos.Pick(replay) + if err != nil { + return ctx, nil, fmt.Errorf("source cache: error parsing replay annotation: %w", err) + } + scope := &v2.SourceCacheScope{} + hasScope, err := respAnnos.Pick(scope) + if err != nil { + return ctx, nil, fmt.Errorf("source cache: error parsing scope annotation: %w", err) + } + if !hasReplay && !hasScope { + return ctx, nil, nil + } + + if !s.sourceCache.enabled { + if hasReplay { + // The connector skipped row generation expecting a replay; with + // source cache disabled there is nothing to replay from. This is + // a connector bug (replay for a scope it never got a lookup hit + // on), not a degradable condition. + return ctx, nil, fmt.Errorf("source cache: connector requested replay for scope %q but source cache is disabled", replay.GetScopeHash()) + } + // Scope annotations without the capability handshake are ignored. + return ctx, nil, nil + } + + page := &sourceCachePage{kind: kind} + switch { + case hasReplay && hasScope: + if replay.GetScopeHash() != scope.GetScopeHash() { + return ctx, nil, fmt.Errorf("source cache: replay scope %q and page scope %q disagree", replay.GetScopeHash(), scope.GetScopeHash()) + } + page.scopeHash = replay.GetScopeHash() + case hasReplay: + page.scopeHash = replay.GetScopeHash() + default: + page.scopeHash = scope.GetScopeHash() + } + if err := sourcecache.ValidateScopeHash(page.scopeHash); err != nil { + return ctx, nil, fmt.Errorf("source cache: %w", err) + } + // Prefer the scope annotation's etag (the freshest validator on + // overlay pages); fall back to the replay's. + page.etag = scope.GetEtag() + if page.etag == "" { + page.etag = replay.GetEtag() + } + // Tombstones may ride either annotation — the replay annotation on a + // round's first page, the scope annotation on every page (so a + // multi-page delta round never buffers deletions). + page.deletedIDs = append(replay.GetDeletedIds(), scope.GetDeletedIds()...) + page.deletedPrincipalIDs = append(replay.GetDeletedPrincipalIds(), scope.GetDeletedPrincipalIds()...) + + if hasReplay { + if s.sourceCache.prev == nil { + // Same invariant violation as the disabled case: the connector + // can only have gotten a lookup hit if a previous source exists. + return ctx, nil, fmt.Errorf("source cache: connector requested replay for scope %q but no previous sync is available", replay.GetScopeHash()) + } + page.replayed = true + if !replay.GetOverlay() && rowCount > 0 { + // The contract says a 304-style replay page is empty, but rows + // arriving here are more data, not less — upsert them on top of + // the replayed base (overlay semantics) rather than failing the + // sync. Kept lenient while the model is proven against real + // providers. + ctxzap.Extract(ctx).Warn("source cache: non-overlay replay returned rows; treating them as an overlay", + zap.String("scope_hash", page.scopeHash), + zap.Int("rows", rowCount), + ) + } + // Advisory check: a well-behaved connector only replays a scope + // whose validator came from this sync's lookup, so a missing + // previous manifest entry is suspicious — but not by itself data + // loss (the stamped rows may still exist, e.g. a partially carried + // file). The hard error below is reserved for a replay that + // produces nothing. + _, entryFound, err := s.sourceCache.prev.LookupSourceCacheEntry(ctx, kind, page.scopeHash) + if err != nil { + return ctx, nil, fmt.Errorf("source cache: error reading previous manifest for scope %q: %w", page.scopeHash, err) + } + if !entryFound { + ctxzap.Extract(ctx).Warn("source cache: replay requested for scope with no previous manifest entry", + zap.String("scope_hash", page.scopeHash)) + } + res, err := s.sourceCache.current.ReplaySourceCache(ctx, s.sourceCache.prevReader, kind, page.scopeHash) + if err != nil { + return ctx, nil, fmt.Errorf("source cache: replay for scope %q failed: %w", page.scopeHash, err) + } + if res.Rows == 0 && !entryFound { + // The connector skipped row generation expecting a base that + // does not exist anywhere in the previous file — this sync + // would silently drop the scope's rows. + return ctx, nil, fmt.Errorf("source cache: replay for scope %q found no previous rows and no manifest entry; the connector replayed a scope it never looked up", page.scopeHash) + } + // Replay bypasses the connector-response path that normally arms + // grant expansion (seeing GrantExpandable on returned rows), so a + // sync whose expandable pages all replay would silently skip the + // expansion phase without this. + if kind == sourcecache.RowKindGrants && res.NeedsExpansion && !s.dontExpandGrants { + s.state.SetNeedsExpansion() + } + ctxzap.Extract(ctx).Debug("source cache replayed scope", + zap.String("row_kind", string(kind)), + zap.String("scope_hash", page.scopeHash), + zap.Int64("rows", res.Rows), + zap.Bool("needs_expansion", res.NeedsExpansion), + zap.Int("deleted_ids", len(page.deletedIDs)), + zap.Int("deleted_principal_ids", len(page.deletedPrincipalIDs)), + ) + } + + return sourcecache.WithScope(ctx, page.scopeHash), page, nil +} + +// finishSourceCachePage runs after the page's rows committed: applies +// delta tombstones and, when the page carried a validator, writes the +// current sync's manifest entry. The entry write is last so a failed page +// can never leave a phantom hit for the next sync. +func (s *syncer) finishSourceCachePage(ctx context.Context, page *sourceCachePage) error { + if page == nil { + return nil + } + if len(page.deletedIDs) > 0 { + if page.kind == sourcecache.RowKindGrants { + // Grant-id tombstones resolve within the scope's own rows so + // connector-custom grant-id shapes (unreachable by the global + // bounded delete) work, and the cost stays bounded by the + // scope's size. + deleted, err := s.sourceCache.current.DeleteSourceCacheGrantsByIDInScope(ctx, page.scopeHash, page.deletedIDs) + if err != nil { + return fmt.Errorf("source cache: error applying grant deletions for scope %q: %w", page.scopeHash, err) + } + ctxzap.Extract(ctx).Debug("source cache applied grant-id deletions", + zap.String("scope_hash", page.scopeHash), + zap.Int("tombstones", len(page.deletedIDs)), + zap.Int64("rows_deleted", deleted), + ) + } else if err := s.sourceCache.current.DeleteSourceCacheRows(ctx, page.kind, page.deletedIDs); err != nil { + return fmt.Errorf("source cache: error applying deletions for scope %q: %w", page.scopeHash, err) + } + } + if len(page.deletedPrincipalIDs) > 0 { + deleted, err := s.sourceCache.current.DeleteSourceCacheRowsInScope(ctx, page.kind, page.scopeHash, page.deletedPrincipalIDs) + if err != nil { + return fmt.Errorf("source cache: error applying scoped deletions for scope %q: %w", page.scopeHash, err) + } + ctxzap.Extract(ctx).Debug("source cache applied scoped deletions", + zap.String("row_kind", string(page.kind)), + zap.String("scope_hash", page.scopeHash), + zap.Int("tombstones", len(page.deletedPrincipalIDs)), + zap.Int64("rows_deleted", deleted), + ) + } + if page.etag != "" { + if err := s.sourceCache.current.PutSourceCacheEntry(ctx, page.kind, page.scopeHash, page.etag); err != nil { + return fmt.Errorf("source cache: error writing manifest entry for scope %q: %w", page.scopeHash, err) + } + } + return nil +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/source_cache_continuation.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/source_cache_continuation.go new file mode 100644 index 00000000..76fb7276 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/source_cache_continuation.go @@ -0,0 +1,248 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// Syncer side of the source-cache lookup continuation (ask/answer). On +// single-shot transports (gRPC-over-Lambda) the connector cannot call the +// lookup service mid-request, so it answers a list RPC with a +// SourceCacheLookupAsk instead of rows; the syncer resolves the queries +// against its LOCAL previous-sync store — the same store replay copies +// from, so lookup and replay can never disagree — and re-invokes the same +// request with SourceCacheLookupAnswers attached. See +// docs/tasks/source-cache-lambda-lookup.md and the annotation contract in +// proto/c1/connector/v2/annotation_source_cache.proto. + +import ( + "context" + "fmt" + stdsync "sync" + + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "go.uber.org/zap" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/sourcecache" +) + +const ( + // sourceCacheBounceCap bounds consecutive asks for the SAME request + // (same page token; only the answers annotation differs between + // re-invokes). A connector that keeps asking without progressing is + // broken (most commonly: swallowing ErrLookupDeferred and re-asking + // for scopes it already "handled"), and silence would be the + // stale-data failure mode — so fail loudly. + // + // Deliberately NOT per action: a multi-page action that asks once per + // page (e.g. a delta planner pre-resolving each planning page's chunk + // scopes) bounces once per page with monotonic progress — every + // NextPageToken advance is a new request and resets the counter. + sourceCacheBounceCap = 4 + + // sourceCacheAnswerBudget caps the FOUND-etag payload attached to one + // re-invoke, keeping the request under single-shot transport payload + // limits (Lambda invokes cap at 6MB; the dual-encoded frame at 5MiB). + // Not-found answers are always complete for the queried set — only + // found answers with large etags are dropped, and a dropped answer is + // ABSENT (re-askable, subject to the cap), never a false not-found. + sourceCacheAnswerBudget = 2 << 20 +) + +// continuationStats accumulates ask/answer counters across a sync for the +// sync-complete log line and bounce-cap diagnostics. Per-op-kind bounce +// counts let a rollout review distinguish planner asks (expected, one per +// planning page) from per-row asks (a batching opportunity). +type continuationStats struct { + mu stdsync.Mutex + requests int // RPCs that bounced at least once + bounces int + bouncesByOp map[string]int + asked int + found int + notFound int + truncated int + capFailures int +} + +func (c *continuationStats) record(op string, bounces, asked, found, notFound, truncated int) { + if c == nil { + return + } + c.mu.Lock() + defer c.mu.Unlock() + if bounces > 0 { + c.requests++ + if c.bouncesByOp == nil { + c.bouncesByOp = map[string]int{} + } + c.bouncesByOp[op] += bounces + } + c.bounces += bounces + c.asked += asked + c.found += found + c.notFound += notFound + c.truncated += truncated +} + +func (c *continuationStats) recordCapFailure() { + if c == nil { + return + } + c.mu.Lock() + defer c.mu.Unlock() + c.capFailures++ +} + +// logTotals emits the continuation counters when any bounces happened. +func (c *continuationStats) logTotals(ctx context.Context) { + if c == nil { + return + } + c.mu.Lock() + defer c.mu.Unlock() + if c.bounces == 0 && c.capFailures == 0 { + return + } + fields := []zap.Field{ + zap.Int("requests_bounced", c.requests), + zap.Int("bounces", c.bounces), + zap.Int("scopes_asked", c.asked), + zap.Int("answered_found", c.found), + zap.Int("answered_not_found", c.notFound), + zap.Int("answers_truncated", c.truncated), + zap.Int("bounce_cap_failures", c.capFailures), + } + for op, n := range c.bouncesByOp { + fields = append(fields, zap.Int("bounces_"+op, n)) + } + ctxzap.Extract(ctx).Info("source-cache lookup continuation totals", fields...) +} + +// listAttempt is one list-RPC attempt as observed by the continuation +// loop: enough of the response to detect and validate an ask. +type listAttempt struct { + annos annotations.Annotations + rows int + nextToken string +} + +// withSourceCacheContinuation drives the ask/answer loop around one list +// RPC. issue performs the RPC with extra request annotations (the lookup +// offer, plus accumulated answers on re-invokes) and reports the response +// surface; the loop returns once a response carries no ask — that final +// response is the one the caller processes. +// +// The offer is attached only when the syncer can actually answer (warm +// previous-sync lookup). Old or cold syncers never send it, and a +// compliant connector never asks without it — that pairing is what makes +// version skew degrade to a cold sync instead of a misread response. +func (s *syncer) withSourceCacheContinuation(ctx context.Context, op string, issue func(extra annotations.Annotations) (listAttempt, error)) error { + l := ctxzap.Extract(ctx) + + warm := s.sourceCache.prev != nil + extra := annotations.Annotations{} + if warm { + extra.Update(&v2.SourceCacheLookupOffer{}) + } + + // Answers accumulate across bounces: the connector re-executes from + // scratch each phase, so every re-invoke must carry the union of all + // resolved queries, in first-resolved order (deterministic requests). + var ordered []sourcecache.Answer + seen := map[sourcecache.Query]bool{} + + asked, found, notFound, truncated := 0, 0, 0, 0 + for bounce := 0; ; bounce++ { + attempt, err := issue(extra) + if err != nil { + return err + } + + ask := &v2.SourceCacheLookupAsk{} + hasAsk, err := attempt.annos.Pick(ask) + if err != nil { + return fmt.Errorf("%s: error parsing source-cache lookup ask: %w", op, err) + } + if !hasAsk { + s.sourceCache.contStats.record(op, bounce, asked, found, notFound, truncated) + return nil + } + + // Ask legality. Failing loudly here is deliberate: every branch + // is a connector bug that would otherwise surface as silently + // wrong data or an unexplained stall. + if !warm { + return fmt.Errorf("%s: connector sent a source-cache lookup ask on a request that carried no offer (connector must gate asks on SourceCacheLookupOffer)", op) + } + if attempt.rows > 0 || attempt.nextToken != "" || + attempt.annos.Contains(&v2.SourceCacheScope{}) || attempt.annos.Contains(&v2.SourceCacheReplay{}) || + attempt.annos.Contains(&v2.SpawnCursors{}) { + return fmt.Errorf("%s: source-cache lookup ask response must carry ONLY the ask: "+ + "no rows, no next page token, no scope/replay annotations, no spawned cursors "+ + "(spawn on the re-invoked request's real response instead)", op) + } + if bounce >= sourceCacheBounceCap { + s.sourceCache.contStats.recordCapFailure() + return fmt.Errorf("%s: source-cache lookup bounce cap (%d) exceeded for one request; "+ + "connector kept asking without progressing (%d scopes still unresolved) — "+ + "check for swallowed ErrLookupDeferred or nondeterministic scope computation", + op, sourceCacheBounceCap, len(ask.GetQueries())) + } + + queries, err := sourcecache.QueriesFromProto(ask) + if err != nil { + return fmt.Errorf("%s: invalid source-cache lookup ask: %w", op, err) + } + + budget := sourceCacheAnswerBudget + for _, a := range ordered { + budget -= len(a.ETag) + } + newAsked, newFound, newNotFound, newTruncated := 0, 0, 0, 0 + for _, q := range queries { + if seen[q] { + continue + } + newAsked++ + entry, ok, err := s.sourceCache.lookup.LookupPreviousSourceCache(ctx, q.RowKind, q.ScopeHash) + if err != nil { + return fmt.Errorf("%s: resolving source-cache lookup ask: %w", op, err) + } + if !ok { + seen[q] = true + ordered = append(ordered, sourcecache.Answer{Query: q, Found: false}) + newNotFound++ + continue + } + if len(entry.ETag) > budget { + // Dropped to budget: the query stays ABSENT from the + // answers (re-askable), never a false not-found. + newTruncated++ + continue + } + budget -= len(entry.ETag) + seen[q] = true + ordered = append(ordered, sourcecache.Answer{Query: q, Found: true, ETag: entry.ETag}) + newFound++ + } + asked += newAsked + found += newFound + notFound += newNotFound + truncated += newTruncated + + if newAsked == 0 { + // Every query was already answered on the request the + // connector just saw; re-invoking cannot make progress. + return fmt.Errorf("%s: connector re-asked only already-answered scopes (%d queries); connector lookup handling is broken", op, len(queries)) + } + + extra.Update(sourcecache.AnswersProto(ordered)) + + l.Debug("source-cache lookup bounce", + zap.String("op", op), + zap.Int("bounce", bounce+1), + zap.Int("asked", newAsked), + zap.Int("found", newFound), + zap.Int("not_found", newNotFound), + zap.Int("truncated_to_budget", newTruncated), + ) + } +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/state.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/state.go index cb1d30ca..bcf89bb7 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sync/state.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/state.go @@ -202,6 +202,15 @@ type Action struct { ResourceID string `json:"resource_id,omitempty"` ParentResourceTypeID string `json:"parent_resource_type_id,omitempty"` ParentResourceID string `json:"parent_resource_id,omitempty"` + + // Spawned marks an action enqueued by a connector's SpawnCursors + // annotation (a sibling cursor) rather than by the syncer's own + // planners. Progress accounting uses it: per-resource grant coverage + // counts a resource once, when its ORIGIN action's chain ends — + // spawned siblings (and their NextPage continuations, which inherit + // the action) don't count, or one resource would count N times. + // omitempty keeps old checkpointed sync tokens decoding unchanged. + Spawned bool `json:"spawned,omitempty"` } var _ State = &state{} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go index 205f4297..d64e99a9 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go @@ -20,6 +20,7 @@ import ( "github.com/conductorone/baton-sdk/pkg/bid" "github.com/conductorone/baton-sdk/pkg/dotc1z" "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/sourcecache" "github.com/conductorone/baton-sdk/pkg/sync/expand" "github.com/conductorone/baton-sdk/pkg/types/entitlement" batonGrant "github.com/conductorone/baton-sdk/pkg/types/grant" @@ -135,6 +136,7 @@ type syncer struct { syncID string skipEGForResourceType syncMap[string, bool] skipEntitlementsForResourceType syncMap[string, bool] + typeScopedGrantsForResourceType syncMap[string, bool] skipEntitlementsAndGrants bool skipGrants bool resourceTypeTraits syncMap[string, []v2.ResourceType_Trait] @@ -145,6 +147,7 @@ type syncer struct { workerCount int // If 1, sync is sequential (default). If > 1, sync operations are done in parallel. metricsHandler metrics.Handler syncIdentity uotel.SyncIdentity + sourceCache syncerSourceCache } var _ Syncer = (*syncer)(nil) @@ -562,6 +565,11 @@ func (s *syncer) Sync(ctx context.Context) error { } } + err = s.configureSourceCache(ctx, resp) + if err != nil { + return err + } + syncResourceTypeMap := make(map[string]bool) if len(s.syncResourceTypes) > 0 { for _, rt := range s.syncResourceTypes { @@ -687,6 +695,7 @@ func (s *syncer) Sync(ctx context.Context) error { return err } + s.sourceCache.contStats.logTotals(ctx) l.Info("Sync complete.") _, err = s.connector.Cleanup(ctx, v2.ConnectorServiceCleanupRequest_builder{ @@ -1042,23 +1051,54 @@ func (s *syncer) SyncResources(ctx context.Context, action *Action) error { // owns a span — the duplicate inflated trace span counts without adding // information. func (s *syncer) syncResources(ctx context.Context, action *Action) error { - req := v2.ResourcesServiceListResourcesRequest_builder{ - ResourceTypeId: action.ResourceTypeID, - PageToken: action.PageToken, - ActiveSyncId: s.getActiveSyncID(), - }.Build() - if action.ParentResourceTypeID != "" && action.ParentResourceID != "" { - req.SetParentResourceId(v2.ResourceId_builder{ - ResourceType: action.ParentResourceTypeID, - Resource: action.ParentResourceID, - }.Build()) + var resp *v2.ResourcesServiceListResourcesResponse + err := s.withSourceCacheContinuation(ctx, "sync-resources", func(extra annotations.Annotations) (listAttempt, error) { + req := v2.ResourcesServiceListResourcesRequest_builder{ + ResourceTypeId: action.ResourceTypeID, + PageToken: action.PageToken, + ActiveSyncId: s.getActiveSyncID(), + Annotations: extra, + }.Build() + if action.ParentResourceTypeID != "" && action.ParentResourceID != "" { + req.SetParentResourceId(v2.ResourceId_builder{ + ResourceType: action.ParentResourceTypeID, + Resource: action.ParentResourceID, + }.Build()) + } + r, err := s.connector.ListResources(ctx, req) + if err != nil { + return listAttempt{}, err + } + resp = r + return listAttempt{ + annos: annotations.Annotations(r.GetAnnotations()), + rows: len(r.GetList()), + nextToken: r.GetNextPageToken(), + }, nil + }) + if err != nil { + return err } - resp, err := s.connector.ListResources(ctx, req) + putCtx, scPage, err := s.beginSourceCachePage(ctx, sourcecache.RowKindResources, annotations.Annotations(resp.GetAnnotations()), len(resp.GetList())) if err != nil { return err } + // On any source-cache-scoped page the "already synced this sync" + // dedupe below is wrong — scoped pages are upsert streams whose rows + // are always authoritative: + // - Replayed rounds copy the previous sync's rows in on the round's + // FIRST page, so an overlay row's identity ALWAYS hits the store, + // on every page of the round — but the stored row is the stale + // base and the overlay row is the update that must overwrite it. + // Keying off the replay annotation alone would only protect page + // one (the annotation fires once per round). + // - Cold delta enumerations may legally return the same object + // multiple times (changed mid-walk), later occurrences + // authoritative; deduping would keep the STALE first occurrence. + pageScoped := scPage != nil + bulkPutResoruces := []*v2.Resource{} for _, r := range resp.GetList() { validatedResource := false @@ -1075,7 +1115,7 @@ func (s *syncer) syncResources(ctx context.Context, action *Action) error { validatedResource = true // We must *ALSO* check if we have any child resources. - if !s.hasChildResources(r) { + if !pageScoped && !s.hasChildResources(r) { // Since we only have the resource type IDs of child resources, // we can't tell if we already have synced those child resources. // Those children may also have their own child resources, @@ -1104,12 +1144,16 @@ func (s *syncer) syncResources(ctx context.Context, action *Action) error { } if len(bulkPutResoruces) > 0 { - err = s.store.PutResources(ctx, bulkPutResoruces...) + err = s.store.PutResources(putCtx, bulkPutResoruces...) if err != nil { return err } } + if err := s.finishSourceCachePage(ctx, scPage); err != nil { + return err + } + s.handleProgress(ctx, action, len(resp.GetList())) s.counts.AddResources(action.ResourceTypeID, len(resp.GetList())) if resp.GetNextPageToken() == "" { @@ -1317,22 +1361,45 @@ func (s *syncer) syncEntitlementsForResource(ctx context.Context, action *Action resource := resourceResponse.GetResource() - resp, err := s.connector.ListEntitlements(ctx, v2.EntitlementsServiceListEntitlementsRequest_builder{ - Resource: resource, - PageToken: action.PageToken, - ActiveSyncId: s.getActiveSyncID(), - }.Build()) + var resp *v2.EntitlementsServiceListEntitlementsResponse + err = s.withSourceCacheContinuation(ctx, "sync-entitlements", func(extra annotations.Annotations) (listAttempt, error) { + r, err := s.connector.ListEntitlements(ctx, v2.EntitlementsServiceListEntitlementsRequest_builder{ + Resource: resource, + PageToken: action.PageToken, + ActiveSyncId: s.getActiveSyncID(), + Annotations: extra, + }.Build()) + if err != nil { + return listAttempt{}, err + } + resp = r + return listAttempt{ + annos: annotations.Annotations(r.GetAnnotations()), + rows: len(r.GetList()), + nextToken: r.GetNextPageToken(), + }, nil + }) if err != nil { return err } if err := s.validateEntitlementExclusionGroups(resp.GetList()); err != nil { return err } - err = s.store.PutEntitlements(ctx, resp.GetList()...) + + putCtx, scPage, err := s.beginSourceCachePage(ctx, sourcecache.RowKindEntitlements, annotations.Annotations(resp.GetAnnotations()), len(resp.GetList())) if err != nil { return err } + err = s.store.PutEntitlements(putCtx, resp.GetList()...) + if err != nil { + return err + } + + if err := s.finishSourceCachePage(ctx, scPage); err != nil { + return err + } + s.handleProgress(ctx, action, len(resp.GetList())) if resp.GetNextPageToken() == "" { s.counts.AddEntitlementsProgress(resourceID.ResourceType, 1) @@ -1758,6 +1825,9 @@ func (s *syncer) fixEntitlementGraphCycles(ctx context.Context, graph *expand.En // SyncGrants fetches the grants for each resource from the connector. It iterates each resource // from the datastore, and pushes a new action to sync the grants for each individual resource. +// Resource types annotated with TypeScopedGrants are excluded from the per-resource fan-out and +// get a single type-scoped action instead (empty ResourceID); the connector enumerates the whole +// type, optionally spawning additional cursors via the SpawnCursors annotation. func (s *syncer) SyncGrants(ctx context.Context, action *Action) error { ctx, span := uotel.StartWithLink(ctx, tracer, "syncer.SyncGrants") uotel.SetSyncIdentityAttrs(ctx, span) @@ -1765,9 +1835,20 @@ func (s *syncer) SyncGrants(ctx context.Context, action *Action) error { defer func() { uotel.EndSpanWithError(span, err) }() if action.ResourceTypeID == "" && action.ResourceID == "" { + actions := make([]Action, 0) if action.PageToken == "" { ctxzap.Extract(ctx).Info("Syncing grants...") s.handleInitialActionForStep(ctx, *action) + + // One type-scoped action per annotated resource type, enqueued + // exactly once (the planner's first page). + typeScoped, err := s.typeScopedGrantsResourceTypes(ctx) + if err != nil { + return fmt.Errorf("sync-grants: error listing type-scoped resource types: %w", err) + } + for _, rtID := range typeScoped { + actions = append(actions, Action{Op: SyncGrantsOp, ResourceTypeID: rtID}) + } } resp, err := s.store.ListResources(ctx, v2.ResourcesServiceListResourcesRequest_builder{ @@ -1778,16 +1859,25 @@ func (s *syncer) SyncGrants(ctx context.Context, action *Action) error { return fmt.Errorf("sync-grants: error listing resources: %w", err) } - actions := make([]Action, 0) for _, r := range resp.GetList() { shouldSkip, err := s.shouldSkipGrants(ctx, r) if err != nil { return err } - if shouldSkip { continue } + + // Types with type-scoped grants are excluded from the + // per-resource fan-out; their single action is enqueued above. + typeScoped, err := s.resourceTypeHasTypeScopedGrants(ctx, r.GetId().GetResourceType()) + if err != nil { + return err + } + if typeScoped { + continue + } + actions = append(actions, Action{Op: SyncGrantsOp, ResourceID: r.GetId().GetResource(), ResourceTypeID: r.GetId().GetResourceType()}) } @@ -1801,27 +1891,107 @@ func (s *syncer) SyncGrants(ctx context.Context, action *Action) error { return nil } +// resourceTypeHasTypeScopedGrants reports (cached per sync) whether the +// resource type carries the TypeScopedGrants annotation. +func (s *syncer) resourceTypeHasTypeScopedGrants(ctx context.Context, resourceTypeID string) (bool, error) { + if v, ok := s.typeScopedGrantsForResourceType.Load(resourceTypeID); ok { + return v, nil + } + rt, err := s.store.GetResourceType(ctx, reader_v2.ResourceTypesReaderServiceGetResourceTypeRequest_builder{ + ResourceTypeId: resourceTypeID, + }.Build()) + if err != nil { + return false, err + } + rtAnnos := annotations.Annotations(rt.GetResourceType().GetAnnotations()) + typeScoped := rtAnnos.Contains(&v2.TypeScopedGrants{}) + s.typeScopedGrantsForResourceType.Store(resourceTypeID, typeScoped) + return typeScoped, nil +} + +// typeScopedGrantsResourceTypes lists every synced resource type annotated +// with TypeScopedGrants. +func (s *syncer) typeScopedGrantsResourceTypes(ctx context.Context) ([]string, error) { + var out []string + pageToken := "" + for { + resp, err := s.store.ListResourceTypes(ctx, v2.ResourceTypesServiceListResourceTypesRequest_builder{ + PageToken: pageToken, + }.Build()) + if err != nil { + return nil, err + } + for _, rt := range resp.GetList() { + rtAnnos := annotations.Annotations(rt.GetAnnotations()) + typeScoped := rtAnnos.Contains(&v2.TypeScopedGrants{}) + s.typeScopedGrantsForResourceType.Store(rt.GetId(), typeScoped) + if typeScoped { + out = append(out, rt.GetId()) + } + } + pageToken = resp.GetNextPageToken() + if pageToken == "" { + return out, nil + } + } +} + // syncGrantsForResource fetches the grants for a specific resource from the connector. +// An action with an empty ResourceID is a TYPE-SCOPED grants cursor: the connector +// enumerates grants for the whole resource type (no single resource backs the call), +// and may spawn sibling cursors via the SpawnCursors response annotation. // No span here: only call site is SyncGrants, which already owns a span. func (s *syncer) syncGrantsForResource(ctx context.Context, action *Action) error { + typeScoped := action.ResourceID == "" resourceID := v2.ResourceId_builder{ ResourceType: action.ResourceTypeID, Resource: action.ResourceID, }.Build() - resourceResponse, err := s.store.GetResource(ctx, reader_v2.ResourcesReaderServiceGetResourceRequest_builder{ - ResourceId: resourceID, - }.Build()) - if err != nil { - return fmt.Errorf("sync-grants-for-resource: error getting resource: %w", err) - } - resource := resourceResponse.GetResource() + var resource *v2.Resource + var reqAnnos annotations.Annotations + if typeScoped { + // Wire validation requires a non-empty resource id, so the stub is + // self-referential ({type, type}) and the request carries the + // TypeScopedGrants annotation as the routing marker. + resource = v2.Resource_builder{ + Id: v2.ResourceId_builder{ + ResourceType: action.ResourceTypeID, + Resource: action.ResourceTypeID, + }.Build(), + }.Build() + reqAnnos.Update(&v2.TypeScopedGrants{}) + } else { + resourceResponse, err := s.store.GetResource(ctx, reader_v2.ResourcesReaderServiceGetResourceRequest_builder{ + ResourceId: resourceID, + }.Build()) + if err != nil { + return fmt.Errorf("sync-grants-for-resource: error getting resource: %w", err) + } + resource = resourceResponse.GetResource() + } - resp, err := s.connector.ListGrants(ctx, v2.GrantsServiceListGrantsRequest_builder{ - Resource: resource, - PageToken: action.PageToken, - ActiveSyncId: s.getActiveSyncID(), - }.Build()) + var resp *v2.GrantsServiceListGrantsResponse + err := s.withSourceCacheContinuation(ctx, "sync-grants-for-resource", func(extra annotations.Annotations) (listAttempt, error) { + annos := make(annotations.Annotations, 0, len(reqAnnos)+len(extra)) + annos = append(annos, reqAnnos...) + annos = append(annos, extra...) + r, err := s.connector.ListGrants(ctx, v2.GrantsServiceListGrantsRequest_builder{ + Resource: resource, + PageToken: action.PageToken, + ActiveSyncId: s.getActiveSyncID(), + Annotations: annos, + }.Build()) + if err != nil { + return listAttempt{}, err + } + resp = r + return listAttempt{ + annos: annotations.Annotations(r.GetAnnotations()), + rows: len(r.GetList()), + nextToken: r.GetNextPageToken(), + }, nil + }) if err != nil { return fmt.Errorf("sync-grants-for-resource: error listing grants: %w", err) } @@ -1833,6 +2003,15 @@ func (s *syncer) syncGrantsForResource(ctx context.Context, action *Action) erro respAnnos := annotations.Annotations(resp.GetAnnotations()) insertResourceGrants := respAnnos.Contains(&v2.InsertResourceGrants{}) + // Source-cache replay/stamping for this grants page. putCtx applies + // ONLY to the PutGrants call below — the related-resource PutResources + // writes in this function are resource rows and must not inherit a + // grants-scope stamp. + putCtx, scPage, err := s.beginSourceCachePage(ctx, sourcecache.RowKindGrants, respAnnos, len(grants)) + if err != nil { + return fmt.Errorf("sync-grants-for-resource: %w", err) + } + // Stamp InsertResourceGrants per-grant so the slim-blob writer's // gate sees it. The annotation is response-level, but the writer // needs it per-row to avoid stripping the Resource this path @@ -1915,19 +2094,79 @@ func (s *syncer) syncGrantsForResource(ctx context.Context, action *Action) erro } } - err = s.store.PutGrants(ctx, grants...) + err = s.store.PutGrants(putCtx, grants...) if err != nil { return fmt.Errorf("sync-grants-for-resource: error putting grants: %w", err) } + if err := s.finishSourceCachePage(ctx, scPage); err != nil { + return fmt.Errorf("sync-grants-for-resource: %w", err) + } + s.handleProgress(ctx, action, len(grants)) - if resp.GetNextPageToken() == "" { + if typeScoped { + // Cursors don't map 1:1 to resources (one cursor can cover many + // groups and may also emit cross-type grants), so the per-resource + // "N of M resources covered" accounting is meaningless here and + // would trip the "more grant resources than resources" warning on + // healthy syncs. Count raw grant rows instead. + s.counts.SetGrantsCountOnly(resourceID.GetResourceType()) + s.counts.AddGrantsProgress(resourceID.GetResourceType(), len(grants)) + s.counts.LogGrantsProgress(ctx, resourceID.GetResourceType()) + } else if resp.GetNextPageToken() == "" && !action.Spawned { + // A resource counts as covered exactly once: when its ORIGIN + // action's page chain ends. Spawned sibling cursors for the same + // resource also end with an empty token but must not count, or a + // resource with N spawned pages would count N times and trip the + // progress anomaly warning on healthy syncs. s.counts.AddGrantsProgress(resourceID.GetResourceType(), 1) s.counts.LogGrantsProgress(ctx, resourceID.GetResourceType()) } - return s.nextPageOrFinishAction(ctx, action, resp.GetNextPageToken()) + // SpawnCursors: the response may enqueue sibling cursors — each runs + // as its own action, scheduled by the worker pool, rate-limited, and + // checkpointed like any other pagination. + // + // - Type-scoped calls: one cursor per connector-defined shard (e.g. + // 50-id delta chunks); actions carry only the resource type. + // - Per-resource calls: parallel page fan-out for page-numbered APIs + // (the connector knows every page URL from page one, so a warm + // sync can revalidate all pages concurrently instead of serially); + // actions carry the resource identity. + // + // Spawned cursors are ORDINARY pages that happen to be enqueued + // eagerly: the SDK assumes nothing about replay — a spawned page may + // hit its lookup and replay, miss and fetch cold (page boundary + // shifted since last sync), and may chain further via NextPageToken. + spawn := &v2.SpawnCursors{} + hasSpawn, err := respAnnos.Pick(spawn) + if err != nil { + return fmt.Errorf("sync-grants-for-resource: error parsing spawn-cursors annotation: %w", err) + } + var spawned []Action + if hasSpawn { + for _, tok := range spawn.GetPageTokens() { + if tok == "" { + continue + } + spawned = append(spawned, Action{ + Op: SyncGrantsOp, + ResourceTypeID: action.ResourceTypeID, + ResourceID: action.ResourceID, // empty on type-scoped actions + PageToken: tok, + Spawned: true, + }) + } + l.Debug("sync-grants-for-resource: spawned sibling grant cursors", + zap.String("resource_type_id", action.ResourceTypeID), + zap.String("resource_id", action.ResourceID), + zap.Bool("type_scoped", typeScoped), + zap.Int("cursors", len(spawned)), + zap.Int64("estimated_total", spawn.GetEstimatedTotal())) + } + + return s.nextPageOrFinishAction(ctx, action, resp.GetNextPageToken(), spawned...) } func (s *syncer) SyncExternalResources(ctx context.Context, action *Action) error { @@ -2735,6 +2974,10 @@ func (s *syncer) Close(ctx context.Context) error { var errs []error + // Detach the source-cache lookup before the stores go away so a late + // connector RPC can't read a store the syncer no longer owns. + s.clearSourceCacheLookup(ctx) + var storeCloseErr error if s.store != nil { storeCloseErr = s.store.Close(finalizeCtx) diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/bucket_plans.go b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/bucket_plans.go index 5b5b1d94..2cd44d8e 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/bucket_plans.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/bucket_plans.go @@ -60,6 +60,34 @@ func buildBucketPlans() []bucketPlan { lower: enginepkg.GrantByNeedsExpansionLowerBound(), upper: enginepkg.GrantByNeedsExpansionUpperBound(), }, + // Source-cache state MUST fold with its rows. Excising these + // ranges alongside the record buckets replaces the base sync's + // manifest/index entries with the applied sync's: scopes the + // applied sync touched keep a consistent (entry, rows) pair, and + // scopes it never touched become clean lookup misses. Leaving + // them out would strand the BASE sync's validators and index + // entries on top of the applied sync's rows — a stale manifest + // hit could then replay rows that no longer belong to the scope. + { + name: "grant_by_source_scope", + lower: enginepkg.GrantBySourceScopeLowerBound(), + upper: enginepkg.GrantBySourceScopeUpperBound(), + }, + { + name: "entitlement_by_source_scope", + lower: enginepkg.EntitlementBySourceScopeLowerBound(), + upper: enginepkg.EntitlementBySourceScopeUpperBound(), + }, + { + name: "resource_by_source_scope", + lower: enginepkg.ResourceBySourceScopeLowerBound(), + upper: enginepkg.ResourceBySourceScopeUpperBound(), + }, + { + name: "source_cache_entry", + lower: enginepkg.SourceCacheEntryLowerBound(), + upper: enginepkg.SourceCacheEntryUpperBound(), + }, { name: "asset", lower: enginepkg.AssetLowerBound(), diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/tasks/local/syncer.go b/vendor/github.com/conductorone/baton-sdk/pkg/tasks/local/syncer.go index 5d469ca5..916b166d 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/tasks/local/syncer.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/tasks/local/syncer.go @@ -24,6 +24,7 @@ type localSyncer struct { o sync.Once tmpDir string externalResourceC1Z string + previousSyncC1Z string externalResourceEntitlementIdFilter string targetedSyncResources []*v2.Resource skipEntitlementsAndGrants bool @@ -47,6 +48,12 @@ func WithExternalResourceC1Z(externalResourceC1Z string) Option { } } +func WithPreviousSyncC1Z(previousSyncC1Z string) Option { + return func(m *localSyncer) { + m.previousSyncC1Z = previousSyncC1Z + } +} + func WithExternalResourceEntitlementIdFilter(entitlementId string) Option { return func(m *localSyncer) { m.externalResourceEntitlementIdFilter = entitlementId @@ -122,6 +129,7 @@ func (m *localSyncer) Process(ctx context.Context, task *v1.Task, cc types.Conne sdkSync.WithC1ZPath(m.dbPath), sdkSync.WithTmpDir(m.tmpDir), sdkSync.WithExternalResourceC1ZPath(m.externalResourceC1Z), + sdkSync.WithPreviousSyncC1ZPath(m.previousSyncC1Z), sdkSync.WithExternalResourceEntitlementIdFilter(m.externalResourceEntitlementIdFilter), sdkSync.WithTargetedSyncResources(m.targetedSyncResources), sdkSync.WithSkipEntitlementsAndGrants(m.skipEntitlementsAndGrants), diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/types/resource/resource.go b/vendor/github.com/conductorone/baton-sdk/pkg/types/resource/resource.go index 16585ed3..49a01e0f 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/types/resource/resource.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/types/resource/resource.go @@ -8,6 +8,7 @@ import ( v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" "github.com/conductorone/baton-sdk/pkg/annotations" "github.com/conductorone/baton-sdk/pkg/pagination" + "github.com/conductorone/baton-sdk/pkg/sourcecache" "github.com/conductorone/baton-sdk/pkg/types/sessions" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -524,9 +525,14 @@ func NewManagedDeviceResource( } type SyncOpAttrs struct { - Session sessions.SessionStore - SyncID string - PageToken pagination.Token + Session sessions.SessionStore + // SourceCache resolves a scope's previous-sync validator (etag / + // delta token) for source-cache replay. Never nil for framework-built + // connectors: when source cache is disabled or degraded the SDK + // supplies sourcecache.NoopLookup, which always misses. + SourceCache sourcecache.Lookup + SyncID string + PageToken pagination.Token } type SyncOpResults struct { diff --git a/vendor/modules.txt b/vendor/modules.txt index 48afa10c..ffcf1ce8 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -273,7 +273,7 @@ github.com/cockroachdb/swiss # github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 ## explicit; go 1.19 github.com/cockroachdb/tokenbucket -# github.com/conductorone/baton-sdk v0.18.2 +# github.com/conductorone/baton-sdk v0.18.2 => ../baton-sdk-2 ## explicit; go 1.25.2 github.com/conductorone/baton-sdk/internal/connector github.com/conductorone/baton-sdk/pb/c1/c1z/v1 @@ -294,6 +294,7 @@ github.com/conductorone/baton-sdk/pkg/bid github.com/conductorone/baton-sdk/pkg/cli github.com/conductorone/baton-sdk/pkg/config github.com/conductorone/baton-sdk/pkg/connectorbuilder +github.com/conductorone/baton-sdk/pkg/connectorclient github.com/conductorone/baton-sdk/pkg/connectorrunner github.com/conductorone/baton-sdk/pkg/connectorstore github.com/conductorone/baton-sdk/pkg/crypto @@ -317,6 +318,7 @@ github.com/conductorone/baton-sdk/pkg/ratelimit github.com/conductorone/baton-sdk/pkg/retry github.com/conductorone/baton-sdk/pkg/sdk github.com/conductorone/baton-sdk/pkg/session +github.com/conductorone/baton-sdk/pkg/sourcecache github.com/conductorone/baton-sdk/pkg/sync github.com/conductorone/baton-sdk/pkg/sync/expand github.com/conductorone/baton-sdk/pkg/sync/expand/scc @@ -1053,3 +1055,4 @@ modernc.org/memory modernc.org/sqlite modernc.org/sqlite/lib modernc.org/sqlite/vtab +# github.com/conductorone/baton-sdk => ../baton-sdk-2