From c8644152b72a2a8716263312ce3c6d15510e7309 Mon Sep 17 00:00:00 2001 From: alexpederneschi Date: Thu, 9 Jul 2026 17:55:59 +0900 Subject: [PATCH 01/20] feat: add easm integration --- .../itds/easm.mia-platform.eu.domains.yaml | 100 ++++++++++++++++ .../itds/easm.mia-platform.eu.endpoints.yaml | 65 ++++++++++ .../easm/itds/easm.mia-platform.eu.hosts.yaml | 51 ++++++++ .../easm/itds/easm.mia-platform.eu.ips.yaml | 59 +++++++++ .../easm.mia-platform.eu.vulnerabilities.yaml | 84 +++++++++++++ docs/examples/easm/mappings/domains.yaml | 15 +++ docs/examples/easm/mappings/endpoints.yaml | 13 ++ docs/examples/easm/mappings/hosts.yaml | 14 +++ docs/examples/easm/mappings/ips.yaml | 13 ++ .../easm/mappings/vulnerabilities.yaml | 20 ++++ internal/cmd/common.go | 6 + internal/source/easm/client.go | 112 ++++++++++++++++++ internal/source/easm/config.go | 68 +++++++++++ internal/source/easm/doc.go | 7 ++ internal/source/easm/source.go | 68 +++++++++++ internal/source/easm/sync.go | 87 ++++++++++++++ 16 files changed, 782 insertions(+) create mode 100644 docs/examples/easm/itds/easm.mia-platform.eu.domains.yaml create mode 100644 docs/examples/easm/itds/easm.mia-platform.eu.endpoints.yaml create mode 100644 docs/examples/easm/itds/easm.mia-platform.eu.hosts.yaml create mode 100644 docs/examples/easm/itds/easm.mia-platform.eu.ips.yaml create mode 100644 docs/examples/easm/itds/easm.mia-platform.eu.vulnerabilities.yaml create mode 100644 docs/examples/easm/mappings/domains.yaml create mode 100644 docs/examples/easm/mappings/endpoints.yaml create mode 100644 docs/examples/easm/mappings/hosts.yaml create mode 100644 docs/examples/easm/mappings/ips.yaml create mode 100644 docs/examples/easm/mappings/vulnerabilities.yaml create mode 100644 internal/source/easm/client.go create mode 100644 internal/source/easm/config.go create mode 100644 internal/source/easm/doc.go create mode 100644 internal/source/easm/source.go create mode 100644 internal/source/easm/sync.go diff --git a/docs/examples/easm/itds/easm.mia-platform.eu.domains.yaml b/docs/examples/easm/itds/easm.mia-platform.eu.domains.yaml new file mode 100644 index 0000000..aba7070 --- /dev/null +++ b/docs/examples/easm/itds/easm.mia-platform.eu.domains.yaml @@ -0,0 +1,100 @@ +# yaml-language-server: $schema=https://cdn.mia-platform.eu/catalog/v1alpha1/mia-platform.eu.v1alpha1.ItemTypeDefinition.json +--- +apiVersion: mia-platform.eu/v1alpha1 +kind: ItemTypeDefinition +metadata: + name: domains.easm.mia-platform.eu +spec: + group: easm.mia-platform.eu + names: + kind: Domain + plural: domains + displayPlural: Domains + scope: Organization + versions: + - deprecated: false + name: v1alpha1 + schema: + openAPIV31Schema: + type: object + properties: + spec: + type: object + properties: + domain: + description: The registered domain name. + type: string + title: Domain + dns: + description: DNS records found for the domain. + type: object + title: DNS + properties: + dmarc: + type: string + title: DMARC + spf: + type: string + title: SPF + mx: + type: array + title: MX + items: + type: string + ns: + type: array + title: NS + items: + type: string + whois: + description: The raw WHOIS record for the domain (irregular structure, stored as-is). + type: object + title: WHOIS + additionalProperties: true + zonetransfer: + description: AXFR zone-transfer lines ("name\ttype\trdata"); absent when blocked. + type: array + title: Zone Transfer + items: + type: string + azure: + description: Azure tenant attribution for the domain (absent when skipped or nothing found). + type: object + title: Azure + properties: + tenant_id: + type: string + title: Tenant ID + namespace_type: + type: string + title: Namespace Type + brand: + type: string + title: Brand + federation_url: + type: string + title: Federation URL + m365: + type: boolean + title: Microsoft 365 + admin_consent_exposed: + type: boolean + title: Admin Consent Exposed + misconfig: + description: Cloud service misconfigurations discovered for the domain. + type: array + title: Misconfigurations + items: + type: object + properties: + service: + type: string + title: Service + url: + type: string + title: URL + status: + type: integer + title: Status + served: true + storage: true diff --git a/docs/examples/easm/itds/easm.mia-platform.eu.endpoints.yaml b/docs/examples/easm/itds/easm.mia-platform.eu.endpoints.yaml new file mode 100644 index 0000000..a06b05b --- /dev/null +++ b/docs/examples/easm/itds/easm.mia-platform.eu.endpoints.yaml @@ -0,0 +1,65 @@ +# yaml-language-server: $schema=https://cdn.mia-platform.eu/catalog/v1alpha1/mia-platform.eu.v1alpha1.ItemTypeDefinition.json +--- +apiVersion: mia-platform.eu/v1alpha1 +kind: ItemTypeDefinition +metadata: + name: endpoints.easm.mia-platform.eu +spec: + group: easm.mia-platform.eu + names: + kind: Endpoint + plural: endpoints + displayPlural: Endpoints + scope: Organization + versions: + - deprecated: false + name: v1alpha1 + schema: + openAPIV31Schema: + type: object + properties: + spec: + type: object + properties: + host: + description: The host the endpoint belongs to. + type: string + title: Host + port: + description: The port the endpoint listens on. + type: integer + title: Port + http: + description: HTTP response details for the endpoint (absent when no HTTP answer). + type: object + title: HTTP + properties: + url: + type: string + title: URL + scheme: + type: string + title: Scheme + status_codes: + type: array + title: Status Codes + items: + type: integer + content_length: + type: integer + title: Content Length + title: + type: string + title: Title + technologies: + type: array + title: Technologies + items: + type: string + tls: + description: TLS report for the endpoint (success shape or an error; stored as-is). + type: object + title: TLS + additionalProperties: true + served: true + storage: true diff --git a/docs/examples/easm/itds/easm.mia-platform.eu.hosts.yaml b/docs/examples/easm/itds/easm.mia-platform.eu.hosts.yaml new file mode 100644 index 0000000..afa81a7 --- /dev/null +++ b/docs/examples/easm/itds/easm.mia-platform.eu.hosts.yaml @@ -0,0 +1,51 @@ +# yaml-language-server: $schema=https://cdn.mia-platform.eu/catalog/v1alpha1/mia-platform.eu.v1alpha1.ItemTypeDefinition.json +--- +apiVersion: mia-platform.eu/v1alpha1 +kind: ItemTypeDefinition +metadata: + name: hosts.easm.mia-platform.eu +spec: + group: easm.mia-platform.eu + names: + kind: Host + plural: hosts + displayPlural: Hosts + scope: Organization + versions: + - deprecated: false + name: v1alpha1 + schema: + openAPIV31Schema: + type: object + properties: + spec: + type: object + properties: + fqdn: + description: The fully qualified domain name of the host. + type: string + title: FQDN + status_code: + description: The DNS response status observed for the host. + type: string + title: Status Code + a: + description: The IPv4 (A) records the host resolves to. + type: array + title: A Records + items: + type: string + aaaa: + description: The IPv6 (AAAA) records the host resolves to. + type: array + title: AAAA Records + items: + type: string + cname: + description: The CNAME records for the host. + type: array + title: CNAME Records + items: + type: string + served: true + storage: true diff --git a/docs/examples/easm/itds/easm.mia-platform.eu.ips.yaml b/docs/examples/easm/itds/easm.mia-platform.eu.ips.yaml new file mode 100644 index 0000000..757b916 --- /dev/null +++ b/docs/examples/easm/itds/easm.mia-platform.eu.ips.yaml @@ -0,0 +1,59 @@ +# yaml-language-server: $schema=https://cdn.mia-platform.eu/catalog/v1alpha1/mia-platform.eu.v1alpha1.ItemTypeDefinition.json +--- +apiVersion: mia-platform.eu/v1alpha1 +kind: ItemTypeDefinition +metadata: + name: ips.easm.mia-platform.eu +spec: + group: easm.mia-platform.eu + names: + kind: Ip + plural: ips + displayPlural: IPs + scope: Organization + versions: + - deprecated: false + name: v1alpha1 + schema: + openAPIV31Schema: + type: object + properties: + spec: + type: object + properties: + ip: + description: The IP address. + type: string + title: IP + ptr: + description: The PTR (reverse DNS) record for the IP. + type: string + title: PTR + ports: + description: The open ports discovered on the IP. + type: array + title: Ports + items: + type: integer + geo: + description: Geographic and network attribution for the IP. + type: object + title: Geo + properties: + provider: + type: string + title: Provider + org: + type: string + title: Organization + asn: + type: string + title: ASN + region: + type: string + title: Region + country: + type: string + title: Country + served: true + storage: true diff --git a/docs/examples/easm/itds/easm.mia-platform.eu.vulnerabilities.yaml b/docs/examples/easm/itds/easm.mia-platform.eu.vulnerabilities.yaml new file mode 100644 index 0000000..5f405e5 --- /dev/null +++ b/docs/examples/easm/itds/easm.mia-platform.eu.vulnerabilities.yaml @@ -0,0 +1,84 @@ +# yaml-language-server: $schema=https://cdn.mia-platform.eu/catalog/v1alpha1/mia-platform.eu.v1alpha1.ItemTypeDefinition.json +--- +apiVersion: mia-platform.eu/v1alpha1 +kind: ItemTypeDefinition +metadata: + name: vulnerabilities.easm.mia-platform.eu +spec: + group: easm.mia-platform.eu + names: + kind: Vulnerability + plural: vulnerabilities + displayPlural: Vulnerabilities + scope: Organization + versions: + - deprecated: false + name: v1alpha1 + schema: + openAPIV31Schema: + type: object + properties: + spec: + type: object + properties: + template_id: + description: Identifier of the detection rule that produced the finding. + type: string + title: Template ID + name: + description: The name of the finding. + type: string + title: Name + severity: + description: The severity of the finding (info, low, medium, high, critical). + type: string + title: Severity + target: + description: The target the finding was matched on. + type: string + title: Target + description: + description: A human-readable description of the finding. + type: string + title: Description + tags: + description: The tags associated with the finding. + type: array + title: Tags + items: + type: string + extracted_results: + description: Values extracted by the detection rule (when any). + type: array + title: Extracted Results + items: + type: string + matcher_name: + description: Name of the matcher that triggered the finding. + type: string + title: Matcher Name + scheme: + description: The scheme the finding was observed on. + type: string + title: Scheme + meta: + description: Additional rule-specific metadata for the finding (free-form). + type: object + title: Meta + additionalProperties: true + owasp: + description: The OWASP Top 10 category associated with the finding. + type: object + title: OWASP + properties: + code: + type: string + title: Code + name: + type: string + title: Name + url: + type: string + title: URL + served: true + storage: true diff --git a/docs/examples/easm/mappings/domains.yaml b/docs/examples/easm/mappings/domains.yaml new file mode 100644 index 0000000..e610cda --- /dev/null +++ b/docs/examples/easm/mappings/domains.yaml @@ -0,0 +1,15 @@ +apiVersion: easm.mia-platform.eu/v1alpha1 +itemFamily: domains +type: domain +syncable: true +mappings: + identifier: "{{ .id | sha256sum }}" + metadata: + title: "{{ get \"domain\" . .id | quote }}" + spec: + domain: "{{ get \"domain\" . \"\" | quote }}" + dns: "{{ get \"dns\" . (object) | toJSON }}" + whois: "{{ get \"whois\" . (object) | toJSON }}" + zonetransfer: "{{ get \"zonetransfer\" . (list) | toJSON }}" + azure: "{{ get \"azure\" . (object) | toJSON }}" + misconfig: "{{ get \"misconfig\" . (list) | toJSON }}" diff --git a/docs/examples/easm/mappings/endpoints.yaml b/docs/examples/easm/mappings/endpoints.yaml new file mode 100644 index 0000000..f128411 --- /dev/null +++ b/docs/examples/easm/mappings/endpoints.yaml @@ -0,0 +1,13 @@ +apiVersion: easm.mia-platform.eu/v1alpha1 +itemFamily: endpoints +type: endpoint +syncable: true +mappings: + identifier: "{{ .id | sha256sum }}" + metadata: + title: "{{ .id | quote }}" + spec: + host: "{{ get \"host\" . \"\" | quote }}" + port: "{{ get \"port\" . 0 }}" + http: "{{ get \"http\" . (object) | toJSON }}" + tls: "{{ get \"tls\" . (object) | toJSON }}" diff --git a/docs/examples/easm/mappings/hosts.yaml b/docs/examples/easm/mappings/hosts.yaml new file mode 100644 index 0000000..2b20fc2 --- /dev/null +++ b/docs/examples/easm/mappings/hosts.yaml @@ -0,0 +1,14 @@ +apiVersion: easm.mia-platform.eu/v1alpha1 +itemFamily: hosts +type: host +syncable: true +mappings: + identifier: "{{ .id | sha256sum }}" + metadata: + title: "{{ get \"fqdn\" . .id | quote }}" + spec: + fqdn: "{{ get \"fqdn\" . \"\" | quote }}" + status_code: "{{ get \"status_code\" . \"\" | quote }}" + a: "{{ get \"a\" . (list) | toJSON }}" + aaaa: "{{ get \"aaaa\" . (list) | toJSON }}" + cname: "{{ get \"cname\" . (list) | toJSON }}" diff --git a/docs/examples/easm/mappings/ips.yaml b/docs/examples/easm/mappings/ips.yaml new file mode 100644 index 0000000..b2551db --- /dev/null +++ b/docs/examples/easm/mappings/ips.yaml @@ -0,0 +1,13 @@ +apiVersion: easm.mia-platform.eu/v1alpha1 +itemFamily: ips +type: ip +syncable: true +mappings: + identifier: "{{ .id | sha256sum }}" + metadata: + title: "{{ get \"ip\" . .id | quote }}" + spec: + ip: "{{ get \"ip\" . \"\" | quote }}" + ptr: "{{ get \"ptr\" . \"\" | quote }}" + ports: "{{ get \"ports\" . (list) | toJSON }}" + geo: "{{ get \"geo\" . (object) | toJSON }}" diff --git a/docs/examples/easm/mappings/vulnerabilities.yaml b/docs/examples/easm/mappings/vulnerabilities.yaml new file mode 100644 index 0000000..febce34 --- /dev/null +++ b/docs/examples/easm/mappings/vulnerabilities.yaml @@ -0,0 +1,20 @@ +apiVersion: easm.mia-platform.eu/v1alpha1 +itemFamily: vulnerabilities +type: vulnerability +syncable: true +mappings: + identifier: "{{ .id | sha256sum }}" + metadata: + title: "{{ get \"name\" . .id | quote }}" + spec: + template_id: "{{ get \"template_id\" . \"\" | quote }}" + name: "{{ get \"name\" . \"\" | quote }}" + severity: "{{ get \"severity\" . \"\" | quote }}" + target: "{{ get \"target\" . \"\" | quote }}" + description: "{{ get \"description\" . \"\" | quote }}" + tags: "{{ get \"tags\" . (list) | toJSON }}" + extracted_results: "{{ get \"extracted_results\" . (list) | toJSON }}" + matcher_name: "{{ get \"matcher_name\" . \"\" | quote }}" + scheme: "{{ get \"scheme\" . \"\" | quote }}" + meta: "{{ get \"meta\" . (object) | toJSON }}" + owasp: "{{ get \"owasp\" . (object) | toJSON }}" diff --git a/internal/cmd/common.go b/internal/cmd/common.go index b0a1fb6..51f02c0 100644 --- a/internal/cmd/common.go +++ b/internal/cmd/common.go @@ -20,6 +20,7 @@ import ( azuredevops "github.com/mia-platform/ibdm/internal/source/azure-devops" "github.com/mia-platform/ibdm/internal/source/bitbucket" "github.com/mia-platform/ibdm/internal/source/console" + "github.com/mia-platform/ibdm/internal/source/easm" "github.com/mia-platform/ibdm/internal/source/gcp" "github.com/mia-platform/ibdm/internal/source/github" "github.com/mia-platform/ibdm/internal/source/gitlab" @@ -36,6 +37,8 @@ const ( bitbucketDescription = "Bitbucket integration" consoleSource = "console" consoleDescription = "Mia Platform Console integration" + easmSource = "easm" + easmDescription = "External Attack Surface Management (EASM) integration" gcpSource = "gcp" gcpDescription = "Google Cloud Platform integration" githubSource = "github" @@ -70,6 +73,7 @@ var ( azureSource: azureDescription, bitbucketSource: bitbucketDescription, consoleSource: consoleDescription, + easmSource: easmDescription, gcpSource: gcpDescription, githubSource: githubDescription, gitlabSource: gitlabDescription, @@ -127,6 +131,8 @@ func sourceFromIntegrationName(integrationName string) (any, error) { return github.NewSource() case consoleSource: return console.NewSource() + case easmSource: + return easm.NewSource() case gitlabSource: return gitlab.NewSource() case nexusSource: diff --git a/internal/source/easm/client.go b/internal/source/easm/client.go new file mode 100644 index 0000000..7ab8e08 --- /dev/null +++ b/internal/source/easm/client.go @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: AGPL-3.0-only or Commercial + +package easm + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" +) + +const ( + // maxErrorBodySize limits how many bytes we read from error response bodies + // to avoid unbounded memory allocation on unexpectedly large payloads. + maxErrorBodySize = 1024 + + // nextCursorHeader carries the cursor for the next page; empty or absent on the last page. + nextCursorHeader = "X-Next-Cursor" + // cursorQueryParam names the query parameter used to request a specific page. + cursorQueryParam = "cursor" +) + +// client wraps an HTTP client with EASM endpoint configuration. +type client struct { + baseURL *url.URL + dataPath string + customer string + token string + + httpClient *http.Client +} + +// newClient creates a client from the given config. +func newClient(cfg config) (*client, error) { + u, err := url.Parse(cfg.BaseURL) + if err != nil { + return nil, fmt.Errorf("%w: invalid EASM_BASE_URL: %w", ErrInvalidEnvVariable, err) + } + + return &client{ + baseURL: u, + dataPath: cfg.DataPath, + customer: cfg.Customer, + token: cfg.Token, + httpClient: &http.Client{ + Timeout: cfg.HTTPTimeout, + }, + }, nil +} + +// dataPage is a single page of the /data response: a flat list of items plus +// the cursor for the next page (empty when this is the last page). +type dataPage struct { + items []map[string]any + nextCursor string +} + +// fetchDataPage retrieves a single page of items from the endpoint. Pass an +// empty cursor for the first page. "Latest completed run" is resolved +// server-side; the client never sees or picks a run id. +func (c *client) fetchDataPage(ctx context.Context, cursor string) (*dataPage, error) { + u := c.baseURL.JoinPath(c.dataPath) + if cursor != "" { + q := u.Query() + q.Set(cursorQueryParam, cursor) + u.RawQuery = q.Encode() + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return nil, err + } + + req.Header.Set("Accept", "application/json") + // Customer scoping travels as a credential. Prod sends a bearer token; the + // no-auth mock sends X-Customer. The request is otherwise identical. + if c.token != "" { + req.Header.Set("Authorization", "Bearer "+c.token) + } + if c.customer != "" { + req.Header.Set("X-Customer", c.customer) + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, readErrorResponse(resp) + } + + var items []map[string]any + if err := json.NewDecoder(resp.Body).Decode(&items); err != nil { + return nil, fmt.Errorf("failed to decode data response: %w", err) + } + + return &dataPage{ + items: items, + nextCursor: resp.Header.Get(nextCursorHeader), + }, nil +} + +// readErrorResponse constructs an error from a non-2xx HTTP response, +// reading at most maxErrorBodySize bytes from the body. +func readErrorResponse(resp *http.Response) error { + body, _ := io.ReadAll(io.LimitReader(resp.Body, maxErrorBodySize)) + return fmt.Errorf("easm API returned status %d: %s", resp.StatusCode, string(body)) +} diff --git a/internal/source/easm/config.go b/internal/source/easm/config.go new file mode 100644 index 0000000..2abaaee --- /dev/null +++ b/internal/source/easm/config.go @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: AGPL-3.0-only or Commercial + +package easm + +import ( + "errors" + "fmt" + "strings" + "time" + + "github.com/caarlos0/env/v11" +) + +var ( + // ErrMissingEnvVariable reports missing mandatory environment variables. + ErrMissingEnvVariable = errors.New("missing environment variable") + // ErrInvalidEnvVariable reports malformed environment variable values. + ErrInvalidEnvVariable = errors.New("invalid environment value") +) + +// config holds the environment-driven EASM settings. +type config struct { + // BaseURL is the FE backend base URL, e.g. http://localhost:8000 (mock) or the product backend (prod). + BaseURL string `env:"EASM_BASE_URL"` + // DataPath is the path of the read endpoint appended to BaseURL. + DataPath string `env:"EASM_DATA_PATH" envDefault:"/data"` + // Customer scopes the request via the X-Customer header (mock / no-auth topology). + Customer string `env:"EASM_CUSTOMER"` + // Token scopes the request via the Authorization: Bearer header (prod topology). + Token string `env:"EASM_TOKEN"` + // HTTPTimeout bounds each request to the endpoint. + HTTPTimeout time.Duration `env:"EASM_HTTP_TIMEOUT" envDefault:"30s"` +} + +// loadConfigFromEnv parses environment variables into a config struct and +// validates that all required fields are present. +func loadConfigFromEnv() (config, error) { + cfg, err := env.ParseAs[config]() + if err != nil { + return config{}, err + } + + if err := validateConfig(cfg); err != nil { + return config{}, err + } + + return cfg, nil +} + +// validateConfig checks that the required config fields are non-empty. Customer +// scoping must travel as a credential, so at least one of Customer or Token is +// required; the endpoint resolves the customer from whichever is provided. +func validateConfig(cfg config) error { + missing := make([]string, 0) + + if cfg.BaseURL == "" { + missing = append(missing, "EASM_BASE_URL") + } + if cfg.Customer == "" && cfg.Token == "" { + missing = append(missing, "EASM_CUSTOMER or EASM_TOKEN") + } + + if len(missing) > 0 { + return fmt.Errorf("%w: %s", ErrMissingEnvVariable, strings.Join(missing, ", ")) + } + + return nil +} diff --git a/internal/source/easm/doc.go b/internal/source/easm/doc.go new file mode 100644 index 0000000..a1d08eb --- /dev/null +++ b/internal/source/easm/doc.go @@ -0,0 +1,7 @@ +// SPDX-License-Identifier: AGPL-3.0-only or Commercial + +// Package easm provides a source implementation that integrates EASM scan +// results into the Catalog. It reads the customer's latest completed run from +// the product backend's /data endpoint as a single paginated list of items, +// each tagged with a "type" discriminator, and emits one source.Data per item. +package easm diff --git a/internal/source/easm/source.go b/internal/source/easm/source.go new file mode 100644 index 0000000..f14a281 --- /dev/null +++ b/internal/source/easm/source.go @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: AGPL-3.0-only or Commercial + +package easm + +import ( + "errors" + "sync" + "time" + + "github.com/mia-platform/ibdm/internal/source" +) + +const ( + loggerName = "ibdm:source:easm" + + domainType = "domain" + hostType = "host" + ipType = "ip" + endpointType = "endpoint" + vulnerabilityType = "vulnerability" +) + +// knownTypes is the set of item types the source can emit. The endpoint tags +// each item with one of these; anything else is skipped. +var knownTypes = map[string]struct{}{ + domainType: {}, + hostType: {}, + ipType: {}, + endpointType: {}, + vulnerabilityType: {}, +} + +var ( + // ErrEASMSource wraps errors emitted by the EASM source implementation. + ErrEASMSource = errors.New("easm source") + + // timeSource is a replaceable function for obtaining the current time. + // Tests override this to produce deterministic timestamps. + timeSource = time.Now +) + +var _ source.SyncableSource = &Source{} + +// Source implements source.SyncableSource for our EASM scan results. +type Source struct { + config config + client *client + + syncLock sync.Mutex +} + +// NewSource creates a new EASM Source reading configuration from environment variables. +func NewSource() (*Source, error) { + cfg, err := loadConfigFromEnv() + if err != nil { + return nil, handleErr(err) + } + + c, err := newClient(cfg) + if err != nil { + return nil, handleErr(err) + } + + return &Source{ + config: cfg, + client: c, + }, nil +} diff --git a/internal/source/easm/sync.go b/internal/source/easm/sync.go new file mode 100644 index 0000000..48bcd00 --- /dev/null +++ b/internal/source/easm/sync.go @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: AGPL-3.0-only or Commercial + +package easm + +import ( + "context" + "errors" + "fmt" + + "github.com/mia-platform/ibdm/internal/logger" + "github.com/mia-platform/ibdm/internal/source" +) + +// StartSyncProcess implements source.SyncableSource. It reads the customer's +// latest completed run from the /data endpoint as a single cursor-paginated +// list and emits one source.Data per item, routed by the item's own "type" +// field. Filtering by typesToSync lets the pipeline sync a subset of types. +func (s *Source) StartSyncProcess(ctx context.Context, typesToSync map[string]source.Extra, results chan<- source.Data) error { + log := logger.FromContext(ctx).WithName(loggerName) + + if !s.syncLock.TryLock() { + log.Debug("sync process already running") + return nil + } + defer s.syncLock.Unlock() + + // Log unknown requested types. + for typeKey := range typesToSync { + if _, ok := knownTypes[typeKey]; !ok { + log.Debug("unknown type requested, skipping", "type", typeKey) + } + } + + cursor := "" + for { + if err := ctx.Err(); err != nil { + return nil + } + + log.Trace("fetching data page", "cursor", cursor) + + page, err := s.client.fetchDataPage(ctx, cursor) + if err != nil { + return handleErr(err) + } + + for _, item := range page.items { + itemType, ok := item["type"].(string) + if !ok || itemType == "" { + log.Debug("item without a type, skipping", "id", item["id"]) + continue + } + + if _, requested := typesToSync[itemType]; !requested { + continue + } + + results <- source.Data{ + Type: itemType, + Operation: source.DataOperationUpsert, + Values: item, + Time: timeSource(), + } + } + + if page.nextCursor == "" { + break + } + cursor = page.nextCursor + } + + return nil +} + +// handleErr wraps non-nil errors with ErrEASMSource, matching the project convention. +// Context cancellation errors are silently swallowed (return nil). +func handleErr(err error) error { + if err == nil { + return nil + } + + if errors.Is(err, context.Canceled) { + return nil + } + + return fmt.Errorf("%w: %w", ErrEASMSource, err) +} From 73d13ac194c5c7211bc61692a8e3e13a81f7f80f Mon Sep 17 00:00:00 2001 From: alexpederneschi Date: Fri, 24 Jul 2026 14:12:28 +0900 Subject: [PATCH 02/20] fix: require EASM_CUSTOMER and update comments --- internal/source/easm/client.go | 5 +++-- internal/source/easm/config.go | 15 +++++++++------ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/internal/source/easm/client.go b/internal/source/easm/client.go index 7ab8e08..15acad0 100644 --- a/internal/source/easm/client.go +++ b/internal/source/easm/client.go @@ -74,8 +74,9 @@ func (c *client) fetchDataPage(ctx context.Context, cursor string) (*dataPage, e } req.Header.Set("Accept", "application/json") - // Customer scoping travels as a credential. Prod sends a bearer token; the - // no-auth mock sends X-Customer. The request is otherwise identical. + // X-Customer scopes the request to a single customer and is always set. + // The bearer token authenticates the caller and is sent only once + // configured — the backend has no auth yet. if c.token != "" { req.Header.Set("Authorization", "Bearer "+c.token) } diff --git a/internal/source/easm/config.go b/internal/source/easm/config.go index 2abaaee..b8a3893 100644 --- a/internal/source/easm/config.go +++ b/internal/source/easm/config.go @@ -24,9 +24,11 @@ type config struct { BaseURL string `env:"EASM_BASE_URL"` // DataPath is the path of the read endpoint appended to BaseURL. DataPath string `env:"EASM_DATA_PATH" envDefault:"/data"` - // Customer scopes the request via the X-Customer header (mock / no-auth topology). + // Customer scopes the request to a single customer via the X-Customer + // header. Always required: it selects whose scan results to read. Customer string `env:"EASM_CUSTOMER"` - // Token scopes the request via the Authorization: Bearer header (prod topology). + // Token authenticates the caller to the backend via Authorization: Bearer. + // Optional for now — the backend has no auth yet; set it once auth lands. Token string `env:"EASM_TOKEN"` // HTTPTimeout bounds each request to the endpoint. HTTPTimeout time.Duration `env:"EASM_HTTP_TIMEOUT" envDefault:"30s"` @@ -48,16 +50,17 @@ func loadConfigFromEnv() (config, error) { } // validateConfig checks that the required config fields are non-empty. Customer -// scoping must travel as a credential, so at least one of Customer or Token is -// required; the endpoint resolves the customer from whichever is provided. +// is always required: it scopes the request to a single customer via the +// X-Customer header. Token is optional for now (the backend has no auth yet) +// and, once set, authenticates the caller via Authorization: Bearer. func validateConfig(cfg config) error { missing := make([]string, 0) if cfg.BaseURL == "" { missing = append(missing, "EASM_BASE_URL") } - if cfg.Customer == "" && cfg.Token == "" { - missing = append(missing, "EASM_CUSTOMER or EASM_TOKEN") + if cfg.Customer == "" { + missing = append(missing, "EASM_CUSTOMER") } if len(missing) > 0 { From 37061e285859fbebcb12ee1ded644e31852390c8 Mon Sep 17 00:00:00 2001 From: alexpederneschi Date: Tue, 18 Aug 2026 16:33:15 +0900 Subject: [PATCH 03/20] add easm source tests and how-to docs, restructure ITDs/mappings per review --- .../easm/how-to/010_easm-source.md | 97 +++++++ .../easm/itds/domains.easm.unguess.io.yaml} | 8 +- .../easm/itds/endpoints.easm.unguess.io.yaml} | 8 +- .../easm/itds/hosts.easm.unguess.io.yaml} | 8 +- .../easm/itds/ips.easm.unguess.io.yaml} | 8 +- .../vulnerabilities.easm.unguess.io.yaml} | 8 +- .../easm/mappings/domains.yaml | 2 +- .../easm/mappings/endpoints.yaml | 2 +- .../easm/mappings/hosts.yaml | 2 +- .../easm/mappings/ips.yaml | 2 +- .../easm/mappings/vulnerabilities.yaml | 2 +- internal/source/easm/client.go | 1 + internal/source/easm/client_test.go | 208 +++++++++++++++ internal/source/easm/config.go | 1 + internal/source/easm/config_test.go | 132 ++++++++++ internal/source/easm/doc.go | 1 + internal/source/easm/helper_test.go | 69 +++++ internal/source/easm/source.go | 1 + internal/source/easm/source_test.go | 68 +++++ internal/source/easm/sync.go | 1 + internal/source/easm/sync_test.go | 247 ++++++++++++++++++ 21 files changed, 856 insertions(+), 20 deletions(-) create mode 100644 docs/external-sources/easm/how-to/010_easm-source.md rename docs/{examples/easm/itds/easm.mia-platform.eu.domains.yaml => external-sources/easm/itds/domains.easm.unguess.io.yaml} (96%) rename docs/{examples/easm/itds/easm.mia-platform.eu.endpoints.yaml => external-sources/easm/itds/endpoints.easm.unguess.io.yaml} (94%) rename docs/{examples/easm/itds/easm.mia-platform.eu.hosts.yaml => external-sources/easm/itds/hosts.easm.unguess.io.yaml} (92%) rename docs/{examples/easm/itds/easm.mia-platform.eu.ips.yaml => external-sources/easm/itds/ips.easm.unguess.io.yaml} (93%) rename docs/{examples/easm/itds/easm.mia-platform.eu.vulnerabilities.yaml => external-sources/easm/itds/vulnerabilities.easm.unguess.io.yaml} (95%) rename docs/{examples => external-sources}/easm/mappings/domains.yaml (92%) rename docs/{examples => external-sources}/easm/mappings/endpoints.yaml (88%) rename docs/{examples => external-sources}/easm/mappings/hosts.yaml (90%) rename docs/{examples => external-sources}/easm/mappings/ips.yaml (88%) rename docs/{examples => external-sources}/easm/mappings/vulnerabilities.yaml (94%) create mode 100644 internal/source/easm/client_test.go create mode 100644 internal/source/easm/config_test.go create mode 100644 internal/source/easm/helper_test.go create mode 100644 internal/source/easm/source_test.go create mode 100644 internal/source/easm/sync_test.go diff --git a/docs/external-sources/easm/how-to/010_easm-source.md b/docs/external-sources/easm/how-to/010_easm-source.md new file mode 100644 index 0000000..53ea9fc --- /dev/null +++ b/docs/external-sources/easm/how-to/010_easm-source.md @@ -0,0 +1,97 @@ +# External Attack Surface Management (EASM) Integration + +The EASM Integration of `ibdm` connects to the EASM backend via its REST API and reads a +customer's latest completed scan run. It supports pull-based sync only. + +## Commands + +### Sync + +```sh +ibdm sync easm --mapping-file +``` + +Performs a one-off synchronisation: fetches the customer's latest completed run from the EASM +`/data` endpoint as a single cursor-paginated list, emits one item per record routed by the +record's own `type` field, and exits. + +## Configuration + +All configuration is read from environment variables. + +### Environment Variables + +| Env Variable | Required | Default | Description | +| --- | --- | --- | --- | +| `EASM_BASE_URL` | Yes | _(empty)_ | Base URL of the EASM backend (e.g. `https://easm.example.com`). | +| `EASM_CUSTOMER` | Yes | _(empty)_ | Customer identifier. Scopes the request to a single customer via the `X-Customer` header — it selects whose scan results to read. | +| `EASM_TOKEN` | No | _(empty)_ | Bearer token authenticating the caller. Sent as `Authorization: Bearer ` when set. Optional for now — the backend has no auth yet; set it once auth lands. | +| `EASM_DATA_PATH` | No | `/data` | Path of the read endpoint appended to `EASM_BASE_URL`. | +| `EASM_HTTP_TIMEOUT` | No | `30s` | Timeout for each HTTP request, parsed as a Go `time.Duration`. | + +## Supported Data Types + +| Type | Sync | +| --- | --- | +| `domain` | ✅ | +| `endpoint` | ✅ | +| `host` | ✅ | +| `ip` | ✅ | +| `vulnerability` | ✅ | + +The endpoint tags each record with one of these types; records with a missing or empty `type` +are skipped. Each emitted item carries the record's fields unchanged, and the pipeline can restrict +a run to a subset of types via the mapping files — only the requested types are emitted. + +### `domain` + +One entry per discovered domain, including its DNS, WHOIS, zone-transfer, Azure, and +misconfiguration data. + +### `endpoint` + +One entry per discovered endpoint. + +### `host` + +One entry per discovered host. + +### `ip` + +One entry per discovered IP address. + +### `vulnerability` + +One entry per discovered vulnerability. + +## Authentication + +The source scopes every request to a single customer with the `X-Customer` header, taken from +`EASM_CUSTOMER` — this is always sent. + +When `EASM_TOKEN` is set, the source authenticates the caller with an +`Authorization: Bearer ` header. The backend has no authentication yet, so the token is +optional for now; set it once auth lands. + +## Example Mapping Files + +Example mapping files are provided in the `docs/external-sources/easm/mappings/` directory: + +- `domains.yaml` — maps domain records to Catalog items. +- `endpoints.yaml` — maps endpoint records to Catalog items. +- `hosts.yaml` — maps host records to Catalog items. +- `ips.yaml` — maps IP records to Catalog items. +- `vulnerabilities.yaml` — maps vulnerability records to Catalog items. + +These files can be used as a starting point for your own mapping configuration. Pass the file +or the folder to the `--mapping-file` flag: + +```sh +ibdm sync easm --mapping-file docs/external-sources/easm/mappings/ +``` + +For local development and debugging, add the `--local-output` flag to send results to stdout: + +```sh +ibdm sync easm --mapping-file docs/external-sources/easm/mappings/ --local-output +``` diff --git a/docs/examples/easm/itds/easm.mia-platform.eu.domains.yaml b/docs/external-sources/easm/itds/domains.easm.unguess.io.yaml similarity index 96% rename from docs/examples/easm/itds/easm.mia-platform.eu.domains.yaml rename to docs/external-sources/easm/itds/domains.easm.unguess.io.yaml index aba7070..2b9c982 100644 --- a/docs/examples/easm/itds/easm.mia-platform.eu.domains.yaml +++ b/docs/external-sources/easm/itds/domains.easm.unguess.io.yaml @@ -3,17 +3,19 @@ apiVersion: mia-platform.eu/v1alpha1 kind: ItemTypeDefinition metadata: - name: domains.easm.mia-platform.eu + name: domains.easm.unguess.io spec: - group: easm.mia-platform.eu + group: easm.unguess.io names: kind: Domain plural: domains + singular: domain displayPlural: Domains + displaySingular: Domain scope: Organization versions: - deprecated: false - name: v1alpha1 + name: v1 schema: openAPIV31Schema: type: object diff --git a/docs/examples/easm/itds/easm.mia-platform.eu.endpoints.yaml b/docs/external-sources/easm/itds/endpoints.easm.unguess.io.yaml similarity index 94% rename from docs/examples/easm/itds/easm.mia-platform.eu.endpoints.yaml rename to docs/external-sources/easm/itds/endpoints.easm.unguess.io.yaml index a06b05b..e67a14f 100644 --- a/docs/examples/easm/itds/easm.mia-platform.eu.endpoints.yaml +++ b/docs/external-sources/easm/itds/endpoints.easm.unguess.io.yaml @@ -3,17 +3,19 @@ apiVersion: mia-platform.eu/v1alpha1 kind: ItemTypeDefinition metadata: - name: endpoints.easm.mia-platform.eu + name: endpoints.easm.unguess.io spec: - group: easm.mia-platform.eu + group: easm.unguess.io names: kind: Endpoint plural: endpoints + singular: endpoint displayPlural: Endpoints + displaySingular: Endpoint scope: Organization versions: - deprecated: false - name: v1alpha1 + name: v1 schema: openAPIV31Schema: type: object diff --git a/docs/examples/easm/itds/easm.mia-platform.eu.hosts.yaml b/docs/external-sources/easm/itds/hosts.easm.unguess.io.yaml similarity index 92% rename from docs/examples/easm/itds/easm.mia-platform.eu.hosts.yaml rename to docs/external-sources/easm/itds/hosts.easm.unguess.io.yaml index afa81a7..5392c89 100644 --- a/docs/examples/easm/itds/easm.mia-platform.eu.hosts.yaml +++ b/docs/external-sources/easm/itds/hosts.easm.unguess.io.yaml @@ -3,17 +3,19 @@ apiVersion: mia-platform.eu/v1alpha1 kind: ItemTypeDefinition metadata: - name: hosts.easm.mia-platform.eu + name: hosts.easm.unguess.io spec: - group: easm.mia-platform.eu + group: easm.unguess.io names: kind: Host plural: hosts + singular: host displayPlural: Hosts + displaySingular: Host scope: Organization versions: - deprecated: false - name: v1alpha1 + name: v1 schema: openAPIV31Schema: type: object diff --git a/docs/examples/easm/itds/easm.mia-platform.eu.ips.yaml b/docs/external-sources/easm/itds/ips.easm.unguess.io.yaml similarity index 93% rename from docs/examples/easm/itds/easm.mia-platform.eu.ips.yaml rename to docs/external-sources/easm/itds/ips.easm.unguess.io.yaml index 757b916..a0fc2a6 100644 --- a/docs/examples/easm/itds/easm.mia-platform.eu.ips.yaml +++ b/docs/external-sources/easm/itds/ips.easm.unguess.io.yaml @@ -3,17 +3,19 @@ apiVersion: mia-platform.eu/v1alpha1 kind: ItemTypeDefinition metadata: - name: ips.easm.mia-platform.eu + name: ips.easm.unguess.io spec: - group: easm.mia-platform.eu + group: easm.unguess.io names: kind: Ip plural: ips + singular: ip displayPlural: IPs + displaySingular: IP scope: Organization versions: - deprecated: false - name: v1alpha1 + name: v1 schema: openAPIV31Schema: type: object diff --git a/docs/examples/easm/itds/easm.mia-platform.eu.vulnerabilities.yaml b/docs/external-sources/easm/itds/vulnerabilities.easm.unguess.io.yaml similarity index 95% rename from docs/examples/easm/itds/easm.mia-platform.eu.vulnerabilities.yaml rename to docs/external-sources/easm/itds/vulnerabilities.easm.unguess.io.yaml index 5f405e5..464e8cd 100644 --- a/docs/examples/easm/itds/easm.mia-platform.eu.vulnerabilities.yaml +++ b/docs/external-sources/easm/itds/vulnerabilities.easm.unguess.io.yaml @@ -3,17 +3,19 @@ apiVersion: mia-platform.eu/v1alpha1 kind: ItemTypeDefinition metadata: - name: vulnerabilities.easm.mia-platform.eu + name: vulnerabilities.easm.unguess.io spec: - group: easm.mia-platform.eu + group: easm.unguess.io names: kind: Vulnerability plural: vulnerabilities + singular: vulnerability displayPlural: Vulnerabilities + displaySingular: Vulnerability scope: Organization versions: - deprecated: false - name: v1alpha1 + name: v1 schema: openAPIV31Schema: type: object diff --git a/docs/examples/easm/mappings/domains.yaml b/docs/external-sources/easm/mappings/domains.yaml similarity index 92% rename from docs/examples/easm/mappings/domains.yaml rename to docs/external-sources/easm/mappings/domains.yaml index e610cda..cd35361 100644 --- a/docs/examples/easm/mappings/domains.yaml +++ b/docs/external-sources/easm/mappings/domains.yaml @@ -1,4 +1,4 @@ -apiVersion: easm.mia-platform.eu/v1alpha1 +apiVersion: easm.unguess.io/v1 itemFamily: domains type: domain syncable: true diff --git a/docs/examples/easm/mappings/endpoints.yaml b/docs/external-sources/easm/mappings/endpoints.yaml similarity index 88% rename from docs/examples/easm/mappings/endpoints.yaml rename to docs/external-sources/easm/mappings/endpoints.yaml index f128411..a413793 100644 --- a/docs/examples/easm/mappings/endpoints.yaml +++ b/docs/external-sources/easm/mappings/endpoints.yaml @@ -1,4 +1,4 @@ -apiVersion: easm.mia-platform.eu/v1alpha1 +apiVersion: easm.unguess.io/v1 itemFamily: endpoints type: endpoint syncable: true diff --git a/docs/examples/easm/mappings/hosts.yaml b/docs/external-sources/easm/mappings/hosts.yaml similarity index 90% rename from docs/examples/easm/mappings/hosts.yaml rename to docs/external-sources/easm/mappings/hosts.yaml index 2b20fc2..c6bbed5 100644 --- a/docs/examples/easm/mappings/hosts.yaml +++ b/docs/external-sources/easm/mappings/hosts.yaml @@ -1,4 +1,4 @@ -apiVersion: easm.mia-platform.eu/v1alpha1 +apiVersion: easm.unguess.io/v1 itemFamily: hosts type: host syncable: true diff --git a/docs/examples/easm/mappings/ips.yaml b/docs/external-sources/easm/mappings/ips.yaml similarity index 88% rename from docs/examples/easm/mappings/ips.yaml rename to docs/external-sources/easm/mappings/ips.yaml index b2551db..e04e6e2 100644 --- a/docs/examples/easm/mappings/ips.yaml +++ b/docs/external-sources/easm/mappings/ips.yaml @@ -1,4 +1,4 @@ -apiVersion: easm.mia-platform.eu/v1alpha1 +apiVersion: easm.unguess.io/v1 itemFamily: ips type: ip syncable: true diff --git a/docs/examples/easm/mappings/vulnerabilities.yaml b/docs/external-sources/easm/mappings/vulnerabilities.yaml similarity index 94% rename from docs/examples/easm/mappings/vulnerabilities.yaml rename to docs/external-sources/easm/mappings/vulnerabilities.yaml index febce34..fd21fe7 100644 --- a/docs/examples/easm/mappings/vulnerabilities.yaml +++ b/docs/external-sources/easm/mappings/vulnerabilities.yaml @@ -1,4 +1,4 @@ -apiVersion: easm.mia-platform.eu/v1alpha1 +apiVersion: easm.unguess.io/v1 itemFamily: vulnerabilities type: vulnerability syncable: true diff --git a/internal/source/easm/client.go b/internal/source/easm/client.go index 15acad0..ab105d8 100644 --- a/internal/source/easm/client.go +++ b/internal/source/easm/client.go @@ -1,3 +1,4 @@ +// Copyright Mia srl // SPDX-License-Identifier: AGPL-3.0-only or Commercial package easm diff --git a/internal/source/easm/client_test.go b/internal/source/easm/client_test.go new file mode 100644 index 0000000..8b25a8b --- /dev/null +++ b/internal/source/easm/client_test.go @@ -0,0 +1,208 @@ +// Copyright Mia srl +// SPDX-License-Identifier: AGPL-3.0-only or Commercial + +package easm + +import ( + "net/http" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNewClient(t *testing.T) { + t.Parallel() + + testCases := map[string]struct { + config config + expectErr error + assertClient func(t *testing.T, c *client) + }{ + "valid base URL": { + config: config{ + BaseURL: "https://easm.example.com", + DataPath: "/data", + Customer: "acme", + Token: "test-token", + HTTPTimeout: 5 * time.Second, + }, + assertClient: func(t *testing.T, c *client) { + t.Helper() + assert.Equal(t, "https://easm.example.com", c.baseURL.String()) + assert.Equal(t, "/data", c.dataPath) + assert.Equal(t, "acme", c.customer) + assert.Equal(t, "test-token", c.token) + assert.Equal(t, 5*time.Second, c.httpClient.Timeout) + }, + }, + "invalid base URL": { + config: config{ + BaseURL: "://invalid", + }, + expectErr: ErrInvalidEnvVariable, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + t.Parallel() + + c, err := newClient(tc.config) + if tc.expectErr != nil { + require.ErrorIs(t, err, tc.expectErr) + return + } + + require.NoError(t, err) + tc.assertClient(t, c) + }) + } +} + +func TestFetchDataPagePagination(t *testing.T) { + t.Parallel() + + testCases := map[string]struct { + body string + nextCursor string + expectedItems int + expectedCursor string + }{ + "first page with next cursor": { + body: `[{"id":"1","type":"domain"},{"id":"2","type":"host"}]`, + nextCursor: "cursor-2", + expectedItems: 2, + expectedCursor: "cursor-2", + }, + "last page without cursor": { + body: `[{"id":"3","type":"ip"}]`, + nextCursor: "", + expectedItems: 1, + expectedCursor: "", + }, + "empty page": { + body: `[]`, + nextCursor: "", + expectedItems: 0, + expectedCursor: "", + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + t.Parallel() + + handler := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if tc.nextCursor != "" { + w.Header().Set(nextCursorHeader, tc.nextCursor) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(tc.body)) + }) + + source := newTestSource(t, handler) + + page, err := source.client.fetchDataPage(t.Context(), "") + require.NoError(t, err) + assert.Len(t, page.items, tc.expectedItems) + assert.Equal(t, tc.expectedCursor, page.nextCursor) + }) + } +} + +func TestFetchDataPageSendsCursor(t *testing.T) { + t.Parallel() + + var gotCursor string + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotCursor = r.URL.Query().Get(cursorQueryParam) + _, _ = w.Write([]byte(`[]`)) + }) + + source := newTestSource(t, handler) + + _, err := source.client.fetchDataPage(t.Context(), "cursor-42") + require.NoError(t, err) + assert.Equal(t, "cursor-42", gotCursor) +} + +func TestFetchDataPageHeaders(t *testing.T) { + t.Parallel() + + t.Run("token and customer set", func(t *testing.T) { + t.Parallel() + + var gotReq *http.Request + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotReq = r.Clone(r.Context()) + _, _ = w.Write([]byte(`[]`)) + }) + source := newTestSource(t, handler) + + _, err := source.client.fetchDataPage(t.Context(), "") + require.NoError(t, err) + assert.Equal(t, "application/json", gotReq.Header.Get("Accept")) + assert.Equal(t, "Bearer test-token", gotReq.Header.Get("Authorization")) + assert.Equal(t, "acme", gotReq.Header.Get("X-Customer")) + }) + + t.Run("empty token omits Authorization header", func(t *testing.T) { + t.Parallel() + + var gotReq *http.Request + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotReq = r.Clone(r.Context()) + _, _ = w.Write([]byte(`[]`)) + }) + source := newTestSource(t, handler) + // Build a client without a token pointing at the same server. + c := &client{ + baseURL: source.client.baseURL, + dataPath: source.client.dataPath, + customer: source.client.customer, + token: "", + httpClient: source.client.httpClient, + } + + _, err := c.fetchDataPage(t.Context(), "") + require.NoError(t, err) + assert.Empty(t, gotReq.Header.Get("Authorization")) + assert.Equal(t, "acme", gotReq.Header.Get("X-Customer")) + }) +} + +func TestFetchDataPageErrors(t *testing.T) { + t.Parallel() + + t.Run("non-200 status", func(t *testing.T) { + t.Parallel() + + handler := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte("boom")) + }) + + source := newTestSource(t, handler) + + _, err := source.client.fetchDataPage(t.Context(), "") + require.Error(t, err) + assert.Contains(t, err.Error(), "500") + assert.Contains(t, err.Error(), "boom") + }) + + t.Run("malformed JSON body", func(t *testing.T) { + t.Parallel() + + handler := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("not json")) + }) + + source := newTestSource(t, handler) + + _, err := source.client.fetchDataPage(t.Context(), "") + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to decode data response") + }) +} diff --git a/internal/source/easm/config.go b/internal/source/easm/config.go index b8a3893..87bcbf2 100644 --- a/internal/source/easm/config.go +++ b/internal/source/easm/config.go @@ -1,3 +1,4 @@ +// Copyright Mia srl // SPDX-License-Identifier: AGPL-3.0-only or Commercial package easm diff --git a/internal/source/easm/config_test.go b/internal/source/easm/config_test.go new file mode 100644 index 0000000..6ce2664 --- /dev/null +++ b/internal/source/easm/config_test.go @@ -0,0 +1,132 @@ +// Copyright Mia srl +// SPDX-License-Identifier: AGPL-3.0-only or Commercial + +package easm + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLoadConfigFromEnv(t *testing.T) { + testCases := map[string]struct { + setupEnv func(t *testing.T) + expectedConfig config + expectedErr error + }{ + "required env vars set, defaults applied": { + setupEnv: func(t *testing.T) { + t.Helper() + t.Setenv("EASM_BASE_URL", "https://easm.example.com") + t.Setenv("EASM_CUSTOMER", "acme") + }, + expectedConfig: config{ + BaseURL: "https://easm.example.com", + DataPath: "/data", + Customer: "acme", + HTTPTimeout: 30 * time.Second, + }, + }, + "all env vars set including optional": { + setupEnv: func(t *testing.T) { + t.Helper() + t.Setenv("EASM_BASE_URL", "https://easm.example.com") + t.Setenv("EASM_CUSTOMER", "acme") + t.Setenv("EASM_TOKEN", "secret") + t.Setenv("EASM_HTTP_TIMEOUT", "10s") + }, + expectedConfig: config{ + BaseURL: "https://easm.example.com", + DataPath: "/data", + Customer: "acme", + Token: "secret", + HTTPTimeout: 10 * time.Second, + }, + }, + "missing EASM_BASE_URL": { + setupEnv: func(t *testing.T) { + t.Helper() + t.Setenv("EASM_CUSTOMER", "acme") + }, + expectedErr: ErrMissingEnvVariable, + }, + "missing EASM_CUSTOMER": { + setupEnv: func(t *testing.T) { + t.Helper() + t.Setenv("EASM_BASE_URL", "https://easm.example.com") + }, + expectedErr: ErrMissingEnvVariable, + }, + "missing all required vars": { + setupEnv: func(t *testing.T) { + t.Helper() + }, + expectedErr: ErrMissingEnvVariable, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + tc.setupEnv(t) + + cfg, err := loadConfigFromEnv() + if tc.expectedErr != nil { + require.ErrorIs(t, err, tc.expectedErr) + return + } + + require.NoError(t, err) + assert.Equal(t, tc.expectedConfig, cfg) + }) + } +} + +func TestValidateConfig(t *testing.T) { + t.Parallel() + + testCases := map[string]struct { + config config + expectErr error + }{ + "valid config": { + config: config{ + BaseURL: "https://easm.example.com", + DataPath: "/data", + Customer: "acme", + HTTPTimeout: 30 * time.Second, + }, + }, + "missing base URL": { + config: config{ + Customer: "acme", + }, + expectErr: ErrMissingEnvVariable, + }, + "missing customer": { + config: config{ + BaseURL: "https://easm.example.com", + }, + expectErr: ErrMissingEnvVariable, + }, + "all missing": { + config: config{}, + expectErr: ErrMissingEnvVariable, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + t.Parallel() + + err := validateConfig(tc.config) + if tc.expectErr != nil { + assert.ErrorIs(t, err, tc.expectErr) + } else { + assert.NoError(t, err) + } + }) + } +} diff --git a/internal/source/easm/doc.go b/internal/source/easm/doc.go index a1d08eb..3606d4f 100644 --- a/internal/source/easm/doc.go +++ b/internal/source/easm/doc.go @@ -1,3 +1,4 @@ +// Copyright Mia srl // SPDX-License-Identifier: AGPL-3.0-only or Commercial // Package easm provides a source implementation that integrates EASM scan diff --git a/internal/source/easm/helper_test.go b/internal/source/easm/helper_test.go new file mode 100644 index 0000000..8cc8b88 --- /dev/null +++ b/internal/source/easm/helper_test.go @@ -0,0 +1,69 @@ +// Copyright Mia srl +// SPDX-License-Identifier: AGPL-3.0-only or Commercial + +package easm + +import ( + "net/http" + "net/http/httptest" + "net/url" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/mia-platform/ibdm/internal/source" +) + +var testTime = time.Date(2025, time.March, 1, 12, 0, 0, 0, time.UTC) + +func init() { + timeSource = func() time.Time { + return testTime + } +} + +// newTestSource builds a Source whose client points at an httptest server +// running the given handler. The server is closed automatically when the test +// finishes. +func newTestSource(t *testing.T, handler http.Handler) *Source { + t.Helper() + + server := httptest.NewServer(handler) + t.Cleanup(server.Close) + + u, err := url.Parse(server.URL) + require.NoError(t, err) + + cfg := config{ + BaseURL: server.URL, + DataPath: "/data", + Customer: "acme", + Token: "test-token", + HTTPTimeout: 5 * time.Second, + } + + return &Source{ + config: cfg, + client: &client{ + baseURL: u, + dataPath: cfg.DataPath, + customer: cfg.Customer, + token: cfg.Token, + httpClient: &http.Client{ + Timeout: cfg.HTTPTimeout, + }, + }, + } +} + +// collectData drains a source.Data channel into a slice. +func collectData(t *testing.T, ch <-chan source.Data) []source.Data { + t.Helper() + + var result []source.Data + for d := range ch { + result = append(result, d) + } + return result +} diff --git a/internal/source/easm/source.go b/internal/source/easm/source.go index f14a281..657b651 100644 --- a/internal/source/easm/source.go +++ b/internal/source/easm/source.go @@ -1,3 +1,4 @@ +// Copyright Mia srl // SPDX-License-Identifier: AGPL-3.0-only or Commercial package easm diff --git a/internal/source/easm/source_test.go b/internal/source/easm/source_test.go new file mode 100644 index 0000000..f930ecb --- /dev/null +++ b/internal/source/easm/source_test.go @@ -0,0 +1,68 @@ +// Copyright Mia srl +// SPDX-License-Identifier: AGPL-3.0-only or Commercial + +package easm + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNewSource(t *testing.T) { + testCases := map[string]struct { + setupEnv func(t *testing.T) + expectErrs []error + assertValid func(t *testing.T, s *Source) + }{ + "valid env": { + setupEnv: func(t *testing.T) { + t.Helper() + t.Setenv("EASM_BASE_URL", "https://easm.example.com") + t.Setenv("EASM_CUSTOMER", "acme") + }, + assertValid: func(t *testing.T, s *Source) { + t.Helper() + assert.Equal(t, "https://easm.example.com", s.config.BaseURL) + assert.Equal(t, "acme", s.config.Customer) + assert.NotNil(t, s.client) + }, + }, + "config error: missing required var": { + setupEnv: func(t *testing.T) { + t.Helper() + t.Setenv("EASM_CUSTOMER", "acme") + }, + expectErrs: []error{ErrEASMSource, ErrMissingEnvVariable}, + }, + "client error: invalid base URL": { + setupEnv: func(t *testing.T) { + t.Helper() + t.Setenv("EASM_BASE_URL", "://invalid") + t.Setenv("EASM_CUSTOMER", "acme") + }, + expectErrs: []error{ErrEASMSource, ErrInvalidEnvVariable}, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + tc.setupEnv(t) + + s, err := NewSource() + if len(tc.expectErrs) > 0 { + require.Error(t, err) + for _, target := range tc.expectErrs { + assert.ErrorIs(t, err, target) + } + assert.Nil(t, s) + return + } + + require.NoError(t, err) + require.NotNil(t, s) + tc.assertValid(t, s) + }) + } +} diff --git a/internal/source/easm/sync.go b/internal/source/easm/sync.go index 48bcd00..cf5dcd6 100644 --- a/internal/source/easm/sync.go +++ b/internal/source/easm/sync.go @@ -1,3 +1,4 @@ +// Copyright Mia srl // SPDX-License-Identifier: AGPL-3.0-only or Commercial package easm diff --git a/internal/source/easm/sync_test.go b/internal/source/easm/sync_test.go new file mode 100644 index 0000000..048127e --- /dev/null +++ b/internal/source/easm/sync_test.go @@ -0,0 +1,247 @@ +// Copyright Mia srl +// SPDX-License-Identifier: AGPL-3.0-only or Commercial + +package easm + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/mia-platform/ibdm/internal/source" +) + +// runSync drives StartSyncProcess to completion and returns the emitted data +// and the returned error. +func runSync(t *testing.T, s *Source, typesToSync map[string]source.Extra) ([]source.Data, error) { + t.Helper() + + ch := make(chan source.Data, 100) + var data []source.Data + + done := make(chan struct{}) + go func() { + defer close(done) + data = collectData(t, ch) + }() + + err := s.StartSyncProcess(t.Context(), typesToSync, ch) + close(ch) + <-done + + return data, err +} + +// writeItems encodes a JSON array of items as the /data response body. +func writeItems(w http.ResponseWriter, items []map[string]any) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(items) +} + +func TestStartSyncProcessRouting(t *testing.T) { + t.Parallel() + + items := []map[string]any{ + {"id": "1", "type": domainType}, + {"id": "2", "type": hostType}, + {"id": "3", "type": ipType}, + {"id": "4", "type": endpointType}, + } + + handler := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + writeItems(w, items) + }) + s := newTestSource(t, handler) + + typesToSync := map[string]source.Extra{ + domainType: {}, + ipType: {}, + } + + data, err := runSync(t, s, typesToSync) + require.NoError(t, err) + + // Only domain and ip are requested; host and endpoint are filtered out. + require.Len(t, data, 2) + for _, d := range data { + assert.Equal(t, source.DataOperationUpsert, d.Operation) + assert.Equal(t, testTime, d.Time) + } + + expected := []source.Data{ + {Type: domainType, Operation: source.DataOperationUpsert, Values: items[0], Time: testTime}, + {Type: ipType, Operation: source.DataOperationUpsert, Values: items[2], Time: testTime}, + } + assert.ElementsMatch(t, expected, data) +} + +func TestStartSyncProcessPagination(t *testing.T) { + t.Parallel() + + page1 := []map[string]any{ + {"id": "1", "type": domainType}, + {"id": "2", "type": hostType}, + } + page2 := []map[string]any{ + {"id": "3", "type": ipType}, + } + + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Query().Get(cursorQueryParam) { + case "": + w.Header().Set(nextCursorHeader, "page2") + writeItems(w, page1) + case "page2": + writeItems(w, page2) + default: + t.Errorf("unexpected cursor %q", r.URL.Query().Get(cursorQueryParam)) + } + }) + s := newTestSource(t, handler) + + typesToSync := map[string]source.Extra{ + domainType: {}, + hostType: {}, + ipType: {}, + } + + data, err := runSync(t, s, typesToSync) + require.NoError(t, err) + + // All three items, across both pages, are collected. + require.Len(t, data, 3) + expected := []source.Data{ + {Type: domainType, Operation: source.DataOperationUpsert, Values: page1[0], Time: testTime}, + {Type: hostType, Operation: source.DataOperationUpsert, Values: page1[1], Time: testTime}, + {Type: ipType, Operation: source.DataOperationUpsert, Values: page2[0], Time: testTime}, + } + assert.ElementsMatch(t, expected, data) +} + +func TestStartSyncProcessSkipsItemsWithoutType(t *testing.T) { + t.Parallel() + + items := []map[string]any{ + {"id": "1"}, // no type field + {"id": "2", "type": ""}, // empty type + {"id": "3", "type": domainType}, // valid + } + + handler := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + writeItems(w, items) + }) + s := newTestSource(t, handler) + + data, err := runSync(t, s, map[string]source.Extra{domainType: {}}) + require.NoError(t, err) + + require.Len(t, data, 1) + assert.Equal(t, domainType, data[0].Type) + assert.Equal(t, items[2], data[0].Values) +} + +func TestStartSyncProcessFetchError(t *testing.T) { + t.Parallel() + + handler := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte("boom")) + }) + s := newTestSource(t, handler) + + data, err := runSync(t, s, map[string]source.Extra{domainType: {}}) + require.Error(t, err) + assert.ErrorIs(t, err, ErrEASMSource) + assert.Empty(t, data) +} + +func TestStartSyncProcessAlreadyRunning(t *testing.T) { + t.Parallel() + + handler := http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) { + t.Fatal("no request should be made when the sync lock is already held") + }) + s := newTestSource(t, handler) + + // Simulate an in-flight sync by holding the lock. + s.syncLock.Lock() + defer s.syncLock.Unlock() + + data, err := runSync(t, s, map[string]source.Extra{domainType: {}}) + assert.NoError(t, err) + assert.Empty(t, data) +} + +func TestContextCancellationInSync(t *testing.T) { + t.Parallel() + + // Handler always advertises another page, so the loop would run forever + // unless the context cancellation breaks it. + handler := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set(nextCursorHeader, "next") + writeItems(w, []map[string]any{{"id": "1", "type": domainType}}) + }) + s := newTestSource(t, handler) + + ctx, cancel := context.WithCancel(t.Context()) + + ch := make(chan source.Data, 100) + done := make(chan error, 1) + go func() { + done <- s.StartSyncProcess(ctx, map[string]source.Extra{domainType: {}}, ch) + close(ch) + }() + + // Read one item, then cancel. + <-ch + cancel() + + err := <-done + assert.NoError(t, err) +} + +func TestHandleErr(t *testing.T) { + t.Parallel() + + testCases := map[string]struct { + err error + expectNil bool + }{ + "nil error": { + err: nil, + expectNil: true, + }, + "context canceled": { + err: context.Canceled, + expectNil: true, + }, + "wrapped context canceled": { + err: fmt.Errorf("fetch failed: %w", context.Canceled), + expectNil: true, + }, + "regular error": { + err: errors.New("something failed"), + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + t.Parallel() + + result := handleErr(tc.err) + if tc.expectNil { + assert.NoError(t, result) + return + } + + require.Error(t, result) + assert.ErrorIs(t, result, ErrEASMSource) + }) + } +} From 25c025af5f63845289a87fdb08fb9da6f785da31 Mon Sep 17 00:00:00 2001 From: alexpederneschi Date: Tue, 18 Aug 2026 18:46:53 +0900 Subject: [PATCH 04/20] correct easm ITD apiVersion and docs --- docs/external-sources/easm/how-to/010_easm-source.md | 5 ++--- .../external-sources/easm/itds/domains.easm.unguess.io.yaml | 2 +- .../easm/itds/endpoints.easm.unguess.io.yaml | 2 +- docs/external-sources/easm/itds/hosts.easm.unguess.io.yaml | 2 +- docs/external-sources/easm/itds/ips.easm.unguess.io.yaml | 2 +- .../easm/itds/vulnerabilities.easm.unguess.io.yaml | 2 +- internal/source/easm/config.go | 6 +++--- 7 files changed, 10 insertions(+), 11 deletions(-) diff --git a/docs/external-sources/easm/how-to/010_easm-source.md b/docs/external-sources/easm/how-to/010_easm-source.md index 53ea9fc..342c56e 100644 --- a/docs/external-sources/easm/how-to/010_easm-source.md +++ b/docs/external-sources/easm/how-to/010_easm-source.md @@ -25,7 +25,7 @@ All configuration is read from environment variables. | --- | --- | --- | --- | | `EASM_BASE_URL` | Yes | _(empty)_ | Base URL of the EASM backend (e.g. `https://easm.example.com`). | | `EASM_CUSTOMER` | Yes | _(empty)_ | Customer identifier. Scopes the request to a single customer via the `X-Customer` header — it selects whose scan results to read. | -| `EASM_TOKEN` | No | _(empty)_ | Bearer token authenticating the caller. Sent as `Authorization: Bearer ` when set. Optional for now — the backend has no auth yet; set it once auth lands. | +| `EASM_TOKEN` | No | _(empty)_ | Bearer token authenticating the caller. When set, it is sent as `Authorization: Bearer `. | | `EASM_DATA_PATH` | No | `/data` | Path of the read endpoint appended to `EASM_BASE_URL`. | | `EASM_HTTP_TIMEOUT` | No | `30s` | Timeout for each HTTP request, parsed as a Go `time.Duration`. | @@ -70,8 +70,7 @@ The source scopes every request to a single customer with the `X-Customer` heade `EASM_CUSTOMER` — this is always sent. When `EASM_TOKEN` is set, the source authenticates the caller with an -`Authorization: Bearer ` header. The backend has no authentication yet, so the token is -optional for now; set it once auth lands. +`Authorization: Bearer ` header. ## Example Mapping Files diff --git a/docs/external-sources/easm/itds/domains.easm.unguess.io.yaml b/docs/external-sources/easm/itds/domains.easm.unguess.io.yaml index 2b9c982..ca03339 100644 --- a/docs/external-sources/easm/itds/domains.easm.unguess.io.yaml +++ b/docs/external-sources/easm/itds/domains.easm.unguess.io.yaml @@ -1,6 +1,6 @@ # yaml-language-server: $schema=https://cdn.mia-platform.eu/catalog/v1alpha1/mia-platform.eu.v1alpha1.ItemTypeDefinition.json --- -apiVersion: mia-platform.eu/v1alpha1 +apiVersion: easm.unguess.io/v1 kind: ItemTypeDefinition metadata: name: domains.easm.unguess.io diff --git a/docs/external-sources/easm/itds/endpoints.easm.unguess.io.yaml b/docs/external-sources/easm/itds/endpoints.easm.unguess.io.yaml index e67a14f..410bc94 100644 --- a/docs/external-sources/easm/itds/endpoints.easm.unguess.io.yaml +++ b/docs/external-sources/easm/itds/endpoints.easm.unguess.io.yaml @@ -1,6 +1,6 @@ # yaml-language-server: $schema=https://cdn.mia-platform.eu/catalog/v1alpha1/mia-platform.eu.v1alpha1.ItemTypeDefinition.json --- -apiVersion: mia-platform.eu/v1alpha1 +apiVersion: easm.unguess.io/v1 kind: ItemTypeDefinition metadata: name: endpoints.easm.unguess.io diff --git a/docs/external-sources/easm/itds/hosts.easm.unguess.io.yaml b/docs/external-sources/easm/itds/hosts.easm.unguess.io.yaml index 5392c89..0a53fb5 100644 --- a/docs/external-sources/easm/itds/hosts.easm.unguess.io.yaml +++ b/docs/external-sources/easm/itds/hosts.easm.unguess.io.yaml @@ -1,6 +1,6 @@ # yaml-language-server: $schema=https://cdn.mia-platform.eu/catalog/v1alpha1/mia-platform.eu.v1alpha1.ItemTypeDefinition.json --- -apiVersion: mia-platform.eu/v1alpha1 +apiVersion: easm.unguess.io/v1 kind: ItemTypeDefinition metadata: name: hosts.easm.unguess.io diff --git a/docs/external-sources/easm/itds/ips.easm.unguess.io.yaml b/docs/external-sources/easm/itds/ips.easm.unguess.io.yaml index a0fc2a6..a209c12 100644 --- a/docs/external-sources/easm/itds/ips.easm.unguess.io.yaml +++ b/docs/external-sources/easm/itds/ips.easm.unguess.io.yaml @@ -1,6 +1,6 @@ # yaml-language-server: $schema=https://cdn.mia-platform.eu/catalog/v1alpha1/mia-platform.eu.v1alpha1.ItemTypeDefinition.json --- -apiVersion: mia-platform.eu/v1alpha1 +apiVersion: easm.unguess.io/v1 kind: ItemTypeDefinition metadata: name: ips.easm.unguess.io diff --git a/docs/external-sources/easm/itds/vulnerabilities.easm.unguess.io.yaml b/docs/external-sources/easm/itds/vulnerabilities.easm.unguess.io.yaml index 464e8cd..7bb1cba 100644 --- a/docs/external-sources/easm/itds/vulnerabilities.easm.unguess.io.yaml +++ b/docs/external-sources/easm/itds/vulnerabilities.easm.unguess.io.yaml @@ -1,6 +1,6 @@ # yaml-language-server: $schema=https://cdn.mia-platform.eu/catalog/v1alpha1/mia-platform.eu.v1alpha1.ItemTypeDefinition.json --- -apiVersion: mia-platform.eu/v1alpha1 +apiVersion: easm.unguess.io/v1 kind: ItemTypeDefinition metadata: name: vulnerabilities.easm.unguess.io diff --git a/internal/source/easm/config.go b/internal/source/easm/config.go index 87bcbf2..fe3955c 100644 --- a/internal/source/easm/config.go +++ b/internal/source/easm/config.go @@ -29,7 +29,7 @@ type config struct { // header. Always required: it selects whose scan results to read. Customer string `env:"EASM_CUSTOMER"` // Token authenticates the caller to the backend via Authorization: Bearer. - // Optional for now — the backend has no auth yet; set it once auth lands. + // Optional: when set, it is sent as the Authorization: Bearer header. Token string `env:"EASM_TOKEN"` // HTTPTimeout bounds each request to the endpoint. HTTPTimeout time.Duration `env:"EASM_HTTP_TIMEOUT" envDefault:"30s"` @@ -52,8 +52,8 @@ func loadConfigFromEnv() (config, error) { // validateConfig checks that the required config fields are non-empty. Customer // is always required: it scopes the request to a single customer via the -// X-Customer header. Token is optional for now (the backend has no auth yet) -// and, once set, authenticates the caller via Authorization: Bearer. +// X-Customer header. Token is optional and, when set, authenticates the caller +// via Authorization: Bearer. func validateConfig(cfg config) error { missing := make([]string, 0) From 42b2cf26432444b98ebf41f22d28a9a207bbb503 Mon Sep 17 00:00:00 2001 From: APagani Date: Mon, 13 Jul 2026 17:30:49 +0200 Subject: [PATCH 05/20] chore: update apiVersion and remove itds from docs (#73) --- .vscode/settings.json | 2 +- cspell.yaml | 2 +- ...evops.mia-platform.eu.gitrepositories.yaml | 53 - .../azuredevops.mia-platform.eu.teams.yaml | 37 - ...ure.mia-platform.eu.cognitiveaccounts.yaml | 51 - .../azure.mia-platform.eu.containerapps.yaml | 67 -- ...azure.mia-platform.eu.managedclusters.yaml | 95 -- .../azure.mia-platform.eu.postgresqldbs.yaml | 59 -- .../azure.mia-platform.eu.resourcegroups.yaml | 43 - ...azure.mia-platform.eu.storageaccounts.yaml | 67 -- .../azure.mia-platform.eu.subscriptions.yaml | 43 - ...azure.mia-platform.eu.virtualmachines.yaml | 55 - ...azure.mia-platform.eu.virtualnetworks.yaml | 68 -- .../itds/azure.mia-platform.eu.websites.yaml | 63 -- .../bitbucket.mia-platform.eu.pipelines.yaml | 73 -- ...itbucket.mia-platform.eu.repositories.yaml | 85 -- .../console.mia-platform.eu.clusters.yaml | 72 -- .../console.mia-platform.eu.projects.yaml | 186 ---- .../console.mia-platform.eu.revisions.yaml | 26 - .../console.mia-platform.eu.services.yaml | 989 ------------------ .../cluster-project-relationships.yaml | 16 - docs/examples/console/mappings/services.yaml | 80 -- .../gcp/itds/gcp.mia-platform.eu.buckets.yaml | 77 -- .../itds/gcp.mia-platform.eu.clusters.yaml | 138 --- .../gcp.mia-platform.eu.computeinstances.yaml | 139 --- .../gcp.mia-platform.eu.firewallrules.yaml | 215 ---- .../gcp/itds/gcp.mia-platform.eu.folders.yaml | 52 - .../gcp/itds/gcp.mia-platform.eu.jobs.yaml | 82 -- .../itds/gcp.mia-platform.eu.networks.yaml | 64 -- .../itds/gcp.mia-platform.eu.projects.yaml | 52 - .../itds/gcp.mia-platform.eu.services.yaml | 73 -- .../gcp.mia-platform.eu.sqlinstances.yaml | 141 --- .../github.mia-platform.eu.repositories.yaml | 108 -- .../github.mia-platform.eu.workflowruns.yaml | 100 -- .../gitlab.mia-platform.eu.accesstokens.yaml | 86 -- .../gitlab.mia-platform.eu.pipelines.yaml | 67 -- .../itds/gitlab.mia-platform.eu.projects.yaml | 248 ----- .../nexus.mia-platform.eu.dockerimages.yaml | 75 -- ...ysdig.mia-platform.eu.vulnerabilities.yaml | 143 --- docs/explanation/20_extra_mappings.md | 6 +- docs/how-to/070_gitlab-source.md | 6 +- docs/how-to/090_nexus-source.md | 6 +- .../azure-devops}/gitrepositories.yaml | 2 +- .../azure-devops}/teams.yaml | 2 +- .../azure}/cognitiveaccounts.yaml | 2 +- .../azure}/containerapps.yaml | 2 +- .../azure}/managedclusters.yaml | 16 +- .../azure}/postgresqldbs.yaml | 2 +- .../azure}/resourcegroups.yaml | 2 +- .../azure}/storageaccounts.yaml | 2 +- .../azure}/subscriptions.yaml | 2 +- .../azure}/virtualmachines.yaml | 2 +- .../azure}/virtualnetworks.yaml | 2 +- .../mappings => mappings/azure}/websites.yaml | 2 +- .../bitbucket}/pipelines.yaml | 16 +- .../bitbucket}/repositories.yaml | 2 +- .../cluster-project-relationships.yaml | 16 + .../console}/clusters.yaml | 2 +- .../console}/projects.yaml | 16 +- .../console}/revisions.yaml | 30 +- docs/mappings/console/services.yaml | 80 ++ .../mappings => mappings/gcp}/buckets.yaml | 2 +- .../mappings => mappings/gcp}/clusters.yaml | 2 +- .../gcp}/computeinstances.yaml | 2 +- .../gcp}/firewallrules.yaml | 2 +- .../mappings => mappings/gcp}/folders.yaml | 2 +- .../gcp/mappings => mappings/gcp}/jobs.yaml | 2 +- .../mappings => mappings/gcp}/networks.yaml | 2 +- .../mappings => mappings/gcp}/projects.yaml | 2 +- .../mappings => mappings/gcp}/services.yaml | 2 +- .../gcp}/sqlinstances.yaml | 2 +- .../github}/repositories.yaml | 2 +- .../github}/workflowruns.yaml | 16 +- .../gitlab}/accesstokens.yaml | 16 +- .../gitlab}/pipelines.yaml | 16 +- .../gitlab}/projects.yaml | 2 +- .../nexus}/dockerimages.yaml | 2 +- .../sysdig}/vulnerabilities.yaml | 16 +- internal/mapper/mapper_test.go | 10 +- internal/pipeline/pipeline_test.go | 14 +- tools/make/release.mk | 6 +- 81 files changed, 219 insertions(+), 4211 deletions(-) delete mode 100644 docs/examples/azure-devops/itds/azuredevops.mia-platform.eu.gitrepositories.yaml delete mode 100644 docs/examples/azure-devops/itds/azuredevops.mia-platform.eu.teams.yaml delete mode 100644 docs/examples/azure/itds/azure.mia-platform.eu.cognitiveaccounts.yaml delete mode 100644 docs/examples/azure/itds/azure.mia-platform.eu.containerapps.yaml delete mode 100644 docs/examples/azure/itds/azure.mia-platform.eu.managedclusters.yaml delete mode 100644 docs/examples/azure/itds/azure.mia-platform.eu.postgresqldbs.yaml delete mode 100644 docs/examples/azure/itds/azure.mia-platform.eu.resourcegroups.yaml delete mode 100644 docs/examples/azure/itds/azure.mia-platform.eu.storageaccounts.yaml delete mode 100644 docs/examples/azure/itds/azure.mia-platform.eu.subscriptions.yaml delete mode 100644 docs/examples/azure/itds/azure.mia-platform.eu.virtualmachines.yaml delete mode 100644 docs/examples/azure/itds/azure.mia-platform.eu.virtualnetworks.yaml delete mode 100644 docs/examples/azure/itds/azure.mia-platform.eu.websites.yaml delete mode 100644 docs/examples/bitbucket/itds/bitbucket.mia-platform.eu.pipelines.yaml delete mode 100644 docs/examples/bitbucket/itds/bitbucket.mia-platform.eu.repositories.yaml delete mode 100644 docs/examples/console/itds/console.mia-platform.eu.clusters.yaml delete mode 100644 docs/examples/console/itds/console.mia-platform.eu.projects.yaml delete mode 100644 docs/examples/console/itds/console.mia-platform.eu.revisions.yaml delete mode 100644 docs/examples/console/itds/console.mia-platform.eu.services.yaml delete mode 100644 docs/examples/console/mappings/cluster-project-relationships.yaml delete mode 100644 docs/examples/console/mappings/services.yaml delete mode 100644 docs/examples/gcp/itds/gcp.mia-platform.eu.buckets.yaml delete mode 100644 docs/examples/gcp/itds/gcp.mia-platform.eu.clusters.yaml delete mode 100644 docs/examples/gcp/itds/gcp.mia-platform.eu.computeinstances.yaml delete mode 100644 docs/examples/gcp/itds/gcp.mia-platform.eu.firewallrules.yaml delete mode 100644 docs/examples/gcp/itds/gcp.mia-platform.eu.folders.yaml delete mode 100644 docs/examples/gcp/itds/gcp.mia-platform.eu.jobs.yaml delete mode 100644 docs/examples/gcp/itds/gcp.mia-platform.eu.networks.yaml delete mode 100644 docs/examples/gcp/itds/gcp.mia-platform.eu.projects.yaml delete mode 100644 docs/examples/gcp/itds/gcp.mia-platform.eu.services.yaml delete mode 100644 docs/examples/gcp/itds/gcp.mia-platform.eu.sqlinstances.yaml delete mode 100644 docs/examples/github/itds/github.mia-platform.eu.repositories.yaml delete mode 100644 docs/examples/github/itds/github.mia-platform.eu.workflowruns.yaml delete mode 100644 docs/examples/gitlab/itds/gitlab.mia-platform.eu.accesstokens.yaml delete mode 100644 docs/examples/gitlab/itds/gitlab.mia-platform.eu.pipelines.yaml delete mode 100644 docs/examples/gitlab/itds/gitlab.mia-platform.eu.projects.yaml delete mode 100644 docs/examples/nexus/itds/nexus.mia-platform.eu.dockerimages.yaml delete mode 100644 docs/examples/sysdig/itds/sysdig.mia-platform.eu.vulnerabilities.yaml rename docs/{examples/azure-devops/mappings => mappings/azure-devops}/gitrepositories.yaml (93%) rename docs/{examples/azure-devops/mappings => mappings/azure-devops}/teams.yaml (84%) rename docs/{examples/azure/mappings => mappings/azure}/cognitiveaccounts.yaml (93%) rename docs/{examples/azure/mappings => mappings/azure}/containerapps.yaml (96%) rename docs/{examples/azure/mappings => mappings/azure}/managedclusters.yaml (72%) rename docs/{examples/azure/mappings => mappings/azure}/postgresqldbs.yaml (94%) rename docs/{examples/azure/mappings => mappings/azure}/resourcegroups.yaml (92%) rename docs/{examples/azure/mappings => mappings/azure}/storageaccounts.yaml (96%) rename docs/{examples/azure/mappings => mappings/azure}/subscriptions.yaml (92%) rename docs/{examples/azure/mappings => mappings/azure}/virtualmachines.yaml (95%) rename docs/{examples/azure/mappings => mappings/azure}/virtualnetworks.yaml (95%) rename docs/{examples/azure/mappings => mappings/azure}/websites.yaml (94%) rename docs/{examples/bitbucket/mappings => mappings/bitbucket}/pipelines.yaml (73%) rename docs/{examples/bitbucket/mappings => mappings/bitbucket}/repositories.yaml (96%) create mode 100644 docs/mappings/console/cluster-project-relationships.yaml rename docs/{examples/console/mappings => mappings/console}/clusters.yaml (93%) rename docs/{examples/console/mappings => mappings/console}/projects.yaml (83%) rename docs/{examples/console/mappings => mappings/console}/revisions.yaml (54%) create mode 100644 docs/mappings/console/services.yaml rename docs/{examples/gcp/mappings => mappings/gcp}/buckets.yaml (96%) rename docs/{examples/gcp/mappings => mappings/gcp}/clusters.yaml (96%) rename docs/{examples/gcp/mappings => mappings/gcp}/computeinstances.yaml (98%) rename docs/{examples/gcp/mappings => mappings/gcp}/firewallrules.yaml (96%) rename docs/{examples/gcp/mappings => mappings/gcp}/folders.yaml (93%) rename docs/{examples/gcp/mappings => mappings/gcp}/jobs.yaml (96%) rename docs/{examples/gcp/mappings => mappings/gcp}/networks.yaml (96%) rename docs/{examples/gcp/mappings => mappings/gcp}/projects.yaml (93%) rename docs/{examples/gcp/mappings => mappings/gcp}/services.yaml (95%) rename docs/{examples/gcp/mappings => mappings/gcp}/sqlinstances.yaml (98%) rename docs/{examples/github/mappings => mappings/github}/repositories.yaml (98%) rename docs/{examples/github/mappings => mappings/github}/workflowruns.yaml (77%) rename docs/{examples/gitlab/mappings => mappings/gitlab}/accesstokens.yaml (74%) rename docs/{examples/gitlab/mappings => mappings/gitlab}/pipelines.yaml (67%) rename docs/{examples/gitlab/mappings => mappings/gitlab}/projects.yaml (98%) rename docs/{examples/nexus/mappings => mappings/nexus}/dockerimages.yaml (96%) rename docs/{examples/sysdig/mappings => mappings/sysdig}/vulnerabilities.yaml (81%) diff --git a/.vscode/settings.json b/.vscode/settings.json index 35fb6be..0cfec25 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,5 @@ { "yaml.schemas": { - "https://cdn.mia-platform.eu/catalog/v1alpha1/mia-platform.eu.v1alpha1.ItemTypeDefinition.json": "docs/example/**/itds/*.{yaml,yml}", + "https://cdn.mia-platform.eu/catalog/v1/mia-platform.eu.v1.ItemTypeDefinition.json": "docs/example/**/itds/*.{yaml,yml}", } } diff --git a/cspell.yaml b/cspell.yaml index aa9a300..9861d58 100644 --- a/cspell.yaml +++ b/cspell.yaml @@ -71,5 +71,5 @@ files: - ".devcontainer/devcontainer.json" - ".github/*.yml" - ".github/**/*.yaml" - - "docs/examples/**/*.yaml" + - "docs/mappings/**/*.yaml" - Makefile diff --git a/docs/examples/azure-devops/itds/azuredevops.mia-platform.eu.gitrepositories.yaml b/docs/examples/azure-devops/itds/azuredevops.mia-platform.eu.gitrepositories.yaml deleted file mode 100644 index 20f677a..0000000 --- a/docs/examples/azure-devops/itds/azuredevops.mia-platform.eu.gitrepositories.yaml +++ /dev/null @@ -1,53 +0,0 @@ -# yaml-language-server: $schema=https://cdn.mia-platform.eu/catalog/v1alpha1/mia-platform.eu.v1alpha1.ItemTypeDefinition.json ---- -apiVersion: mia-platform.eu/v1alpha1 -kind: ItemTypeDefinition -metadata: - name: gitrepositories.azuredevops.mia-platform.eu -spec: - group: azuredevops.mia-platform.eu - names: - kind: GitRepository - plural: gitrepositories - displayPlural: GitRepositories - scope: Organization - versions: - - deprecated: false - name: v1alpha1 - schema: - openAPIV31Schema: - type: object - properties: - spec: - type: object - properties: - name: - description: The name of the item. - type: string - title: Name - defaultBranch: - description: The default branch of the repository. - type: string - title: Default Branch Name - isDisabled: - description: Whether the repository is disabled. - type: boolean - title: Repo Disabled - isAFork: - description: Whether the repository is a fork. - type: boolean - title: Forked Repo - isArchived: - description: Whether the repository is archived. - type: boolean - title: Archived Repo - sshGitUrl: - description: The SSH URL of the repository. - type: string - title: SSH Git URL - httpGitUrl: - description: The HTTP URL of the repository. - type: string - title: HTTP Git URL - served: true - storage: true diff --git a/docs/examples/azure-devops/itds/azuredevops.mia-platform.eu.teams.yaml b/docs/examples/azure-devops/itds/azuredevops.mia-platform.eu.teams.yaml deleted file mode 100644 index a089ff0..0000000 --- a/docs/examples/azure-devops/itds/azuredevops.mia-platform.eu.teams.yaml +++ /dev/null @@ -1,37 +0,0 @@ -# yaml-language-server: $schema=https://cdn.mia-platform.eu/catalog/v1alpha1/mia-platform.eu.v1alpha1.ItemTypeDefinition.json ---- -apiVersion: mia-platform.eu/v1alpha1 -kind: ItemTypeDefinition -metadata: - name: teams.azuredevops.mia-platform.eu -spec: - group: azuredevops.mia-platform.eu - names: - kind: Team - plural: teams - displayPlural: Teams - scope: Organization - versions: - - deprecated: false - name: v1alpha1 - schema: - openAPIV31Schema: - type: object - properties: - spec: - type: object - properties: - name: - description: The name of the item. - type: string - title: Name - projectName: - description: The name of the project the team belongs to. - type: string - title: Project Name - description: - description: The description of the team. - type: string - title: Description - served: true - storage: true diff --git a/docs/examples/azure/itds/azure.mia-platform.eu.cognitiveaccounts.yaml b/docs/examples/azure/itds/azure.mia-platform.eu.cognitiveaccounts.yaml deleted file mode 100644 index 69c7b60..0000000 --- a/docs/examples/azure/itds/azure.mia-platform.eu.cognitiveaccounts.yaml +++ /dev/null @@ -1,51 +0,0 @@ -# yaml-language-server: $schema=https://cdn.mia-platform.eu/catalog/v1alpha1/mia-platform.eu.v1alpha1.ItemTypeDefinition.json ---- -apiVersion: mia-platform.eu/v1alpha1 -kind: ItemTypeDefinition -metadata: - name: cognitiveaccounts.azure.mia-platform.eu -spec: - group: azure.mia-platform.eu - names: - kind: CognitiveAccount - plural: cognitiveaccounts - displayPlural: CognitiveAccounts - scope: Organization - versions: - - deprecated: false - name: v1alpha1 - schema: - openAPIV31Schema: - type: object - properties: - spec: - type: object - properties: - name: - description: The name of the item. - type: string - title: Name - location: - description: Location of the item. - type: string - title: Azure Location - provisioningState: - description: The provisioning state of the item. - type: string - title: Provisioning State - kind: - description: The kind of the cognitive account item. - type: string - title: Cognitive Account Kind - endpoint: - description: The endpoint of the cognitive account. - type: string - title: Account Endpoint - tags: - description: User-provided tags in key-value pairs. - type: object - title: Tags - additionalProperties: - type: string - served: true - storage: true diff --git a/docs/examples/azure/itds/azure.mia-platform.eu.containerapps.yaml b/docs/examples/azure/itds/azure.mia-platform.eu.containerapps.yaml deleted file mode 100644 index 2594c3c..0000000 --- a/docs/examples/azure/itds/azure.mia-platform.eu.containerapps.yaml +++ /dev/null @@ -1,67 +0,0 @@ -# yaml-language-server: $schema=https://cdn.mia-platform.eu/catalog/v1alpha1/mia-platform.eu.v1alpha1.ItemTypeDefinition.json ---- -apiVersion: mia-platform.eu/v1alpha1 -kind: ItemTypeDefinition -metadata: - name: containerapps.azure.mia-platform.eu -spec: - group: azure.mia-platform.eu - names: - kind: ContainerApp - plural: containerapps - displayPlural: ContainerApps - scope: Organization - versions: - - deprecated: false - name: v1alpha1 - schema: - openAPIV31Schema: - type: object - properties: - spec: - type: object - properties: - name: - description: The name of the item. - type: string - title: Name - location: - description: Location of the item. - type: string - title: Azure Location - provisioningState: - description: The provisioning state of the item. - type: string - title: Provisioning State - runningStatus: - description: The running status of the container app. - type: string - title: Container Running Status - fqdn: - description: The fully qualified domain name of the container app. - type: string - title: Container Fully Qualified Domain Name - external: - description: Indicates if the container app is externally accessible. - type: boolean - title: Container Is Exposed Externally - targetPort: - description: The target port of the container app. - type: integer - title: Container Target Port - minReplicas: - description: The minimum number of replicas for the container app. - type: integer - title: Minimum Replicas - maxReplicas: - description: The maximum number of replicas for the container app. - type: integer - title: Maximum Replicas - tags: - description: User-provided tags in key-value pairs. - type: object - title: Tags - additionalProperties: - type: string - served: true - storage: true diff --git a/docs/examples/azure/itds/azure.mia-platform.eu.managedclusters.yaml b/docs/examples/azure/itds/azure.mia-platform.eu.managedclusters.yaml deleted file mode 100644 index c5caebd..0000000 --- a/docs/examples/azure/itds/azure.mia-platform.eu.managedclusters.yaml +++ /dev/null @@ -1,95 +0,0 @@ -# yaml-language-server: $schema=https://cdn.mia-platform.eu/catalog/v1alpha1/mia-platform.eu.v1alpha1.ItemTypeDefinition.json ---- -apiVersion: mia-platform.eu/v1alpha1 -kind: ItemTypeDefinition -metadata: - name: managedclusters.azure.mia-platform.eu -spec: - group: azure.mia-platform.eu - names: - kind: ManagedCluster - plural: managedclusters - displayPlural: ManagedClusters - scope: Organization - versions: - - deprecated: false - name: v1alpha1 - schema: - openAPIV31Schema: - type: object - properties: - spec: - type: object - properties: - name: - description: The name of the item. - type: string - title: Name - location: - description: Location of the item. - type: string - title: Azure Location - provisioningState: - description: The provisioning state of the item. - type: string - title: Provisioning State - currentKubernetesVersion: - description: The current Kubernetes version of the cluster. - type: string - title: Current Kubernetes API-Server Version - supportPlan: - description: The support plan of the cluster. - type: string - title: AKS Support Plan - publicNetworkAccess: - description: Indicates whether public network access is enabled or disabled. - type: string - title: API-Server Public Network Access - disableLocalAccounts: - description: Indicates whether local accounts are disabled. - type: boolean - title: Local Accounts Disabled - dnsPrefix: - description: The DNS prefix for the cluster. - type: string - title: Cluster DNS Prefix - fqdn: - description: The fully qualified domain name for the api-server of the cluster. - type: string - title: API-Server Fully Qualified Domain Name - enableRBAC: - description: Indicates whether RBAC is enabled for the cluster. - type: boolean - title: RBAC Enabled - skuTier: - description: The SKU tier of the cluster. - type: string - title: AKS SKU Tier - podCidrs: - description: The pod CIDRs for the cluster. - type: array - title: Cluster Pods CIDRs - items: - type: string - serviceCidrs: - description: The service CIDRs for the cluster. - type: array - title: Cluster Services CIDRs - items: - type: string - networkPlugin: - description: The network plugin used by the cluster. - type: string - title: AKS Network Plugin - outboundType: - description: The outbound type of the cluster. - type: string - title: Cluster Outbound Type - tags: - description: User-provided tags in key-value pairs. - type: object - title: Tags - additionalProperties: - type: string - served: true - storage: true diff --git a/docs/examples/azure/itds/azure.mia-platform.eu.postgresqldbs.yaml b/docs/examples/azure/itds/azure.mia-platform.eu.postgresqldbs.yaml deleted file mode 100644 index f20e14e..0000000 --- a/docs/examples/azure/itds/azure.mia-platform.eu.postgresqldbs.yaml +++ /dev/null @@ -1,59 +0,0 @@ -# yaml-language-server: $schema=https://cdn.mia-platform.eu/catalog/v1alpha1/mia-platform.eu.v1alpha1.ItemTypeDefinition.json ---- -apiVersion: mia-platform.eu/v1alpha1 -kind: ItemTypeDefinition -metadata: - name: postgresqldbs.azure.mia-platform.eu -spec: - group: azure.mia-platform.eu - names: - kind: PostgreSQLDB - plural: postgresqldbs - displayPlural: PostgreSQLDBs - scope: Organization - versions: - - deprecated: false - name: v1alpha1 - schema: - openAPIV31Schema: - type: object - properties: - spec: - type: object - properties: - name: - description: The name of the item. - type: string - title: Name - location: - description: Location of the item. - type: string - title: Azure Location - state: - description: The state of the db. - type: string - title: DB State - version: - description: The major version of the server. - type: number - title: PostgreSQL Major Version - minorVersion: - description: The minor version of the server. - type: number - title: PostgreSQL Minor Version - fullyQualifiedDomainName: - description: The fully qualified domain name of the server. - type: string - title: DB Fully Qualified Domain Name - publicNetworkAccess: - description: Whether public network access is allowed for this server. - type: string - title: Public Access Enabled - tags: - description: User-provided tags in key-value pairs. - type: object - title: Tags - additionalProperties: - type: string - served: true - storage: true diff --git a/docs/examples/azure/itds/azure.mia-platform.eu.resourcegroups.yaml b/docs/examples/azure/itds/azure.mia-platform.eu.resourcegroups.yaml deleted file mode 100644 index 45aa570..0000000 --- a/docs/examples/azure/itds/azure.mia-platform.eu.resourcegroups.yaml +++ /dev/null @@ -1,43 +0,0 @@ -# yaml-language-server: $schema=https://cdn.mia-platform.eu/catalog/v1alpha1/mia-platform.eu.v1alpha1.ItemTypeDefinition.json ---- -apiVersion: mia-platform.eu/v1alpha1 -kind: ItemTypeDefinition -metadata: - name: resourcegroups.azure.mia-platform.eu -spec: - group: azure.mia-platform.eu - names: - kind: ResourceGroup - plural: resourcegroups - displayPlural: ResourceGroups - scope: Organization - versions: - - deprecated: false - name: v1alpha1 - schema: - openAPIV31Schema: - type: object - properties: - spec: - type: object - properties: - name: - description: The name of the item. - type: string - title: Name - location: - description: Location of the item. - type: string - title: Azure Location - provisioningState: - description: The provisioning state of the item. - type: string - title: Provisioning State - tags: - description: User-provided tags in key-value pairs. - type: object - title: Tags - additionalProperties: - type: string - served: true - storage: true diff --git a/docs/examples/azure/itds/azure.mia-platform.eu.storageaccounts.yaml b/docs/examples/azure/itds/azure.mia-platform.eu.storageaccounts.yaml deleted file mode 100644 index 714f135..0000000 --- a/docs/examples/azure/itds/azure.mia-platform.eu.storageaccounts.yaml +++ /dev/null @@ -1,67 +0,0 @@ -# yaml-language-server: $schema=https://cdn.mia-platform.eu/catalog/v1alpha1/mia-platform.eu.v1alpha1.ItemTypeDefinition.json ---- -apiVersion: mia-platform.eu/v1alpha1 -kind: ItemTypeDefinition -metadata: - name: storageaccounts.azure.mia-platform.eu -spec: - group: azure.mia-platform.eu - names: - kind: StorageAccount - plural: storageaccounts - displayPlural: StorageAccounts - scope: Organization - versions: - - deprecated: false - name: v1alpha1 - schema: - openAPIV31Schema: - type: object - properties: - spec: - type: object - properties: - name: - description: The name of the item. - type: string - title: Name - provisioningState: - description: The provisioning state of the item. - type: string - title: Provisioning State - allowBlobPublicAccess: - description: Indicates whether data in the storage account may be accessed publicly. - type: boolean - title: Blob Public Access Enabled - publicNetworkAccess: - description: Specifies whether public network access is allowed for this storage account. - type: string - title: Storage Account Public Access Enabled - isHnsEnabled: - description: Indicates whether Hierarchical Namespace is enabled for the storage account. - type: boolean - title: Hierarchical Namespace Enabled - primaryLocation: - description: The primary location of the item. - type: string - title: Storage Account Primary Location - statusOfPrimaryLocation: - description: The status of the primary location of the item. - type: string - title: Primary Location Status - secondaryLocation: - description: The secondary location of the item if available. - type: string - title: Storage Account Secondary Location - statusOfSecondaryLocation: - description: The status of the secondary location of the item if available. - type: string - title: Secondary Location Status - tags: - description: User-provided tags in key-value pairs. - type: object - title: Tags - additionalProperties: - type: string - served: true - storage: true diff --git a/docs/examples/azure/itds/azure.mia-platform.eu.subscriptions.yaml b/docs/examples/azure/itds/azure.mia-platform.eu.subscriptions.yaml deleted file mode 100644 index 5ea8cdc..0000000 --- a/docs/examples/azure/itds/azure.mia-platform.eu.subscriptions.yaml +++ /dev/null @@ -1,43 +0,0 @@ -# yaml-language-server: $schema=https://cdn.mia-platform.eu/catalog/v1alpha1/mia-platform.eu.v1alpha1.ItemTypeDefinition.json ---- -apiVersion: mia-platform.eu/v1alpha1 -kind: ItemTypeDefinition -metadata: - name: subscriptions.azure.mia-platform.eu -spec: - group: azure.mia-platform.eu - names: - kind: Subscription - plural: subscriptions - displayPlural: Subscriptions - scope: Organization - versions: - - deprecated: false - name: v1alpha1 - schema: - openAPIV31Schema: - type: object - properties: - spec: - type: object - properties: - name: - description: The name of the item. - type: string - title: Name - displayName: - description: A custom name for the subscription. - type: string - title: Display Name - state: - description: The current state of the subscription. - type: string - title: Subscription State - tags: - description: User-provided tags in key-value pairs. - type: object - title: Tags - additionalProperties: - type: string - served: true - storage: true diff --git a/docs/examples/azure/itds/azure.mia-platform.eu.virtualmachines.yaml b/docs/examples/azure/itds/azure.mia-platform.eu.virtualmachines.yaml deleted file mode 100644 index 5082201..0000000 --- a/docs/examples/azure/itds/azure.mia-platform.eu.virtualmachines.yaml +++ /dev/null @@ -1,55 +0,0 @@ -# yaml-language-server: $schema=https://cdn.mia-platform.eu/catalog/v1alpha1/mia-platform.eu.v1alpha1.ItemTypeDefinition.json ---- -apiVersion: mia-platform.eu/v1alpha1 -kind: ItemTypeDefinition -metadata: - name: virtualmachines.azure.mia-platform.eu -spec: - group: azure.mia-platform.eu - names: - kind: VirtualMachine - plural: virtualmachines - displayPlural: VirtualMachines - scope: Organization - versions: - - deprecated: false - name: v1alpha1 - schema: - openAPIV31Schema: - type: object - properties: - spec: - type: object - properties: - name: - description: The name of the item. - type: string - title: Name - location: - description: Location of the item. - type: string - title: Azure Location - provisioningState: - description: The provisioning state of the item. - type: string - title: Provisioning State - vmSize: - description: The size of the virtual machine. - type: string - title: VM Size - osType: - description: The operating system type. - type: string - title: OS Type - osVersion: - description: The operating system version. - type: string - title: OS Version - tags: - description: User-provided tags in key-value pairs. - type: object - title: Tags - additionalProperties: - type: string - served: true - storage: true diff --git a/docs/examples/azure/itds/azure.mia-platform.eu.virtualnetworks.yaml b/docs/examples/azure/itds/azure.mia-platform.eu.virtualnetworks.yaml deleted file mode 100644 index 9f32f47..0000000 --- a/docs/examples/azure/itds/azure.mia-platform.eu.virtualnetworks.yaml +++ /dev/null @@ -1,68 +0,0 @@ -# yaml-language-server: $schema=https://cdn.mia-platform.eu/catalog/v1alpha1/mia-platform.eu.v1alpha1.ItemTypeDefinition.json ---- -apiVersion: mia-platform.eu/v1alpha1 -kind: ItemTypeDefinition -metadata: - name: virtualnetworks.azure.mia-platform.eu -spec: - group: azure.mia-platform.eu - names: - kind: VirtualNetwork - plural: virtualnetworks - displayPlural: VirtualNetworks - scope: Organization - versions: - - deprecated: false - name: v1alpha1 - schema: - openAPIV31Schema: - type: object - properties: - spec: - type: object - properties: - name: - description: The name of the item. - type: string - title: Name - location: - description: Location of the item. - type: string - title: Azure Location - provisioningState: - description: The provisioning state of the item. - type: string - title: Provisioning State - addressPrefixes: - description: The address prefixes of the virtual network. - type: array - title: Network Address Prefixes - items: - type: string - subnets: - description: The subnets of the virtual network. - type: array - title: Subnets - items: - type: object - properties: - name: - description: The name of the subnet. - type: string - title: Subnet Name - addressPrefix: - description: The address prefix of the subnet. - type: string - title: Subnet Address Prefix - provisioningState: - description: The provisioning state of the subnet. - type: string - title: Subnet Provisioning State - tags: - description: User-provided tags in key-value pairs. - type: object - title: Tags - additionalProperties: - type: string - served: true - storage: true diff --git a/docs/examples/azure/itds/azure.mia-platform.eu.websites.yaml b/docs/examples/azure/itds/azure.mia-platform.eu.websites.yaml deleted file mode 100644 index cfca98e..0000000 --- a/docs/examples/azure/itds/azure.mia-platform.eu.websites.yaml +++ /dev/null @@ -1,63 +0,0 @@ -# yaml-language-server: $schema=https://cdn.mia-platform.eu/catalog/v1alpha1/mia-platform.eu.v1alpha1.ItemTypeDefinition.json ---- -apiVersion: mia-platform.eu/v1alpha1 -kind: ItemTypeDefinition -metadata: - name: websites.azure.mia-platform.eu -spec: - group: azure.mia-platform.eu - names: - kind: WebSite - plural: websites - displayPlural: WebSites - scope: Organization - versions: - - deprecated: false - name: v1alpha1 - schema: - openAPIV31Schema: - type: object - properties: - spec: - type: object - properties: - name: - description: The name of the item. - type: string - title: Name - location: - description: Location of the item. - type: string - title: Azure Location - state: - description: The state of the item. - type: string - title: Service State - kind: - description: The kind of the item, for example if it is an app or a function. - type: string - title: App Service Kind - enabled: - description: Indicates whether the website is enabled. - type: boolean - title: Enabled - availabilityState: - description: The availability state of the website. - type: string - title: Availability State - defaultHostname: - description: The default hostname of the website. - type: string - title: Default Hostname - serverFarmId: - description: The ID of the App Service Plan associated with the website. - type: string - title: App Service Plan - tags: - description: User-provided tags in key-value pairs. - type: object - title: tags - additionalProperties: - type: string - served: true - storage: true diff --git a/docs/examples/bitbucket/itds/bitbucket.mia-platform.eu.pipelines.yaml b/docs/examples/bitbucket/itds/bitbucket.mia-platform.eu.pipelines.yaml deleted file mode 100644 index d854e9f..0000000 --- a/docs/examples/bitbucket/itds/bitbucket.mia-platform.eu.pipelines.yaml +++ /dev/null @@ -1,73 +0,0 @@ -# yaml-language-server: $schema=https://cdn.mia-platform.eu/catalog/v1alpha1/mia-platform.eu.v1alpha1.ItemTypeDefinition.json ---- -apiVersion: mia-platform.eu/v1alpha1 -kind: ItemTypeDefinition -metadata: - name: pipelines.bitbucket.mia-platform.eu -spec: - group: bitbucket.mia-platform.eu - names: - kind: Pipeline - plural: pipelines - singular: pipeline - displayPlural: Bitbucket Pipelines - displaySingular: Bitbucket Pipeline - scope: Organization - versions: - - deprecated: false - name: v1alpha1 - schema: - openAPIV31Schema: - type: object - properties: - spec: - type: object - properties: - uuid: - description: The unique UUID of the pipeline run. - type: string - title: UUID - buildNumber: - description: The sequential build number of the pipeline. - type: integer - title: Build Number - state: - description: "The current state of the pipeline (e.g. COMPLETED, IN_PROGRESS)." - type: string - title: State - result: - description: "The result of the pipeline when completed (e.g. SUCCESSFUL, FAILED)." - type: string - title: Result - createdOn: - description: The creation time of the pipeline (ISO-8601). - type: string - format: date-time - title: Created On - completedOn: - description: The completion time of the pipeline (ISO-8601). - type: string - format: date-time - title: Completed On - durationInSeconds: - description: The total duration of the pipeline run in seconds. - type: integer - title: Duration In Seconds - buildSecondsUsed: - description: The build seconds consumed by the pipeline run. - type: integer - title: Build Seconds Used - targetRef: - description: The branch or tag name the pipeline ran on. - type: string - title: Target Ref - repositoryFullName: - description: The full name (workspace/repo) of the repository the pipeline belongs to. - type: string - title: Repository - repositoryHtmlUrl: - description: The HTML URL of the repository the pipeline belongs to. - type: string - title: Repository HTML URL - served: true - storage: true diff --git a/docs/examples/bitbucket/itds/bitbucket.mia-platform.eu.repositories.yaml b/docs/examples/bitbucket/itds/bitbucket.mia-platform.eu.repositories.yaml deleted file mode 100644 index 45d550a..0000000 --- a/docs/examples/bitbucket/itds/bitbucket.mia-platform.eu.repositories.yaml +++ /dev/null @@ -1,85 +0,0 @@ -# yaml-language-server: $schema=https://cdn.mia-platform.eu/catalog/v1alpha1/mia-platform.eu.v1alpha1.ItemTypeDefinition.json ---- -apiVersion: mia-platform.eu/v1alpha1 -kind: ItemTypeDefinition -metadata: - name: repositories.bitbucket.mia-platform.eu -spec: - group: bitbucket.mia-platform.eu - names: - kind: Repository - plural: repositories - singular: repository - displayPlural: Bitbucket Repositories - displaySingular: Bitbucket Repository - scope: Organization - versions: - - deprecated: false - name: v1alpha1 - schema: - openAPIV31Schema: - type: object - properties: - spec: - type: object - properties: - fullName: - description: The full name of the repository (workspace/repo). - type: string - title: Full Name - uuid: - description: The UUID of the repository. - type: string - title: UUID - name: - description: The name of the repository. - type: string - title: Name - slug: - description: The URL-safe name of the repository. - type: string - title: Slug - description: - description: A short description of the repository. - type: string - title: Description - isPrivate: - description: Whether the repository is private. - type: boolean - title: Private - scm: - description: The source control type (e.g. git). - type: string - title: SCM - language: - description: The primary programming language of the repository. - type: string - title: Language - htmlUrl: - description: The URL of the repository on Bitbucket. - type: string - title: HTML URL - defaultBranch: - description: The default branch of the repository. - type: string - title: Default Branch - createdOn: - description: The creation time of the repository (ISO-8601). - type: string - format: date-time - title: Created On - updatedOn: - description: The last time the repository metadata was updated (ISO-8601). - type: string - format: date-time - title: Updated On - projectName: - description: The name of the Bitbucket project the repository belongs to. - type: string - title: Project Name - workspaceName: - description: The name of the workspace the repository belongs to. - type: string - title: Workspace Name - served: true - storage: true diff --git a/docs/examples/console/itds/console.mia-platform.eu.clusters.yaml b/docs/examples/console/itds/console.mia-platform.eu.clusters.yaml deleted file mode 100644 index 3ab9990..0000000 --- a/docs/examples/console/itds/console.mia-platform.eu.clusters.yaml +++ /dev/null @@ -1,72 +0,0 @@ -# yaml-language-server: $schema=https://cdn.mia-platform.eu/catalog/v1alpha1/mia-platform.eu.v1alpha1.ItemTypeDefinition.json ---- -apiVersion: mia-platform.eu/v1alpha1 -kind: ItemTypeDefinition -metadata: - name: clusters.console.mia-platform.eu - description: A Kubernetes cluster registered in the Mia-Platform Console -spec: - group: console.mia-platform.eu - names: - kind: Cluster - plural: clusters - singular: cluster - displayPlural: Clusters - displaySingular: Cluster - scope: Organization - versions: - - name: v1alpha1 - served: true - schema: - openAPIV31Schema: - type: object - properties: - spec: - type: object - properties: - clusterId: - title: Cluster ID - description: The unique identifier of the cluster as defined in Console. - type: string - tenantId: - title: Tenant ID - description: The tenant that owns this cluster. - type: string - vendor: - title: Vendor - description: The cloud or infrastructure vendor (e.g. Azure, GCP, AWS). - type: string - distribution: - title: Distribution - description: The Kubernetes distribution (e.g. AKS, GKE, EKS). - type: string - connection: - title: Connection - description: Connection details for the cluster. - type: object - properties: - url: - title: API Server URL - description: The URL of the Kubernetes API server. - type: string - runtimeInfo: - title: Runtime Info - description: Runtime resource information reported by the cluster. - type: object - properties: - cpuCores: - title: CPU Cores - description: Total number of CPU cores available in the cluster. - type: number - nodesCount: - title: Nodes Count - description: Total number of nodes in the cluster. - type: integer - ramMiB: - title: RAM (MiB) - description: Total RAM available in the cluster, expressed in mebibytes. - type: number - version: - title: Kubernetes Version - description: The Kubernetes version running on the cluster. - type: string diff --git a/docs/examples/console/itds/console.mia-platform.eu.projects.yaml b/docs/examples/console/itds/console.mia-platform.eu.projects.yaml deleted file mode 100644 index f5406ac..0000000 --- a/docs/examples/console/itds/console.mia-platform.eu.projects.yaml +++ /dev/null @@ -1,186 +0,0 @@ -# yaml-language-server: $schema=https://cdn.mia-platform.eu/catalog/v1alpha1/mia-platform.eu.v1alpha1.ItemTypeDefinition.json ---- -apiVersion: mia-platform.eu/v1alpha1 -kind: ItemTypeDefinition -metadata: - name: projects.console.mia-platform.eu -spec: - group: console.mia-platform.eu - names: - kind: Project - plural: projects - displayPlural: Projects - scope: Organization - versions: - - deprecated: false - name: v1alpha1 - schema: - openAPIV31Schema: - type: object - properties: - spec: - type: object - properties: - projectUniqueId: - description: The unique identifier for the project. - type: string - projectId: - description: The project ID used to identify the project in Console - type: string - name: - description: The user visible name for the project. - type: string - tenantId: - description: The tenant ID used to identify the tenant in Console - type: string - tenantName: - description: The user visible name for the tenant. - type: string - repository: - description: The repository information for the project. - type: object - additionalProperties: - type: string - repositoryUrl: - description: The URL of the git repository associated with the project. - type: string - availableNamespaces: - description: The list of namespaces available for the project. - type: array - items: - type: object - additionalProperties: - type: string - configurationGitPath: - description: The git path where the project configuration is stored. - type: string - containerRegistries: - description: The container registries associated with the project. - type: array - items: - type: object - additionalProperties: true - defaultBranch: - description: The default branch for the project. - type: string - deploy: - description: The deployment settings for the project. - type: object - additionalProperties: true - description: - description: The user visible description for the project. - type: string - dockerImageNameSuggestion: - description: Suggested name for the docker image associated with the project. - type: object - additionalProperties: true - enabledSecurityFeatures: - description: The enabled security features for the project. - type: object - additionalProperties: true - enabledServices: - description: The enabled services for the project. - type: object - additionalProperties: true - environments: - description: The list of environments associated with the project. - type: array - items: - type: object - properties: - label: - type: string - envId: - type: string - envPrefix: - type: string - hosts: - type: array - items: - type: object - properties: - host: - type: string - isBackoffice: - type: boolean - scheme: - type: string - description: an url schema, usually one of 'http' or 'https' - description: - type: string - isProduction: - type: boolean - default: false - cluster: - type: object - properties: - namespace: - type: string - clusterId: - type: string - deploy: - type: object - properties: - type: - type: string - environmentsVariables: - description: The environment variables associated with the project. - type: object - additionalProperties: true - flavor: - description: The flavor of the project. - type: string - info: - description: Additional information about the project. - type: object - properties: - createdAt: - type: string - format: date-time - projectOwner: - type: string - teamContact: - type: string - additionalProperties: true - imagePullSecretNames: - description: The list of image pull secret names associated with the project. - type: array - items: - type: string - links: - description: The list of links associated with the project. - type: object - additionalProperties: true - logicalScopeLayers: - description: The logical scope layers associated with the project. - type: array - items: - type: object - additionalProperties: true - originalTemplate: - description: The original template used for the project. - type: object - properties: - id: - type: string - name: - type: string - additionalProperties: true - pipelines: - description: The pipelines associated with the project. - type: object - properties: - providerId: - type: string - type: - type: string - additionalProperties: true - projectNamespaceVariable: - description: The project namespace variable. - type: string - lastUpdate: - description: Time when the project was last updated. - type: string - format: date-time - served: true - storage: true diff --git a/docs/examples/console/itds/console.mia-platform.eu.revisions.yaml b/docs/examples/console/itds/console.mia-platform.eu.revisions.yaml deleted file mode 100644 index 66dbcf2..0000000 --- a/docs/examples/console/itds/console.mia-platform.eu.revisions.yaml +++ /dev/null @@ -1,26 +0,0 @@ -# yaml-language-server: $schema=https://cdn.mia-platform.eu/catalog/v1alpha1/mia-platform.eu.v1alpha1.ItemTypeDefinition.json ---- -apiVersion: mia-platform.eu/v1alpha1 -kind: ItemTypeDefinition -metadata: - name: revisions.console.mia-platform.eu -spec: - group: console.mia-platform.eu - names: - kind: Revision - plural: revisions - displayPlural: Revisions - scope: Organization - versions: - - deprecated: false - name: v1alpha1 - schema: - openAPIV31Schema: - type: object - properties: - spec: - type: object - properties: - additionalProperties: false - served: true - storage: true diff --git a/docs/examples/console/itds/console.mia-platform.eu.services.yaml b/docs/examples/console/itds/console.mia-platform.eu.services.yaml deleted file mode 100644 index a835f97..0000000 --- a/docs/examples/console/itds/console.mia-platform.eu.services.yaml +++ /dev/null @@ -1,989 +0,0 @@ -# yaml-language-server: $schema=https://cdn.mia-platform.eu/catalog/v1alpha1/mia-platform.eu.v1alpha1.ItemTypeDefinition.json ---- -apiVersion: mia-platform.eu/v1alpha1 -kind: ItemTypeDefinition -metadata: - name: services.console.mia-platform.eu - description: The configuration of a microservice in the Mia-Platform Console -spec: - group: console.mia-platform.eu - names: - kind: Service - plural: services - displayPlural: Services - scope: Organization - versions: - - name: v1alpha1 - served: true - schema: - openAPIV31Schema: - type: object - properties: - spec: - type: object - properties: - projectUniqueId: - title: Console project unique ID - description: The unique identifier for the project. - type: string - projectId: - title: Console project ID - description: The project ID used to identify the project in Console. - type: string - tenantId: - title: Console tenant ID - description: The tenant ID used to identify the tenant in Console. - type: string - revisionName: - title: Console project revision name - description: The user visible name for the revision. - type: string - configuration: - title: Configuration - description: The configuration of the service as defined in the Design area of the Console. - type: object - properties: - name: - type: string - minLength: 1 - description: - type: string - tags: - type: array - items: - type: string - type: - type: string - enum: - - custom - advanced: - type: boolean - enum: - - false - dockerImage: - type: string - repoUrl: - type: string - sshUrl: - type: string - links: - type: array - items: - type: object - properties: - label: - type: string - enableIf: - type: string - targetSection: - type: string - hidePrefix: - type: boolean - dockerImagePullSecrets: - type: array - items: - type: object - properties: - name: - type: string - required: - - name - replicas: - anyOf: - - type: number - minimum: 0 - - type: string - serviceAccountName: - type: string - maxLength: 253 - productionReplicas: - type: object - properties: - min: - type: string - max: - type: string - cpuThreshold: - type: string - generate: - type: boolean - deleted: - type: boolean - logParser: - type: string - additionalContainers: - type: array - items: - type: object - properties: - name: - type: string - minLength: 1 - description: - type: string - repoUrl: - type: string - dockerImage: - type: string - imagePullPolicy: - type: string - environment: - type: array - items: - oneOf: - - type: object - properties: - name: - type: string - minLength: 1 - readOnly: - type: boolean - managedBy: - type: string - description: - type: string - valueType: - type: string - enum: - - plain - value: - type: string - required: - - name - - value - - valueType - - type: object - properties: - name: - type: string - minLength: 1 - readOnly: - type: boolean - managedBy: - type: string - description: - type: string - valueType: - type: string - enum: - - secret - secretName: - type: string - secretKey: - type: string - required: - - name - - secretName - - secretKey - - valueType - - type: object - properties: - name: - type: string - minLength: 1 - readOnly: - type: boolean - managedBy: - type: string - description: - type: string - valueType: - type: string - enum: - - configmap - configMapName: - type: string - configMapFileName: - type: string - required: - - name - - valueType - - configMapName - - configMapFileName - - anyOf: - - type: object - required: - - name - - valueType - - fieldPath - - containerName - properties: - name: - type: string - minLength: 1 - readOnly: - type: boolean - managedBy: - type: string - description: - type: string - valueType: - type: string - enum: - - downwardAPI - fieldPath: - type: string - enum: - - resource.limits.cpu - - resource.requests.cpu - - resource.limits.memory - - resource.requests.memory - - resource.limits.ephemeral-storage - - resource.requests.ephemeral-storage - containerName: - type: string - - type: object - required: - - name - - valueType - - fieldPath - properties: - name: - type: string - minLength: 1 - readOnly: - type: boolean - managedBy: - type: string - description: - type: string - valueType: - type: string - enum: - - downwardAPI - fieldPath: - oneOf: - - type: string - enum: - - metadata.name - - metadata.namespace - - metadata.uid - - spec.serviceAccountName - - spec.nodeName - - status.hostIP - - status.podIP - - status.podIPs - - type: string - - type: string - annotations: - type: array - items: - type: object - properties: - name: - type: string - value: - type: string - description: - type: string - readOnly: - type: boolean - required: - - name - - value - labels: - type: array - items: - type: object - properties: - name: - type: string - value: - type: string - description: - type: string - readOnly: - type: boolean - isSelector: - type: boolean - required: - - name - - value - resources: - type: object - properties: - cpuLimits: - type: object - properties: - max: - type: string - min: - type: string - memoryLimits: - type: object - properties: - max: - type: string - min: - type: string - probes: - type: object - properties: - liveness: - type: object - properties: - path: - type: string - port: - type: string - cmd: - type: array - items: - type: string - initialDelaySeconds: - type: number - periodSeconds: - type: number - timeoutSeconds: - type: number - successThreshold: - type: number - failureThreshold: - type: number - readiness: - type: object - properties: - path: - type: string - port: - type: string - cmd: - type: array - items: - type: string - initialDelaySeconds: - type: number - periodSeconds: - type: number - timeoutSeconds: - type: number - successThreshold: - type: number - failureThreshold: - type: number - startup: - type: object - properties: - path: - type: string - port: - type: string - cmd: - type: array - items: - type: string - initialDelaySeconds: - type: number - periodSeconds: - type: number - timeoutSeconds: - type: number - successThreshold: - type: number - failureThreshold: - type: number - tags: - type: array - items: - type: string - generatedFrom: - type: object - properties: - _id: - type: string - swaggerPath: - type: string - sshUrl: - type: string - configMaps: - type: array - items: - type: object - properties: - name: - type: string - mountPath: - type: string - subPaths: - type: array - items: - type: string - viewAsReadOnly: - type: boolean - link: - type: object - properties: - targetSection: - type: string - required: - - name - - mountPath - secrets: - type: array - items: - type: object - properties: - name: - type: string - mountPath: - type: string - emptyDirMounts: - type: array - items: - type: object - properties: - name: - type: string - mountPath: - type: string - sourceComponentId: - type: string - sourceMarketplaceItem: - type: object - properties: - itemId: - type: string - version: - type: string - tenantId: - type: string - detached: - type: boolean - required: - - itemId - - version - - tenantId - mapEnvVarToMountPath: - type: object - additionalProperties: - type: object - required: - - type - - envName - properties: - type: - type: string - enum: - - file - - folder - envName: - type: string - links: - type: array - items: - type: object - properties: - label: - type: string - enableIf: - type: string - targetSection: - type: string - hidePrefix: - type: boolean - createdAt: - type: string - owners: - type: array - items: - type: object - required: - - owner - additionalProperties: true - properties: - owner: - type: string - containerPorts: - type: array - items: - type: object - properties: - name: - type: string - from: - anyOf: - - type: integer - minimum: 0 - maximum: 65535 - - type: string - to: - anyOf: - - type: integer - minimum: 0 - maximum: 65535 - - type: string - protocol: - type: string - required: - - name - - from - monitoring: - type: object - properties: - endpoints: - type: array - items: - type: object - properties: - path: - type: string - port: - type: string - interval: - type: string - required: - - interval - - port - - path - deleted: - type: boolean - execPreStop: - type: array - items: - type: string - args: - type: array - items: - type: string - terminationGracePeriodSeconds: - type: number - exclusiveServiceExposure: - type: boolean - containerRegistryId: - type: string - required: - - name - - dockerImage - emptyDirs: - type: object - additionalProperties: - type: object - required: - - type - properties: - type: - type: string - enum: - - default - - memory - size: - type: string - imagePullPolicy: - type: string - environment: - type: array - items: - oneOf: - - type: object - properties: - name: - type: string - minLength: 1 - readOnly: - type: boolean - managedBy: - type: string - description: - type: string - valueType: - type: string - enum: - - plain - value: - type: string - required: - - name - - value - - valueType - - type: object - properties: - name: - type: string - minLength: 1 - readOnly: - type: boolean - managedBy: - type: string - description: - type: string - valueType: - type: string - enum: - - secret - secretName: - type: string - secretKey: - type: string - required: - - name - - secretName - - secretKey - - valueType - - type: object - properties: - name: - type: string - minLength: 1 - readOnly: - type: boolean - managedBy: - type: string - description: - type: string - valueType: - type: string - enum: - - configmap - configMapName: - type: string - configMapFileName: - type: string - required: - - name - - valueType - - configMapName - - configMapFileName - - anyOf: - - type: object - required: - - name - - valueType - - fieldPath - - containerName - properties: - name: - type: string - minLength: 1 - readOnly: - type: boolean - managedBy: - type: string - description: - type: string - valueType: - type: string - enum: - - downwardAPI - fieldPath: - type: string - enum: - - resource.limits.cpu - - resource.requests.cpu - - resource.limits.memory - - resource.requests.memory - - resource.limits.ephemeral-storage - - resource.requests.ephemeral-storage - containerName: - type: string - - type: object - required: - - name - - valueType - - fieldPath - properties: - name: - type: string - minLength: 1 - readOnly: - type: boolean - managedBy: - type: string - description: - type: string - valueType: - type: string - enum: - - downwardAPI - fieldPath: - oneOf: - - type: string - enum: - - metadata.name - - metadata.namespace - - metadata.uid - - spec.serviceAccountName - - spec.nodeName - - status.hostIP - - status.podIP - - status.podIPs - - type: string - - type: string - annotations: - type: array - items: - type: object - properties: - name: - type: string - value: - type: string - description: - type: string - readOnly: - type: boolean - required: - - name - - value - labels: - type: array - items: - type: object - properties: - name: - type: string - value: - type: string - description: - type: string - readOnly: - type: boolean - isSelector: - type: boolean - required: - - name - - value - resources: - type: object - properties: - cpuLimits: - type: object - properties: - max: - type: string - min: - type: string - memoryLimits: - type: object - properties: - max: - type: string - min: - type: string - probes: - type: object - properties: - liveness: - type: object - properties: - path: - type: string - port: - type: string - cmd: - type: array - items: - type: string - initialDelaySeconds: - type: number - periodSeconds: - type: number - timeoutSeconds: - type: number - successThreshold: - type: number - failureThreshold: - type: number - readiness: - type: object - properties: - path: - type: string - port: - type: string - cmd: - type: array - items: - type: string - initialDelaySeconds: - type: number - periodSeconds: - type: number - timeoutSeconds: - type: number - successThreshold: - type: number - failureThreshold: - type: number - startup: - type: object - properties: - path: - type: string - port: - type: string - cmd: - type: array - items: - type: string - initialDelaySeconds: - type: number - periodSeconds: - type: number - timeoutSeconds: - type: number - successThreshold: - type: number - failureThreshold: - type: number - generatedFrom: - type: object - properties: - _id: - type: string - swaggerPath: - type: string - configMaps: - type: array - items: - type: object - properties: - name: - type: string - mountPath: - type: string - subPaths: - type: array - items: - type: string - viewAsReadOnly: - type: boolean - link: - type: object - properties: - targetSection: - type: string - required: - - name - - mountPath - secrets: - type: array - items: - type: object - properties: - name: - type: string - mountPath: - type: string - emptyDirMounts: - type: array - items: - type: object - properties: - name: - type: string - mountPath: - type: string - sourceComponentId: - type: string - sourceMarketplaceItem: - type: object - properties: - itemId: - type: string - version: - type: string - tenantId: - type: string - detached: - type: boolean - required: - - itemId - - version - - tenantId - mapEnvVarToMountPath: - type: object - additionalProperties: - type: object - required: - - type - - envName - properties: - type: - type: string - enum: - - file - - folder - envName: - type: string - createdAt: - type: string - owners: - type: array - items: - type: object - required: - - owner - additionalProperties: true - properties: - owner: - type: string - containerPorts: - type: array - items: - type: object - properties: - name: - type: string - from: - anyOf: - - type: integer - minimum: 0 - maximum: 65535 - - type: string - to: - anyOf: - - type: integer - minimum: 0 - maximum: 65535 - - type: string - protocol: - type: string - required: - - name - - from - monitoring: - type: object - properties: - endpoints: - type: array - items: - type: object - properties: - path: - type: string - port: - type: string - interval: - type: string - required: - - interval - - port - - path - deleted: - type: boolean - execPreStop: - type: array - items: - type: string - args: - type: array - items: - type: string - terminationGracePeriodSeconds: - type: number - exclusiveServiceExposure: - type: boolean - containerRegistryId: - type: string - required: - - name - - advanced - - type - - dockerImage - selectableFields: - - jsonPath: spec.projectUniqueId - - jsonPath: spec.projectId - - jsonPath: spec.tenantId - - jsonPath: spec.revisionName - - jsonPath: spec.configuration.name - - jsonPath: spec.configuration.dockerImage diff --git a/docs/examples/console/mappings/cluster-project-relationships.yaml b/docs/examples/console/mappings/cluster-project-relationships.yaml deleted file mode 100644 index 1856362..0000000 --- a/docs/examples/console/mappings/cluster-project-relationships.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: mia-platform.eu/v1alpha1 -itemFamily: relationships -type: clusterProjectRelationship -syncable: true -mappings: - identifier: |- - {{ $src := printf "urn:mia-platform-catalog:console.mia-platform.eu:v1alpha1:Project:%s" .project._id -}} - {{- $type := "urn:mia-platform-catalog:mia-platform.eu:v1alpha1:RelationshipType:part-of.mia-platform.eu" -}} - {{- $tgt := printf "urn:mia-platform-catalog:console.mia-platform.eu:v1alpha1:Cluster:%s" (printf "%s" (replace "https://" "" .cluster.connection.url) | sha256sum) -}} - {{- printf "%s-%s-%s" $src $type $tgt | sha256sum }} - spec: - sourceRef: |- - urn:mia-platform-catalog:console.mia-platform.eu:v1alpha1:Project:{{ .project._id }} - targetRef: |- - urn:mia-platform-catalog:console.mia-platform.eu:v1alpha1:Cluster:{{ printf "%s" (replace "https://" "" .cluster.connection.url) | sha256sum }} - typeRef: "urn:mia-platform-catalog:mia-platform.eu:v1alpha1:RelationshipType:part-of.mia-platform.eu" diff --git a/docs/examples/console/mappings/services.yaml b/docs/examples/console/mappings/services.yaml deleted file mode 100644 index dab01b9..0000000 --- a/docs/examples/console/mappings/services.yaml +++ /dev/null @@ -1,80 +0,0 @@ -apiVersion: console.mia-platform.eu/v1alpha1 -itemFamily: services -type: service -syncable: true -mappings: - identifier: |- - {{ printf "%s-%s-%s" .project._id .revision.name .service.name | sha256sum }} - metadata: - title: "{{ printf \"%s\" .service.name }}" - spec: - projectUniqueId: "{{ .project._id }}" - projectId: "{{ get \"projectId\" .project \"\" | quote }}" - tenantId: "{{ .project.tenantId }}" - revisionName: "{{ .revision.name }}" - configuration: "{{ .service | toJSON }}" - extra: - - apiVersion: mia-platform.eu/v1alpha1 - itemFamily: relationships - deletePolicy: "cascade" - identifier: |- - {{ $src := printf "urn:mia-platform-catalog:console.mia-platform.eu:v1alpha1:Service:%s" (printf "%s-%s-%s" .project._id .revision.name .service.name | sha256sum) -}} - {{- $type := "urn:mia-platform-catalog:mia-platform.eu:v1alpha1:RelationshipType:part-of.mia-platform.eu" -}} - {{- $tgt := printf "urn:mia-platform-catalog:console.mia-platform.eu:v1alpha1:Project:%s" .project._id -}} - {{- printf "%s-%s-%s" $src $type $tgt | sha256sum }} - sourceRef: |- - urn:mia-platform-catalog:console.mia-platform.eu:v1alpha1:Service:{{ printf "%s-%s-%s" .project._id .revision.name .service.name | sha256sum }} - targetRef: |- - urn:mia-platform-catalog:console.mia-platform.eu:v1alpha1:Project:{{ .project._id }} - typeRef: "urn:mia-platform-catalog:mia-platform.eu:v1alpha1:RelationshipType:part-of.mia-platform.eu" - - apiVersion: mia-platform.eu/v1alpha1 - itemFamily: relationships - deletePolicy: "cascade" - identifier: |- - {{ $src := printf "urn:mia-platform-catalog:console.mia-platform.eu:v1alpha1:Service:%s" (printf "%s-%s-%s" .project._id .revision.name .service.name | sha256sum) -}} - {{- $type := "urn:mia-platform-catalog:mia-platform.eu:v1alpha1:RelationshipType:part-of.mia-platform.eu" -}} - {{- $tgt := printf "urn:mia-platform-catalog:console.mia-platform.eu:v1alpha1:Revision:%s" (printf "%s-%s" .project._id .revision.name | sha256sum) -}} - {{- printf "%s-%s-%s" $src $type $tgt | sha256sum }} - sourceRef: |- - urn:mia-platform-catalog:console.mia-platform.eu:v1alpha1:Service:{{ printf "%s-%s-%s" .project._id .revision.name .service.name | sha256sum }} - targetRef: |- - urn:mia-platform-catalog:console.mia-platform.eu:v1alpha1:Revision:{{ printf "%s-%s" .project._id .revision.name | sha256sum }} - typeRef: "urn:mia-platform-catalog:mia-platform.eu:v1alpha1:RelationshipType:part-of.mia-platform.eu" - - apiVersion: mia-platform.eu/v1alpha1 - itemFamily: relationships - deletePolicy: "cascade" - createIf: |- - {{ $info := (get "info" .project (object)) -}} - {{- $teamContact := (get "teamContact" $info nil) -}} - {{- if $teamContact -}} - true - {{- else -}} - false - {{- end }} - identifier: |- - {{ $info := (get "info" .project (object)) -}} - {{- $teamContact := (get "teamContact" $info "") -}} - {{- $src := printf "urn:mia-platform-catalog:iam.mia-platform.eu:v1alpha1:User:%s" ($teamContact | sha256sum) -}} - {{- $type := "urn:mia-platform-catalog:mia-platform.eu:v1alpha1:RelationshipType:ownership.mia-platform.eu" -}} - {{- $tgt := printf "urn:mia-platform-catalog:console.mia-platform.eu:v1alpha1:Service:%s" (printf "%s-%s-%s" .project._id .revision.name .service.name | sha256sum) -}} - {{- printf "%s-%s-%s" $src $type $tgt | sha256sum }} - sourceRef: |- - {{ $info := (get "info" .project (object)) -}} - {{- $teamContact := (get "teamContact" $info "") -}} - urn:mia-platform-catalog:iam.mia-platform.eu:v1alpha1:User:{{ $teamContact | sha256sum }} - targetRef: |- - urn:mia-platform-catalog:console.mia-platform.eu:v1alpha1:Service:{{ printf "%s-%s-%s" .project._id .revision.name .service.name | sha256sum }} - typeRef: "urn:mia-platform-catalog:mia-platform.eu:v1alpha1:RelationshipType:ownership.mia-platform.eu" - - apiVersion: mia-platform.eu/v1alpha1 - itemFamily: relationships - deletePolicy: "none" - identifier: |- - {{ $src := printf "urn:mia-platform-catalog:console.mia-platform.eu:v1alpha1:Service:%s" (printf "%s-%s-%s" .project._id .revision.name .service.name | sha256sum) -}} - {{- $type := "urn:mia-platform-catalog:mia-platform.eu:v1alpha1:RelationshipType:dependency.mia-platform.eu" -}} - {{- $tgt := printf "urn:mia-platform-catalog:nexus.mia-platform.eu:v1alpha1:DockerImage:%s" (printf "%s" .service.dockerImage | sha256sum) -}} - {{- printf "%s-%s-%s" $src $type $tgt | sha256sum }} - sourceRef: |- - urn:mia-platform-catalog:console.mia-platform.eu:v1alpha1:Service:{{ printf "%s-%s-%s" .project._id .revision.name .service.name | sha256sum }} - targetRef: |- - urn:mia-platform-catalog:nexus.mia-platform.eu:v1alpha1:DockerImage:{{ printf "%s" .service.dockerImage | sha256sum }} - typeRef: "urn:mia-platform-catalog:mia-platform.eu:v1alpha1:RelationshipType:dependency.mia-platform.eu" diff --git a/docs/examples/gcp/itds/gcp.mia-platform.eu.buckets.yaml b/docs/examples/gcp/itds/gcp.mia-platform.eu.buckets.yaml deleted file mode 100644 index 4ecc62a..0000000 --- a/docs/examples/gcp/itds/gcp.mia-platform.eu.buckets.yaml +++ /dev/null @@ -1,77 +0,0 @@ -# yaml-language-server: $schema=https://cdn.mia-platform.eu/catalog/v1alpha1/mia-platform.eu.v1alpha1.ItemTypeDefinition.json ---- -apiVersion: mia-platform.eu/v1alpha1 -kind: ItemTypeDefinition -metadata: - name: buckets.gcp.mia-platform.eu -spec: - group: gcp.mia-platform.eu - names: - kind: Bucket - plural: buckets - displayPlural: Storage Buckets - scope: Organization - versions: - - deprecated: false - name: v1alpha1 - schema: - openAPIV31Schema: - type: object - properties: - spec: - type: object - properties: - name: - description: The name of the bucket. - type: string - title: Name - location: - description: Location of the bucket (region, dual-region or multi-region). - type: string - title: GCP Location - locationType: - description: Type of location the bucket resides in (possible options can be 'region', 'dual-region', 'multi-region'). - type: string - title: GCP Location Type - storageClass: - description: The bucket's default storage class (possible options can be 'standard', 'nearline', 'coldline', or 'archive'). - type: string - title: Storage Class - autoclassEnabled: - description: Whether Autoclass is enabled for the bucket or not. - type: boolean - title: Storage Autoclass Enabled - versioningEnabled: - description: Indicate if the versioning configuration for the bucket is turned on. - type: boolean - title: Versioning Enabled - hierarchicalNamespaceEnabled: - description: Whether hierarchical namespace is enabled for the bucket or not. - type: boolean - title: Hierarchical Namespace Enabled - publicAccess: - description: The public access configuration for the bucket. Possible values are 'inherited', 'enforced' and 'unspecified'. - type: string - title: Bucket Public Access - retentionPeriod: - description: The period of time, in seconds, that objects in the bucket cannot be deleted or modified. - type: integer - title: Retention Period (seconds) - created: - description: The creation time of the bucket (RFC 3339). - type: string - format: date-time - title: Created At - lastUpdate: - description: Time when the bucket metadata or IAM policy was last updated (RFC 3339). - type: string - format: date-time - title: Last Update - labels: - description: User-provided labels in key-value pairs. - type: object - title: Labels - additionalProperties: - type: string - served: true - storage: true diff --git a/docs/examples/gcp/itds/gcp.mia-platform.eu.clusters.yaml b/docs/examples/gcp/itds/gcp.mia-platform.eu.clusters.yaml deleted file mode 100644 index a06da05..0000000 --- a/docs/examples/gcp/itds/gcp.mia-platform.eu.clusters.yaml +++ /dev/null @@ -1,138 +0,0 @@ -# yaml-language-server: $schema=https://cdn.mia-platform.eu/catalog/v1alpha1/mia-platform.eu.v1alpha1.ItemTypeDefinition.json ---- -apiVersion: mia-platform.eu/v1alpha1 -kind: ItemTypeDefinition -metadata: - name: clusters.gcp.mia-platform.eu -spec: - group: gcp.mia-platform.eu - names: - kind: Cluster - plural: clusters - singular: cluster - scope: Organization - versions: - - deprecated: false - name: v1alpha1 - schema: - openAPIV31Schema: - type: object - properties: - spec: - type: object - properties: - name: - description: The cluster name - type: string - title: Name - description: - description: Human-readable description of the cluster - type: string - title: Description - autopilot: - description: Whether the cluster is an Autopilot one - type: boolean - title: Autopilot Enabled - apiServerVersion: - description: The current kubernetes version of the cluster API server - type: string - title: Current API-Server Version - network: - description: Network resource the cluster uses - type: string - title: GCP Network - subnetwork: - description: Subnetwork resource the cluster uses - type: string - title: GCP Subnetwork - podsIpv4Cidr: - description: The IP address range for the pods in this cluster - type: string - title: Cluster Pods CIDR - servicesIpv4Cidr: - description: The IP address range for the services in this cluster - type: string - title: Cluster Services CIDR - clusterEndpoints: - description: Configuration of the cluster's API server endpoints - type: object - title: API-Server Endpoints - properties: - dnsEndpointConfig: - description: configuration for the DNS endpoint of the api server - type: object - title: DNS Endpoint - properties: - allowExternalTraffic: - type: boolean - title: Allow External Traffic - enableK8sCertsViaDns: - type: boolean - title: Enable Kubernetes Certificates via DNS - enableK8sTokensViaDns: - type: boolean - title: Enable Kubernetes Tokens via DNS - endpoint: - type: string - title: Endpoint - ipEndpointsConfig: - description: configuration for the IP endpoint of the api server - type: object - title: IP Endpoint - properties: - authorizedNetworksConfig: - type: object - title: Authorized Networks - properties: - cidrBlocks: - type: array - title: CIDR Blocks - items: - type: object - properties: - cidrBlock: - title: CIDR - type: string - displayName: - title: Name - type: string - enabled: - type: boolean - title: Authorized Networks Enabled - gcpPublicCidrsAccessEnabled: - type: boolean - title: GCP Public CIDRs Access Enabled - privateEndpointEnforcementEnabled: - type: boolean - title: Private Endpoint Enabled - enablePublicEndpoint: - type: boolean - title: Enable Public Endpoint - enabled: - type: boolean - title: IP Endpoint Enabled - globalAccess: - type: boolean - title: Global Access Enabled - privateEndpoint: - type: string - title: Private IP Endpoint - publicEndpoint: - title: Public IP Endpoint - type: string - status: - description: Cluster status (possible options can be PROVISIONING, RUNNING, etc.) - type: string - title: Cluster Status - location: - description: Primary location for the cluster - type: string - title: Cluster GCP Location - labels: - description: User-provided labels in key-value pairs. - type: object - title: Labels - additionalProperties: - type: string - served: true - storage: true diff --git a/docs/examples/gcp/itds/gcp.mia-platform.eu.computeinstances.yaml b/docs/examples/gcp/itds/gcp.mia-platform.eu.computeinstances.yaml deleted file mode 100644 index 8131bb4..0000000 --- a/docs/examples/gcp/itds/gcp.mia-platform.eu.computeinstances.yaml +++ /dev/null @@ -1,139 +0,0 @@ -# yaml-language-server: $schema=https://cdn.mia-platform.eu/catalog/v1alpha1/mia-platform.eu.v1alpha1.ItemTypeDefinition.json ---- -apiVersion: mia-platform.eu/v1alpha1 -kind: ItemTypeDefinition -metadata: - name: computeinstances.gcp.mia-platform.eu -spec: - group: gcp.mia-platform.eu - names: - kind: ComputeInstance - plural: computeinstances - displayPlural: ComputeInstances - scope: Organization - versions: - - deprecated: false - name: v1alpha1 - schema: - openAPIV31Schema: - type: object - properties: - spec: - type: object - properties: - name: - description: The GCP compute instance name - type: string - title: Name - description: - description: Human-readable description - type: string - title: Description - cpuPlatform: - description: The CPU platform string - type: string - title: CPU Platform - location: - description: The region zone where the instance is deployed - type: string - title: GCP Location - status: - description: Current status of the instance, possible values are 'provisioning', 'staging', 'running', 'stopping', 'suspending', 'suspended', or 'terminated' - type: string - title: VM Status - deletionProtection: - description: Whether the instance deletion protection is enabled - type: boolean - title: Deletion Protection Enabled - integrityMonitoringEnabled: - description: Whether integrity monitoring is enabled - type: boolean - title: Integrity Monitoring Enabled - secureBootEnabled: - description: Whether secure boot is enabled - type: boolean - title: Secure Boot Enabled - vtpmEnabled: - description: Whether vTPM is enabled - type: boolean - title: vTPM Enabled - confidentialInstanceConfigEnabled: - description: Whether confidential instance config is enabled - type: boolean - title: Confidential Instance Config Enabled - lastStart: - description: RFC 3339 timestamp for the last start time - type: string - format: date-time - title: VM Last Start Time - machineType: - description: The machine type of the instance - type: string - title: Virtual Machine Type - instancesIPs: - description: List of the instance IPs. One or more IPs can be attached to the instance - type: array - title: Instance IPs - items: - type: string - disks: - description: Instance disks details - type: array - title: Instance Disks - items: - type: object - properties: - architecture: - type: string - title: Disk Architecture - autoDelete: - type: boolean - title: Auto Delete Enabled - boot: - type: boolean - title: Boot Disk - deviceName: - type: string - title: Disk Device Name - diskSizeGb: - type: string - title: Disk Size (GB) - mode: - type: string - title: Disk Mode - type: - type: string - title: Disk Type - network: - description: Instance network details - type: array - title: Virtual Machine Network - items: - type: object - properties: - name: - type: string - title: GCP Network Name - network: - type: string - title: GCP Network URL - networkIP: - type: string - title: Network IP - subnetwork: - type: string - title: GCP Subnetwork URL - labels: - description: User-provided labels in key-value pairs. - type: object - title: Labels - additionalProperties: - type: string - tags: - description: Network tags attached to the instance - type: array - title: Network Tags - items: - type: string - served: true - storage: true diff --git a/docs/examples/gcp/itds/gcp.mia-platform.eu.firewallrules.yaml b/docs/examples/gcp/itds/gcp.mia-platform.eu.firewallrules.yaml deleted file mode 100644 index c31e79c..0000000 --- a/docs/examples/gcp/itds/gcp.mia-platform.eu.firewallrules.yaml +++ /dev/null @@ -1,215 +0,0 @@ -# yaml-language-server: $schema=https://cdn.mia-platform.eu/catalog/v1alpha1/mia-platform.eu.v1alpha1.ItemTypeDefinition.json ---- -apiVersion: mia-platform.eu/v1alpha1 -kind: ItemTypeDefinition -metadata: - name: firewallrules.gcp.mia-platform.eu -spec: - group: gcp.mia-platform.eu - names: - kind: FirewallRule - plural: firewallrules - displayPlural: FirewallRules - scope: Organization - versions: - - deprecated: false - name: v1alpha1 - schema: - openAPIV31Schema: - type: object - properties: - spec: - type: object - properties: - name: - description: The user-assigned name for the firewall rule - type: string - title: Name - description: - description: Human-readable description of the firewall rule - type: string - title: Description - disabled: - description: Whether the firewall rule is disabled - type: boolean - title: Rule Disabled - network: - description: The network self link this firewall rule applies to - type: string - title: GCP Network URL - priority: - description: Priority of the firewall rule - type: integer - title: Rule Priority - logEnabled: - description: Whether logging is enabled for this firewall rule - type: boolean - title: Logging Enabled - ingress: - description: If the rule is an ingress, this property will contains the firewall rule ingress configuration - type: object - title: Ingress Configuration - properties: - allowed: - description: List of allowed protocols and ports - type: array - title: Allowed Rule - items: - type: object - properties: - IPProtocol: - description: The IP protocol of this allowed rule - type: string - title: IP Protocol - ports: - description: List of ports or port ranges - type: array - title: Ports - items: - type: string - denied: - description: List of denied protocols and ports - type: array - title: Denied Rule - items: - type: object - properties: - IPProtocol: - description: The IP protocol of this denied rule - type: string - title: IP Protocol - ports: - description: List of ports or port ranges - type: array - title: Ports - items: - type: string - sourceRanges: - description: List of source IP address ranges - type: array - title: Source IP Ranges - items: - type: string - sourceTags: - description: List of source tags - type: array - title: Source Tags - items: - type: string - sourceServiceAccounts: - description: List of source service accounts - type: array - title: Source Service Accounts - items: - type: string - targetRanges: - description: List of target IP address ranges - type: array - title: Target IP Ranges - items: - type: string - targetTags: - description: List of target tags - type: array - title: Target Network Tags - items: - type: string - targetServiceAccounts: - description: List of target service accounts - type: array - title: Target Service Accounts - items: - type: string - destinationRanges: - description: List of destination IP address ranges - type: array - title: Destination IP Ranges - items: - type: string - egress: - description: If the rule is an egress, this property will contains the firewall rule egress configuration - type: object - title: Egress Configuration - properties: - allowed: - description: List of allowed protocols and ports - type: array - title: Allowed Rule - items: - type: object - properties: - IPProtocol: - description: The IP protocol of this allowed rule - type: string - title: IP Protocol - ports: - description: List of ports or port ranges - type: array - title: Ports - items: - type: string - denied: - description: List of denied protocols and ports - type: array - title: Denied Rule - items: - type: object - properties: - IPProtocol: - description: The IP protocol of this denied rule - type: string - title: IP Protocol - ports: - description: List of ports or port ranges - type: array - title: Ports - items: - type: string - sourceRanges: - description: List of source IP address ranges - type: array - title: Source IP Ranges - items: - type: string - sourceTags: - description: List of source tags - type: array - title: Source Tags - items: - type: string - sourceServiceAccounts: - description: List of source service accounts - type: array - title: Source Service Accounts - items: - type: string - targetRanges: - description: List of target IP address ranges - type: array - title: Target IP Ranges - items: - type: string - targetTags: - description: List of target tags - type: array - title: Target Network Tags - items: - type: string - targetServiceAccounts: - description: List of target service accounts - type: array - title: Target Service Accounts - items: - type: string - destinationRanges: - description: List of destination IP address ranges - type: array - title: Destination IP Ranges - items: - type: string - location: - description: Location of the firewall rule. It will be always 'global' - type: string - title: GCP Location - served: true - storage: true diff --git a/docs/examples/gcp/itds/gcp.mia-platform.eu.folders.yaml b/docs/examples/gcp/itds/gcp.mia-platform.eu.folders.yaml deleted file mode 100644 index efd16b1..0000000 --- a/docs/examples/gcp/itds/gcp.mia-platform.eu.folders.yaml +++ /dev/null @@ -1,52 +0,0 @@ -# yaml-language-server: $schema=https://cdn.mia-platform.eu/catalog/v1alpha1/mia-platform.eu.v1alpha1.ItemTypeDefinition.json ---- -apiVersion: mia-platform.eu/v1alpha1 -kind: ItemTypeDefinition -metadata: - name: folders.gcp.mia-platform.eu -spec: - group: gcp.mia-platform.eu - names: - kind: Folder - plural: folders - displayPlural: Folders - scope: Organization - versions: - - deprecated: false - name: v1alpha1 - schema: - openAPIV31Schema: - type: object - properties: - spec: - type: object - properties: - name: - description: The folder id used to identify the folder in GCP - type: string - title: ID - displayName: - description: The user visible name for the project. - type: string - title: Name - parent: - description: The parent resource identifier (for example, 'folders/{folder_id}' or 'organizations/{org_id}') - type: string - title: Parent Resource - state: - description: Current lifecycle state of the folder. Possible values are 'state_unspecified', 'active', or 'delete_requested'. - type: string - title: Folder State - createTime: - description: RFC 3339 timestamp for when the folder was created - type: string - format: date-time - title: Creation Time - labels: - description: User-provided labels in key-value pairs. - type: object - title: Labels - additionalProperties: - type: string - served: true - storage: true diff --git a/docs/examples/gcp/itds/gcp.mia-platform.eu.jobs.yaml b/docs/examples/gcp/itds/gcp.mia-platform.eu.jobs.yaml deleted file mode 100644 index a53d576..0000000 --- a/docs/examples/gcp/itds/gcp.mia-platform.eu.jobs.yaml +++ /dev/null @@ -1,82 +0,0 @@ -# yaml-language-server: $schema=https://cdn.mia-platform.eu/catalog/v1alpha1/mia-platform.eu.v1alpha1.ItemTypeDefinition.json ---- -apiVersion: mia-platform.eu/v1alpha1 -kind: ItemTypeDefinition -metadata: - name: jobs.gcp.mia-platform.eu -spec: - group: gcp.mia-platform.eu - names: - kind: Job - plural: jobs - displayPlural: Jobs - scope: Organization - versions: - - deprecated: false - name: v1alpha1 - schema: - openAPIV31Schema: - type: object - properties: - spec: - type: object - properties: - apiVersion: - description: API version of the item - type: string - title: API Version - kind: - description: The item kind - type: string - title: Kind - name: - description: The name of the Job item - type: string - title: Name - image: - description: The main container image used for the Job - type: string - title: Container Image - latestExecution: - description: Details of the latest execution of the Job - type: object - title: Latest Job Execution - properties: - name: - description: The name of the executed job - type: string - title: Name - creationTimeStamp: - description: Start time of the execution - type: string - title: Creation Date - completionTimestamp: - description: End time of the execution - type: string - title: Completion Date - completionStatus: - description: Status of the execution - type: string - title: Completion Status - maxRetries: - description: Maximum number of retries for failed executions - type: integer - title: Max Retries - parallelism: - description: Number of parallel executions allowed. If set to 0, there is no limit - type: integer - title: Parallelism - serviceAccountName: - description: The service account email to run the Job - type: string - title: Service Account Name - timeoutSeconds: - description: Maximum execution time in seconds before task termination - type: integer - title: Job Timeout (seconds) - location: - description: Location/region for the Job item - type: string - title: GCP Location - served: true - storage: true diff --git a/docs/examples/gcp/itds/gcp.mia-platform.eu.networks.yaml b/docs/examples/gcp/itds/gcp.mia-platform.eu.networks.yaml deleted file mode 100644 index 86cd632..0000000 --- a/docs/examples/gcp/itds/gcp.mia-platform.eu.networks.yaml +++ /dev/null @@ -1,64 +0,0 @@ -# yaml-language-server: $schema=https://cdn.mia-platform.eu/catalog/v1alpha1/mia-platform.eu.v1alpha1.ItemTypeDefinition.json ---- -apiVersion: mia-platform.eu/v1alpha1 -kind: ItemTypeDefinition -metadata: - name: networks.gcp.mia-platform.eu -spec: - group: gcp.mia-platform.eu - names: - kind: Network - plural: networks - singular: network - scope: Organization - versions: - - deprecated: false - name: v1alpha1 - schema: - openAPIV31Schema: - type: object - properties: - spec: - type: object - properties: - name: - description: The name of the network - type: string - title: Name - description: - description: Human-readable description of the network - type: string - title: Description - mtu: - description: Maximum transmission unit in bytes - type: integer - title: MTU (bytes) - routingMode: - description: Routing mode for the network, possible values are 'regional', or 'global' - type: string - title: Routing Mode - location: - description: Location of the network. It will always be 'global' for GCP networks. - type: string - title: GCP Location - autoCreateSubnetworks: - description: Whether the subnets of the network are created automatically by GCP. - type: boolean - title: Auto Created Subnetworks - subnetworks: - description: List of subnetworks associated with the network. - type: array - title: Subnetworks - items: - type: object - properties: - name: - description: Name of the subnetwork. - type: string - title: Subnetwork Name - region: - description: Region of the subnetwork. - type: string - title: Subnetwork GCP Location - served: true - storage: true diff --git a/docs/examples/gcp/itds/gcp.mia-platform.eu.projects.yaml b/docs/examples/gcp/itds/gcp.mia-platform.eu.projects.yaml deleted file mode 100644 index 5ee7a3c..0000000 --- a/docs/examples/gcp/itds/gcp.mia-platform.eu.projects.yaml +++ /dev/null @@ -1,52 +0,0 @@ -# yaml-language-server: $schema=https://cdn.mia-platform.eu/catalog/v1alpha1/mia-platform.eu.v1alpha1.ItemTypeDefinition.json ---- -apiVersion: mia-platform.eu/v1alpha1 -kind: ItemTypeDefinition -metadata: - name: projects.gcp.mia-platform.eu -spec: - group: gcp.mia-platform.eu - names: - kind: Project - plural: projects - displayPlural: Projects - scope: Organization - versions: - - deprecated: false - name: v1alpha1 - schema: - openAPIV31Schema: - type: object - properties: - spec: - type: object - properties: - projectId: - description: The project ID used to identify the project in GCP - type: string - title: ID - name: - description: The user visible name for the project. - type: string - title: Name - parent: - description: The parent resource identifier (for example, 'folders/{folder_id}' or 'organizations/{org_id}') - type: string - title: Parent Resource - state: - description: Current lifecycle state of the project. Possible values are 'state_unspecified', 'active', or 'delete_requested'. - type: string - title: Project State - createTime: - description: RFC 3339 timestamp for when the project was created - type: string - format: date-time - title: Creation Date - labels: - description: User-provided labels in key-value pairs. - type: object - title: Labels - additionalProperties: - type: string - served: true - storage: true diff --git a/docs/examples/gcp/itds/gcp.mia-platform.eu.services.yaml b/docs/examples/gcp/itds/gcp.mia-platform.eu.services.yaml deleted file mode 100644 index f7d0b53..0000000 --- a/docs/examples/gcp/itds/gcp.mia-platform.eu.services.yaml +++ /dev/null @@ -1,73 +0,0 @@ -# yaml-language-server: $schema=https://cdn.mia-platform.eu/catalog/v1alpha1/mia-platform.eu.v1alpha1.ItemTypeDefinition.json ---- -apiVersion: mia-platform.eu/v1alpha1 -kind: ItemTypeDefinition -metadata: - name: services.gcp.mia-platform.eu -spec: - group: gcp.mia-platform.eu - names: - kind: Service - plural: services - displayPlural: Services - scope: Organization - versions: - - deprecated: false - name: v1alpha1 - schema: - openAPIV31Schema: - type: object - properties: - spec: - type: object - properties: - apiVersion: - description: API version of the item - type: string - title: API Version - kind: - description: The item kind - type: string - title: Kind - name: - description: The name of the Service - type: string - title: Name - traffic: - description: Traffic routing configuration for the Service - type: array - title: Network Traffic Configuration - items: - type: object - properties: - latestRevision: - type: boolean - title: Latest Revision - percent: - type: integer - title: Percentage - revisionName: - type: string - title: Revision Name - url: - description: URL of the Service - type: string - title: Service Endpoint - serviceAccountName: - description: The email of the IAM service account associated with the Service - type: string - title: Service Account Name - latestCreatedRevisionName: - description: Name of the latest created revision - type: string - title: Latest Created Revision Name - latestReadyRevisionName: - description: Name of the latest ready revision - type: string - title: Latest Ready Revision Name - location: - description: Region where the service is deployed - type: string - title: GCP Location - served: true - storage: true diff --git a/docs/examples/gcp/itds/gcp.mia-platform.eu.sqlinstances.yaml b/docs/examples/gcp/itds/gcp.mia-platform.eu.sqlinstances.yaml deleted file mode 100644 index 16387fe..0000000 --- a/docs/examples/gcp/itds/gcp.mia-platform.eu.sqlinstances.yaml +++ /dev/null @@ -1,141 +0,0 @@ -# yaml-language-server: $schema=https://cdn.mia-platform.eu/catalog/v1alpha1/mia-platform.eu.v1alpha1.ItemTypeDefinition.json ---- -apiVersion: mia-platform.eu/v1alpha1 -kind: ItemTypeDefinition -metadata: - name: sqlinstances.gcp.mia-platform.eu -spec: - group: gcp.mia-platform.eu - names: - kind: SQLInstance - plural: sqlinstances - displayPlural: SQLInstances - scope: Organization - versions: - - deprecated: false - name: v1alpha1 - schema: - openAPIV31Schema: - type: object - properties: - spec: - type: object - properties: - name: - description: The name of the Cloud SQL instance - type: string - title: DB Name - region: - description: Region where the instance is located - type: string - title: GCP Location - location: - description: The GCP zone of the region where the primary instance is located - type: string - title: Primary GCP Location - dataDiskSizeGb: - description: Instance data disk size in GB - type: string - title: Data Disk Size (GB) - dataDiskType: - description: The type of data disk, possible values are 'ssd', or 'hdd'. - type: string - title: Data Disk Type - storageAutoResize: - description: Whether storage auto resize is enabled - type: boolean - title: Storage Auto Resize - storageAutoResizeLimit: - description: The maximum size to which storage capacity can be automatically increased. The default value is 0, which specifies that there is no limit. - type: integer - title: Storage Auto Resize Limit (GB) - installedVersion: - description: The installed database patch/version - type: string - title: DB Version - availabilityType: - description: The availability type of the instance, possible values are 'zonal' and 'regional'. - type: string - title: Availability Type - serviceAccount: - description: The service account email address assigned to the instance - type: string - title: Service Account Name - deletionProtectionEnabled: - description: Whether the deletion protection is enabled - type: boolean - title: Deletion Protection Enabled - state: - description: Instance current state - type: string - title: DB State - connection: - description: Connection settings for the instance - type: object - title: Connection Settings - properties: - requireSSL: - type: boolean - title: Require SSL - sslMode: - type: string - title: SSL Mode - serverCAMode: - type: string - title: CA Mode - serverCertificateRotationMode: - type: string - title: Server Certificate Rotation Mode - ipv4Enabled: - type: boolean - title: IPv4 Enabled - authorizedNetworks: - type: array - title: Authorized Networks - items: - type: object - properties: - name: - type: string - title: Name - value: - type: string - title: CIDR Range - dns: - type: string - title: DNS - dnsNames: - type: array - title: DNS Names - items: - type: object - properties: - connectionType: - type: string - title: Connection Type - dnsScope: - type: string - title: DNS Scope - name: - type: string - title: Name - ipAddresses: - type: array - title: IP Addresses - items: - type: object - properties: - type: - type: string - title: Type - ipAddress: - type: string - title: Address - labels: - description: User labels assigned to the instance - type: object - title: Labels - additionalProperties: - type: string - served: true - storage: true diff --git a/docs/examples/github/itds/github.mia-platform.eu.repositories.yaml b/docs/examples/github/itds/github.mia-platform.eu.repositories.yaml deleted file mode 100644 index d960b66..0000000 --- a/docs/examples/github/itds/github.mia-platform.eu.repositories.yaml +++ /dev/null @@ -1,108 +0,0 @@ -# yaml-language-server: $schema=https://cdn.mia-platform.eu/catalog/v1alpha1/mia-platform.eu.v1alpha1.ItemTypeDefinition.json ---- -apiVersion: mia-platform.eu/v1alpha1 -kind: ItemTypeDefinition -metadata: - name: repositories.github.mia-platform.eu -spec: - group: github.mia-platform.eu - names: - kind: Repository - plural: repositories - displayPlural: GitHub Repositories - scope: Organization - versions: - - deprecated: false - name: v1alpha1 - schema: - openAPIV31Schema: - type: object - properties: - spec: - type: object - properties: - id: - description: The unique numeric ID of the repository. - type: integer - title: ID - name: - description: The name of the repository. - type: string - title: Name - fullName: - description: The full name of the repository (org/repo). - type: string - title: Full Name - description: - description: A short description of the repository. - type: string - title: Description - htmlUrl: - description: The URL of the repository on GitHub. - type: string - title: HTML URL - language: - description: The primary programming language of the repository. - type: string - title: Language - languages: - description: Percentage breakdown of programming languages used in the repository. - type: object - title: Languages - additionalProperties: - type: number - visibility: - description: "Repository visibility: public, private, or internal." - type: string - title: Visibility - defaultBranch: - description: The default branch of the repository. - type: string - title: Default Branch - archived: - description: Whether the repository is archived. - type: boolean - title: Archived - fork: - description: Whether the repository is a fork. - type: boolean - title: Fork - stargazersCount: - description: Number of stars on the repository. - type: integer - title: Stars - forksCount: - description: Number of forks of the repository. - type: integer - title: Forks - openIssuesCount: - description: Number of open issues and pull requests. - type: integer - title: Open Issues - license: - description: The license of the repository, serialized as JSON. - type: string - title: License - topics: - description: The list of topics associated with the project. - type: array - title: Topics - items: - type: string - created: - description: The creation time of the repository (RFC 3339). - type: string - format: date-time - title: Created At - lastUpdate: - description: The last time the repository metadata was updated (RFC 3339). - type: string - format: date-time - title: Last Update - lastPush: - description: The last time a push was made to the repository (RFC 3339). - type: string - format: date-time - title: Last Push - served: true - storage: true diff --git a/docs/examples/github/itds/github.mia-platform.eu.workflowruns.yaml b/docs/examples/github/itds/github.mia-platform.eu.workflowruns.yaml deleted file mode 100644 index f726ed5..0000000 --- a/docs/examples/github/itds/github.mia-platform.eu.workflowruns.yaml +++ /dev/null @@ -1,100 +0,0 @@ -# yaml-language-server: $schema=https://cdn.mia-platform.eu/catalog/v1alpha1/mia-platform.eu.v1alpha1.ItemTypeDefinition.json ---- -apiVersion: mia-platform.eu/v1alpha1 -kind: ItemTypeDefinition -metadata: - name: workflowruns.github.mia-platform.eu -spec: - group: github.mia-platform.eu - names: - kind: WorkflowRun - plural: workflowruns - displayPlural: GitHub Workflow Runs - scope: Organization - versions: - - deprecated: false - name: v1alpha1 - schema: - openAPIV31Schema: - type: object - properties: - spec: - type: object - properties: - id: - description: The unique numeric ID of the workflow run. - type: integer - title: ID - name: - description: The name of the workflow run. - type: string - title: Name - displayTitle: - description: The display title of the workflow run. - type: string - title: Display Title - status: - description: "The current status of the workflow run: queued, in_progress, or completed." - type: string - title: Status - conclusion: - description: "The outcome of the completed run: success, failure, cancelled, skipped, etc." - type: string - title: Conclusion - workflowId: - description: The ID of the workflow definition that this run belongs to. - type: integer - title: Workflow ID - workflowPath: - description: The path of the workflow definition file within the repository. - type: string - title: Workflow Path - repositoryId: - description: The numeric ID of the repository the run belongs to. - type: integer - title: Repository ID - repositoryFullName: - description: The full name (owner/repo) of the repository the run belongs to. - type: string - title: Repository - headBranch: - description: The branch the run was triggered on. - type: string - title: Branch - headSha: - description: The commit SHA that triggered the run. - type: string - title: Commit SHA - event: - description: The event that triggered the workflow run. - type: string - title: Trigger Event - runNumber: - description: The sequential run number within the repository. - type: integer - title: Run Number - runAttempt: - description: The attempt number of the run (re-runs increment this). - type: integer - title: Run Attempt - htmlUrl: - description: The URL of the workflow run on GitHub. - type: string - title: HTML URL - created: - description: The time the run was created (RFC 3339). - type: string - format: date-time - title: Created At - updated: - description: The time the run was last updated (RFC 3339). - type: string - format: date-time - title: Updated At - runStartedAt: - description: The time the run actually started executing (RFC 3339). - type: string - format: date-time - title: Started At - served: true - storage: true diff --git a/docs/examples/gitlab/itds/gitlab.mia-platform.eu.accesstokens.yaml b/docs/examples/gitlab/itds/gitlab.mia-platform.eu.accesstokens.yaml deleted file mode 100644 index 35d9f23..0000000 --- a/docs/examples/gitlab/itds/gitlab.mia-platform.eu.accesstokens.yaml +++ /dev/null @@ -1,86 +0,0 @@ -# yaml-language-server: $schema=https://cdn.mia-platform.eu/catalog/v1alpha1/mia-platform.eu.v1alpha1.ItemTypeDefinition.json ---- -apiVersion: mia-platform.eu/v1alpha1 -kind: ItemTypeDefinition -metadata: - name: accesstokens.gitlab.mia-platform.eu -spec: - group: gitlab.mia-platform.eu - names: - kind: AccessToken - plural: accesstokens - displayPlural: Access Tokens - scope: Organization - versions: - - deprecated: false - name: v1alpha1 - schema: - openAPIV31Schema: - type: object - properties: - spec: - type: object - properties: - id: - description: The unique identifier of the access token. - type: integer - title: ID - name: - description: The name of the access token. - type: string - title: Name - revoked: - description: Whether the access token has been revoked. - type: boolean - title: Revoked - createdAt: - description: The date and time when the access token was created. - type: string - title: Created At - format: date-time - description: - description: An optional description of the access token. - type: string - title: Description - scopes: - description: The list of scopes granted to the access token. - type: array - title: Scopes - items: - type: string - userId: - description: The ID of the user associated with the access token. - type: integer - title: User ID - lastUsedAt: - description: The date and time the access token was last used. - type: string - title: Last Used At - format: date-time - active: - description: Whether the access token is currently active. - type: boolean - title: Active - expiresAt: - description: The date when the access token expires. - type: string - title: Expires At - format: date - accessLevel: - description: The access level granted by the token (e.g. 40 for Maintainer). - type: integer - title: Access Level - resourceType: - description: The type of resource the token belongs to (project or group). - type: string - title: Resource Type - resourceId: - description: The ID of the resource (project or group) the token belongs to. - type: integer - title: Resource ID - resourceName: - description: The name of the resource (project or group) the token belongs to. - type: string - title: Resource Name - served: true - storage: true diff --git a/docs/examples/gitlab/itds/gitlab.mia-platform.eu.pipelines.yaml b/docs/examples/gitlab/itds/gitlab.mia-platform.eu.pipelines.yaml deleted file mode 100644 index b1cc5b9..0000000 --- a/docs/examples/gitlab/itds/gitlab.mia-platform.eu.pipelines.yaml +++ /dev/null @@ -1,67 +0,0 @@ -# yaml-language-server: $schema=https://cdn.mia-platform.eu/catalog/v1alpha1/mia-platform.eu.v1alpha1.ItemTypeDefinition.json ---- -apiVersion: mia-platform.eu/v1alpha1 -kind: ItemTypeDefinition -metadata: - name: pipelines.gitlab.mia-platform.eu -spec: - group: gitlab.mia-platform.eu - names: - kind: Pipeline - plural: pipelines - displayPlural: Pipelines - scope: Organization - versions: - - deprecated: false - name: v1alpha1 - schema: - openAPIV31Schema: - type: object - properties: - spec: - type: object - properties: - id: - description: The unique identifier of the pipeline. - type: integer - title: ID - created_at: - description: The date and time when the pipeline was created. - type: string - title: Created At - format: date-time - finished_at: - description: The date and time when the pipeline finished. - type: string - title: Finished At - format: date-time - projectId: - description: The ID of the project this pipeline belongs to. - type: integer - title: Project ID - ref: - description: The branch or tag the pipeline was triggered for. - type: string - title: Ref - sha: - description: The commit SHA the pipeline was triggered for. - type: string - title: SHA - source: - description: The source that triggered the pipeline (e.g. push, web, schedule). - type: string - title: Source - status: - description: The current status of the pipeline (e.g. running, success, failed). - type: string - title: Status - userId: - description: The ID of the user who triggered the pipeline. - type: integer - title: User ID - url: - description: The web URL of the pipeline. - type: string - title: URL - served: true - storage: true diff --git a/docs/examples/gitlab/itds/gitlab.mia-platform.eu.projects.yaml b/docs/examples/gitlab/itds/gitlab.mia-platform.eu.projects.yaml deleted file mode 100644 index 2390f51..0000000 --- a/docs/examples/gitlab/itds/gitlab.mia-platform.eu.projects.yaml +++ /dev/null @@ -1,248 +0,0 @@ -# yaml-language-server: $schema=https://cdn.mia-platform.eu/catalog/v1alpha1/mia-platform.eu.v1alpha1.ItemTypeDefinition.json ---- -apiVersion: mia-platform.eu/v1alpha1 -kind: ItemTypeDefinition -metadata: - name: projects.gitlab.mia-platform.eu -spec: - group: gitlab.mia-platform.eu - names: - kind: Project - plural: projects - displayPlural: Projects - scope: Organization - versions: - - deprecated: false - name: v1alpha1 - schema: - openAPIV31Schema: - type: object - properties: - spec: - type: object - properties: - id: - description: The unique identifier of the project. - type: integer - title: ID - name: - description: The name of the project. - type: string - title: Name - nameWithNamespace: - description: The name of the project including its namespace. - type: string - title: Name With Namespace - description: - description: A description of the project. - type: string - title: Description - descriptionHtml: - description: The HTML rendered description of the project. - type: string - title: Description HTML - createdAt: - description: The date and time when the project was created. - type: string - title: Created At - format: date-time - defaultBranch: - description: The default branch of the project. - type: string - title: Default Branch - tagList: - description: The list of tags associated with the project. - type: array - title: Tag List - items: - type: string - topics: - description: The list of topics associated with the project. - type: array - title: Topics - items: - type: string - sshUrlToRepo: - description: The SSH URL to clone the repository. - type: string - title: SSH URL to Repo - httpUrlToRepo: - description: The HTTP URL to clone the repository. - type: string - title: HTTP URL to Repo - webUrl: - description: The web URL of the project. - type: string - title: Web URL - readmeUrl: - description: The URL of the project's README file. - type: string - title: README URL - forksCount: - description: The number of forks of the project. - type: integer - title: Forks Count - starCount: - description: The number of stars of the project. - type: integer - title: Star Count - lastActivityAt: - description: The date and time of the last activity on the project. - type: string - title: Last Activity At - format: date-time - packagesEnabled: - description: Whether packages are enabled for the project. - type: boolean - title: Packages Enabled - emptyRepo: - description: Whether the repository is empty. - type: boolean - title: Empty Repo - archived: - description: Whether the project is archived. - type: boolean - title: Archived - visibility: - description: The visibility level of the project (public, internal, private). - type: string - title: Visibility - issuesEnabled: - description: Whether issues are enabled for the project. - type: boolean - title: Issues Enabled - mergeRequestsEnabled: - description: Whether merge requests are enabled for the project. - type: boolean - title: Merge Requests Enabled - wikiEnabled: - description: Whether the wiki is enabled for the project. - type: boolean - title: Wiki Enabled - jobsEnabled: - description: Whether CI/CD jobs are enabled for the project. - type: boolean - title: Jobs Enabled - snippetsEnabled: - description: Whether snippets are enabled for the project. - type: boolean - title: Snippets Enabled - containerRegistryEnabled: - description: Whether the container registry is enabled for the project. - type: boolean - title: Container Registry Enabled - serviceDesksEnabled: - description: Whether service desks are enabled for the project. - type: boolean - title: Service Desks Enabled - canCreateMergeRequestIn: - description: Whether a merge request can be created for the current user in this project. - type: boolean - title: Can Create Merge Request In - issuesAccessLevel: - description: The access level for issues. - type: string - title: Issues Access Level - repositoryAccessLevel: - description: The access level for the repository. - type: string - title: Repository Access Level - mergeRequestsAccessLevel: - description: The access level for merge requests. - type: string - title: Merge Requests Access Level - forkingAccessLevel: - description: The access level for forking. - type: string - title: Forking Access Level - wikiAccessLevel: - description: The access level for the wiki. - type: string - title: Wiki Access Level - buildsAccessLevel: - description: The access level for builds. - type: string - title: Builds Access Level - snippetsAccessLevel: - description: The access level for snippets. - type: string - title: Snippets Access Level - pagesAccessLevel: - description: The access level for pages. - type: string - title: Pages Access Level - securityAndComplianceAccessLevel: - description: The access level for security and compliance features. - type: string - title: Security And Compliance Access Level - releasesAccessLevel: - description: The access level for releases. - type: string - title: Releases Access Level - environmentsAccessLevel: - description: The access level for environments. - type: string - title: Environments Access Level - featureFlagsAccessLevel: - description: The access level for feature flags. - type: string - title: Feature Flags Access Level - infrastructureAccessLevel: - description: The access level for infrastructure. - type: string - title: Infrastructure Access Level - creatorId: - description: The ID of the user who created the project. - type: integer - title: Creator ID - publicJobs: - description: Whether jobs are visible to the public. - type: boolean - title: Public Jobs - onlyAllowMergeIfPipelineSucceeds: - description: Whether merges are only allowed when all pipeline jobs succeed. - type: boolean - title: Only Allow Merge If Pipeline Succeeds - allowMergeOnSkippedPipeline: - description: Whether merges are allowed when the pipeline is skipped. - type: boolean - title: Allow Merge On Skipped Pipeline - requestAccessEnabled: - description: Whether users can request access to the project. - type: boolean - title: Request Access Enabled - mergeMethod: - description: The merge method for the project (merge, rebase_merge, ff). - type: string - title: Merge Method - license: - description: The license information for the project. - type: object - title: License - additionalProperties: true - owner: - description: The owner user of the project (for personal projects). - type: object - title: Owner - additionalProperties: true - path: - description: The path of the project. - type: string - title: Path - pathWithNamespace: - description: The path of the project including its namespace. - type: string - title: Path With Namespace - updatedAt: - description: The date and time when the project was last updated. - type: string - title: Updated At - format: date-time - languages: - description: The programming languages used in the project and their percentage. - type: object - title: Languages - additionalProperties: - type: number - served: true - storage: true diff --git a/docs/examples/nexus/itds/nexus.mia-platform.eu.dockerimages.yaml b/docs/examples/nexus/itds/nexus.mia-platform.eu.dockerimages.yaml deleted file mode 100644 index 760f82b..0000000 --- a/docs/examples/nexus/itds/nexus.mia-platform.eu.dockerimages.yaml +++ /dev/null @@ -1,75 +0,0 @@ -# yaml-language-server: $schema=https://cdn.mia-platform.eu/catalog/v1alpha1/mia-platform.eu.v1alpha1.ItemTypeDefinition.json ---- -apiVersion: mia-platform.eu/v1alpha1 -kind: ItemTypeDefinition -metadata: - name: dockerimages.nexus.mia-platform.eu -spec: - group: nexus.mia-platform.eu - names: - kind: DockerImage - plural: dockerimages - displayPlural: Docker Images - scope: Organization - versions: - - deprecated: false - name: v1alpha1 - schema: - openAPIV31Schema: - type: object - properties: - spec: - type: object - properties: - host: - description: The registry host of the Docker image. - type: string - title: Host - name: - description: The name of the Docker image. - type: string - title: Name - version: - description: The version tag of the Docker image. - type: string - title: Version - repository: - description: The repository containing this Docker image. - type: string - title: Repository - format: - description: The component format (e.g. docker). - type: string - title: Format - tags: - description: Tags associated with this Docker image. - type: array - title: Tags - additionalProperties: true - assets: - description: The list of assets belonging to this Docker image component. - type: array - title: Assets - items: - type: object - properties: - sha256: - description: SHA-256 checksum of the asset. - type: string - title: SHA-256 - downloadUrl: - description: URL to download the asset. - type: string - title: Download URL - lastModified: - description: Timestamp of the last modification to the asset. - type: string - format: date-time - title: Last Modified - lastDownloaded: - description: Timestamp of the last download of the asset. - type: string - format: date-time - title: Last Downloaded - served: true - storage: true diff --git a/docs/examples/sysdig/itds/sysdig.mia-platform.eu.vulnerabilities.yaml b/docs/examples/sysdig/itds/sysdig.mia-platform.eu.vulnerabilities.yaml deleted file mode 100644 index cfd870e..0000000 --- a/docs/examples/sysdig/itds/sysdig.mia-platform.eu.vulnerabilities.yaml +++ /dev/null @@ -1,143 +0,0 @@ -# yaml-language-server: $schema=https://cdn.mia-platform.eu/catalog/v1alpha1/mia-platform.eu.v1alpha1.ItemTypeDefinition.json ---- -apiVersion: mia-platform.eu/v1alpha1 -kind: ItemTypeDefinition -metadata: - name: vulnerabilities.sysdig.mia-platform.eu -spec: - group: sysdig.mia-platform.eu - names: - kind: Vulnerability - plural: vulnerabilities - singular: vulnerability - displayPlural: Sysdig vulnerabilities - displaySingular: Sysdig vulnerability - scope: Organization - versions: - - deprecated: false - name: v1alpha1 - schema: - openAPIV31Schema: - type: object - properties: - spec: - type: object - properties: - createdAt: - description: The timestamp when the vulnerability record was created. - type: string - format: date-time - title: Created At - cvssScore: - description: The CVSS score of the vulnerability. - type: number - title: CVSS Score - cvssSource: - description: The source of the CVSS score. - type: string - title: CVSS Source - cvssVector: - description: The CVSS vector string. - type: string - title: CVSS Vector - cvssVersion: - description: The version of the CVSS scoring system used. - type: string - title: CVSS Version - firstSeen: - description: The timestamp when the vulnerability was first seen. - type: string - format: date-time - title: First Seen - fixDate: - description: The date when a fix was made available. - type: string - format: date-time - title: Fix Date - fixedInVersion: - description: The package version that contains the fix. - type: string - title: Fixed In Version - globalId: - description: The global identifier of the vulnerability (e.g. CVE ID). - type: string - title: Global ID - hasExploit: - description: Whether a known exploit exists for this vulnerability. - type: [boolean, "null"] - title: Has Exploit - hasFix: - description: Whether a fix is available for this vulnerability. - type: [boolean, "null"] - title: Has Fix - hash: - description: A hash uniquely identifying the vulnerability record. - type: string - title: Hash - inUse: - description: Whether the vulnerable package is currently in use. - type: [boolean, "null"] - title: In Use - knownRansomwareCampaignUse: - description: Whether this vulnerability is known to be used in ransomware campaigns. - type: [boolean, "null"] - title: Known Ransomware Campaign Use - lastModified: - description: The timestamp when the vulnerability record was last modified. - type: string - format: date-time - title: Last Modified - name: - description: The name of the vulnerability (e.g. CVE identifier). - type: string - title: Name - namespace: - description: The namespace of the vulnerability source feed. - type: array - title: Namespace - items: - type: string - packageInfo: - description: Additional information about the affected package. - type: string - title: Package Info - packageName: - description: The name of the affected package. - type: string - title: Package Name - packagePath: - description: The file-system path of the affected package. - type: string - title: Package Path - packageType: - description: The type of the affected package (e.g. rpm, deb, npm). - type: string - title: Package Type - packageVersion: - description: The version of the affected package. - type: string - title: Package Version - publicationDate: - description: The date when the vulnerability was publicly disclosed. - type: string - format: date-time - title: Publication Date - purl: - description: The package URL (purl) of the affected package. - type: string - title: Package URL - severity: - description: The severity label of the vulnerability (e.g. Critical, High, Medium, Low). - type: string - title: Severity - solutionDate: - description: The date when a solution was provided. - type: string - format: date-time - title: Solution Date - suggestedFix: - description: The suggested remediation or fix for the vulnerability. - type: string - title: Suggested Fix - served: true - storage: true diff --git a/docs/explanation/20_extra_mappings.md b/docs/explanation/20_extra_mappings.md index 375be91..37474b5 100644 --- a/docs/explanation/20_extra_mappings.md +++ b/docs/explanation/20_extra_mappings.md @@ -53,7 +53,7 @@ will map the sent relationship anyway. ``` yaml extra: - - apiVersion: mia-platform.eu/v1alpha1 + - apiVersion: mia-platform.eu/v1 itemFamily: relationships deletePolicy: "cascade" createIf: |- @@ -69,12 +69,12 @@ extra: {{- $otherValue := (get "otherValue" $value nil) -}} {{- printf "relationship-%s-%s-example-type" $otherValue .directValue | sha256sum}} sourceRef: - apiVersion: "mia-platform.eu/v1alpha1" + apiVersion: "mia-platform.eu/v1" family: "family-example" name: |- {{- printf "family-example-%s-example" .anotherId | sha256sum}} typeRef: - apiVersion: "mia-platform.eu/v1alpha1" + apiVersion: "mia-platform.eu/v1" family: "relationship-types" name: "example-type.mia-platform.eu" ``` diff --git a/docs/how-to/070_gitlab-source.md b/docs/how-to/070_gitlab-source.md index 07294df..fe2c9bb 100644 --- a/docs/how-to/070_gitlab-source.md +++ b/docs/how-to/070_gitlab-source.md @@ -80,7 +80,7 @@ synchronize. ## Example Mapping Files -Example mapping files are provided in the `docs/examples/gitlab/mappings/` directory: +Example mapping files are provided in the `docs/mappings/gitlab/` directory: - `projects.yaml` — maps GitLab projects to Catalog items. - `pipelines.yaml` — maps pipelines to Catalog items. @@ -90,11 +90,11 @@ These files can be used as a starting point for your own mapping configurations. or a specific file to the `--mapping-file` flag: ```sh -ibdm sync gitlab --mapping-file docs/examples/gitlab/mappings/ +ibdm sync gitlab --mapping-file docs/mappings/gitlab/ ``` For local development and debugging, add the `--local-output` flag to send results to stdout: ```sh -ibdm sync gitlab --mapping-file docs/examples/gitlab/mappings/ --local-output +ibdm sync gitlab --mapping-file docs/mappings/gitlab/ --local-output ``` diff --git a/docs/how-to/090_nexus-source.md b/docs/how-to/090_nexus-source.md index 9a83c23..124a8c6 100644 --- a/docs/how-to/090_nexus-source.md +++ b/docs/how-to/090_nexus-source.md @@ -96,7 +96,7 @@ in the Nexus administration UI. ## Example Mapping Files -Example mapping files are provided in the `docs/examples/nexus/mappings/` directory: +Example mapping files are provided in the `docs/mappings/nexus/` directory: - `dockerimages.yaml` — maps Docker image assets to Catalog items. @@ -104,11 +104,11 @@ This file can be used as a starting point for your own mapping configuration. Pa or the folder to the `--mapping-file` flag: ```sh -ibdm sync nexus --mapping-file docs/examples/nexus/mappings/ +ibdm sync nexus --mapping-file docs/mappings/nexus/ ``` For local development and debugging, add the `--local-output` flag to send results to stdout: ```sh -ibdm sync nexus --mapping-file docs/examples/nexus/mappings/ --local-output +ibdm sync nexus --mapping-file docs/mappings/nexus/ --local-output ``` diff --git a/docs/examples/azure-devops/mappings/gitrepositories.yaml b/docs/mappings/azure-devops/gitrepositories.yaml similarity index 93% rename from docs/examples/azure-devops/mappings/gitrepositories.yaml rename to docs/mappings/azure-devops/gitrepositories.yaml index 0d996b1..6ff2e1f 100644 --- a/docs/examples/azure-devops/mappings/gitrepositories.yaml +++ b/docs/mappings/azure-devops/gitrepositories.yaml @@ -1,4 +1,4 @@ -apiVersion: azuredevops.mia-platform.eu/v1alpha1 +apiVersion: azuredevops.mia-platform.eu/v1 itemFamily: gitrepositories type: gitrepository syncable: true diff --git a/docs/examples/azure-devops/mappings/teams.yaml b/docs/mappings/azure-devops/teams.yaml similarity index 84% rename from docs/examples/azure-devops/mappings/teams.yaml rename to docs/mappings/azure-devops/teams.yaml index 8413765..1d181e7 100644 --- a/docs/examples/azure-devops/mappings/teams.yaml +++ b/docs/mappings/azure-devops/teams.yaml @@ -1,4 +1,4 @@ -apiVersion: azuredevops.mia-platform.eu/v1alpha1 +apiVersion: azuredevops.mia-platform.eu/v1 itemFamily: teams type: team syncable: true diff --git a/docs/examples/azure/mappings/cognitiveaccounts.yaml b/docs/mappings/azure/cognitiveaccounts.yaml similarity index 93% rename from docs/examples/azure/mappings/cognitiveaccounts.yaml rename to docs/mappings/azure/cognitiveaccounts.yaml index 45bf1c4..0ab5e14 100644 --- a/docs/examples/azure/mappings/cognitiveaccounts.yaml +++ b/docs/mappings/azure/cognitiveaccounts.yaml @@ -1,4 +1,4 @@ -apiVersion: azure.mia-platform.eu/v1alpha1 +apiVersion: azure.mia-platform.eu/v1 itemFamily: cognitiveaccounts type: Microsoft.CognitiveServices/accounts extra: diff --git a/docs/examples/azure/mappings/containerapps.yaml b/docs/mappings/azure/containerapps.yaml similarity index 96% rename from docs/examples/azure/mappings/containerapps.yaml rename to docs/mappings/azure/containerapps.yaml index fc70176..1202e34 100644 --- a/docs/examples/azure/mappings/containerapps.yaml +++ b/docs/mappings/azure/containerapps.yaml @@ -1,4 +1,4 @@ -apiVersion: azure.mia-platform.eu/v1alpha1 +apiVersion: azure.mia-platform.eu/v1 itemFamily: containerapps type: Microsoft.App/containerApps extra: diff --git a/docs/examples/azure/mappings/managedclusters.yaml b/docs/mappings/azure/managedclusters.yaml similarity index 72% rename from docs/examples/azure/mappings/managedclusters.yaml rename to docs/mappings/azure/managedclusters.yaml index 79ada4c..490ea24 100644 --- a/docs/examples/azure/mappings/managedclusters.yaml +++ b/docs/mappings/azure/managedclusters.yaml @@ -1,4 +1,4 @@ -apiVersion: azure.mia-platform.eu/v1alpha1 +apiVersion: azure.mia-platform.eu/v1 itemFamily: managedclusters type: Microsoft.ContainerService/managedClusters extra: @@ -32,16 +32,16 @@ mappings: {{- object | toJSON -}} {{- end }} extra: - - apiVersion: mia-platform.eu/v1alpha1 + - apiVersion: mia-platform.eu/v1 itemFamily: relationships deletePolicy: "cascade" identifier: |- - {{ $src := printf "urn:mia-platform-catalog:console.mia-platform.eu:v1alpha1:Cluster:%s" (printf "%s" .properties.fqdn | sha256sum) -}} - {{- $type := "urn:mia-platform-catalog:mia-platform.eu:v1alpha1:RelationshipType:dependency.mia-platform.eu" -}} - {{- $tgt := printf "urn:mia-platform-catalog:azure.mia-platform.eu:v1alpha1:ManagedCluster:%s" (printf "%s" .id | sha256sum) -}} + {{ $src := printf "urn:mia-platform-catalog:console.mia-platform.eu:v1:Cluster:%s" (printf "%s" .properties.fqdn | sha256sum) -}} + {{- $type := "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:dependency.mia-platform.eu" -}} + {{- $tgt := printf "urn:mia-platform-catalog:azure.mia-platform.eu:v1:ManagedCluster:%s" (printf "%s" .id | sha256sum) -}} {{- printf "%s-%s-%s" $src $type $tgt | sha256sum }} sourceRef: |- - urn:mia-platform-catalog:console.mia-platform.eu:v1alpha1:Cluster:{{ printf "%s" .properties.fqdn | sha256sum }} + urn:mia-platform-catalog:console.mia-platform.eu:v1:Cluster:{{ printf "%s" .properties.fqdn | sha256sum }} targetRef: |- - urn:mia-platform-catalog:azure.mia-platform.eu:v1alpha1:ManagedCluster:{{ (printf "%s" .id | sha256sum) }} - typeRef: "urn:mia-platform-catalog:mia-platform.eu:v1alpha1:RelationshipType:dependency.mia-platform.eu" + urn:mia-platform-catalog:azure.mia-platform.eu:v1:ManagedCluster:{{ (printf "%s" .id | sha256sum) }} + typeRef: "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:dependency.mia-platform.eu" diff --git a/docs/examples/azure/mappings/postgresqldbs.yaml b/docs/mappings/azure/postgresqldbs.yaml similarity index 94% rename from docs/examples/azure/mappings/postgresqldbs.yaml rename to docs/mappings/azure/postgresqldbs.yaml index 7f070dc..1762c71 100644 --- a/docs/examples/azure/mappings/postgresqldbs.yaml +++ b/docs/mappings/azure/postgresqldbs.yaml @@ -1,4 +1,4 @@ -apiVersion: azure.mia-platform.eu/v1alpha1 +apiVersion: azure.mia-platform.eu/v1 itemFamily: postgresqldbs type: Microsoft.DBforPostgreSQL/flexibleServers extra: diff --git a/docs/examples/azure/mappings/resourcegroups.yaml b/docs/mappings/azure/resourcegroups.yaml similarity index 92% rename from docs/examples/azure/mappings/resourcegroups.yaml rename to docs/mappings/azure/resourcegroups.yaml index cad271c..3e6f103 100644 --- a/docs/examples/azure/mappings/resourcegroups.yaml +++ b/docs/mappings/azure/resourcegroups.yaml @@ -1,4 +1,4 @@ -apiVersion: azure.mia-platform.eu/v1alpha1 +apiVersion: azure.mia-platform.eu/v1 itemFamily: resourcegroups type: Microsoft.Resources/resourceGroups extra: diff --git a/docs/examples/azure/mappings/storageaccounts.yaml b/docs/mappings/azure/storageaccounts.yaml similarity index 96% rename from docs/examples/azure/mappings/storageaccounts.yaml rename to docs/mappings/azure/storageaccounts.yaml index 6c79773..8904d78 100644 --- a/docs/examples/azure/mappings/storageaccounts.yaml +++ b/docs/mappings/azure/storageaccounts.yaml @@ -1,4 +1,4 @@ -apiVersion: azure.mia-platform.eu/v1alpha1 +apiVersion: azure.mia-platform.eu/v1 itemFamily: storageaccounts type: Microsoft.Storage/storageAccounts extra: diff --git a/docs/examples/azure/mappings/subscriptions.yaml b/docs/mappings/azure/subscriptions.yaml similarity index 92% rename from docs/examples/azure/mappings/subscriptions.yaml rename to docs/mappings/azure/subscriptions.yaml index f626a44..02fbd11 100644 --- a/docs/examples/azure/mappings/subscriptions.yaml +++ b/docs/mappings/azure/subscriptions.yaml @@ -1,4 +1,4 @@ -apiVersion: azure.mia-platform.eu/v1alpha1 +apiVersion: azure.mia-platform.eu/v1 itemFamily: subscriptions type: Microsoft.Resources/subscriptions extra: diff --git a/docs/examples/azure/mappings/virtualmachines.yaml b/docs/mappings/azure/virtualmachines.yaml similarity index 95% rename from docs/examples/azure/mappings/virtualmachines.yaml rename to docs/mappings/azure/virtualmachines.yaml index 411fd52..0e66cf7 100644 --- a/docs/examples/azure/mappings/virtualmachines.yaml +++ b/docs/mappings/azure/virtualmachines.yaml @@ -1,4 +1,4 @@ -apiVersion: azure.mia-platform.eu/v1alpha1 +apiVersion: azure.mia-platform.eu/v1 itemFamily: virtualmachines type: Microsoft.Compute/virtualMachines extra: diff --git a/docs/examples/azure/mappings/virtualnetworks.yaml b/docs/mappings/azure/virtualnetworks.yaml similarity index 95% rename from docs/examples/azure/mappings/virtualnetworks.yaml rename to docs/mappings/azure/virtualnetworks.yaml index 0747dd3..1da91a1 100644 --- a/docs/examples/azure/mappings/virtualnetworks.yaml +++ b/docs/mappings/azure/virtualnetworks.yaml @@ -1,4 +1,4 @@ -apiVersion: azure.mia-platform.eu/v1alpha1 +apiVersion: azure.mia-platform.eu/v1 itemFamily: virtualnetworks type: Microsoft.Network/virtualNetworks extra: diff --git a/docs/examples/azure/mappings/websites.yaml b/docs/mappings/azure/websites.yaml similarity index 94% rename from docs/examples/azure/mappings/websites.yaml rename to docs/mappings/azure/websites.yaml index 86c1365..d9b022b 100644 --- a/docs/examples/azure/mappings/websites.yaml +++ b/docs/mappings/azure/websites.yaml @@ -1,4 +1,4 @@ -apiVersion: azure.mia-platform.eu/v1alpha1 +apiVersion: azure.mia-platform.eu/v1 itemFamily: websites type: Microsoft.Web/sites extra: diff --git a/docs/examples/bitbucket/mappings/pipelines.yaml b/docs/mappings/bitbucket/pipelines.yaml similarity index 73% rename from docs/examples/bitbucket/mappings/pipelines.yaml rename to docs/mappings/bitbucket/pipelines.yaml index c897c71..aaf8eae 100644 --- a/docs/examples/bitbucket/mappings/pipelines.yaml +++ b/docs/mappings/bitbucket/pipelines.yaml @@ -1,4 +1,4 @@ -apiVersion: bitbucket.mia-platform.eu/v1alpha1 +apiVersion: bitbucket.mia-platform.eu/v1 itemFamily: pipelines type: pipeline syncable: true @@ -34,16 +34,16 @@ mappings: repositoryFullName: "{{ .repository.full_name }}" repositoryHtmlUrl: "{{ .repository.links.html.href }}" extra: - - apiVersion: mia-platform.eu/v1alpha1 + - apiVersion: mia-platform.eu/v1 itemFamily: relationships deletePolicy: "cascade" identifier: |- - {{ $src := printf "urn:mia-platform-catalog:bitbucket.mia-platform.eu:v1alpha1:Pipeline:%s" (.pipeline.uuid | trimPrefix "{" | trimSuffix "}" | sha256sum) -}} - {{- $type := "urn:mia-platform-catalog:mia-platform.eu:v1alpha1:RelationshipType:part-of.mia-platform.eu" -}} - {{- $tgt := printf "urn:mia-platform-catalog:bitbucket.mia-platform.eu:v1alpha1:Repository:%s" (.repository.links.html.href | sha256sum) -}} + {{ $src := printf "urn:mia-platform-catalog:bitbucket.mia-platform.eu:v1:Pipeline:%s" (.pipeline.uuid | trimPrefix "{" | trimSuffix "}" | sha256sum) -}} + {{- $type := "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:part-of.mia-platform.eu" -}} + {{- $tgt := printf "urn:mia-platform-catalog:bitbucket.mia-platform.eu:v1:Repository:%s" (.repository.links.html.href | sha256sum) -}} {{- printf "%s-%s-%s" $src $type $tgt | sha256sum }} sourceRef: |- - urn:mia-platform-catalog:bitbucket.mia-platform.eu:v1alpha1:Pipeline:{{ .pipeline.uuid | trimPrefix "{" | trimSuffix "}" | sha256sum }} + urn:mia-platform-catalog:bitbucket.mia-platform.eu:v1:Pipeline:{{ .pipeline.uuid | trimPrefix "{" | trimSuffix "}" | sha256sum }} targetRef: |- - urn:mia-platform-catalog:bitbucket.mia-platform.eu:v1alpha1:Repository:{{ .repository.links.html.href | sha256sum }} - typeRef: "urn:mia-platform-catalog:mia-platform.eu:v1alpha1:RelationshipType:part-of.mia-platform.eu" + urn:mia-platform-catalog:bitbucket.mia-platform.eu:v1:Repository:{{ .repository.links.html.href | sha256sum }} + typeRef: "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:part-of.mia-platform.eu" diff --git a/docs/examples/bitbucket/mappings/repositories.yaml b/docs/mappings/bitbucket/repositories.yaml similarity index 96% rename from docs/examples/bitbucket/mappings/repositories.yaml rename to docs/mappings/bitbucket/repositories.yaml index 6488465..84a1861 100644 --- a/docs/examples/bitbucket/mappings/repositories.yaml +++ b/docs/mappings/bitbucket/repositories.yaml @@ -1,4 +1,4 @@ -apiVersion: bitbucket.mia-platform.eu/v1alpha1 +apiVersion: bitbucket.mia-platform.eu/v1 itemFamily: repositories type: repository syncable: true diff --git a/docs/mappings/console/cluster-project-relationships.yaml b/docs/mappings/console/cluster-project-relationships.yaml new file mode 100644 index 0000000..3785ec3 --- /dev/null +++ b/docs/mappings/console/cluster-project-relationships.yaml @@ -0,0 +1,16 @@ +apiVersion: mia-platform.eu/v1 +itemFamily: relationships +type: clusterProjectRelationship +syncable: true +mappings: + identifier: |- + {{ $src := printf "urn:mia-platform-catalog:console.mia-platform.eu:v1:Project:%s" .project._id -}} + {{- $type := "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:part-of.mia-platform.eu" -}} + {{- $tgt := printf "urn:mia-platform-catalog:console.mia-platform.eu:v1:Cluster:%s" (printf "%s" (replace "https://" "" .cluster.connection.url) | sha256sum) -}} + {{- printf "%s-%s-%s" $src $type $tgt | sha256sum }} + spec: + sourceRef: |- + urn:mia-platform-catalog:console.mia-platform.eu:v1:Project:{{ .project._id }} + targetRef: |- + urn:mia-platform-catalog:console.mia-platform.eu:v1:Cluster:{{ printf "%s" (replace "https://" "" .cluster.connection.url) | sha256sum }} + typeRef: "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:part-of.mia-platform.eu" diff --git a/docs/examples/console/mappings/clusters.yaml b/docs/mappings/console/clusters.yaml similarity index 93% rename from docs/examples/console/mappings/clusters.yaml rename to docs/mappings/console/clusters.yaml index 14574a2..6810825 100644 --- a/docs/examples/console/mappings/clusters.yaml +++ b/docs/mappings/console/clusters.yaml @@ -1,4 +1,4 @@ -apiVersion: console.mia-platform.eu/v1alpha1 +apiVersion: console.mia-platform.eu/v1 itemFamily: clusters type: cluster syncable: true diff --git a/docs/examples/console/mappings/projects.yaml b/docs/mappings/console/projects.yaml similarity index 83% rename from docs/examples/console/mappings/projects.yaml rename to docs/mappings/console/projects.yaml index 02025ea..13b0bec 100644 --- a/docs/examples/console/mappings/projects.yaml +++ b/docs/mappings/console/projects.yaml @@ -1,4 +1,4 @@ -apiVersion: console.mia-platform.eu/v1alpha1 +apiVersion: console.mia-platform.eu/v1 itemFamily: projects type: project syncable: true @@ -36,7 +36,7 @@ mappings: projectNamespaceVariable: "{{ get \"projectNamespaceVariable\" .project \"\" | quote }}" lastUpdate: "{{ get \"lastUpdate\" .project (now) }}" extra: - - apiVersion: mia-platform.eu/v1alpha1 + - apiVersion: mia-platform.eu/v1 itemFamily: relationships deletePolicy: "cascade" createIf: |- @@ -50,14 +50,14 @@ mappings: identifier: |- {{ $info := (get "info" .project (object)) -}} {{- $teamContact := (get "teamContact" $info "") -}} - {{- $src := printf "urn:mia-platform-catalog:iam.mia-platform.eu:v1alpha1:User:%s" ($teamContact | sha256sum) -}} - {{- $type := "urn:mia-platform-catalog:mia-platform.eu:v1alpha1:RelationshipType:ownership.mia-platform.eu" -}} - {{- $tgt := printf "urn:mia-platform-catalog:console.mia-platform.eu:v1alpha1:Project:%s" (printf "%s" .project._id) -}} + {{- $src := printf "urn:mia-platform-catalog:iam.mia-platform.eu:v1:User:%s" ($teamContact | sha256sum) -}} + {{- $type := "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:ownership.mia-platform.eu" -}} + {{- $tgt := printf "urn:mia-platform-catalog:console.mia-platform.eu:v1:Project:%s" (printf "%s" .project._id) -}} {{- printf "%s-%s-%s" $src $type $tgt | sha256sum }} sourceRef: |- {{ $info := (get "info" .project (object)) -}} {{- $teamContact := (get "teamContact" $info "") -}} - urn:mia-platform-catalog:iam.mia-platform.eu:v1alpha1:User:{{ $teamContact | sha256sum }} + urn:mia-platform-catalog:iam.mia-platform.eu:v1:User:{{ $teamContact | sha256sum }} targetRef: |- - urn:mia-platform-catalog:console.mia-platform.eu:v1alpha1:Project:{{ printf "%s" .project._id }} - typeRef: "urn:mia-platform-catalog:mia-platform.eu:v1alpha1:RelationshipType:ownership.mia-platform.eu" + urn:mia-platform-catalog:console.mia-platform.eu:v1:Project:{{ printf "%s" .project._id }} + typeRef: "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:ownership.mia-platform.eu" diff --git a/docs/examples/console/mappings/revisions.yaml b/docs/mappings/console/revisions.yaml similarity index 54% rename from docs/examples/console/mappings/revisions.yaml rename to docs/mappings/console/revisions.yaml index 394b656..853442c 100644 --- a/docs/examples/console/mappings/revisions.yaml +++ b/docs/mappings/console/revisions.yaml @@ -1,4 +1,4 @@ -apiVersion: console.mia-platform.eu/v1alpha1 +apiVersion: console.mia-platform.eu/v1 itemFamily: revisions type: revision syncable: true @@ -9,20 +9,20 @@ mappings: title: "{{ printf \"%s\" .revision.name }}" spec: {} extra: - - apiVersion: mia-platform.eu/v1alpha1 + - apiVersion: mia-platform.eu/v1 itemFamily: relationships deletePolicy: "cascade" identifier: |- - {{ $src := printf "urn:mia-platform-catalog:console.mia-platform.eu:v1alpha1:Revision:%s" (printf "%s-%s" .project._id .revision.name | sha256sum) -}} - {{- $type := "urn:mia-platform-catalog:mia-platform.eu:v1alpha1:RelationshipType:part-of.mia-platform.eu" -}} - {{- $tgt := printf "urn:mia-platform-catalog:console.mia-platform.eu:v1alpha1:Project:%s" .project._id -}} + {{ $src := printf "urn:mia-platform-catalog:console.mia-platform.eu:v1:Revision:%s" (printf "%s-%s" .project._id .revision.name | sha256sum) -}} + {{- $type := "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:part-of.mia-platform.eu" -}} + {{- $tgt := printf "urn:mia-platform-catalog:console.mia-platform.eu:v1:Project:%s" .project._id -}} {{- printf "%s-%s-%s" $src $type $tgt | sha256sum }} sourceRef: |- - urn:mia-platform-catalog:console.mia-platform.eu:v1alpha1:Revision:{{ printf "%s-%s" .project._id .revision.name | sha256sum }} + urn:mia-platform-catalog:console.mia-platform.eu:v1:Revision:{{ printf "%s-%s" .project._id .revision.name | sha256sum }} targetRef: |- - urn:mia-platform-catalog:console.mia-platform.eu:v1alpha1:Project:{{ .project._id }} - typeRef: "urn:mia-platform-catalog:mia-platform.eu:v1alpha1:RelationshipType:part-of.mia-platform.eu" - - apiVersion: mia-platform.eu/v1alpha1 + urn:mia-platform-catalog:console.mia-platform.eu:v1:Project:{{ .project._id }} + typeRef: "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:part-of.mia-platform.eu" + - apiVersion: mia-platform.eu/v1 itemFamily: relationships deletePolicy: "cascade" createIf: |- @@ -36,14 +36,14 @@ mappings: identifier: |- {{ $info := (get "info" .project (object)) -}} {{- $teamContact := (get "teamContact" $info "") -}} - {{- $src := printf "urn:mia-platform-catalog:iam.mia-platform.eu:v1alpha1:User:%s" ($teamContact | sha256sum) -}} - {{- $type := "urn:mia-platform-catalog:mia-platform.eu:v1alpha1:RelationshipType:ownership.mia-platform.eu" -}} - {{- $tgt := printf "urn:mia-platform-catalog:console.mia-platform.eu:v1alpha1:Revision:%s" (printf "%s-%s" .project._id .revision.name | sha256sum) -}} + {{- $src := printf "urn:mia-platform-catalog:iam.mia-platform.eu:v1:User:%s" ($teamContact | sha256sum) -}} + {{- $type := "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:ownership.mia-platform.eu" -}} + {{- $tgt := printf "urn:mia-platform-catalog:console.mia-platform.eu:v1:Revision:%s" (printf "%s-%s" .project._id .revision.name | sha256sum) -}} {{- printf "%s-%s-%s" $src $type $tgt | sha256sum }} sourceRef: |- {{ $info := (get "info" .project (object)) -}} {{- $teamContact := (get "teamContact" $info "") -}} - urn:mia-platform-catalog:iam.mia-platform.eu:v1alpha1:User:{{ $teamContact | sha256sum }} + urn:mia-platform-catalog:iam.mia-platform.eu:v1:User:{{ $teamContact | sha256sum }} targetRef: |- - urn:mia-platform-catalog:console.mia-platform.eu:v1alpha1:Revision:{{ printf "%s-%s" .project._id .revision.name | sha256sum }} - typeRef: "urn:mia-platform-catalog:mia-platform.eu:v1alpha1:RelationshipType:ownership.mia-platform.eu" + urn:mia-platform-catalog:console.mia-platform.eu:v1:Revision:{{ printf "%s-%s" .project._id .revision.name | sha256sum }} + typeRef: "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:ownership.mia-platform.eu" diff --git a/docs/mappings/console/services.yaml b/docs/mappings/console/services.yaml new file mode 100644 index 0000000..0cf543c --- /dev/null +++ b/docs/mappings/console/services.yaml @@ -0,0 +1,80 @@ +apiVersion: console.mia-platform.eu/v1 +itemFamily: services +type: service +syncable: true +mappings: + identifier: |- + {{ printf "%s-%s-%s" .project._id .revision.name .service.name | sha256sum }} + metadata: + title: "{{ printf \"%s\" .service.name }}" + spec: + projectUniqueId: "{{ .project._id }}" + projectId: "{{ get \"projectId\" .project \"\" | quote }}" + tenantId: "{{ .project.tenantId }}" + revisionName: "{{ .revision.name }}" + configuration: "{{ .service | toJSON }}" + extra: + - apiVersion: mia-platform.eu/v1 + itemFamily: relationships + deletePolicy: "cascade" + identifier: |- + {{ $src := printf "urn:mia-platform-catalog:console.mia-platform.eu:v1:Service:%s" (printf "%s-%s-%s" .project._id .revision.name .service.name | sha256sum) -}} + {{- $type := "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:part-of.mia-platform.eu" -}} + {{- $tgt := printf "urn:mia-platform-catalog:console.mia-platform.eu:v1:Project:%s" .project._id -}} + {{- printf "%s-%s-%s" $src $type $tgt | sha256sum }} + sourceRef: |- + urn:mia-platform-catalog:console.mia-platform.eu:v1:Service:{{ printf "%s-%s-%s" .project._id .revision.name .service.name | sha256sum }} + targetRef: |- + urn:mia-platform-catalog:console.mia-platform.eu:v1:Project:{{ .project._id }} + typeRef: "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:part-of.mia-platform.eu" + - apiVersion: mia-platform.eu/v1 + itemFamily: relationships + deletePolicy: "cascade" + identifier: |- + {{ $src := printf "urn:mia-platform-catalog:console.mia-platform.eu:v1:Service:%s" (printf "%s-%s-%s" .project._id .revision.name .service.name | sha256sum) -}} + {{- $type := "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:part-of.mia-platform.eu" -}} + {{- $tgt := printf "urn:mia-platform-catalog:console.mia-platform.eu:v1:Revision:%s" (printf "%s-%s" .project._id .revision.name | sha256sum) -}} + {{- printf "%s-%s-%s" $src $type $tgt | sha256sum }} + sourceRef: |- + urn:mia-platform-catalog:console.mia-platform.eu:v1:Service:{{ printf "%s-%s-%s" .project._id .revision.name .service.name | sha256sum }} + targetRef: |- + urn:mia-platform-catalog:console.mia-platform.eu:v1:Revision:{{ printf "%s-%s" .project._id .revision.name | sha256sum }} + typeRef: "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:part-of.mia-platform.eu" + - apiVersion: mia-platform.eu/v1 + itemFamily: relationships + deletePolicy: "cascade" + createIf: |- + {{ $info := (get "info" .project (object)) -}} + {{- $teamContact := (get "teamContact" $info nil) -}} + {{- if $teamContact -}} + true + {{- else -}} + false + {{- end }} + identifier: |- + {{ $info := (get "info" .project (object)) -}} + {{- $teamContact := (get "teamContact" $info "") -}} + {{- $src := printf "urn:mia-platform-catalog:iam.mia-platform.eu:v1:User:%s" ($teamContact | sha256sum) -}} + {{- $type := "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:ownership.mia-platform.eu" -}} + {{- $tgt := printf "urn:mia-platform-catalog:console.mia-platform.eu:v1:Service:%s" (printf "%s-%s-%s" .project._id .revision.name .service.name | sha256sum) -}} + {{- printf "%s-%s-%s" $src $type $tgt | sha256sum }} + sourceRef: |- + {{ $info := (get "info" .project (object)) -}} + {{- $teamContact := (get "teamContact" $info "") -}} + urn:mia-platform-catalog:iam.mia-platform.eu:v1:User:{{ $teamContact | sha256sum }} + targetRef: |- + urn:mia-platform-catalog:console.mia-platform.eu:v1:Service:{{ printf "%s-%s-%s" .project._id .revision.name .service.name | sha256sum }} + typeRef: "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:ownership.mia-platform.eu" + - apiVersion: mia-platform.eu/v1 + itemFamily: relationships + deletePolicy: "none" + identifier: |- + {{ $src := printf "urn:mia-platform-catalog:console.mia-platform.eu:v1:Service:%s" (printf "%s-%s-%s" .project._id .revision.name .service.name | sha256sum) -}} + {{- $type := "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:dependency.mia-platform.eu" -}} + {{- $tgt := printf "urn:mia-platform-catalog:nexus.mia-platform.eu:v1:DockerImage:%s" (printf "%s" .service.dockerImage | sha256sum) -}} + {{- printf "%s-%s-%s" $src $type $tgt | sha256sum }} + sourceRef: |- + urn:mia-platform-catalog:console.mia-platform.eu:v1:Service:{{ printf "%s-%s-%s" .project._id .revision.name .service.name | sha256sum }} + targetRef: |- + urn:mia-platform-catalog:nexus.mia-platform.eu:v1:DockerImage:{{ printf "%s" .service.dockerImage | sha256sum }} + typeRef: "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:dependency.mia-platform.eu" diff --git a/docs/examples/gcp/mappings/buckets.yaml b/docs/mappings/gcp/buckets.yaml similarity index 96% rename from docs/examples/gcp/mappings/buckets.yaml rename to docs/mappings/gcp/buckets.yaml index 3e62e3a..4f40825 100644 --- a/docs/examples/gcp/mappings/buckets.yaml +++ b/docs/mappings/gcp/buckets.yaml @@ -1,4 +1,4 @@ -apiVersion: gcp.mia-platform.eu/v1alpha1 +apiVersion: gcp.mia-platform.eu/v1 itemFamily: buckets type: storage.googleapis.com/Bucket syncable: true diff --git a/docs/examples/gcp/mappings/clusters.yaml b/docs/mappings/gcp/clusters.yaml similarity index 96% rename from docs/examples/gcp/mappings/clusters.yaml rename to docs/mappings/gcp/clusters.yaml index 0e024de..7f30215 100644 --- a/docs/examples/gcp/mappings/clusters.yaml +++ b/docs/mappings/gcp/clusters.yaml @@ -1,4 +1,4 @@ -apiVersion: gcp.mia-platform.eu/v1alpha1 +apiVersion: gcp.mia-platform.eu/v1 itemFamily: clusters type: container.googleapis.com/Cluster syncable: true diff --git a/docs/examples/gcp/mappings/computeinstances.yaml b/docs/mappings/gcp/computeinstances.yaml similarity index 98% rename from docs/examples/gcp/mappings/computeinstances.yaml rename to docs/mappings/gcp/computeinstances.yaml index a3f15f1..e9c8151 100644 --- a/docs/examples/gcp/mappings/computeinstances.yaml +++ b/docs/mappings/gcp/computeinstances.yaml @@ -1,4 +1,4 @@ -apiVersion: gcp.mia-platform.eu/v1alpha1 +apiVersion: gcp.mia-platform.eu/v1 itemFamily: virtualmachines type: compute.googleapis.com/Instance syncable: true diff --git a/docs/examples/gcp/mappings/firewallrules.yaml b/docs/mappings/gcp/firewallrules.yaml similarity index 96% rename from docs/examples/gcp/mappings/firewallrules.yaml rename to docs/mappings/gcp/firewallrules.yaml index 70c4fbd..92eab4e 100644 --- a/docs/examples/gcp/mappings/firewallrules.yaml +++ b/docs/mappings/gcp/firewallrules.yaml @@ -1,4 +1,4 @@ -apiVersion: gcp.mia-platform.eu/v1alpha1 +apiVersion: gcp.mia-platform.eu/v1 itemFamily: firewallrules type: compute.googleapis.com/Firewall syncable: true diff --git a/docs/examples/gcp/mappings/folders.yaml b/docs/mappings/gcp/folders.yaml similarity index 93% rename from docs/examples/gcp/mappings/folders.yaml rename to docs/mappings/gcp/folders.yaml index 14c9309..41ea5b8 100644 --- a/docs/examples/gcp/mappings/folders.yaml +++ b/docs/mappings/gcp/folders.yaml @@ -1,4 +1,4 @@ -apiVersion: gcp.mia-platform.eu/v1alpha1 +apiVersion: gcp.mia-platform.eu/v1 itemFamily: folders type: cloudresourcemanager.googleapis.com/Folder syncable: true diff --git a/docs/examples/gcp/mappings/jobs.yaml b/docs/mappings/gcp/jobs.yaml similarity index 96% rename from docs/examples/gcp/mappings/jobs.yaml rename to docs/mappings/gcp/jobs.yaml index 0a9ab98..4685183 100644 --- a/docs/examples/gcp/mappings/jobs.yaml +++ b/docs/mappings/gcp/jobs.yaml @@ -1,4 +1,4 @@ -apiVersion: gcp.mia-platform.eu/v1alpha1 +apiVersion: gcp.mia-platform.eu/v1 itemFamily: jobs type: run.googleapis.com/Job syncable: true diff --git a/docs/examples/gcp/mappings/networks.yaml b/docs/mappings/gcp/networks.yaml similarity index 96% rename from docs/examples/gcp/mappings/networks.yaml rename to docs/mappings/gcp/networks.yaml index 2e0566a..35085f8 100644 --- a/docs/examples/gcp/mappings/networks.yaml +++ b/docs/mappings/gcp/networks.yaml @@ -1,4 +1,4 @@ -apiVersion: gcp.mia-platform.eu/v1alpha1 +apiVersion: gcp.mia-platform.eu/v1 itemFamily: networks type: compute.googleapis.com/Network syncable: true diff --git a/docs/examples/gcp/mappings/projects.yaml b/docs/mappings/gcp/projects.yaml similarity index 93% rename from docs/examples/gcp/mappings/projects.yaml rename to docs/mappings/gcp/projects.yaml index 7908721..5684db9 100644 --- a/docs/examples/gcp/mappings/projects.yaml +++ b/docs/mappings/gcp/projects.yaml @@ -1,4 +1,4 @@ -apiVersion: gcp.mia-platform.eu/v1alpha1 +apiVersion: gcp.mia-platform.eu/v1 itemFamily: projects type: cloudresourcemanager.googleapis.com/Project syncable: true diff --git a/docs/examples/gcp/mappings/services.yaml b/docs/mappings/gcp/services.yaml similarity index 95% rename from docs/examples/gcp/mappings/services.yaml rename to docs/mappings/gcp/services.yaml index a69f846..36de742 100644 --- a/docs/examples/gcp/mappings/services.yaml +++ b/docs/mappings/gcp/services.yaml @@ -1,4 +1,4 @@ -apiVersion: gcp.mia-platform.eu/v1alpha1 +apiVersion: gcp.mia-platform.eu/v1 itemFamily: services type: run.googleapis.com/Service syncable: true diff --git a/docs/examples/gcp/mappings/sqlinstances.yaml b/docs/mappings/gcp/sqlinstances.yaml similarity index 98% rename from docs/examples/gcp/mappings/sqlinstances.yaml rename to docs/mappings/gcp/sqlinstances.yaml index ed5cf65..78f6cd2 100644 --- a/docs/examples/gcp/mappings/sqlinstances.yaml +++ b/docs/mappings/gcp/sqlinstances.yaml @@ -1,4 +1,4 @@ -apiVersion: gcp.mia-platform.eu/v1alpha1 +apiVersion: gcp.mia-platform.eu/v1 itemFamily: sqlinstances type: sqladmin.googleapis.com/Instance syncable: true diff --git a/docs/examples/github/mappings/repositories.yaml b/docs/mappings/github/repositories.yaml similarity index 98% rename from docs/examples/github/mappings/repositories.yaml rename to docs/mappings/github/repositories.yaml index 3b34eaf..922be46 100644 --- a/docs/examples/github/mappings/repositories.yaml +++ b/docs/mappings/github/repositories.yaml @@ -1,4 +1,4 @@ -apiVersion: github.mia-platform.eu/v1alpha1 +apiVersion: github.mia-platform.eu/v1 itemFamily: repositories type: repository extra: diff --git a/docs/examples/github/mappings/workflowruns.yaml b/docs/mappings/github/workflowruns.yaml similarity index 77% rename from docs/examples/github/mappings/workflowruns.yaml rename to docs/mappings/github/workflowruns.yaml index ea7c62c..ae1966b 100644 --- a/docs/examples/github/mappings/workflowruns.yaml +++ b/docs/mappings/github/workflowruns.yaml @@ -1,4 +1,4 @@ -apiVersion: github.mia-platform.eu/v1alpha1 +apiVersion: github.mia-platform.eu/v1 itemFamily: workflowruns type: workflow_run extra: @@ -69,16 +69,16 @@ mappings: {{- "" | quote -}} {{- end }} extra: - - apiVersion: mia-platform.eu/v1alpha1 + - apiVersion: mia-platform.eu/v1 itemFamily: relationships deletePolicy: "none" identifier: |- - {{ $src := printf "urn:mia-platform-catalog:github.mia-platform.eu:v1alpha1:WorkflowRun:%s" (printf "%.0f-%s" .workflow_run.id .workflow_run.html_url | sha256sum) -}} - {{- $type := "urn:mia-platform-catalog:mia-platform.eu:v1alpha1:RelationshipType:part-of.mia-platform.eu" -}} - {{- $tgt := printf "urn:mia-platform-catalog:github.mia-platform.eu:v1alpha1:Repository:%s" (printf "%.0f-%s" .workflow_run.repository.id .workflow_run.repository.html_url | sha256sum) -}} + {{ $src := printf "urn:mia-platform-catalog:github.mia-platform.eu:v1:WorkflowRun:%s" (printf "%.0f-%s" .workflow_run.id .workflow_run.html_url | sha256sum) -}} + {{- $type := "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:part-of.mia-platform.eu" -}} + {{- $tgt := printf "urn:mia-platform-catalog:github.mia-platform.eu:v1:Repository:%s" (printf "%.0f-%s" .workflow_run.repository.id .workflow_run.repository.html_url | sha256sum) -}} {{- printf "%s-%s-%s" $src $type $tgt | sha256sum }} sourceRef: |- - urn:mia-platform-catalog:github.mia-platform.eu:v1alpha1:WorkflowRun:{{ printf "%.0f-%s" .workflow_run.id .workflow_run.html_url | sha256sum }} + urn:mia-platform-catalog:github.mia-platform.eu:v1:WorkflowRun:{{ printf "%.0f-%s" .workflow_run.id .workflow_run.html_url | sha256sum }} targetRef: |- - urn:mia-platform-catalog:github.mia-platform.eu:v1alpha1:Repository:{{ printf "%.0f-%s" .workflow_run.repository.id .workflow_run.repository.html_url | sha256sum }} - typeRef: "urn:mia-platform-catalog:mia-platform.eu:v1alpha1:RelationshipType:part-of.mia-platform.eu" + urn:mia-platform-catalog:github.mia-platform.eu:v1:Repository:{{ printf "%.0f-%s" .workflow_run.repository.id .workflow_run.repository.html_url | sha256sum }} + typeRef: "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:part-of.mia-platform.eu" diff --git a/docs/examples/gitlab/mappings/accesstokens.yaml b/docs/mappings/gitlab/accesstokens.yaml similarity index 74% rename from docs/examples/gitlab/mappings/accesstokens.yaml rename to docs/mappings/gitlab/accesstokens.yaml index ee009fb..1b0e3b2 100644 --- a/docs/examples/gitlab/mappings/accesstokens.yaml +++ b/docs/mappings/gitlab/accesstokens.yaml @@ -1,4 +1,4 @@ -apiVersion: gitlab.mia-platform.eu/v1alpha1 +apiVersion: gitlab.mia-platform.eu/v1 itemFamily: accesstokens type: accesstoken syncable: true @@ -41,7 +41,7 @@ mappings: {{- printf "%s" .group.full_name -}} {{- end }} extra: - - apiVersion: mia-platform.eu/v1alpha1 + - apiVersion: mia-platform.eu/v1 itemFamily: relationships deletePolicy: "none" createIf: |- @@ -52,12 +52,12 @@ mappings: false {{- end }} identifier: |- - {{ $src := printf "urn:mia-platform-catalog:gitlab.mia-platform.eu:v1alpha1:AccessToken:%s" (printf "%s" .token.id | sha256sum) -}} - {{- $type := "urn:mia-platform-catalog:mia-platform.eu:v1alpha1:RelationshipType:part-of.mia-platform.eu" -}} - {{- $tgt := printf "urn:mia-platform-catalog:gitlab.mia-platform.eu:v1alpha1:Project:%s" (printf "%s" .project.id | sha256sum) -}} + {{ $src := printf "urn:mia-platform-catalog:gitlab.mia-platform.eu:v1:AccessToken:%s" (printf "%s" .token.id | sha256sum) -}} + {{- $type := "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:part-of.mia-platform.eu" -}} + {{- $tgt := printf "urn:mia-platform-catalog:gitlab.mia-platform.eu:v1:Project:%s" (printf "%s" .project.id | sha256sum) -}} {{- printf "%s-%s-%s" $src $type $tgt | sha256sum }} sourceRef: |- - urn:mia-platform-catalog:gitlab.mia-platform.eu:v1alpha1:AccessToken:{{ printf "%s" .token.id | sha256sum }} + urn:mia-platform-catalog:gitlab.mia-platform.eu:v1:AccessToken:{{ printf "%s" .token.id | sha256sum }} targetRef: |- - urn:mia-platform-catalog:gitlab.mia-platform.eu:v1alpha1:Project:{{ printf "%s" .project.id | sha256sum }} - typeRef: "urn:mia-platform-catalog:mia-platform.eu:v1alpha1:RelationshipType:part-of.mia-platform.eu" + urn:mia-platform-catalog:gitlab.mia-platform.eu:v1:Project:{{ printf "%s" .project.id | sha256sum }} + typeRef: "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:part-of.mia-platform.eu" diff --git a/docs/examples/gitlab/mappings/pipelines.yaml b/docs/mappings/gitlab/pipelines.yaml similarity index 67% rename from docs/examples/gitlab/mappings/pipelines.yaml rename to docs/mappings/gitlab/pipelines.yaml index e1992f1..b478e78 100644 --- a/docs/examples/gitlab/mappings/pipelines.yaml +++ b/docs/mappings/gitlab/pipelines.yaml @@ -1,4 +1,4 @@ -apiVersion: gitlab.mia-platform.eu/v1alpha1 +apiVersion: gitlab.mia-platform.eu/v1 itemFamily: pipelines type: pipeline syncable: true @@ -32,16 +32,16 @@ mappings: {{- "" | quote -}} {{- end }} extra: - - apiVersion: mia-platform.eu/v1alpha1 + - apiVersion: mia-platform.eu/v1 itemFamily: relationships deletePolicy: "cascade" identifier: |- - {{ $src := printf "urn:mia-platform-catalog:gitlab.mia-platform.eu:v1alpha1:Pipeline:%s" (printf "%s" .pipeline.id | sha256sum) -}} - {{- $type := "urn:mia-platform-catalog:mia-platform.eu:v1alpha1:RelationshipType:part-of.mia-platform.eu" -}} - {{- $tgt := printf "urn:mia-platform-catalog:gitlab.mia-platform.eu:v1alpha1:Project:%s" (printf "%s" .project.id | sha256sum) -}} + {{ $src := printf "urn:mia-platform-catalog:gitlab.mia-platform.eu:v1:Pipeline:%s" (printf "%s" .pipeline.id | sha256sum) -}} + {{- $type := "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:part-of.mia-platform.eu" -}} + {{- $tgt := printf "urn:mia-platform-catalog:gitlab.mia-platform.eu:v1:Project:%s" (printf "%s" .project.id | sha256sum) -}} {{- printf "%s-%s-%s" $src $type $tgt | sha256sum }} sourceRef: |- - urn:mia-platform-catalog:gitlab.mia-platform.eu:v1alpha1:Pipeline:{{ printf "%s" .pipeline.id | sha256sum }} + urn:mia-platform-catalog:gitlab.mia-platform.eu:v1:Pipeline:{{ printf "%s" .pipeline.id | sha256sum }} targetRef: |- - urn:mia-platform-catalog:gitlab.mia-platform.eu:v1alpha1:Project:{{ printf "%s" .project.id | sha256sum }} - typeRef: "urn:mia-platform-catalog:mia-platform.eu:v1alpha1:RelationshipType:part-of.mia-platform.eu" + urn:mia-platform-catalog:gitlab.mia-platform.eu:v1:Project:{{ printf "%s" .project.id | sha256sum }} + typeRef: "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:part-of.mia-platform.eu" diff --git a/docs/examples/gitlab/mappings/projects.yaml b/docs/mappings/gitlab/projects.yaml similarity index 98% rename from docs/examples/gitlab/mappings/projects.yaml rename to docs/mappings/gitlab/projects.yaml index baaefbb..ca7736f 100644 --- a/docs/examples/gitlab/mappings/projects.yaml +++ b/docs/mappings/gitlab/projects.yaml @@ -1,4 +1,4 @@ -apiVersion: gitlab.mia-platform.eu/v1alpha1 +apiVersion: gitlab.mia-platform.eu/v1 itemFamily: projects type: project syncable: true diff --git a/docs/examples/nexus/mappings/dockerimages.yaml b/docs/mappings/nexus/dockerimages.yaml similarity index 96% rename from docs/examples/nexus/mappings/dockerimages.yaml rename to docs/mappings/nexus/dockerimages.yaml index 56f337c..fc5f312 100644 --- a/docs/examples/nexus/mappings/dockerimages.yaml +++ b/docs/mappings/nexus/dockerimages.yaml @@ -1,4 +1,4 @@ -apiVersion: nexus.mia-platform.eu/v1alpha1 +apiVersion: nexus.mia-platform.eu/v1 itemFamily: dockerimages type: dockerimage syncable: true diff --git a/docs/examples/sysdig/mappings/vulnerabilities.yaml b/docs/mappings/sysdig/vulnerabilities.yaml similarity index 81% rename from docs/examples/sysdig/mappings/vulnerabilities.yaml rename to docs/mappings/sysdig/vulnerabilities.yaml index c3f7f2d..d5d3717 100644 --- a/docs/examples/sysdig/mappings/vulnerabilities.yaml +++ b/docs/mappings/sysdig/vulnerabilities.yaml @@ -1,4 +1,4 @@ -apiVersion: sysdig.mia-platform.eu/v1alpha1 +apiVersion: sysdig.mia-platform.eu/v1 itemFamily: vulnerabilities type: vulnerability syncable: true @@ -67,16 +67,16 @@ mappings: {{- "" | quote -}} {{- end }} extra: - - apiVersion: mia-platform.eu/v1alpha1 + - apiVersion: mia-platform.eu/v1 itemFamily: relationships deletePolicy: "none" identifier: |- - {{ $src := printf "urn:mia-platform-catalog:sysdig.mia-platform.eu:v1alpha1:Vulnerability:%s" (printf "%s" .vuln.name | sha256sum) -}} - {{- $type := "urn:mia-platform-catalog:mia-platform.eu:v1alpha1:RelationshipType:affect.mia-platform.eu" -}} - {{- $tgt := printf "urn:mia-platform-catalog:nexus.mia-platform.eu:v1alpha1:DockerImage:%s" (printf "%s" .img.imageReference | sha256sum) -}} + {{ $src := printf "urn:mia-platform-catalog:sysdig.mia-platform.eu:v1:Vulnerability:%s" (printf "%s" .vuln.name | sha256sum) -}} + {{- $type := "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:affect.mia-platform.eu" -}} + {{- $tgt := printf "urn:mia-platform-catalog:nexus.mia-platform.eu:v1:DockerImage:%s" (printf "%s" .img.imageReference | sha256sum) -}} {{- printf "%s-%s-%s" $src $type $tgt | sha256sum }} sourceRef: |- - urn:mia-platform-catalog:sysdig.mia-platform.eu:v1alpha1:Vulnerability:{{ printf "%s" .vuln.name | sha256sum }} + urn:mia-platform-catalog:sysdig.mia-platform.eu:v1:Vulnerability:{{ printf "%s" .vuln.name | sha256sum }} targetRef: |- - urn:mia-platform-catalog:nexus.mia-platform.eu:v1alpha1:DockerImage:{{ printf "%s" .img.imageReference | sha256sum }} - typeRef: "urn:mia-platform-catalog:mia-platform.eu:v1alpha1:RelationshipType:affect.mia-platform.eu" + urn:mia-platform-catalog:nexus.mia-platform.eu:v1:DockerImage:{{ printf "%s" .img.imageReference | sha256sum }} + typeRef: "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:affect.mia-platform.eu" diff --git a/internal/mapper/mapper_test.go b/internal/mapper/mapper_test.go index b2fc430..4568fa3 100644 --- a/internal/mapper/mapper_test.go +++ b/internal/mapper/mapper_test.go @@ -374,7 +374,7 @@ func TestMapper(t *testing.T) { "identifier": "{{ .name }}", "sourceRef": "urn:mia-platform-catalog:resource.custom-platform:v1:Family1:null:{{ .otherKey.value }}", "targetRef": "urn:mia-platform-catalog:resource.custom-platform:v1:Family1:null:{{ .name }}", - "typeRef": "urn:mia-platform-catalog:mia-platform.eu:v1alpha1:RelationshipType:null:dependency", + "typeRef": "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:null:dependency", }, { "apiVersion": "api/v1", @@ -384,7 +384,7 @@ func TestMapper(t *testing.T) { "identifier": "{{ .name }}", "sourceRef": "urn:mia-platform-catalog:resource.custom-platform:v1:Family1:null:{{ .otherKey.value }}", "targetRef": "urn:mia-platform-catalog:resource.custom-platform:v1:Family1:null:{{ .name }}", - "typeRef": "urn:mia-platform-catalog:mia-platform.eu:v1alpha1:RelationshipType:null:dependency", + "typeRef": "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:null:dependency", }, { "apiVersion": "api/v1", @@ -394,7 +394,7 @@ func TestMapper(t *testing.T) { "identifier": "{{ .name }}-create", "sourceRef": "urn:mia-platform-catalog:resource.custom-platform:v1:Family1:null:{{ .otherKey.value }}", "targetRef": "urn:mia-platform-catalog:resource.custom-platform:v1:Family1:null:{{ .name }}", - "typeRef": "urn:mia-platform-catalog:mia-platform.eu:v1alpha1:RelationshipType:null:dependency", + "typeRef": "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:null:dependency", }, }, ) @@ -442,7 +442,7 @@ func TestMapper(t *testing.T) { Spec: map[string]any{ "sourceRef": "urn:mia-platform-catalog:resource.custom-platform:v1:Family1:null:42", "targetRef": "urn:mia-platform-catalog:resource.custom-platform:v1:Family1:null:example", - "typeRef": "urn:mia-platform-catalog:mia-platform.eu:v1alpha1:RelationshipType:null:dependency", + "typeRef": "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:null:dependency", }, }, { @@ -452,7 +452,7 @@ func TestMapper(t *testing.T) { Spec: map[string]any{ "sourceRef": "urn:mia-platform-catalog:resource.custom-platform:v1:Family1:null:42", "targetRef": "urn:mia-platform-catalog:resource.custom-platform:v1:Family1:null:example", - "typeRef": "urn:mia-platform-catalog:mia-platform.eu:v1alpha1:RelationshipType:null:dependency", + "typeRef": "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:null:dependency", }, }, }, diff --git a/internal/pipeline/pipeline_test.go b/internal/pipeline/pipeline_test.go index 7e07c27..da0e29a 100644 --- a/internal/pipeline/pipeline_test.go +++ b/internal/pipeline/pipeline_test.go @@ -132,7 +132,7 @@ func getExtra(tb testing.TB, deletePolicy string, id int) config.Extra { "identifier": `{{ printf "relationship--%s--%s--dependency` + idString + `" .field1 .field2 }}`, "sourceRef": "urn:mia-platform-catalog:resource.custom-platform:v1:Family1:null:{{ .field2 }}" + idString, "targetRef": "urn:mia-platform-catalog:mia-platform.eu:v1:Family:null:{{ .id }}", - "typeRef": "urn:mia-platform-catalog:mia-platform.eu:v1alpha1:RelationshipType:null:dependency", + "typeRef": "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:null:dependency", } } @@ -233,7 +233,7 @@ func TestStreamPipeline(t *testing.T) { Data: map[string]any{ "sourceRef": "urn:mia-platform-catalog:resource.custom-platform:v1:Family1:null:value2", "targetRef": "urn:mia-platform-catalog:mia-platform.eu:v1:Family:null:item1", - "typeRef": "urn:mia-platform-catalog:mia-platform.eu:v1alpha1:RelationshipType:null:dependency", + "typeRef": "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:null:dependency", }, OperationTime: "2024-06-01T12:00:00Z", }, @@ -293,7 +293,7 @@ func TestStreamPipeline(t *testing.T) { Data: map[string]any{ "sourceRef": "urn:mia-platform-catalog:resource.custom-platform:v1:Family1:null:value2", "targetRef": "urn:mia-platform-catalog:mia-platform.eu:v1:Family:null:item1", - "typeRef": "urn:mia-platform-catalog:mia-platform.eu:v1alpha1:RelationshipType:null:dependency", + "typeRef": "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:null:dependency", }, OperationTime: "2024-06-01T12:00:00Z", }, @@ -304,7 +304,7 @@ func TestStreamPipeline(t *testing.T) { Data: map[string]any{ "sourceRef": "urn:mia-platform-catalog:resource.custom-platform:v1:Family1:null:value2-1", "targetRef": "urn:mia-platform-catalog:mia-platform.eu:v1:Family:null:item1", - "typeRef": "urn:mia-platform-catalog:mia-platform.eu:v1alpha1:RelationshipType:null:dependency", + "typeRef": "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:null:dependency", }, OperationTime: "2024-06-01T12:00:00Z", }, @@ -444,7 +444,7 @@ func TestStreamPipelineWebhook(t *testing.T) { Data: map[string]any{ "sourceRef": "urn:mia-platform-catalog:resource.custom-platform:v1:Family1:null:value2", "targetRef": "urn:mia-platform-catalog:mia-platform.eu:v1:Family:null:item1", - "typeRef": "urn:mia-platform-catalog:mia-platform.eu:v1alpha1:RelationshipType:null:dependency", + "typeRef": "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:null:dependency", }, OperationTime: "2024-06-01T12:00:00Z", }, @@ -513,7 +513,7 @@ func TestStreamPipelineWebhook(t *testing.T) { Data: map[string]any{ "sourceRef": "urn:mia-platform-catalog:resource.custom-platform:v1:Family1:null:value2", "targetRef": "urn:mia-platform-catalog:mia-platform.eu:v1:Family:null:item1", - "typeRef": "urn:mia-platform-catalog:mia-platform.eu:v1alpha1:RelationshipType:null:dependency", + "typeRef": "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:null:dependency", }, OperationTime: "2024-06-01T12:00:00Z", }, @@ -524,7 +524,7 @@ func TestStreamPipelineWebhook(t *testing.T) { Data: map[string]any{ "sourceRef": "urn:mia-platform-catalog:resource.custom-platform:v1:Family1:null:value2-1", "targetRef": "urn:mia-platform-catalog:mia-platform.eu:v1:Family:null:item1", - "typeRef": "urn:mia-platform-catalog:mia-platform.eu:v1alpha1:RelationshipType:null:dependency", + "typeRef": "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:null:dependency", }, OperationTime: "2024-06-01T12:00:00Z", }, diff --git a/tools/make/release.mk b/tools/make/release.mk index 24febd1..2ec5a0b 100644 --- a/tools/make/release.mk +++ b/tools/make/release.mk @@ -60,9 +60,9 @@ $(RELEASE_DIR)/%.sigstore.json: $(RELEASE_DIR)/% $(COSIGN_PATH) $(info Signing $* with cosign...) $(COSIGN_PATH) sign-blob $< --key $(COSIGN_PRIVATE_KEY) --bundle $@ --yes -$(RELEASE_DIR)/default-itds-mappings.tar.gz: docs/examples/* - $(info Creating the default ItemTypeDefinitions and mappings tarball...) - tar -czf $@ -C docs/examples . +$(RELEASE_DIR)/default-itds-mappings.tar.gz: docs/mappings/* + $(info Creating the default mappings tarball...) + tar -czf $@ -C docs/mappings . $(TOOLS_BIN)/cosign: $(TOOLS_DIR)/COSIGN_VERSION $(eval COSIGN_VERSION:= $(shell cat $<)) From fddf16a4d075046acf31d771c7e47d2035a8ec43 Mon Sep 17 00:00:00 2001 From: APagani Date: Tue, 14 Jul 2026 16:14:07 +0200 Subject: [PATCH 06/20] feat: new metadata owner (#74) --- docs/mappings/console/projects.yaml | 35 +++++++--------------------- docs/mappings/console/revisions.yaml | 34 +++++++-------------------- docs/mappings/console/services.yaml | 34 +++++++-------------------- internal/config/mappings.go | 1 + 4 files changed, 28 insertions(+), 76 deletions(-) diff --git a/docs/mappings/console/projects.yaml b/docs/mappings/console/projects.yaml index 13b0bec..302366c 100644 --- a/docs/mappings/console/projects.yaml +++ b/docs/mappings/console/projects.yaml @@ -7,6 +7,15 @@ mappings: {{ printf "%s" .project._id }} metadata: title: "{{ printf \"%s\" .project.name }}" + owner: |- + {{ $info := (get "info" .project (object)) -}} + {{- $teamContact := (get "teamContact" $info nil) -}} + {{- if $teamContact -}} + {{- $owner := object -}} + {{- $owner = set "type" "email" $owner -}} + {{- $owner = set "ref" (get "teamContact" $info "") $owner -}} + {{- $owner | toJSON -}} + {{- end }} spec: projectUniqueId: "{{ .project._id }}" projectId: "{{ .project.projectId }}" @@ -35,29 +44,3 @@ mappings: pipelines: "{{ get \"pipelines\" .project (object) | toJSON }}" projectNamespaceVariable: "{{ get \"projectNamespaceVariable\" .project \"\" | quote }}" lastUpdate: "{{ get \"lastUpdate\" .project (now) }}" - extra: - - apiVersion: mia-platform.eu/v1 - itemFamily: relationships - deletePolicy: "cascade" - createIf: |- - {{ $info := (get "info" .project (object)) -}} - {{- $teamContact := (get "teamContact" $info nil) -}} - {{- if $teamContact -}} - true - {{- else -}} - false - {{- end }} - identifier: |- - {{ $info := (get "info" .project (object)) -}} - {{- $teamContact := (get "teamContact" $info "") -}} - {{- $src := printf "urn:mia-platform-catalog:iam.mia-platform.eu:v1:User:%s" ($teamContact | sha256sum) -}} - {{- $type := "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:ownership.mia-platform.eu" -}} - {{- $tgt := printf "urn:mia-platform-catalog:console.mia-platform.eu:v1:Project:%s" (printf "%s" .project._id) -}} - {{- printf "%s-%s-%s" $src $type $tgt | sha256sum }} - sourceRef: |- - {{ $info := (get "info" .project (object)) -}} - {{- $teamContact := (get "teamContact" $info "") -}} - urn:mia-platform-catalog:iam.mia-platform.eu:v1:User:{{ $teamContact | sha256sum }} - targetRef: |- - urn:mia-platform-catalog:console.mia-platform.eu:v1:Project:{{ printf "%s" .project._id }} - typeRef: "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:ownership.mia-platform.eu" diff --git a/docs/mappings/console/revisions.yaml b/docs/mappings/console/revisions.yaml index 853442c..972a6fd 100644 --- a/docs/mappings/console/revisions.yaml +++ b/docs/mappings/console/revisions.yaml @@ -7,6 +7,15 @@ mappings: {{ printf "%s-%s" .project._id .revision.name | sha256sum }} metadata: title: "{{ printf \"%s\" .revision.name }}" + owner: |- + {{ $info := (get "info" .project (object)) -}} + {{- $teamContact := (get "teamContact" $info nil) -}} + {{- if $teamContact -}} + {{- $owner := object -}} + {{- $owner = set "type" "email" $owner -}} + {{- $owner = set "ref" (get "teamContact" $info "") $owner -}} + {{- $owner | toJSON -}} + {{- end }} spec: {} extra: - apiVersion: mia-platform.eu/v1 @@ -22,28 +31,3 @@ mappings: targetRef: |- urn:mia-platform-catalog:console.mia-platform.eu:v1:Project:{{ .project._id }} typeRef: "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:part-of.mia-platform.eu" - - apiVersion: mia-platform.eu/v1 - itemFamily: relationships - deletePolicy: "cascade" - createIf: |- - {{ $info := (get "info" .project (object)) -}} - {{- $teamContact := (get "teamContact" $info nil) -}} - {{- if $teamContact -}} - true - {{- else -}} - false - {{- end }} - identifier: |- - {{ $info := (get "info" .project (object)) -}} - {{- $teamContact := (get "teamContact" $info "") -}} - {{- $src := printf "urn:mia-platform-catalog:iam.mia-platform.eu:v1:User:%s" ($teamContact | sha256sum) -}} - {{- $type := "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:ownership.mia-platform.eu" -}} - {{- $tgt := printf "urn:mia-platform-catalog:console.mia-platform.eu:v1:Revision:%s" (printf "%s-%s" .project._id .revision.name | sha256sum) -}} - {{- printf "%s-%s-%s" $src $type $tgt | sha256sum }} - sourceRef: |- - {{ $info := (get "info" .project (object)) -}} - {{- $teamContact := (get "teamContact" $info "") -}} - urn:mia-platform-catalog:iam.mia-platform.eu:v1:User:{{ $teamContact | sha256sum }} - targetRef: |- - urn:mia-platform-catalog:console.mia-platform.eu:v1:Revision:{{ printf "%s-%s" .project._id .revision.name | sha256sum }} - typeRef: "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:ownership.mia-platform.eu" diff --git a/docs/mappings/console/services.yaml b/docs/mappings/console/services.yaml index 0cf543c..aa1a84e 100644 --- a/docs/mappings/console/services.yaml +++ b/docs/mappings/console/services.yaml @@ -7,6 +7,15 @@ mappings: {{ printf "%s-%s-%s" .project._id .revision.name .service.name | sha256sum }} metadata: title: "{{ printf \"%s\" .service.name }}" + owner: |- + {{ $info := (get "info" .project (object)) -}} + {{- $teamContact := (get "teamContact" $info nil) -}} + {{- if $teamContact -}} + {{- $owner := object -}} + {{- $owner = set "type" "email" $owner -}} + {{- $owner = set "ref" (get "teamContact" $info "") $owner -}} + {{- $owner | toJSON -}} + {{- end }} spec: projectUniqueId: "{{ .project._id }}" projectId: "{{ get \"projectId\" .project \"\" | quote }}" @@ -40,31 +49,6 @@ mappings: targetRef: |- urn:mia-platform-catalog:console.mia-platform.eu:v1:Revision:{{ printf "%s-%s" .project._id .revision.name | sha256sum }} typeRef: "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:part-of.mia-platform.eu" - - apiVersion: mia-platform.eu/v1 - itemFamily: relationships - deletePolicy: "cascade" - createIf: |- - {{ $info := (get "info" .project (object)) -}} - {{- $teamContact := (get "teamContact" $info nil) -}} - {{- if $teamContact -}} - true - {{- else -}} - false - {{- end }} - identifier: |- - {{ $info := (get "info" .project (object)) -}} - {{- $teamContact := (get "teamContact" $info "") -}} - {{- $src := printf "urn:mia-platform-catalog:iam.mia-platform.eu:v1:User:%s" ($teamContact | sha256sum) -}} - {{- $type := "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:ownership.mia-platform.eu" -}} - {{- $tgt := printf "urn:mia-platform-catalog:console.mia-platform.eu:v1:Service:%s" (printf "%s-%s-%s" .project._id .revision.name .service.name | sha256sum) -}} - {{- printf "%s-%s-%s" $src $type $tgt | sha256sum }} - sourceRef: |- - {{ $info := (get "info" .project (object)) -}} - {{- $teamContact := (get "teamContact" $info "") -}} - urn:mia-platform-catalog:iam.mia-platform.eu:v1:User:{{ $teamContact | sha256sum }} - targetRef: |- - urn:mia-platform-catalog:console.mia-platform.eu:v1:Service:{{ printf "%s-%s-%s" .project._id .revision.name .service.name | sha256sum }} - typeRef: "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:ownership.mia-platform.eu" - apiVersion: mia-platform.eu/v1 itemFamily: relationships deletePolicy: "none" diff --git a/internal/config/mappings.go b/internal/config/mappings.go index dc32cfa..c5aaf94 100644 --- a/internal/config/mappings.go +++ b/internal/config/mappings.go @@ -70,6 +70,7 @@ type MetadataTemplate struct { Labels string `json:"labels,omitempty" yaml:"labels,omitempty"` Links string `json:"links,omitempty" yaml:"links,omitempty"` Name string `json:"name,omitempty" yaml:"name,omitempty"` + Owner string `json:"owner,omitempty" yaml:"owner,omitempty"` Tags string `json:"tags,omitempty" yaml:"tags,omitempty"` Title string `json:"title,omitempty" yaml:"title,omitempty"` UID string `json:"uid,omitempty" yaml:"uid,omitempty"` From 60895b39737ae51744677818f1849775f70101f0 Mon Sep 17 00:00:00 2001 From: APagani Date: Thu, 16 Jul 2026 18:14:21 +0200 Subject: [PATCH 07/20] feat: add oauth2 support for catalog destination (#75) --- .gitignore | 3 + docs/how-to/020_destinations.md | 41 +- go.mod | 11 + go.sum | 22 + internal/destination/catalog/catalog.go | 148 +++- internal/destination/catalog/catalog_test.go | 504 +++++++++++++ internal/destination/catalog/helper_test.go | 19 + internal/destination/catalog/transport.go | 22 +- .../destination/catalog/transport_test.go | 68 ++ internal/jwk/keys.go | 108 +++ internal/jwk/keys_test.go | 116 +++ internal/tokensource/doc.go | 7 + internal/tokensource/oauth2source/doc.go | 17 + .../tokensource/oauth2source/oauth2source.go | 323 ++++++++ .../oauth2source/oauth2source_test.go | 690 ++++++++++++++++++ internal/tokensource/source.go | 13 + 16 files changed, 2080 insertions(+), 32 deletions(-) create mode 100644 internal/destination/catalog/helper_test.go create mode 100644 internal/destination/catalog/transport_test.go create mode 100644 internal/jwk/keys.go create mode 100644 internal/jwk/keys_test.go create mode 100644 internal/tokensource/doc.go create mode 100644 internal/tokensource/oauth2source/doc.go create mode 100644 internal/tokensource/oauth2source/oauth2source.go create mode 100644 internal/tokensource/oauth2source/oauth2source_test.go create mode 100644 internal/tokensource/source.go diff --git a/.gitignore b/.gitignore index 235bc51..fe38c25 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,9 @@ bin/ local/ +.local/ +tmp/ +.tmp/ release/ coverage.txt .github/agents diff --git a/docs/how-to/020_destinations.md b/docs/how-to/020_destinations.md index 1e83702..5bc57d1 100644 --- a/docs/how-to/020_destinations.md +++ b/docs/how-to/020_destinations.md @@ -10,17 +10,46 @@ To enable this destination no flags are needed but only a series of environment - `MIA_CATALOG_ENDPOINT`: the destination endpoint for the data, available during the integration setup +- `MIA_CATALOG_TOKEN`: a static bearer token used to authenticate requests to the Mia-Platform + Catalog. It cannot be combined with `MIA_CATALOG_CLIENT_ID`, `MIA_CATALOG_CLIENT_SECRET` or + `MIA_CATALOG_PRIVATE_KEY_PATH` - `MIA_CATALOG_CLIENT_ID`: the client id for validating the requests to the Mia-Platform Catalog, - availbale during the integration setup -- `MIA_CATALOG_CLIENT_SECRET`: the client id for validating the requests to the Mia-Platform Catalog, - availbale during the integration setup -- `MIA_CATALOG_AUTH_ENDPOINT`: a custom endpoint for authentication, if you don’t set the variable - the endpoint used will be the host present in MIA_CATALOG_ENDPOINT with the `/oauth/token` - endpoint + available during the integration setup +- `MIA_CATALOG_CLIENT_SECRET`: the client secret for validating the requests to the Mia-Platform + Catalog, available during the integration setup +- `MIA_CATALOG_PRIVATE_KEY_PATH`: path to a PEM-encoded private key file, used together with + `MIA_CATALOG_CLIENT_ID`, to authenticate via private-key JWT client authentication + (RFC 7523 section 2.2) instead of `MIA_CATALOG_CLIENT_SECRET`. When this variable is set you + must also configure at least one of `MIA_CATALOG_ISSUER`, `MIA_CATALOG_ISSUER_METADATA` or + `MIA_CATALOG_TOKEN_ENDPOINT` +- `MIA_CATALOG_AUTH_ENDPOINT`: the token endpoint used by the client-credentials flow + (`MIA_CATALOG_CLIENT_ID` + `MIA_CATALOG_CLIENT_SECRET`), if you don’t set the variable the + endpoint used will be the host present in MIA_CATALOG_ENDPOINT with the `/oauth/token` endpoint +- `MIA_CATALOG_ISSUER`: the OIDC issuer URL used as the discovery base and expected issuer when + authenticating via private-key JWT client authentication, the discovery document is looked up + relative to this value +- `MIA_CATALOG_ISSUER_METADATA`: a custom URL for the OIDC discovery document used to resolve the + token endpoint when authenticating via private-key JWT client authentication, if you don’t set + the variable the discovery document is looked up relative to `MIA_CATALOG_ISSUER` +- `MIA_CATALOG_TOKEN_ENDPOINT`: a custom token endpoint used when authenticating via private-key + JWT client authentication, if you set the variable OIDC discovery is skipped entirely and this + endpoint is used directly +- `MIA_CATALOG_CUSTOM_SCOPE`: a custom scope requested during the token exchange when authenticating + via private-key JWT client authentication, if you don’t set the variable no scope is sent If you don’t set any variables the destination will try to connect to `http://localhost:8080/api/publish/` without any authentication. +### OIDC discovery path + +The following variable is not specific to this destination but applies to any OIDC discovery +performed during private-key JWT client authentication: + +- `OIDC_DISCOVERY_PATH`: the well-known path suffix joined to `MIA_CATALOG_ISSUER` to fetch the OIDC + discovery document, it defaults to `.well-known/openid-configuration` and should only be changed + when the issuer serves its discovery document at a non-standard path. It has no effect when + `MIA_CATALOG_ISSUER_METADATA` or `MIA_CATALOG_TOKEN_ENDPOINT` is set, as those skip discovery + ## Local Output `ibdm` can also be used to validate custom mappings or custom source implementations with a diff --git a/go.mod b/go.mod index 072ff88..c3e448b 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( github.com/gofiber/fiber/v2 v2.52.14 github.com/google/uuid v1.6.0 github.com/hashicorp/go-hclog v1.6.3 + github.com/lestrrat-go/jwx/v3 v3.1.1 github.com/spf13/cobra v1.10.2 github.com/stretchr/testify v1.11.1 golang.org/x/oauth2 v0.36.0 @@ -45,10 +46,12 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/clipperhouse/uax29/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 // indirect github.com/fatih/color v1.19.0 // indirect github.com/felixge/httpsnoop v1.1.0 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/goccy/go-json v0.10.6 // indirect github.com/golang-jwt/jwt/v5 v5.3.1 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/s2a-go v0.1.9 // indirect @@ -57,14 +60,22 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/klauspost/compress v1.19.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect + github.com/lestrrat-go/blackmagic v1.0.4 // indirect + github.com/lestrrat-go/dsig v1.2.1 // indirect + github.com/lestrrat-go/dsig-secp256k1 v1.0.0 // indirect + github.com/lestrrat-go/httpcc v1.0.1 // indirect + github.com/lestrrat-go/httprc/v3 v3.0.5 // indirect + github.com/lestrrat-go/option/v2 v2.0.0 // indirect github.com/mattn/go-colorable v0.1.15 // indirect github.com/mattn/go-isatty v0.0.22 // indirect github.com/mattn/go-runewidth v0.0.24 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/segmentio/asm v1.2.1 // indirect github.com/spf13/pflag v1.0.10 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasthttp v1.72.0 // indirect + github.com/valyala/fastjson v1.6.10 // indirect go.einride.tech/aip v0.86.3 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect diff --git a/go.sum b/go.sum index 94d6714..dcd0963 100644 --- a/go.sum +++ b/go.sum @@ -75,6 +75,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6N github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 h1:5RVFMOWjMyRy8cARdy79nAmgYw3hK/4HUq48LQ6Wwqo= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -96,6 +98,8 @@ github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU= +github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gofiber/fiber/v2 v2.52.14 h1:Of3L+9qVFaQNwPlcmEdl5IIodHz8BSE0j37R7rWu4pE= github.com/gofiber/fiber/v2 v2.52.14/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw= github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= @@ -151,6 +155,20 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/lestrrat-go/blackmagic v1.0.4 h1:IwQibdnf8l2KoO+qC3uT4OaTWsW7tuRQXy9TRN9QanA= +github.com/lestrrat-go/blackmagic v1.0.4/go.mod h1:6AWFyKNNj0zEXQYfTMPfZrAXUWUfTIZ5ECEUEJaijtw= +github.com/lestrrat-go/dsig v1.2.1 h1:MwxzZhE4+4fguHi+uDALKVlC3Cn+O1QU1Q/F8D7hVIc= +github.com/lestrrat-go/dsig v1.2.1/go.mod h1:RD2eOaidyPvpc7IJQoO3Qq52RWdy8ZcJs8lrOnoa1Kc= +github.com/lestrrat-go/dsig-secp256k1 v1.0.0 h1:JpDe4Aybfl0soBvoVwjqDbp+9S1Y2OM7gcrVVMFPOzY= +github.com/lestrrat-go/dsig-secp256k1 v1.0.0/go.mod h1:CxUgAhssb8FToqbL8NjSPoGQlnO4w3LG1P0qPWQm/NU= +github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE= +github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E= +github.com/lestrrat-go/httprc/v3 v3.0.5 h1:S+Mb4L2I+bM6JGTibLmxExhyTOqnXjqx+zi9MoXw/TM= +github.com/lestrrat-go/httprc/v3 v3.0.5/go.mod h1:mSMtkZW92Z98M5YoNNztbRGxbXHql7tSitCvaxvo9l0= +github.com/lestrrat-go/jwx/v3 v3.1.1 h1:yd9AdPmZ4INnQ7k42IrzXYpnEG803+SrQ6hdMvzHJzw= +github.com/lestrrat-go/jwx/v3 v3.1.1/go.mod h1:uw/MN2M/Xiu4FhwcIwH11Zsh9JWx9SWzgALl7/uIEkU= +github.com/lestrrat-go/option/v2 v2.0.0 h1:XxrcaJESE1fokHy3FpaQ/cXW8ZsIdWcdFzzLOcID3Ss= +github.com/lestrrat-go/option/v2 v2.0.0/go.mod h1:oSySsmzMoR0iRzCDCaUfsCzxQHUEuhOViQObyy7S6Vg= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY= @@ -171,6 +189,8 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/segmentio/asm v1.2.1 h1:DTNbBqs57ioxAD4PrArqftgypG4/qNpXoJx8TVXxPR0= +github.com/segmentio/asm v1.2.1/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= @@ -189,6 +209,8 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.72.0 h1:R7kYdoWhn1ye1fVpP+cDHDJwYm3NkwLliwgzJ/Abg7M= github.com/valyala/fasthttp v1.72.0/go.mod h1:zsbLTYqcpIktdQytlVBwIjY9La5d6bs990nBxWg8efk= +github.com/valyala/fastjson v1.6.10 h1:/yjJg8jaVQdYR3arGxPE2X5z89xrlhS0eGXdv+ADTh4= +github.com/valyala/fastjson v1.6.10/go.mod h1:e6FubmQouUNP73jtMLmcbxS6ydWIpOfhz34TSfO3JaE= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= go.einride.tech/aip v0.86.3 h1:jg80Ec4XBPYg1i7avzrl3MJol/dUwmMMLHtcmEMyxgM= diff --git a/internal/destination/catalog/catalog.go b/internal/destination/catalog/catalog.go index 916c981..dc9bd30 100644 --- a/internal/destination/catalog/catalog.go +++ b/internal/destination/catalog/catalog.go @@ -17,12 +17,16 @@ import ( "github.com/mia-platform/ibdm/internal/destination" "github.com/mia-platform/ibdm/internal/info" + "github.com/mia-platform/ibdm/internal/jwk" ) var ( - errMultipleAuthMethods = errors.New("MIA_CATALOG_TOKEN cannot be used with MIA_CATALOG_CLIENT_ID or MIA_CATALOG_CLIENT_SECRET") - errMissingClientID = errors.New("MIA_CATALOG_CLIENT_ID is required when MIA_CATALOG_CLIENT_SECRET is set") - errMissingClientSecret = errors.New("MIA_CATALOG_CLIENT_SECRET is required when MIA_CATALOG_CLIENT_ID is set") + errMultipleAuthMethods = errors.New("MIA_CATALOG_TOKEN cannot be used with MIA_CATALOG_CLIENT_ID, MIA_CATALOG_CLIENT_SECRET or MIA_CATALOG_PRIVATE_KEY") + errMissingClientID = errors.New("MIA_CATALOG_CLIENT_ID is required when MIA_CATALOG_CLIENT_SECRET is set") + errMissingClientSecret = errors.New("MIA_CATALOG_CLIENT_SECRET is required when MIA_CATALOG_CLIENT_ID is set") + errMissingClientIDForPrivKey = errors.New("MIA_CATALOG_CLIENT_ID is required when MIA_CATALOG_PRIVATE_KEY is set") + errPrivateKeyWithClientSecret = errors.New("MIA_CATALOG_PRIVATE_KEY cannot be used with MIA_CATALOG_CLIENT_SECRET") + errMissingIssuerConfig = errors.New("private-key JWT authentication requires one of MIA_CATALOG_ISSUER, MIA_CATALOG_ISSUER_METADATA or MIA_CATALOG_TOKEN_ENDPOINT") ) var _ destination.Sender = &catalogDestination{} @@ -51,12 +55,42 @@ func (e *CatalogError) Is(target error) bool { // catalogDestination implements destination.Sender against the Mia-Platform Catalog API. type catalogDestination struct { + // CatalogEndpoint is the base URL of the Mia-Platform Catalog API that receives sent data. CatalogEndpoint string `env:"MIA_CATALOG_ENDPOINT,required"` - Token string `env:"MIA_CATALOG_TOKEN"` - ClientID string `env:"MIA_CATALOG_CLIENT_ID"` - ClientSecret string `env:"MIA_CATALOG_CLIENT_SECRET"` - AuthEndpoint string `env:"MIA_CATALOG_AUTH_ENDPOINT"` - + // Token, when set, is used as a static bearer token for authentication. It cannot be combined + // with MIA_CATALOG_CLIENT_ID, MIA_CATALOG_CLIENT_SECRET or MIA_CATALOG_PRIVATE_KEY_PATH. + Token string `env:"MIA_CATALOG_TOKEN"` + // ClientID is the OAuth2 client identifier used for either the client-credentials flow (with + // MIA_CATALOG_CLIENT_SECRET) or private-key JWT authentication (with MIA_CATALOG_PRIVATE_KEY_PATH). + ClientID string `env:"MIA_CATALOG_CLIENT_ID"` + // ClientSecret is the OAuth2 client secret used together with MIA_CATALOG_CLIENT_ID for the + // client-credentials flow. It cannot be combined with MIA_CATALOG_PRIVATE_KEY_PATH. + ClientSecret string `env:"MIA_CATALOG_CLIENT_SECRET"` + // PrivateKeyPath is the filesystem path to the private key used for private-key JWT + // authentication together with MIA_CATALOG_CLIENT_ID. + PrivateKeyPath string `env:"MIA_CATALOG_PRIVATE_KEY_PATH"` + // AuthEndpoint is the token endpoint used by the client-credentials flow. It is only meaningful + // together with MIA_CATALOG_CLIENT_ID and MIA_CATALOG_CLIENT_SECRET. When unset it defaults to + // the host of MIA_CATALOG_ENDPOINT with the /oauth/token path. + AuthEndpoint string `env:"MIA_CATALOG_AUTH_ENDPOINT"` + // Issuer is the OIDC issuer URL used as the discovery base and expected issuer for private-key + // JWT authentication. It is only meaningful together with MIA_CATALOG_CLIENT_ID and + // MIA_CATALOG_PRIVATE_KEY_PATH. + Issuer string `env:"MIA_CATALOG_ISSUER"` + // IssuerMetadata, when set, is fetched verbatim as the OIDC discovery document instead of the + // URL derived from MIA_CATALOG_ISSUER, to resolve the token endpoint for private-key JWT + // authentication. It is only meaningful together with MIA_CATALOG_CLIENT_ID and + // MIA_CATALOG_PRIVATE_KEY_PATH. + IssuerMetadata string `env:"MIA_CATALOG_ISSUER_METADATA"` + // TokenEndpoint, when set, is used directly as the token endpoint for private-key JWT + // authentication, skipping OIDC discovery entirely. It is only meaningful together with + // MIA_CATALOG_CLIENT_ID and MIA_CATALOG_PRIVATE_KEY_PATH. + TokenEndpoint string `env:"MIA_CATALOG_TOKEN_ENDPOINT"` + // CustomScope, when set, is used as the scope for private-key JWT authentication. It is only + // meaningful together with MIA_CATALOG_CLIENT_ID and MIA_CATALOG_PRIVATE_KEY_PATH. + CustomScope string `env:"MIA_CATALOG_CUSTOM_SCOPE"` + + keys *jwk.Keys client atomic.Pointer[http.Client] } @@ -72,13 +106,16 @@ func NewDestination() (destination.Sender, error) { return nil, handleError(fmt.Errorf("invalid MIA_CATALOG_ENDPOINT: %w", err)) } - switch { - case len(destination.Token) > 0 && (len(destination.ClientID) > 0 || len(destination.ClientSecret) > 0): - return nil, handleError(errMultipleAuthMethods) - case len(destination.ClientID) > 0 && len(destination.ClientSecret) == 0: - return nil, handleError(errMissingClientSecret) - case len(destination.ClientSecret) > 0 && len(destination.ClientID) == 0: - return nil, handleError(errMissingClientID) + if err := destination.validateAuthConfig(); err != nil { + return nil, handleError(err) + } + + if len(destination.PrivateKeyPath) > 0 { + keys, err := jwk.LoadKeys(destination.PrivateKeyPath) + if err != nil { + return nil, handleError(err) + } + destination.keys = keys } if len(destination.AuthEndpoint) == 0 { @@ -91,9 +128,69 @@ func NewDestination() (destination.Sender, error) { } } + if len(destination.Issuer) > 0 { + _, err := url.Parse(destination.Issuer) + if err != nil { + return nil, handleError(fmt.Errorf("invalid MIA_CATALOG_ISSUER: %w", err)) + } + } + + if len(destination.IssuerMetadata) > 0 { + _, err := url.Parse(destination.IssuerMetadata) + if err != nil { + return nil, handleError(fmt.Errorf("invalid MIA_CATALOG_ISSUER_METADATA: %w", err)) + } + } + + if len(destination.TokenEndpoint) > 0 { + _, err := url.Parse(destination.TokenEndpoint) + if err != nil { + return nil, handleError(fmt.Errorf("invalid MIA_CATALOG_TOKEN_ENDPOINT: %w", err)) + } + } + return destination, nil } +// validateAuthConfig ensures that at most one authentication method is configured and that each +// configured method has all of its required environment variables set. +func (d *catalogDestination) validateAuthConfig() error { + hasToken := len(d.Token) > 0 + hasClientID := len(d.ClientID) > 0 + hasClientSecret := len(d.ClientSecret) > 0 + hasPrivateKey := len(d.PrivateKeyPath) > 0 + + switch { + case hasToken && (hasClientID || hasClientSecret || hasPrivateKey): + return errMultipleAuthMethods + case hasPrivateKey: + return d.validatePrivateKeyAuthConfig() + case hasClientID && !hasClientSecret: + return errMissingClientSecret + case hasClientSecret && !hasClientID: + return errMissingClientID + } + + return nil +} + +// validatePrivateKeyAuthConfig validates the environment variables required by the private-key +// JWT authentication method. It assumes MIA_CATALOG_PRIVATE_KEY_PATH is set. +func (d *catalogDestination) validatePrivateKeyAuthConfig() error { + hasIssuerSource := len(d.Issuer) > 0 || len(d.IssuerMetadata) > 0 || len(d.TokenEndpoint) > 0 + + switch { + case len(d.ClientSecret) > 0: + return errPrivateKeyWithClientSecret + case len(d.ClientID) == 0: + return errMissingClientIDForPrivKey + case !hasIssuerSource: + return errMissingIssuerConfig + } + + return nil +} + // SendData implements destination.Sender. func (d *catalogDestination) SendData(ctx context.Context, data *destination.Data) error { return d.handleRequest(ctx, http.MethodPost, data) @@ -126,7 +223,12 @@ func (d *catalogDestination) handleRequest(ctx context.Context, method string, d } //nolint:contextcheck // need a new context because it will be used in token requests - resp, err := d.getClient(context.Background()).Do(request) + client, err := d.getClient(context.Background()) + if err != nil { + return handleError(err) + } + + resp, err := client.Do(request) if err != nil { return handleError(err) } @@ -173,14 +275,18 @@ func handleError(err error) error { } } -func (d *catalogDestination) getClient(ctx context.Context) *http.Client { +func (d *catalogDestination) getClient(ctx context.Context) (*http.Client, error) { client := d.client.Load() if client != nil { - return client + return client, nil + } + + transport, err := NewTransport(ctx, d.Token, d.AuthEndpoint, d.ClientID, d.ClientSecret, d.Issuer, d.IssuerMetadata, d.TokenEndpoint, d.CustomScope, d.keys) + if err != nil { + return nil, err } - client = &http.Client{} - client.Transport = NewTransport(ctx, d.Token, d.AuthEndpoint, d.ClientID, d.ClientSecret) + client = &http.Client{Transport: transport} d.client.Store(client) - return client + return client, nil } diff --git a/internal/destination/catalog/catalog_test.go b/internal/destination/catalog/catalog_test.go index fc1c54a..f0d17e1 100644 --- a/internal/destination/catalog/catalog_test.go +++ b/internal/destination/catalog/catalog_test.go @@ -5,22 +5,58 @@ package catalog import ( "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" "encoding/json" + "encoding/pem" "errors" "net/http" "net/http/httptest" "net/url" + "path/filepath" + "sync/atomic" "testing" "time" + lestrratjwk "github.com/lestrrat-go/jwx/v3/jwk" + "github.com/caarlos0/env/v11" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/mia-platform/ibdm/internal/destination" "github.com/mia-platform/ibdm/internal/info" + "github.com/mia-platform/ibdm/internal/jwk" ) +const rsaKeyBits = 4096 + +// generateTestRSAKey creates a fresh RSA key pair to be used as fictional test material. It is +// never used outside of this test file. +func generateTestRSAKey(t *testing.T) *rsa.PrivateKey { + t.Helper() + + key, err := rsa.GenerateKey(rand.Reader, rsaKeyBits) + require.NoError(t, err) + return key +} + +// encodePKCS8PEM PEM-encodes key using the PKCS8 container, matching the format most identity +// providers expect for a "private key" credential. +func encodePKCS8PEM(t *testing.T, key *rsa.PrivateKey) string { + t.Helper() + + der, err := x509.MarshalPKCS8PrivateKey(key) + require.NoError(t, err) + + block := &pem.Block{ + Type: "PRIVATE KEY", + Bytes: der, + } + return string(pem.EncodeToMemory(block)) +} + func TestInitialization(t *testing.T) { t.Run("without envs", func(t *testing.T) { dest, err := NewDestination() @@ -129,6 +165,156 @@ func TestInitialization(t *testing.T) { assert.ErrorIs(t, err, url.EscapeError("%41")) assert.Nil(t, dest) }) + + t.Run("invalid issuer", func(t *testing.T) { + t.Setenv("MIA_CATALOG_ENDPOINT", "http://localhost:8080/custom-catalog") + t.Setenv("MIA_CATALOG_ISSUER", "http://%41:8080/") // invalid URL + dest, err := NewDestination() + assert.ErrorIs(t, err, url.EscapeError("%41")) + assert.Nil(t, dest) + }) + + t.Run("invalid issuer metadata", func(t *testing.T) { + t.Setenv("MIA_CATALOG_ENDPOINT", "http://localhost:8080/custom-catalog") + t.Setenv("MIA_CATALOG_ISSUER_METADATA", "http://%41:8080/") // invalid URL + dest, err := NewDestination() + assert.ErrorIs(t, err, url.EscapeError("%41")) + assert.Nil(t, dest) + }) + + t.Run("invalid token endpoint", func(t *testing.T) { + t.Setenv("MIA_CATALOG_ENDPOINT", "http://localhost:8080/custom-catalog") + t.Setenv("MIA_CATALOG_TOKEN_ENDPOINT", "http://%41:8080/") // invalid URL + dest, err := NewDestination() + assert.ErrorIs(t, err, url.EscapeError("%41")) + assert.Nil(t, dest) + }) + + t.Run("with private key and client id", func(t *testing.T) { + keyPath := filepath.Join(t.TempDir(), "private-key.pem") + writeTestFile(t, keyPath, encodePKCS8PEM(t, generateTestRSAKey(t))) + + t.Setenv("MIA_CATALOG_ENDPOINT", "http://localhost:8080/custom-catalog") + t.Setenv("MIA_CATALOG_CLIENT_ID", "client-id") + t.Setenv("MIA_CATALOG_PRIVATE_KEY_PATH", keyPath) + t.Setenv("MIA_CATALOG_ISSUER", "http://localhost:8081/issuer") + dest, err := NewDestination() + require.NoError(t, err) + catalogDestination, ok := dest.(*catalogDestination) + require.True(t, ok) + + assert.Equal(t, "client-id", catalogDestination.ClientID) + assert.Equal(t, keyPath, catalogDestination.PrivateKeyPath) + assert.Empty(t, catalogDestination.ClientSecret) + assert.Equal(t, "http://localhost:8081/issuer", catalogDestination.Issuer) + // AuthEndpoint keeps its client-credentials default even in private-key mode, where it is + // unused: it is not consulted by the private-key JWT branch. + assert.Equal(t, "http://localhost:8080/oauth/token", catalogDestination.AuthEndpoint) + require.NotNil(t, catalogDestination.keys) + assert.NotNil(t, catalogDestination.keys.PrivateKey) + }) + + t.Run("private key without issuer config", func(t *testing.T) { + keyPath := filepath.Join(t.TempDir(), "private-key.pem") + writeTestFile(t, keyPath, encodePKCS8PEM(t, generateTestRSAKey(t))) + + t.Setenv("MIA_CATALOG_ENDPOINT", "http://localhost:8080/custom-catalog") + t.Setenv("MIA_CATALOG_CLIENT_ID", "client-id") + t.Setenv("MIA_CATALOG_PRIVATE_KEY_PATH", keyPath) + dest, err := NewDestination() + assert.ErrorIs(t, err, errMissingIssuerConfig) + assert.Nil(t, dest) + }) + + t.Run("with custom scope", func(t *testing.T) { + keyPath := filepath.Join(t.TempDir(), "private-key.pem") + writeTestFile(t, keyPath, encodePKCS8PEM(t, generateTestRSAKey(t))) + + t.Setenv("MIA_CATALOG_ENDPOINT", "http://localhost:8080/custom-catalog") + t.Setenv("MIA_CATALOG_CLIENT_ID", "client-id") + t.Setenv("MIA_CATALOG_PRIVATE_KEY_PATH", keyPath) + t.Setenv("MIA_CATALOG_ISSUER", "http://localhost:8081/issuer") + t.Setenv("MIA_CATALOG_CUSTOM_SCOPE", "organization:custom") + dest, err := NewDestination() + require.NoError(t, err) + catalogDestination, ok := dest.(*catalogDestination) + require.True(t, ok) + + assert.Equal(t, "organization:custom", catalogDestination.CustomScope) + }) + + t.Run("without custom scope", func(t *testing.T) { + keyPath := filepath.Join(t.TempDir(), "private-key.pem") + writeTestFile(t, keyPath, encodePKCS8PEM(t, generateTestRSAKey(t))) + + t.Setenv("MIA_CATALOG_ENDPOINT", "http://localhost:8080/custom-catalog") + t.Setenv("MIA_CATALOG_CLIENT_ID", "client-id") + t.Setenv("MIA_CATALOG_PRIVATE_KEY_PATH", keyPath) + t.Setenv("MIA_CATALOG_ISSUER", "http://localhost:8081/issuer") + dest, err := NewDestination() + require.NoError(t, err) + catalogDestination, ok := dest.(*catalogDestination) + require.True(t, ok) + + // MIA_CATALOG_CUSTOM_SCOPE has no default: when unset the scope must remain empty. + assert.Empty(t, catalogDestination.CustomScope) + }) + + t.Run("with private key, client id and custom issuer metadata/token endpoint", func(t *testing.T) { + keyPath := filepath.Join(t.TempDir(), "private-key.pem") + writeTestFile(t, keyPath, encodePKCS8PEM(t, generateTestRSAKey(t))) + + t.Setenv("MIA_CATALOG_ENDPOINT", "http://localhost:8080/custom-catalog") + t.Setenv("MIA_CATALOG_CLIENT_ID", "client-id") + t.Setenv("MIA_CATALOG_PRIVATE_KEY_PATH", keyPath) + t.Setenv("MIA_CATALOG_ISSUER_METADATA", "http://localhost:8081/custom/metadata") + t.Setenv("MIA_CATALOG_TOKEN_ENDPOINT", "http://localhost:8081/custom/token") + dest, err := NewDestination() + require.NoError(t, err) + catalogDestination, ok := dest.(*catalogDestination) + require.True(t, ok) + + assert.Equal(t, "http://localhost:8081/custom/metadata", catalogDestination.IssuerMetadata) + assert.Equal(t, "http://localhost:8081/custom/token", catalogDestination.TokenEndpoint) + }) + + t.Run("private key with unreadable file", func(t *testing.T) { + t.Setenv("MIA_CATALOG_ENDPOINT", "http://localhost:8080/custom-catalog") + t.Setenv("MIA_CATALOG_CLIENT_ID", "client-id") + t.Setenv("MIA_CATALOG_PRIVATE_KEY_PATH", filepath.Join(t.TempDir(), "does-not-exist.pem")) + t.Setenv("MIA_CATALOG_ISSUER", "http://localhost:8081/issuer") + dest, err := NewDestination() + assert.ErrorContains(t, err, "cannot read private key from") + assert.Nil(t, dest) + }) + + t.Run("private key without client id", func(t *testing.T) { + t.Setenv("MIA_CATALOG_ENDPOINT", "http://localhost:8080/custom-catalog") + t.Setenv("MIA_CATALOG_PRIVATE_KEY_PATH", "fictional-private-key-path") + t.Setenv("MIA_CATALOG_ISSUER", "http://localhost:8081/issuer") + dest, err := NewDestination() + assert.ErrorIs(t, err, errMissingClientIDForPrivKey) + assert.Nil(t, dest) + }) + + t.Run("private key with client secret", func(t *testing.T) { + t.Setenv("MIA_CATALOG_ENDPOINT", "http://localhost:8080/custom-catalog") + t.Setenv("MIA_CATALOG_CLIENT_ID", "client-id") + t.Setenv("MIA_CATALOG_CLIENT_SECRET", "client-secret") + t.Setenv("MIA_CATALOG_PRIVATE_KEY_PATH", "fictional-private-key-path") + dest, err := NewDestination() + assert.ErrorIs(t, err, errPrivateKeyWithClientSecret) + assert.Nil(t, dest) + }) + + t.Run("private key with fixed token", func(t *testing.T) { + t.Setenv("MIA_CATALOG_ENDPOINT", "http://localhost:8080/custom-catalog") + t.Setenv("MIA_CATALOG_TOKEN", "test-token") + t.Setenv("MIA_CATALOG_PRIVATE_KEY_PATH", "fictional-private-key-path") + dest, err := NewDestination() + assert.ErrorIs(t, err, errMultipleAuthMethods) + assert.Nil(t, dest) + }) } func TestSendData(t *testing.T) { @@ -418,3 +604,321 @@ func TestClientCredentialFlow(t *testing.T) { err := dest.SendData(ctx, &destination.Data{}) assert.NoError(t, err) } + +// newTestPrivateKeyFor wraps key into a jwk.Keys usable as catalogDestination.keys, to be used as +// fictional test material. It is never used outside of this test file. +func newTestPrivateKeyFor(t *testing.T, key *rsa.PrivateKey) *jwk.Keys { + t.Helper() + + jwkKey, err := lestrratjwk.Import(key) + require.NoError(t, err) + + return &jwk.Keys{PrivateKey: jwkKey} +} + +// TestPrivateKeyJWTFlowWithExplicitTokenEndpoint verifies that a catalogDestination configured +// with MIA_CATALOG_TOKEN_ENDPOINT reaches oauth2source.NewSource with that value, skipping OIDC +// discovery entirely: the discovery endpoint is registered but never hit, and the token exchange +// goes straight to the configured token endpoint. +func TestPrivateKeyJWTFlowWithExplicitTokenEndpoint(t *testing.T) { + t.Parallel() + + key := generateTestRSAKey(t) + var discoveryHits atomic.Int32 + + testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Body != nil { + defer r.Body.Close() + } + + switch { + case r.Method == http.MethodGet && r.URL.Path == "/.well-known/openid-configuration": + discoveryHits.Add(1) + http.NotFound(w, r) + case r.Method == http.MethodPost && r.URL.Path == "/custom/token": + require.NoError(t, r.ParseForm()) + assert.Equal(t, "test-client-id", r.FormValue("client_id")) + assert.Equal(t, "private_key_jwt", r.FormValue("token_endpoint_auth_method")) + + w.Header().Set("Content-Type", "application/json") + err := json.NewEncoder(w).Encode(map[string]any{ + "access_token": "generated-jwt-bearer-token", + "token_type": "Bearer", + "expires_in": 3600, + }) + require.NoError(t, err) + case r.Method == http.MethodPost && r.URL.Path == "/": + assert.Equal(t, "Bearer generated-jwt-bearer-token", r.Header.Get("Authorization")) + w.WriteHeader(http.StatusNoContent) + default: + http.NotFound(w, r) + } + })) + defer testServer.Close() + + ctx, cancel := context.WithTimeout(t.Context(), 1*time.Second) + defer cancel() + + dest := &catalogDestination{ + CatalogEndpoint: testServer.URL + "/", + ClientID: "test-client-id", + Issuer: testServer.URL, + TokenEndpoint: testServer.URL + "/custom/token", + keys: newTestPrivateKeyFor(t, key), + } + + err := dest.SendData(ctx, &destination.Data{}) + require.NoError(t, err) + assert.Equal(t, int32(0), discoveryHits.Load()) +} + +// TestPrivateKeyJWTFlowWithCustomScope verifies that a catalogDestination configured with a custom +// scope forwards it through NewTransport to the token request as the "scope" form field, and that +// when no custom scope is configured the token request carries no scope (the default was removed). +func TestPrivateKeyJWTFlowWithCustomScope(t *testing.T) { + t.Parallel() + + testCases := map[string]struct { + customScope string + expectedScope string + }{ + "with custom scope": { + customScope: "organization:custom", + expectedScope: "organization:custom", + }, + "without custom scope": { + customScope: "", + expectedScope: "", + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + t.Parallel() + + key := generateTestRSAKey(t) + + testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Body != nil { + defer r.Body.Close() + } + + switch { + case r.Method == http.MethodPost && r.URL.Path == "/custom/token": + require.NoError(t, r.ParseForm()) + assert.Equal(t, "test-client-id", r.FormValue("client_id")) + assert.Equal(t, tc.expectedScope, r.FormValue("scope")) + + w.Header().Set("Content-Type", "application/json") + err := json.NewEncoder(w).Encode(map[string]any{ + "access_token": "generated-jwt-bearer-token", + "token_type": "Bearer", + "expires_in": 3600, + }) + require.NoError(t, err) + case r.Method == http.MethodPost && r.URL.Path == "/": + assert.Equal(t, "Bearer generated-jwt-bearer-token", r.Header.Get("Authorization")) + w.WriteHeader(http.StatusNoContent) + default: + http.NotFound(w, r) + } + })) + defer testServer.Close() + + ctx, cancel := context.WithTimeout(t.Context(), 1*time.Second) + defer cancel() + + dest := &catalogDestination{ + CatalogEndpoint: testServer.URL + "/", + ClientID: "test-client-id", + Issuer: testServer.URL, + TokenEndpoint: testServer.URL + "/custom/token", + CustomScope: tc.customScope, + keys: newTestPrivateKeyFor(t, key), + } + + err := dest.SendData(ctx, &destination.Data{}) + require.NoError(t, err) + }) + } +} + +// TestPrivateKeyJWTFlowWithCustomDiscoveryMetadata verifies that a catalogDestination configured +// with MIA_CATALOG_ISSUER_METADATA reaches oauth2source.NewSource with that value, so discovery is +// fetched from the custom URL rather than the default well-known path. +func TestPrivateKeyJWTFlowWithCustomDiscoveryMetadata(t *testing.T) { + t.Parallel() + + key := generateTestRSAKey(t) + var defaultDiscoveryHits atomic.Int32 + + var testServer *httptest.Server + testServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Body != nil { + defer r.Body.Close() + } + + switch { + case r.Method == http.MethodGet && r.URL.Path == "/.well-known/openid-configuration": + defaultDiscoveryHits.Add(1) + http.NotFound(w, r) + case r.Method == http.MethodGet && r.URL.Path == "/custom/metadata": + w.Header().Set("Content-Type", "application/json") + err := json.NewEncoder(w).Encode(map[string]string{ + "issuer": testServer.URL, + "token_endpoint": testServer.URL + "/oauth/token", + }) + require.NoError(t, err) + case r.Method == http.MethodPost && r.URL.Path == "/oauth/token": + require.NoError(t, r.ParseForm()) + assert.Equal(t, "test-client-id", r.FormValue("client_id")) + + w.Header().Set("Content-Type", "application/json") + err := json.NewEncoder(w).Encode(map[string]any{ + "access_token": "generated-jwt-bearer-token", + "token_type": "Bearer", + "expires_in": 3600, + }) + require.NoError(t, err) + case r.Method == http.MethodPost && r.URL.Path == "/": + assert.Equal(t, "Bearer generated-jwt-bearer-token", r.Header.Get("Authorization")) + w.WriteHeader(http.StatusNoContent) + default: + http.NotFound(w, r) + } + })) + defer testServer.Close() + + ctx, cancel := context.WithTimeout(t.Context(), 1*time.Second) + defer cancel() + + dest := &catalogDestination{ + CatalogEndpoint: testServer.URL + "/", + ClientID: "test-client-id", + Issuer: testServer.URL, + IssuerMetadata: testServer.URL + "/custom/metadata", + keys: newTestPrivateKeyFor(t, key), + } + + err := dest.SendData(ctx, &destination.Data{}) + require.NoError(t, err) + assert.Equal(t, int32(0), defaultDiscoveryHits.Load()) +} + +// TestPrivateKeyJWTFlowWithMetadataOnlyNoIssuer verifies that a catalogDestination configured with +// MIA_CATALOG_ISSUER_METADATA but without MIA_CATALOG_ISSUER still completes the private-key JWT +// flow: with no configured issuer there is no expected issuer to validate the discovery document +// against, so the discovery document's issuer (which here has no relationship to the test server +// at all) must not cause a failure. +func TestPrivateKeyJWTFlowWithMetadataOnlyNoIssuer(t *testing.T) { + t.Parallel() + + key := generateTestRSAKey(t) + + var testServer *httptest.Server + testServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Body != nil { + defer r.Body.Close() + } + + switch { + case r.Method == http.MethodGet && r.URL.Path == "/custom/metadata": + w.Header().Set("Content-Type", "application/json") + err := json.NewEncoder(w).Encode(map[string]string{ + "issuer": "https://unrelated-issuer.example.com", + "token_endpoint": testServer.URL + "/oauth/token", + }) + require.NoError(t, err) + case r.Method == http.MethodPost && r.URL.Path == "/oauth/token": + require.NoError(t, r.ParseForm()) + assert.Equal(t, "test-client-id", r.FormValue("client_id")) + + w.Header().Set("Content-Type", "application/json") + err := json.NewEncoder(w).Encode(map[string]any{ + "access_token": "generated-jwt-bearer-token", + "token_type": "Bearer", + "expires_in": 3600, + }) + require.NoError(t, err) + case r.Method == http.MethodPost && r.URL.Path == "/": + assert.Equal(t, "Bearer generated-jwt-bearer-token", r.Header.Get("Authorization")) + w.WriteHeader(http.StatusNoContent) + default: + http.NotFound(w, r) + } + })) + defer testServer.Close() + + ctx, cancel := context.WithTimeout(t.Context(), 1*time.Second) + defer cancel() + + dest := &catalogDestination{ + CatalogEndpoint: testServer.URL + "/", + ClientID: "test-client-id", + IssuerMetadata: testServer.URL + "/custom/metadata", + keys: newTestPrivateKeyFor(t, key), + } + + err := dest.SendData(ctx, &destination.Data{}) + require.NoError(t, err) +} + +// TestPrivateKeyJWTFlowThroughNewDestination exercises the whole private-key JWT stack built by +// NewDestination from environment variables (rather than a hand-assembled struct), for the case +// where only MIA_CATALOG_ISSUER is configured: OIDC discovery is performed against the issuer's +// well-known path, the resolved token endpoint mints a bearer token, and the publish request +// carries it. This guards the construction path that wires MIA_CATALOG_ISSUER into oauth2source. +func TestPrivateKeyJWTFlowThroughNewDestination(t *testing.T) { + // Not parallel: mutates the process environment via t.Setenv. + var testServer *httptest.Server + testServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Body != nil { + defer r.Body.Close() + } + + switch { + case r.Method == http.MethodGet && r.URL.Path == "/.well-known/openid-configuration": + w.Header().Set("Content-Type", "application/json") + err := json.NewEncoder(w).Encode(map[string]string{ + "issuer": testServer.URL, + "token_endpoint": testServer.URL + "/oauth/token", + }) + require.NoError(t, err) + case r.Method == http.MethodPost && r.URL.Path == "/oauth/token": + require.NoError(t, r.ParseForm()) + assert.Equal(t, "test-client-id", r.FormValue("client_id")) + assert.Equal(t, "private_key_jwt", r.FormValue("token_endpoint_auth_method")) + + w.Header().Set("Content-Type", "application/json") + err := json.NewEncoder(w).Encode(map[string]any{ + "access_token": "generated-jwt-bearer-token", + "token_type": "Bearer", + "expires_in": 3600, + }) + require.NoError(t, err) + case r.Method == http.MethodPost && r.URL.Path == "/": + assert.Equal(t, "Bearer generated-jwt-bearer-token", r.Header.Get("Authorization")) + w.WriteHeader(http.StatusNoContent) + default: + http.NotFound(w, r) + } + })) + defer testServer.Close() + + keyPath := filepath.Join(t.TempDir(), "private-key.pem") + writeTestFile(t, keyPath, encodePKCS8PEM(t, generateTestRSAKey(t))) + + t.Setenv("MIA_CATALOG_ENDPOINT", testServer.URL+"/") + t.Setenv("MIA_CATALOG_CLIENT_ID", "test-client-id") + t.Setenv("MIA_CATALOG_PRIVATE_KEY_PATH", keyPath) + t.Setenv("MIA_CATALOG_ISSUER", testServer.URL) + + dest, err := NewDestination() + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(t.Context(), 1*time.Second) + defer cancel() + + err = dest.SendData(ctx, &destination.Data{}) + require.NoError(t, err) +} diff --git a/internal/destination/catalog/helper_test.go b/internal/destination/catalog/helper_test.go new file mode 100644 index 0000000..b2f07f0 --- /dev/null +++ b/internal/destination/catalog/helper_test.go @@ -0,0 +1,19 @@ +// Copyright Mia srl +// SPDX-License-Identifier: AGPL-3.0-only or Commercial + +package catalog + +import ( + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +// writeTestFile writes content to path, failing the test immediately on error. It is used to +// materialize fictional key material on disk for tests that exercise file-based configuration. +func writeTestFile(t *testing.T, path, content string) { + t.Helper() + + require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) +} diff --git a/internal/destination/catalog/transport.go b/internal/destination/catalog/transport.go index 58dc802..665ffb1 100644 --- a/internal/destination/catalog/transport.go +++ b/internal/destination/catalog/transport.go @@ -9,10 +9,16 @@ import ( "golang.org/x/oauth2" "golang.org/x/oauth2/clientcredentials" + + "github.com/mia-platform/ibdm/internal/jwk" + "github.com/mia-platform/ibdm/internal/tokensource/oauth2source" ) -// NewTransport creates an HTTP transport configured with either a static token or a client-credentials flow. -func NewTransport(ctx context.Context, token, tokenURL, clientID, clientSecret string) http.RoundTripper { +// NewTransport creates an HTTP transport configured with either a static token, private-key JWT +// client authentication, or client-credentials flow. authEndpoint is the token URL used by the +// client-credentials flow. issuer, issuerMetadata and tokenEndpoint are only used by the +// private-key JWT branch: see oauth2source.NewSource for their meaning. +func NewTransport(ctx context.Context, token, authEndpoint, clientID, clientSecret, issuer, issuerMetadata, tokenEndpoint, customScope string, keys *jwk.Keys) (http.RoundTripper, error) { var source oauth2.TokenSource switch { case len(token) > 0: @@ -24,18 +30,24 @@ func NewTransport(ctx context.Context, token, tokenURL, clientID, clientSecret s config := clientcredentials.Config{ ClientID: clientID, ClientSecret: clientSecret, - TokenURL: tokenURL, + TokenURL: authEndpoint, AuthStyle: oauth2.AuthStyleInHeader, } source = config.TokenSource(ctx) + case len(clientID) > 0 && keys != nil && keys.PrivateKey != nil: + oauth2Source, err := oauth2source.NewSource(ctx, clientID, issuer, issuerMetadata, tokenEndpoint, customScope, keys.PrivateKey) + if err != nil { + return nil, err + } + source = oauth2Source } if source == nil { - return http.DefaultTransport + return http.DefaultTransport, nil } return &oauth2.Transport{ Source: source, - } + }, nil } diff --git a/internal/destination/catalog/transport_test.go b/internal/destination/catalog/transport_test.go new file mode 100644 index 0000000..617dac2 --- /dev/null +++ b/internal/destination/catalog/transport_test.go @@ -0,0 +1,68 @@ +// Copyright Mia srl +// SPDX-License-Identifier: AGPL-3.0-only or Commercial + +package catalog + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNewTransportWithoutCredentials(t *testing.T) { + t.Parallel() + + transport, err := NewTransport(t.Context(), "", "", "", "", "", "", "", "", nil) + assert.NoError(t, err) + assert.Same(t, http.DefaultTransport, transport) +} + +// TestNewTransportPrivateKeyJWTWiring verifies that NewTransport wires the client-ID/private-key +// case to an oauth2source provider by exercising a full token exchange plus authenticated +// request through the resulting transport. The JWT assertion contents and provider-level failure +// modes are covered by the oauth2source package's own tests. +func TestNewTransportPrivateKeyJWTWiring(t *testing.T) { + t.Parallel() + + key := generateTestRSAKey(t) + const clientID = "jwt-bearer-client" + + testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/oauth/token": + require.NoError(t, r.ParseForm()) + assert.Equal(t, clientID, r.FormValue("client_id")) + assert.Equal(t, "organization:*", r.FormValue("scope")) + + w.Header().Set("Content-Type", "application/json") + err := json.NewEncoder(w).Encode(map[string]any{ + "access_token": "generated-jwt-bearer-token", + "token_type": "Bearer", + "expires_in": 3600, + }) + require.NoError(t, err) + case "/": + assert.Equal(t, "Bearer generated-jwt-bearer-token", r.Header.Get("Authorization")) + w.WriteHeader(http.StatusNoContent) + default: + http.NotFound(w, r) + } + })) + defer testServer.Close() + + transport, err := NewTransport(t.Context(), "", "", clientID, "", "", "", testServer.URL+"/oauth/token", "organization:*", newTestPrivateKeyFor(t, key)) + require.NoError(t, err) + + client := &http.Client{ + Transport: transport, + } + + resp, err := client.Get(testServer.URL + "/") + require.NoError(t, err) + defer resp.Body.Close() + assert.Equal(t, http.StatusNoContent, resp.StatusCode) +} diff --git a/internal/jwk/keys.go b/internal/jwk/keys.go new file mode 100644 index 0000000..20214d2 --- /dev/null +++ b/internal/jwk/keys.go @@ -0,0 +1,108 @@ +// Copyright Mia srl +// SPDX-License-Identifier: AGPL-3.0-only or Commercial + +package jwk + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/caarlos0/env/v11" + "github.com/lestrrat-go/jwx/v3/jwk" +) + +// Keys holds the private key used to sign JWT assertions and the corresponding public JWKS +// document, derived from it by LoadKeys. +type Keys struct { + // PrivateKey is used to sign the JWT assertion sent to the token endpoint. + PrivateKey jwk.Key + // JWKSBytes is the JSON-serialized public JWK set matching PrivateKey. + JWKSBytes []byte +} + +// config holds the environment-driven settings consumed by LoadKeys. +type config struct { + // CustomKeyID is an optional operator-provided key ID (kid). When empty, LoadKeys derives the + // key ID automatically from the key material. + CustomKeyID string `env:"CUSTOM_KEY_ID"` +} + +// loadConfigFromEnv parses the LoadKeys configuration from environment variables. +func loadConfigFromEnv() (*config, error) { + cfg := new(config) + if err := env.Parse(cfg); err != nil { + return nil, fmt.Errorf("cannot parse key configuration from environment: %w", err) + } + return cfg, nil +} + +// readKeyMaterial resolves the raw PEM bytes from the configured source. +// "file": reads from the path mounted as a Kubernetes Secret volume. +func readKeyMaterial(privateKeyPath string) ([]byte, error) { + data, err := os.ReadFile(privateKeyPath) + if err != nil { + return nil, fmt.Errorf("cannot read private key from %q: %w", privateKeyPath, err) + } + return data, nil +} + +// LoadKeys reads the private key from privateKeyPath, builds the corresponding public JWKS, and +// returns the result. When a custom key ID is configured in the environment (see config) it is +// used as the key ID (kid) for both the public and private key; otherwise a key ID is derived +// automatically from the key material. It fails fast — any missing file, malformed PEM, or +// serialization error is returned as an error. +func LoadKeys(privateKeyPath string) (*Keys, error) { + pemBytes, err := readKeyMaterial(privateKeyPath) + if err != nil { + return nil, err + } + + privateKey, err := jwk.ParseKey(pemBytes, jwk.WithPEM(true)) + if err != nil { + return nil, fmt.Errorf("cannot parse private key PEM: %w", err) + } + + publicKey, err := jwk.PublicKeyOf(privateKey) + if err != nil { + return nil, fmt.Errorf("cannot derive public key from private key: %w", err) + } + + if err := publicKey.Set(jwk.KeyUsageKey, "sig"); err != nil { + return nil, fmt.Errorf("cannot set key usage on public key: %w", err) + } + + cfg, err := loadConfigFromEnv() + if err != nil { + return nil, err + } + + // Use the operator-provided key ID when set, otherwise derive one automatically from the key. + if len(cfg.CustomKeyID) > 0 { + if err := publicKey.Set(jwk.KeyIDKey, cfg.CustomKeyID); err != nil { + return nil, fmt.Errorf("cannot set custom key ID on public key: %w", err) + } + } else if err := jwk.AssignKeyID(publicKey); err != nil { + return nil, fmt.Errorf("cannot assign key ID to public key: %w", err) + } + + kid, _ := publicKey.KeyID() + if err := privateKey.Set(jwk.KeyIDKey, kid); err != nil { + return nil, fmt.Errorf("cannot set key ID on private key: %w", err) + } + + set := jwk.NewSet() + if err := set.AddKey(publicKey); err != nil { + return nil, fmt.Errorf("cannot add public key to JWK set: %w", err) + } + + jwksBytes, err := json.Marshal(set) + if err != nil { + return nil, fmt.Errorf("cannot serialize JWKS to JSON: %w", err) + } + + return &Keys{ + PrivateKey: privateKey, + JWKSBytes: jwksBytes, + }, nil +} diff --git a/internal/jwk/keys_test.go b/internal/jwk/keys_test.go new file mode 100644 index 0000000..e925bde --- /dev/null +++ b/internal/jwk/keys_test.go @@ -0,0 +1,116 @@ +// Copyright Mia srl +// SPDX-License-Identifier: AGPL-3.0-only or Commercial + +package jwk + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/json" + "encoding/pem" + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +// MockRSAPEM generates a test-only private key PEM. +func MockRSAPEM(tb testing.TB) string { + tb.Helper() + + privateKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + tb.Fatalf("cannot generate mock RSA key: %v", err) + } + + pkcs8Bytes, err := x509.MarshalPKCS8PrivateKey(privateKey) + if err != nil { + tb.Fatalf("cannot marshal mock RSA key: %v", err) + } + + pemBytes := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: pkcs8Bytes}) + if pemBytes == nil { + tb.Fatal("cannot encode mock RSA key to PEM") + } + + return string(pemBytes) +} + +func TestLoadKeys(t *testing.T) { + t.Run("loads RSA key from file", func(t *testing.T) { + f, err := os.CreateTemp(t.TempDir(), "key-*.pem") + require.NoError(t, err) + _, err = f.WriteString(MockRSAPEM(t)) + require.NoError(t, err) + require.NoError(t, f.Close()) + + keys, err := LoadKeys(f.Name()) + require.NoError(t, err) + + require.NotNil(t, keys.PrivateKey) + require.NotEmpty(t, keys.JWKSBytes) + + // JWKS must be valid JSON with a "keys" array. + var jwks map[string]any + require.NoError(t, json.Unmarshal(keys.JWKSBytes, &jwks)) + keysArr, ok := jwks["keys"].([]any) + require.True(t, ok, "JWKS must have a 'keys' array") + require.Len(t, keysArr, 1) + + // The single key must carry a kid and have use=sig. + entry := keysArr[0].(map[string]any) + require.NotEmpty(t, entry["kid"], "public key must have a kid") + require.Equal(t, "sig", entry["use"]) + + // The private key must have the same kid as the public key in the JWKS. + kid, ok := keys.PrivateKey.KeyID() + require.True(t, ok) + require.Equal(t, entry["kid"], kid) + }) + + t.Run("uses custom key ID when provided", func(t *testing.T) { + f, err := os.CreateTemp(t.TempDir(), "key-*.pem") + require.NoError(t, err) + _, err = f.WriteString(MockRSAPEM(t)) + require.NoError(t, err) + require.NoError(t, f.Close()) + + const customKID = "my-custom-kid" + t.Setenv("CUSTOM_KEY_ID", customKID) + + keys, err := LoadKeys(f.Name()) + require.NoError(t, err) + + // The public key in the JWKS must carry the custom kid. + var jwks map[string]any + require.NoError(t, json.Unmarshal(keys.JWKSBytes, &jwks)) + keysArr, ok := jwks["keys"].([]any) + require.True(t, ok, "JWKS must have a 'keys' array") + require.Len(t, keysArr, 1) + + entry := keysArr[0].(map[string]any) + require.Equal(t, customKID, entry["kid"]) + + // The private key must carry the same custom kid. + kid, ok := keys.PrivateKey.KeyID() + require.True(t, ok) + require.Equal(t, customKID, kid) + }) + + t.Run("missing file returns error", func(t *testing.T) { + _, err := LoadKeys("/nonexistent/key.pem") + require.ErrorContains(t, err, "cannot read private key") + }) + + t.Run("invalid PEM returns error", func(t *testing.T) { + f, err := os.CreateTemp(t.TempDir(), "key-*.pem") + require.NoError(t, err) + _, err = f.WriteString("not a valid pem") + require.NoError(t, err) + require.NoError(t, f.Close()) + + _, err = LoadKeys(f.Name()) + require.ErrorContains(t, err, "cannot parse private key PEM") + }) +} diff --git a/internal/tokensource/doc.go b/internal/tokensource/doc.go new file mode 100644 index 0000000..0837a3e --- /dev/null +++ b/internal/tokensource/doc.go @@ -0,0 +1,7 @@ +// Copyright Mia srl +// SPDX-License-Identifier: AGPL-3.0-only or Commercial + +// Package tokensource defines the common Provider interface implemented by every +// client-authentication strategy used to obtain OAuth2 access tokens on behalf of ibdm +// destinations. Concrete strategies live in their own sub-packages. +package tokensource diff --git a/internal/tokensource/oauth2source/doc.go b/internal/tokensource/oauth2source/doc.go new file mode 100644 index 0000000..40877c2 --- /dev/null +++ b/internal/tokensource/oauth2source/doc.go @@ -0,0 +1,17 @@ +// Copyright Mia srl +// SPDX-License-Identifier: AGPL-3.0-only or Commercial + +// Package oauth2source implements tokensource.Source using the private_key_jwt +// client authentication method defined in RFC 7523 section 2.2: a JWT assertion is signed with +// a private key and exchanged for an access token via the client_credentials grant. +// +// The token endpoint and the JWT audience are not configured directly; instead, they are +// resolved via OIDC discovery (the ".well-known/openid-configuration" document, overridable via +// the OIDC_DISCOVERY_PATH environment variable) against a configured issuer URL, following an +// OAuth2/OIDC provider setup such as Keycloak. The resolved token endpoint is cached for the +// lifetime of the source, and discovery is retried on the next call whenever a previous attempt +// failed. +// +// The oauth2.TokenSource returned by NewSource automatically reuses tokens until they are near +// expiry, via oauth2.ReuseTokenSource. +package oauth2source diff --git a/internal/tokensource/oauth2source/oauth2source.go b/internal/tokensource/oauth2source/oauth2source.go new file mode 100644 index 0000000..8210eb3 --- /dev/null +++ b/internal/tokensource/oauth2source/oauth2source.go @@ -0,0 +1,323 @@ +// Copyright Mia srl +// SPDX-License-Identifier: AGPL-3.0-only or Commercial + +package oauth2source + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "sync" + "time" + + "github.com/caarlos0/env/v11" + "github.com/google/uuid" + "github.com/lestrrat-go/jwx/v3/jwa" + "github.com/lestrrat-go/jwx/v3/jwk" + "github.com/lestrrat-go/jwx/v3/jwt" + "golang.org/x/oauth2" + + "github.com/mia-platform/ibdm/internal/tokensource" +) + +const ( + // clientCredentialsGrantType is the standard OAuth2 grant type used together with private-key + // JWT client authentication, as defined by RFC 7523 section 2.2. + clientCredentialsGrantType = "client_credentials" + // jwtBearerClientAssertionType identifies a JWT bearer assertion used for client + // authentication, as defined by RFC 7523 section 2.2. + jwtBearerClientAssertionType = "urn:ietf:params:oauth:client-assertion-type:jwt-bearer" //nolint:gosec + // privateKeyJWTAuthMethod identifies the client authentication method used when exchanging a + // JWT assertion signed with a private key for an access token. + privateKeyJWTAuthMethod = "private_key_jwt" + // jwtAssertionLifetime is the validity window of the JWT assertion sent to the token endpoint. + jwtAssertionLifetime = 5 * time.Minute + // tokenRequestTimeout bounds how long a single token exchange request or discovery request is + // allowed to take. + tokenRequestTimeout = 30 * time.Second + // maxTokenErrorBodyBytes caps how much of a non-2xx token response body is read into memory. + maxTokenErrorBodyBytes = 1024 +) + +var ( + // ErrTokenExchange wraps failures encountered while exchanging a JWT assertion for an access token. + ErrTokenExchange = errors.New("oauth2source token exchange") + + // ErrDiscovery wraps failures encountered while resolving the token endpoint via OIDC discovery. + ErrDiscovery = errors.New("oauth2source discovery") + + // ErrConfig wraps failures encountered while loading oauth2source configuration from the environment. + ErrConfig = errors.New("oauth2source config") +) + +// source implements tokensource.Source by authenticating with a JWT assertion signed with +// a private key, following the private_key_jwt client authentication method defined in RFC 7523 +// section 2.2. The token endpoint and the JWT audience are resolved lazily via OIDC discovery +// against issuerURL. +// +// The oauth2.TokenSource interface does not accept a context on Token(), so the context used for +// outgoing token and discovery requests is captured once at construction time, mirroring the +// behaviour of golang.org/x/oauth2/clientcredentials.Config.TokenSource. +type source struct { + ctx context.Context //nolint:containedctx // Token() has no context parameter, see doc comment above. + clientID string + issuerURL string + privateKey jwk.Key + discoveryPath string + discoveryURL string + customScope string + httpClient *http.Client + + mu sync.Mutex + // tokenEndpoint caches the token endpoint resolved via OIDC discovery. When the caller supplies + // one explicitly at construction time, it is stored here directly, so that resolveTokenEndpoint's + // existing cache check short-circuits and OIDC discovery is never performed. + tokenEndpoint string +} + +var _ tokensource.Source = &source{} + +// config holds the environment-driven settings for oauth2source. +type config struct { + // DiscoveryPath is the well-known path suffix used to discover OIDC provider metadata, as + // defined by the OpenID Connect Discovery specification. It is configurable to accommodate + // issuers that serve their discovery document at a non-standard path. + DiscoveryPath string `env:"OIDC_DISCOVERY_PATH" envDefault:".well-known/openid-configuration"` +} + +// loadConfigFromEnv parses the oauth2source configuration from environment variables. +func loadConfigFromEnv() (*config, error) { + cfg := new(config) + if err := env.Parse(cfg); err != nil { + return nil, fmt.Errorf("%w: %w", ErrConfig, err) + } + return cfg, nil +} + +// NewSource returns a tokensource.Source that signs a JWT assertion with privateKey and +// exchanges it, via the private_key_jwt method defined in RFC 7523 section 2.2, for an access +// token authenticating as clientID. The returned source automatically reuses tokens until near +// expiry, see oauth2.ReuseTokenSource. Configuration is validated at construction time. +// +// The token endpoint and the JWT audience are normally resolved via OIDC discovery against +// issuerURL. Callers may override this in two ways, both optional (this package itself does not +// read environment variables for either value — they arrive as explicit parameters): +// - discoveryURL, when non-empty, is fetched verbatim as the OIDC discovery document instead of +// the URL computed from issuerURL and the configured discovery path. +// - tokenEndpoint, when non-empty, is used directly as the token endpoint and skips OIDC +// discovery entirely. It takes precedence over discoveryURL. +// +// issuerURL itself is optional: when non-empty, the fetched discovery document's issuer must +// match it or discovery fails, guarding against a misconfigured or unexpected discovery endpoint. +// When issuerURL is empty (for example, only discoveryURL was supplied), there is nothing to +// validate the document against, so this check is skipped. +func NewSource(ctx context.Context, clientID, issuerURL, discoveryURL, tokenEndpoint, customScope string, privateKey jwk.Key) (tokensource.Source, error) { + cfg, err := loadConfigFromEnv() + if err != nil { + return nil, err + } + + inner := &source{ + ctx: ctx, + clientID: clientID, + issuerURL: issuerURL, + privateKey: privateKey, + discoveryPath: cfg.DiscoveryPath, + discoveryURL: discoveryURL, + customScope: customScope, + tokenEndpoint: tokenEndpoint, + httpClient: &http.Client{ + Timeout: tokenRequestTimeout, + }, + } + + return oauth2.ReuseTokenSource(nil, inner), nil +} + +// Token implements oauth2.TokenSource by signing a JWT assertion with the private key and +// exchanging it with the token endpoint for an access token. The token endpoint is resolved via +// OIDC discovery against issuerURL on first use and cached for subsequent calls. +func (p *source) Token() (*oauth2.Token, error) { + now := time.Now() + + tokenEndpoint, err := p.resolveTokenEndpoint(p.ctx) + if err != nil { + return nil, err + } + + assertion, err := p.signedAssertion(now, tokenEndpoint) + if err != nil { + return nil, err + } + + form := url.Values{} + form.Set("grant_type", clientCredentialsGrantType) + form.Set("client_assertion_type", jwtBearerClientAssertionType) + form.Set("client_assertion", assertion) + form.Set("client_id", p.clientID) + form.Set("token_endpoint_auth_method", privateKeyJWTAuthMethod) + if p.customScope != "" { + form.Set("scope", p.customScope) + } + + req, err := http.NewRequestWithContext(p.ctx, http.MethodPost, tokenEndpoint, strings.NewReader(form.Encode())) + if err != nil { + return nil, fmt.Errorf("%w: failed to build token request: %w", ErrTokenExchange, err) + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + resp, err := p.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("%w: failed to exchange jwt assertion: %w", ErrTokenExchange, err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(io.LimitReader(resp.Body, maxTokenErrorBodyBytes)) + return nil, fmt.Errorf("%w: upstream token exchange failed: status %s: %s", ErrTokenExchange, resp.Status, body) + } + + var tokenResp struct { + AccessToken string `json:"access_token"` //nolint:tagliatelle // OAuth2 token response uses snake_case + ExpiresIn int64 `json:"expires_in"` //nolint:tagliatelle // OAuth2 token response uses snake_case + RefreshExpiresIn int64 `json:"refresh_expires_in"` //nolint:tagliatelle // OAuth2 token response uses snake_case + TokenType string `json:"token_type"` //nolint:tagliatelle // OAuth2 token response uses snake_case + NotBeforePolicy int64 `json:"not-before-policy"` //nolint:tagliatelle + Scope string `json:"scope"` + } + if err := json.NewDecoder(resp.Body).Decode(&tokenResp); err != nil { + return nil, fmt.Errorf("%w: failed to decode token response: %w", ErrTokenExchange, err) + } + + return &oauth2.Token{ + AccessToken: tokenResp.AccessToken, + TokenType: tokenResp.TokenType, + Expiry: now.Add(time.Duration(tokenResp.ExpiresIn) * time.Second), + }, nil +} + +// signedAssertion builds and signs the JWT assertion sent to tokenEndpoint, using the +// signature algorithm advertised by the key, or defaulting to RS256 when the key does not +// declare one. tokenEndpoint is used as the assertion's audience, as required by RFC 7523 +// section 3. +func (p *source) signedAssertion(now time.Time, tokenEndpoint string) (string, error) { + jti, err := uuid.NewRandom() + if err != nil { + return "", fmt.Errorf("%w: failed to generate jti: %w", ErrTokenExchange, err) + } + + signAlg := jwa.RS256() + if alg, ok := p.privateKey.Algorithm(); ok { + if sa, ok := jwa.LookupSignatureAlgorithm(alg.String()); ok { + signAlg = sa + } + } + + tok, err := jwt.NewBuilder(). + Issuer(p.clientID). + Subject(p.clientID). + Audience([]string{tokenEndpoint}). + JwtID(jti.String()). + IssuedAt(now). + Expiration(now.Add(jwtAssertionLifetime)). + Build() + if err != nil { + return "", fmt.Errorf("%w: failed to build token payload: %w", ErrTokenExchange, err) + } + + signed, err := jwt.Sign(tok, jwt.WithKey(signAlg, p.privateKey)) + if err != nil { + return "", fmt.Errorf("%w: failed to sign token: %w", ErrTokenExchange, err) + } + + return string(signed), nil +} + +// discoveryDocument is the minimal subset of OIDC provider metadata, as defined by the OpenID +// Connect Discovery specification, that this package relies on. +type discoveryDocument struct { + Issuer string `json:"issuer"` + TokenEndpoint string `json:"token_endpoint"` //nolint:tagliatelle // OIDC discovery document uses snake_case +} + +// resolveTokenEndpoint returns the token endpoint advertised by the issuer's OIDC discovery +// document, fetching and validating it on first use and caching the result for subsequent calls. +// A failed discovery is never cached, so a later call retries it from scratch. +func (p *source) resolveTokenEndpoint(ctx context.Context) (string, error) { + p.mu.Lock() + defer p.mu.Unlock() + + if p.tokenEndpoint != "" { + return p.tokenEndpoint, nil + } + + discoveryURL := p.discoveryURL + if discoveryURL == "" { + joined, err := url.JoinPath(p.issuerURL, p.discoveryPath) + if err != nil { + return "", fmt.Errorf("%w: failed to build discovery url: %w", ErrDiscovery, err) + } + discoveryURL = joined + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, discoveryURL, nil) + if err != nil { + return "", fmt.Errorf("%w: failed to build discovery request: %w", ErrDiscovery, err) + } + + resp, err := p.httpClient.Do(req) + if err != nil { + return "", fmt.Errorf("%w: failed to fetch discovery document: %w", ErrDiscovery, err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(io.LimitReader(resp.Body, maxTokenErrorBodyBytes)) + return "", fmt.Errorf("%w: upstream discovery failed: status %s: %s", ErrDiscovery, resp.Status, body) + } + + var doc discoveryDocument + if err := json.NewDecoder(resp.Body).Decode(&doc); err != nil { + return "", fmt.Errorf("%w: failed to decode discovery document: %w", ErrDiscovery, err) + } + + // The issuer check is only meaningful when the caller configured an expected issuer: when + // p.issuerURL is empty (e.g. only a discoveryURL or tokenEndpoint override was supplied), there + // is nothing to validate the discovery document against, so it is skipped rather than rejecting + // an otherwise valid document. + if p.issuerURL != "" && normalizeIssuer(doc.Issuer) != normalizeIssuer(p.issuerURL) { + return "", fmt.Errorf("%w: issuer mismatch: expected %q, got %q", ErrDiscovery, p.issuerURL, doc.Issuer) + } + + if doc.TokenEndpoint == "" { + return "", fmt.Errorf("%w: discovery document is missing token_endpoint", ErrDiscovery) + } + + p.tokenEndpoint = doc.TokenEndpoint + + return p.tokenEndpoint, nil +} + +// normalizeIssuer canonicalizes an issuer identifier for the OIDC issuer check, tolerating the +// differences that commonly arise between a configured issuer URL and the value advertised in a +// discovery document without weakening the check: a trailing slash and differing case in the +// scheme or host, none of which are significant per RFC 3986. The path is left case-sensitive, +// as OIDC issuer paths (such as Keycloak realm names) are. If the value does not parse as a URL +// it is compared verbatim, minus any trailing slash. +func normalizeIssuer(raw string) string { + parsed, err := url.Parse(raw) + if err != nil { + return strings.TrimRight(raw, "/") + } + + parsed.Scheme = strings.ToLower(parsed.Scheme) + parsed.Host = strings.ToLower(parsed.Host) + parsed.Path = strings.TrimRight(parsed.Path, "/") + + return parsed.String() +} diff --git a/internal/tokensource/oauth2source/oauth2source_test.go b/internal/tokensource/oauth2source/oauth2source_test.go new file mode 100644 index 0000000..549f444 --- /dev/null +++ b/internal/tokensource/oauth2source/oauth2source_test.go @@ -0,0 +1,690 @@ +// Copyright Mia srl +// SPDX-License-Identifier: AGPL-3.0-only or Commercial + +package oauth2source + +import ( + "crypto/rand" + "crypto/rsa" + "encoding/json" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + + "github.com/lestrrat-go/jwx/v3/jwa" + "github.com/lestrrat-go/jwx/v3/jwk" + "github.com/lestrrat-go/jwx/v3/jwt" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/oauth2" +) + +const ( + rsaKeyBits = 4096 + // clientID is the fictional client identifier used across the tests in this package. + clientID = "jwt-bearer-client" +) + +// generateTestKey creates a fresh RSA key pair and wraps it into a jwk.Key, to be used as +// fictional test material. It is never used outside of this test file. +func generateTestKey(t *testing.T) (*rsa.PrivateKey, jwk.Key) { + t.Helper() + + key, err := rsa.GenerateKey(rand.Reader, rsaKeyBits) + require.NoError(t, err) + + jwkKey, err := jwk.Import(key) + require.NoError(t, err) + + return key, jwkKey +} + +// newHappyPathServer starts a test server that serves a valid OIDC discovery document and a +// valid token response with the given expiresIn (in seconds), together with two protected +// resources that require a bearer token. It returns the server together with atomic counters +// tracking how many times the discovery and token endpoints were hit. +func newHappyPathServer(t *testing.T, key *rsa.PrivateKey, expiresIn int) (*httptest.Server, *atomic.Int32, *atomic.Int32) { + t.Helper() + + var discoveryHits, tokenHits atomic.Int32 + var testServer *httptest.Server + + testServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/.well-known/openid-configuration": + discoveryHits.Add(1) + w.Header().Set("Content-Type", "application/json") + err := json.NewEncoder(w).Encode(map[string]string{ + "issuer": testServer.URL, + "token_endpoint": testServer.URL + "/oauth/token", + }) + require.NoError(t, err) + case "/oauth/token": + tokenHits.Add(1) + require.NoError(t, r.ParseForm()) + assert.Equal(t, "client_credentials", r.FormValue("grant_type")) + assert.Equal(t, "urn:ietf:params:oauth:client-assertion-type:jwt-bearer", r.FormValue("client_assertion_type")) + assert.Equal(t, clientID, r.FormValue("client_id")) + assert.Equal(t, "private_key_jwt", r.FormValue("token_endpoint_auth_method")) + + assertion := r.FormValue("client_assertion") + require.NotEmpty(t, assertion) + + parsed, err := jwt.Parse([]byte(assertion), jwt.WithKey(jwa.RS256(), key.Public())) + require.NoError(t, err) + + issuer, ok := parsed.Issuer() + require.True(t, ok) + assert.Equal(t, clientID, issuer) + + subject, ok := parsed.Subject() + require.True(t, ok) + assert.Equal(t, clientID, subject) + + audience, ok := parsed.Audience() + require.True(t, ok) + assert.Equal(t, []string{testServer.URL + "/oauth/token"}, audience) + + jti, ok := parsed.JwtID() + require.True(t, ok) + assert.NotEmpty(t, jti) + + w.Header().Set("Content-Type", "application/json") + err = json.NewEncoder(w).Encode(map[string]any{ + "access_token": "generated-jwt-bearer-token", + "token_type": "Bearer", + "expires_in": expiresIn, + }) + require.NoError(t, err) + case "/protected-a", "/protected-b": + assert.Equal(t, "Bearer generated-jwt-bearer-token", r.Header.Get("Authorization")) + w.WriteHeader(http.StatusNoContent) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(testServer.Close) + + return testServer, &discoveryHits, &tokenHits +} + +// newDiscoveryOnlyServer starts a test server that serves a valid OIDC discovery document +// pointing at "/oauth/token", delegating requests to that path to tokenHandler. It is used by +// tests that only need to exercise the token-exchange failure path, discovery having already +// succeeded. +func newDiscoveryOnlyServer(t *testing.T, tokenHandler http.HandlerFunc) *httptest.Server { + t.Helper() + + var testServer *httptest.Server + + testServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/.well-known/openid-configuration": + w.Header().Set("Content-Type", "application/json") + err := json.NewEncoder(w).Encode(map[string]string{ + "issuer": testServer.URL, + "token_endpoint": testServer.URL + "/oauth/token", + }) + require.NoError(t, err) + case "/oauth/token": + tokenHandler(w, r) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(testServer.Close) + + return testServer +} + +func TestPrivateKeyFlow(t *testing.T) { + t.Parallel() + + key, jwkKey := generateTestKey(t) + + testServer, discoveryHits, tokenHits := newHappyPathServer(t, key, 3600) + + source, err := NewSource(t.Context(), clientID, testServer.URL, "", "", "", jwkKey) + require.NoError(t, err) + + client := &http.Client{ + Transport: &oauth2.Transport{ + Source: source, + }, + } + + resp, err := client.Get(testServer.URL + "/protected-a") + require.NoError(t, err) + defer resp.Body.Close() + assert.Equal(t, http.StatusNoContent, resp.StatusCode) + + assert.Equal(t, int32(1), discoveryHits.Load()) + assert.Equal(t, int32(1), tokenHits.Load()) +} + +func TestNewSourceCustomDiscoveryPath(t *testing.T) { + // Not parallel: mutates the process environment via t.Setenv. + key, jwkKey := generateTestKey(t) + + const customDiscoveryPath = "custom/discovery/document" + t.Setenv("OIDC_DISCOVERY_PATH", customDiscoveryPath) + + var testServer *httptest.Server + testServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/" + customDiscoveryPath: + w.Header().Set("Content-Type", "application/json") + err := json.NewEncoder(w).Encode(map[string]string{ + "issuer": testServer.URL, + "token_endpoint": testServer.URL + "/oauth/token", + }) + require.NoError(t, err) + case "/oauth/token": + require.NoError(t, r.ParseForm()) + parsed, err := jwt.Parse([]byte(r.FormValue("client_assertion")), jwt.WithKey(jwa.RS256(), key.Public())) + require.NoError(t, err) + audience, ok := parsed.Audience() + require.True(t, ok) + assert.Equal(t, []string{testServer.URL + "/oauth/token"}, audience) + + w.Header().Set("Content-Type", "application/json") + err = json.NewEncoder(w).Encode(map[string]any{ + "access_token": "generated-jwt-bearer-token", + "token_type": "Bearer", + "expires_in": 3600, + }) + require.NoError(t, err) + case "/protected": + assert.Equal(t, "Bearer generated-jwt-bearer-token", r.Header.Get("Authorization")) + w.WriteHeader(http.StatusNoContent) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(testServer.Close) + + source, err := NewSource(t.Context(), clientID, testServer.URL, "", "", "", jwkKey) + require.NoError(t, err) + + client := &http.Client{ + Transport: &oauth2.Transport{ + Source: source, + }, + } + + resp, err := client.Get(testServer.URL + "/protected") + require.NoError(t, err) + defer resp.Body.Close() + assert.Equal(t, http.StatusNoContent, resp.StatusCode) +} + +func TestNewSourceCustomDiscoveryURL(t *testing.T) { + t.Parallel() + + key, jwkKey := generateTestKey(t) + + // customDiscoveryURL points at a path that is neither the default + // ".well-known/openid-configuration" nor a value derivable via OIDC_DISCOVERY_PATH, proving + // that a fully custom discoveryURL is honoured verbatim. + const customDiscoveryPath = "/custom/metadata/endpoint" + + var defaultDiscoveryHits atomic.Int32 + var testServer *httptest.Server + testServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/.well-known/openid-configuration": + defaultDiscoveryHits.Add(1) + http.NotFound(w, r) + case customDiscoveryPath: + w.Header().Set("Content-Type", "application/json") + err := json.NewEncoder(w).Encode(map[string]string{ + "issuer": testServer.URL, + "token_endpoint": testServer.URL + "/oauth/token", + }) + require.NoError(t, err) + case "/oauth/token": + require.NoError(t, r.ParseForm()) + parsed, err := jwt.Parse([]byte(r.FormValue("client_assertion")), jwt.WithKey(jwa.RS256(), key.Public())) + require.NoError(t, err) + audience, ok := parsed.Audience() + require.True(t, ok) + assert.Equal(t, []string{testServer.URL + "/oauth/token"}, audience) + + w.Header().Set("Content-Type", "application/json") + err = json.NewEncoder(w).Encode(map[string]any{ + "access_token": "generated-jwt-bearer-token", + "token_type": "Bearer", + "expires_in": 3600, + }) + require.NoError(t, err) + case "/protected": + assert.Equal(t, "Bearer generated-jwt-bearer-token", r.Header.Get("Authorization")) + w.WriteHeader(http.StatusNoContent) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(testServer.Close) + + source, err := NewSource(t.Context(), clientID, testServer.URL, testServer.URL+customDiscoveryPath, "", "", jwkKey) + require.NoError(t, err) + + client := &http.Client{ + Transport: &oauth2.Transport{ + Source: source, + }, + } + + resp, err := client.Get(testServer.URL + "/protected") + require.NoError(t, err) + defer resp.Body.Close() + assert.Equal(t, http.StatusNoContent, resp.StatusCode) + + assert.Equal(t, int32(0), defaultDiscoveryHits.Load()) +} + +func TestNewSourceExplicitTokenEndpoint(t *testing.T) { + t.Parallel() + + key, jwkKey := generateTestKey(t) + + var discoveryHits atomic.Int32 + var tokenHits atomic.Int32 + var testServer *httptest.Server + testServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/.well-known/openid-configuration": + discoveryHits.Add(1) + http.NotFound(w, r) + case "/oauth/token": + tokenHits.Add(1) + require.NoError(t, r.ParseForm()) + parsed, err := jwt.Parse([]byte(r.FormValue("client_assertion")), jwt.WithKey(jwa.RS256(), key.Public())) + require.NoError(t, err) + audience, ok := parsed.Audience() + require.True(t, ok) + assert.Equal(t, []string{testServer.URL + "/oauth/token"}, audience) + + w.Header().Set("Content-Type", "application/json") + err = json.NewEncoder(w).Encode(map[string]any{ + "access_token": "generated-jwt-bearer-token", + "token_type": "Bearer", + "expires_in": 3600, + }) + require.NoError(t, err) + case "/protected": + assert.Equal(t, "Bearer generated-jwt-bearer-token", r.Header.Get("Authorization")) + w.WriteHeader(http.StatusNoContent) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(testServer.Close) + + // issuerURL is deliberately left pointing at the test server so that, if discovery were ever + // attempted, it would hit the (404-returning) default discovery path above rather than the + // discoveryURL used by other tests, making an accidental discovery call detectable. + source, err := NewSource(t.Context(), clientID, testServer.URL, "", testServer.URL+"/oauth/token", "", jwkKey) + require.NoError(t, err) + + client := &http.Client{ + Transport: &oauth2.Transport{ + Source: source, + }, + } + + resp, err := client.Get(testServer.URL + "/protected") + require.NoError(t, err) + defer resp.Body.Close() + assert.Equal(t, http.StatusNoContent, resp.StatusCode) + + assert.Equal(t, int32(0), discoveryHits.Load()) + assert.Equal(t, int32(1), tokenHits.Load()) +} + +func TestDiscoveryCaching(t *testing.T) { + t.Parallel() + + key, jwkKey := generateTestKey(t) + + // expiresIn = 0 forces the wrapping oauth2.ReuseTokenSource to request a fresh token on the + // second call, so that a discovery hit count of 1 proves resolveTokenEndpoint's own cache + // rather than the outer token cache. + testServer, discoveryHits, tokenHits := newHappyPathServer(t, key, 0) + + source, err := NewSource(t.Context(), clientID, testServer.URL, "", "", "", jwkKey) + require.NoError(t, err) + + client := &http.Client{ + Transport: &oauth2.Transport{ + Source: source, + }, + } + + resp1, err := client.Get(testServer.URL + "/protected-a") + require.NoError(t, err) + defer resp1.Body.Close() + assert.Equal(t, http.StatusNoContent, resp1.StatusCode) + + resp2, err := client.Get(testServer.URL + "/protected-b") + require.NoError(t, err) + defer resp2.Body.Close() + assert.Equal(t, http.StatusNoContent, resp2.StatusCode) + + assert.Equal(t, int32(1), discoveryHits.Load()) + assert.Equal(t, int32(2), tokenHits.Load()) +} + +func TestTokenReuse(t *testing.T) { + t.Parallel() + + key, jwkKey := generateTestKey(t) + + testServer, discoveryHits, tokenHits := newHappyPathServer(t, key, 3600) + + source, err := NewSource(t.Context(), clientID, testServer.URL, "", "", "", jwkKey) + require.NoError(t, err) + + client := &http.Client{ + Transport: &oauth2.Transport{ + Source: source, + }, + } + + resp1, err := client.Get(testServer.URL + "/protected-a") + require.NoError(t, err) + defer resp1.Body.Close() + assert.Equal(t, http.StatusNoContent, resp1.StatusCode) + + resp2, err := client.Get(testServer.URL + "/protected-b") + require.NoError(t, err) + defer resp2.Body.Close() + assert.Equal(t, http.StatusNoContent, resp2.StatusCode) + + assert.Equal(t, int32(1), discoveryHits.Load()) + assert.Equal(t, int32(1), tokenHits.Load()) +} + +func TestPrivateKeyFlowTokenEndpointError(t *testing.T) { + t.Parallel() + + _, jwkKey := generateTestKey(t) + + testServer := newDiscoveryOnlyServer(t, func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "boom", http.StatusInternalServerError) + }) + + source, err := NewSource(t.Context(), "client-id", testServer.URL, "", "", "", jwkKey) + require.NoError(t, err) + + client := &http.Client{ + Transport: &oauth2.Transport{ + Source: source, + }, + } + + resp, err := client.Get(testServer.URL + "/") + if resp != nil { + defer resp.Body.Close() + } + require.Error(t, err) + assert.ErrorIs(t, err, ErrTokenExchange) + assert.ErrorContains(t, err, "upstream token exchange failed") + assert.ErrorContains(t, err, "boom") +} + +func TestPrivateKeyFlowMalformedTokenResponse(t *testing.T) { + t.Parallel() + + _, jwkKey := generateTestKey(t) + + testServer := newDiscoveryOnlyServer(t, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, err := w.Write([]byte("not-json")) + require.NoError(t, err) + }) + + source, err := NewSource(t.Context(), "client-id", testServer.URL, "", "", "", jwkKey) + require.NoError(t, err) + + client := &http.Client{ + Transport: &oauth2.Transport{ + Source: source, + }, + } + + resp, err := client.Get(testServer.URL + "/") + if resp != nil { + defer resp.Body.Close() + } + require.Error(t, err) + assert.ErrorIs(t, err, ErrTokenExchange) + assert.ErrorContains(t, err, "failed to decode token response") +} + +func TestResolveTokenEndpointDiscoveryFailures(t *testing.T) { + t.Parallel() + + testCases := map[string]struct { + handler func(t *testing.T) http.HandlerFunc + wantErrContains string + }{ + "non-200 status": { + handler: func(*testing.T) http.HandlerFunc { + return func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "discovery unavailable", http.StatusInternalServerError) + } + }, + wantErrContains: "upstream discovery failed", + }, + "malformed json": { + handler: func(t *testing.T) http.HandlerFunc { + t.Helper() + return func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, err := w.Write([]byte("not-json")) + require.NoError(t, err) + } + }, + wantErrContains: "failed to decode discovery document", + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + t.Parallel() + + _, jwkKey := generateTestKey(t) + handler := tc.handler(t) + + testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/.well-known/openid-configuration" { + http.NotFound(w, r) + return + } + handler(w, r) + })) + t.Cleanup(testServer.Close) + + source, err := NewSource(t.Context(), "client-id", testServer.URL, "", "", "", jwkKey) + require.NoError(t, err) + + client := &http.Client{ + Transport: &oauth2.Transport{ + Source: source, + }, + } + + resp, err := client.Get(testServer.URL + "/") + if resp != nil { + defer resp.Body.Close() + } + require.Error(t, err) + assert.ErrorIs(t, err, ErrDiscovery) + assert.ErrorContains(t, err, tc.wantErrContains) + }) + } +} + +func TestResolveTokenEndpointMissingTokenEndpoint(t *testing.T) { + t.Parallel() + + _, jwkKey := generateTestKey(t) + + testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/.well-known/openid-configuration" { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + err := json.NewEncoder(w).Encode(map[string]string{ + "issuer": "http://" + r.Host, + }) + require.NoError(t, err) + })) + t.Cleanup(testServer.Close) + + source, err := NewSource(t.Context(), "client-id", testServer.URL, "", "", "", jwkKey) + require.NoError(t, err) + + client := &http.Client{ + Transport: &oauth2.Transport{ + Source: source, + }, + } + + resp, err := client.Get(testServer.URL + "/") + if resp != nil { + defer resp.Body.Close() + } + require.Error(t, err) + assert.ErrorIs(t, err, ErrDiscovery) + assert.ErrorContains(t, err, "missing token_endpoint") +} + +func TestResolveTokenEndpointIssuerTrailingSlash(t *testing.T) { + t.Parallel() + + key, jwkKey := generateTestKey(t) + + // The discovery document reports the issuer without a trailing slash (as Keycloak does), + // while the caller configures the issuer URL with one. The two denote the same issuer and + // must be accepted. + testServer, _, tokenHits := newHappyPathServer(t, key, 3600) + + source, err := NewSource(t.Context(), clientID, testServer.URL+"/", "", "", "", jwkKey) + require.NoError(t, err) + + client := &http.Client{ + Transport: &oauth2.Transport{ + Source: source, + }, + } + + resp, err := client.Get(testServer.URL + "/protected-a") + require.NoError(t, err) + defer resp.Body.Close() + assert.Equal(t, http.StatusNoContent, resp.StatusCode) + assert.Equal(t, int32(1), tokenHits.Load()) +} + +func TestResolveTokenEndpointNoIssuerConfiguredSkipsMismatchCheck(t *testing.T) { + t.Parallel() + + key, jwkKey := generateTestKey(t) + + // The discovery document reports an issuer that has no relationship whatsoever to the test + // server, proving that the mismatch check is skipped entirely when the caller configures no + // issuerURL, rather than being loosened to tolerate a near match. + const customDiscoveryPath = "/custom/metadata/endpoint" + + var testServer *httptest.Server + testServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case customDiscoveryPath: + w.Header().Set("Content-Type", "application/json") + err := json.NewEncoder(w).Encode(map[string]string{ + "issuer": "https://unrelated-issuer.example.com", + "token_endpoint": testServer.URL + "/oauth/token", + }) + require.NoError(t, err) + case "/oauth/token": + require.NoError(t, r.ParseForm()) + parsed, err := jwt.Parse([]byte(r.FormValue("client_assertion")), jwt.WithKey(jwa.RS256(), key.Public())) + require.NoError(t, err) + audience, ok := parsed.Audience() + require.True(t, ok) + assert.Equal(t, []string{testServer.URL + "/oauth/token"}, audience) + + w.Header().Set("Content-Type", "application/json") + err = json.NewEncoder(w).Encode(map[string]any{ + "access_token": "generated-jwt-bearer-token", + "token_type": "Bearer", + "expires_in": 3600, + }) + require.NoError(t, err) + case "/protected": + assert.Equal(t, "Bearer generated-jwt-bearer-token", r.Header.Get("Authorization")) + w.WriteHeader(http.StatusNoContent) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(testServer.Close) + + // issuerURL is deliberately left empty, as it would be when only a discovery metadata URL is + // configured and no issuer/auth endpoint is set. + source, err := NewSource(t.Context(), clientID, "", testServer.URL+customDiscoveryPath, "", "", jwkKey) + require.NoError(t, err) + + client := &http.Client{ + Transport: &oauth2.Transport{ + Source: source, + }, + } + + resp, err := client.Get(testServer.URL + "/protected") + require.NoError(t, err) + defer resp.Body.Close() + assert.Equal(t, http.StatusNoContent, resp.StatusCode) +} + +func TestResolveTokenEndpointIssuerMismatch(t *testing.T) { + t.Parallel() + + _, jwkKey := generateTestKey(t) + + testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/.well-known/openid-configuration" { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + err := json.NewEncoder(w).Encode(map[string]string{ + "issuer": "https://impostor.example.com", + "token_endpoint": "http://" + r.Host + "/oauth/token", + }) + require.NoError(t, err) + })) + t.Cleanup(testServer.Close) + + source, err := NewSource(t.Context(), "client-id", testServer.URL, "", "", "", jwkKey) + require.NoError(t, err) + + client := &http.Client{ + Transport: &oauth2.Transport{ + Source: source, + }, + } + + resp, err := client.Get(testServer.URL + "/") + if resp != nil { + defer resp.Body.Close() + } + require.Error(t, err) + assert.ErrorIs(t, err, ErrDiscovery) + assert.ErrorContains(t, err, "issuer mismatch") +} diff --git a/internal/tokensource/source.go b/internal/tokensource/source.go new file mode 100644 index 0000000..6c40c91 --- /dev/null +++ b/internal/tokensource/source.go @@ -0,0 +1,13 @@ +// Copyright Mia srl +// SPDX-License-Identifier: AGPL-3.0-only or Commercial + +package tokensource + +import "golang.org/x/oauth2" + +// Source is implemented by every client-authentication strategy able to produce an +// oauth2.TokenSource for authenticating outgoing requests. Each strategy lives in its own +// sub-package under internal/tokensource and exposes a constructor returning a Source. +type Source interface { + oauth2.TokenSource +} From 6c19353243d29ae0ec7614b3b1fec97b9384c477 Mon Sep 17 00:00:00 2001 From: APagani Date: Mon, 20 Jul 2026 15:12:03 +0200 Subject: [PATCH 08/20] feat: remove bearer token support for catalog destination (#76) --- docs/how-to/020_destinations.md | 3 -- internal/destination/catalog/catalog.go | 19 +++---- internal/destination/catalog/catalog_test.go | 54 +++++-------------- internal/destination/catalog/transport.go | 11 ++-- .../destination/catalog/transport_test.go | 4 +- 5 files changed, 25 insertions(+), 66 deletions(-) diff --git a/docs/how-to/020_destinations.md b/docs/how-to/020_destinations.md index 5bc57d1..cc97e32 100644 --- a/docs/how-to/020_destinations.md +++ b/docs/how-to/020_destinations.md @@ -10,9 +10,6 @@ To enable this destination no flags are needed but only a series of environment - `MIA_CATALOG_ENDPOINT`: the destination endpoint for the data, available during the integration setup -- `MIA_CATALOG_TOKEN`: a static bearer token used to authenticate requests to the Mia-Platform - Catalog. It cannot be combined with `MIA_CATALOG_CLIENT_ID`, `MIA_CATALOG_CLIENT_SECRET` or - `MIA_CATALOG_PRIVATE_KEY_PATH` - `MIA_CATALOG_CLIENT_ID`: the client id for validating the requests to the Mia-Platform Catalog, available during the integration setup - `MIA_CATALOG_CLIENT_SECRET`: the client secret for validating the requests to the Mia-Platform diff --git a/internal/destination/catalog/catalog.go b/internal/destination/catalog/catalog.go index dc9bd30..ae0c7c9 100644 --- a/internal/destination/catalog/catalog.go +++ b/internal/destination/catalog/catalog.go @@ -21,7 +21,6 @@ import ( ) var ( - errMultipleAuthMethods = errors.New("MIA_CATALOG_TOKEN cannot be used with MIA_CATALOG_CLIENT_ID, MIA_CATALOG_CLIENT_SECRET or MIA_CATALOG_PRIVATE_KEY") errMissingClientID = errors.New("MIA_CATALOG_CLIENT_ID is required when MIA_CATALOG_CLIENT_SECRET is set") errMissingClientSecret = errors.New("MIA_CATALOG_CLIENT_SECRET is required when MIA_CATALOG_CLIENT_ID is set") errMissingClientIDForPrivKey = errors.New("MIA_CATALOG_CLIENT_ID is required when MIA_CATALOG_PRIVATE_KEY is set") @@ -57,9 +56,6 @@ func (e *CatalogError) Is(target error) bool { type catalogDestination struct { // CatalogEndpoint is the base URL of the Mia-Platform Catalog API that receives sent data. CatalogEndpoint string `env:"MIA_CATALOG_ENDPOINT,required"` - // Token, when set, is used as a static bearer token for authentication. It cannot be combined - // with MIA_CATALOG_CLIENT_ID, MIA_CATALOG_CLIENT_SECRET or MIA_CATALOG_PRIVATE_KEY_PATH. - Token string `env:"MIA_CATALOG_TOKEN"` // ClientID is the OAuth2 client identifier used for either the client-credentials flow (with // MIA_CATALOG_CLIENT_SECRET) or private-key JWT authentication (with MIA_CATALOG_PRIVATE_KEY_PATH). ClientID string `env:"MIA_CATALOG_CLIENT_ID"` @@ -152,17 +148,14 @@ func NewDestination() (destination.Sender, error) { return destination, nil } -// validateAuthConfig ensures that at most one authentication method is configured and that each -// configured method has all of its required environment variables set. +// validateAuthConfig ensures that the configured authentication method has all of its required +// environment variables set. func (d *catalogDestination) validateAuthConfig() error { - hasToken := len(d.Token) > 0 hasClientID := len(d.ClientID) > 0 hasClientSecret := len(d.ClientSecret) > 0 hasPrivateKey := len(d.PrivateKeyPath) > 0 switch { - case hasToken && (hasClientID || hasClientSecret || hasPrivateKey): - return errMultipleAuthMethods case hasPrivateKey: return d.validatePrivateKeyAuthConfig() case hasClientID && !hasClientSecret: @@ -235,8 +228,10 @@ func (d *catalogDestination) handleRequest(ctx context.Context, method string, d defer resp.Body.Close() switch resp.StatusCode { - case http.StatusForbidden, http.StatusUnauthorized: - return handleError(errors.New("invalid token or insufficient permissions")) + case http.StatusUnauthorized: + return handleError(errors.New("invalid credentials")) + case http.StatusForbidden: + return handleError(errors.New("insufficient permissions")) case http.StatusNotFound: return handleError(errors.New("integration registration not found")) case http.StatusNoContent: @@ -281,7 +276,7 @@ func (d *catalogDestination) getClient(ctx context.Context) (*http.Client, error return client, nil } - transport, err := NewTransport(ctx, d.Token, d.AuthEndpoint, d.ClientID, d.ClientSecret, d.Issuer, d.IssuerMetadata, d.TokenEndpoint, d.CustomScope, d.keys) + transport, err := NewTransport(ctx, d.AuthEndpoint, d.ClientID, d.ClientSecret, d.Issuer, d.IssuerMetadata, d.TokenEndpoint, d.CustomScope, d.keys) if err != nil { return nil, err } diff --git a/internal/destination/catalog/catalog_test.go b/internal/destination/catalog/catalog_test.go index f0d17e1..8f1449d 100644 --- a/internal/destination/catalog/catalog_test.go +++ b/internal/destination/catalog/catalog_test.go @@ -72,22 +72,6 @@ func TestInitialization(t *testing.T) { require.True(t, ok) assert.Equal(t, "http://localhost:8080/custom-catalog", catalogDestination.CatalogEndpoint) - assert.Empty(t, catalogDestination.Token) - assert.Empty(t, catalogDestination.ClientID) - assert.Empty(t, catalogDestination.ClientSecret) - assert.Equal(t, "http://localhost:8080/oauth/token", catalogDestination.AuthEndpoint) - }) - - t.Run("with all envs", func(t *testing.T) { - t.Setenv("MIA_CATALOG_TOKEN", "test-token2") - t.Setenv("MIA_CATALOG_ENDPOINT", "http://localhost:8080/custom-catalog") - dest, err := NewDestination() - require.NoError(t, err) - catalogDestination, ok := dest.(*catalogDestination) - require.True(t, ok) - - assert.Equal(t, "http://localhost:8080/custom-catalog", catalogDestination.CatalogEndpoint) - assert.Equal(t, "test-token2", catalogDestination.Token) assert.Empty(t, catalogDestination.ClientID) assert.Empty(t, catalogDestination.ClientSecret) assert.Equal(t, "http://localhost:8080/oauth/token", catalogDestination.AuthEndpoint) @@ -103,7 +87,6 @@ func TestInitialization(t *testing.T) { require.True(t, ok) assert.Equal(t, "http://localhost:8080/custom-catalog", catalogDestination.CatalogEndpoint) - assert.Empty(t, catalogDestination.Token) assert.Equal(t, "client-id", catalogDestination.ClientID) assert.Equal(t, "client-secret", catalogDestination.ClientSecret) assert.Equal(t, "http://localhost:8080/oauth/token", catalogDestination.AuthEndpoint) @@ -120,7 +103,6 @@ func TestInitialization(t *testing.T) { require.True(t, ok) assert.Equal(t, "http://localhost:8080/custom-catalog", catalogDestination.CatalogEndpoint) - assert.Empty(t, catalogDestination.Token) assert.Equal(t, "client-id", catalogDestination.ClientID) assert.Equal(t, "client-secret", catalogDestination.ClientSecret) assert.Equal(t, "http://localhost:8081/custom/auth", catalogDestination.AuthEndpoint) @@ -133,15 +115,6 @@ func TestInitialization(t *testing.T) { assert.Nil(t, dest) }) - t.Run("with both env for fixed token and client credentials", func(t *testing.T) { - t.Setenv("MIA_CATALOG_ENDPOINT", "http://localhost:8080/custom-catalog") - t.Setenv("MIA_CATALOG_TOKEN", "test-token") - t.Setenv("MIA_CATALOG_CLIENT_ID", "client-id") - dest, err := NewDestination() - assert.ErrorIs(t, err, errMultipleAuthMethods) - assert.Nil(t, dest) - }) - t.Run("missing secret with client id", func(t *testing.T) { t.Setenv("MIA_CATALOG_ENDPOINT", "http://localhost:8080/custom-catalog") t.Setenv("MIA_CATALOG_CLIENT_ID", "client-id") @@ -306,15 +279,6 @@ func TestInitialization(t *testing.T) { assert.ErrorIs(t, err, errPrivateKeyWithClientSecret) assert.Nil(t, dest) }) - - t.Run("private key with fixed token", func(t *testing.T) { - t.Setenv("MIA_CATALOG_ENDPOINT", "http://localhost:8080/custom-catalog") - t.Setenv("MIA_CATALOG_TOKEN", "test-token") - t.Setenv("MIA_CATALOG_PRIVATE_KEY_PATH", "fictional-private-key-path") - dest, err := NewDestination() - assert.ErrorIs(t, err, errMultipleAuthMethods) - assert.Nil(t, dest) - }) } func TestSendData(t *testing.T) { @@ -358,7 +322,14 @@ func TestSendData(t *testing.T) { data: &destination.Data{ APIVersion: "v1", }, - expectedError: &CatalogError{err: errors.New("invalid token or insufficient permissions")}, + expectedError: &CatalogError{err: errors.New("invalid credentials")}, + }, + "forbidden send": { + endpoint: "/forbidden-endpoint", + data: &destination.Data{ + APIVersion: "v1", + }, + expectedError: &CatalogError{err: errors.New("insufficient permissions")}, }, "not found send": { endpoint: "/not-found-endpoint", @@ -391,8 +362,10 @@ func TestSendData(t *testing.T) { // check headers assert.Equal(t, "application/json", r.Header.Get("Content-Type")) - assert.Equal(t, "Bearer test-token", r.Header.Get("Authorization")) assert.Equal(t, info.AppName+"/"+info.Version, r.Header.Get("User-Agent")) + // with no authentication configured the destination must not send an + // Authorization header (unauthenticated fallback). + assert.Empty(t, r.Header.Get("Authorization")) switch r.RequestURI { case "/valid-endpoint": @@ -409,6 +382,8 @@ func TestSendData(t *testing.T) { http.Error(w, "unexpected error", http.StatusBadGateway) case "/unauthorized-endpoint": http.Error(w, "unauthorized", http.StatusUnauthorized) + case "/forbidden-endpoint": + http.Error(w, "forbidden", http.StatusForbidden) default: errCode := http.StatusInternalServerError w.WriteHeader(errCode) @@ -430,7 +405,6 @@ func TestSendData(t *testing.T) { dest := &catalogDestination{ CatalogEndpoint: testServer.URL + tc.endpoint, - Token: "test-token", } err := dest.SendData(ctx, tc.data) @@ -490,7 +464,6 @@ func TestDeleteData(t *testing.T) { // check headers assert.Equal(t, "application/json", r.Header.Get("Content-Type")) - assert.Equal(t, "Bearer test-token", r.Header.Get("Authorization")) assert.Equal(t, info.AppName+"/"+info.Version, r.Header.Get("User-Agent")) switch r.RequestURI { @@ -523,7 +496,6 @@ func TestDeleteData(t *testing.T) { dest := &catalogDestination{ CatalogEndpoint: testServer.URL + tc.endpoint, - Token: "test-token", } err := dest.DeleteData(ctx, tc.data) diff --git a/internal/destination/catalog/transport.go b/internal/destination/catalog/transport.go index 665ffb1..aa9aa72 100644 --- a/internal/destination/catalog/transport.go +++ b/internal/destination/catalog/transport.go @@ -14,18 +14,13 @@ import ( "github.com/mia-platform/ibdm/internal/tokensource/oauth2source" ) -// NewTransport creates an HTTP transport configured with either a static token, private-key JWT -// client authentication, or client-credentials flow. authEndpoint is the token URL used by the +// NewTransport creates an HTTP transport configured with either private-key JWT client +// authentication or the client-credentials flow. authEndpoint is the token URL used by the // client-credentials flow. issuer, issuerMetadata and tokenEndpoint are only used by the // private-key JWT branch: see oauth2source.NewSource for their meaning. -func NewTransport(ctx context.Context, token, authEndpoint, clientID, clientSecret, issuer, issuerMetadata, tokenEndpoint, customScope string, keys *jwk.Keys) (http.RoundTripper, error) { +func NewTransport(ctx context.Context, authEndpoint, clientID, clientSecret, issuer, issuerMetadata, tokenEndpoint, customScope string, keys *jwk.Keys) (http.RoundTripper, error) { var source oauth2.TokenSource switch { - case len(token) > 0: - source = oauth2.StaticTokenSource(&oauth2.Token{ - AccessToken: token, - TokenType: "Bearer", - }) case len(clientID) > 0 && len(clientSecret) > 0: config := clientcredentials.Config{ ClientID: clientID, diff --git a/internal/destination/catalog/transport_test.go b/internal/destination/catalog/transport_test.go index 617dac2..8c63b2b 100644 --- a/internal/destination/catalog/transport_test.go +++ b/internal/destination/catalog/transport_test.go @@ -16,7 +16,7 @@ import ( func TestNewTransportWithoutCredentials(t *testing.T) { t.Parallel() - transport, err := NewTransport(t.Context(), "", "", "", "", "", "", "", "", nil) + transport, err := NewTransport(t.Context(), "", "", "", "", "", "", "", nil) assert.NoError(t, err) assert.Same(t, http.DefaultTransport, transport) } @@ -54,7 +54,7 @@ func TestNewTransportPrivateKeyJWTWiring(t *testing.T) { })) defer testServer.Close() - transport, err := NewTransport(t.Context(), "", "", clientID, "", "", "", testServer.URL+"/oauth/token", "organization:*", newTestPrivateKeyFor(t, key)) + transport, err := NewTransport(t.Context(), "", clientID, "", "", "", testServer.URL+"/oauth/token", "organization:*", newTestPrivateKeyFor(t, key)) require.NoError(t, err) client := &http.Client{ From 50acf55b2f3d52ca5fe11fe87d69650fb21fa2c3 Mon Sep 17 00:00:00 2001 From: APagani Date: Mon, 20 Jul 2026 15:45:54 +0200 Subject: [PATCH 09/20] chore(deps): bump go from 1.26.4 to 1.26.5 and upgrade deps (#77) --- .devcontainer/devcontainer.json | 2 +- .go-version | 2 +- go.mod | 53 ++++++++--------- go.sum | 102 ++++++++++++++++---------------- 4 files changed, 78 insertions(+), 81 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 9d8640d..b98c8b4 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -3,7 +3,7 @@ "image": "mcr.microsoft.com/devcontainers/base:2.1.2-trixie", "features": { "ghcr.io/devcontainers/features/go:1.3.3": { - "version": "1.26.4", + "version": "1.26.5", "golangciLintVersion": "2.11.2" } }, diff --git a/.go-version b/.go-version index ea0928c..8fe00a5 100644 --- a/.go-version +++ b/.go-version @@ -1 +1 @@ -1.26.4 +1.26.5 diff --git a/go.mod b/go.mod index c3e448b..e1db2b0 100644 --- a/go.mod +++ b/go.mod @@ -2,11 +2,11 @@ module github.com/mia-platform/ibdm go 1.26 -toolchain go1.26.4 +toolchain go1.26.5 require ( - cloud.google.com/go/asset v1.27.0 - cloud.google.com/go/pubsub/v2 v2.6.0 + cloud.google.com/go/asset v1.28.0 + cloud.google.com/go/pubsub/v2 v2.6.1 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0 github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs/v2 v2.0.2 @@ -23,29 +23,29 @@ require ( github.com/spf13/cobra v1.10.2 github.com/stretchr/testify v1.11.1 golang.org/x/oauth2 v0.36.0 - google.golang.org/api v0.287.0 - google.golang.org/grpc v1.82.0 + google.golang.org/api v0.289.0 + google.golang.org/grpc v1.82.1 google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v3 v3.0.1 ) require ( cloud.google.com/go v0.123.0 // indirect - cloud.google.com/go/accesscontextmanager v1.14.0 // indirect - cloud.google.com/go/auth v0.20.0 // indirect + cloud.google.com/go/accesscontextmanager v1.15.0 // indirect + cloud.google.com/go/auth v0.22.0 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect cloud.google.com/go/compute/metadata v0.9.0 // indirect - cloud.google.com/go/iam v1.11.0 // indirect - cloud.google.com/go/longrunning v1.1.0 // indirect + cloud.google.com/go/iam v1.12.0 // indirect + cloud.google.com/go/longrunning v1.2.0 // indirect cloud.google.com/go/orgpolicy v1.20.0 // indirect - cloud.google.com/go/osconfig v1.21.0 // indirect + cloud.google.com/go/osconfig v1.22.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect github.com/Azure/go-amqp v1.7.0 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 // indirect github.com/andybalholm/brotli v1.2.2 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/clipperhouse/uax29/v2 v2.2.0 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/clipperhouse/uax29/v2 v2.7.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 // indirect github.com/fatih/color v1.19.0 // indirect github.com/felixge/httpsnoop v1.1.0 // indirect @@ -56,27 +56,26 @@ require ( github.com/google/go-cmp v0.7.0 // indirect github.com/google/s2a-go v0.1.9 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.18 // indirect - github.com/googleapis/gax-go/v2 v2.22.0 // indirect + github.com/googleapis/gax-go/v2 v2.23.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/klauspost/compress v1.19.0 // indirect + github.com/klauspost/compress v1.19.1 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/lestrrat-go/blackmagic v1.0.4 // indirect - github.com/lestrrat-go/dsig v1.2.1 // indirect + github.com/lestrrat-go/dsig v1.3.0 // indirect github.com/lestrrat-go/dsig-secp256k1 v1.0.0 // indirect github.com/lestrrat-go/httpcc v1.0.1 // indirect - github.com/lestrrat-go/httprc/v3 v3.0.5 // indirect + github.com/lestrrat-go/httprc/v3 v3.0.6 // indirect github.com/lestrrat-go/option/v2 v2.0.0 // indirect github.com/mattn/go-colorable v0.1.15 // indirect - github.com/mattn/go-isatty v0.0.22 // indirect + github.com/mattn/go-isatty v0.0.23 // indirect github.com/mattn/go-runewidth v0.0.24 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/segmentio/asm v1.2.1 // indirect github.com/spf13/pflag v1.0.10 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasthttp v1.72.0 // indirect github.com/valyala/fastjson v1.6.10 // indirect - go.einride.tech/aip v0.86.3 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.69.0 // indirect @@ -85,13 +84,13 @@ require ( go.opentelemetry.io/otel/metric v1.44.0 // indirect go.opentelemetry.io/otel/sdk v1.44.0 // indirect go.opentelemetry.io/otel/trace v1.44.0 // indirect - golang.org/x/crypto v0.53.0 // indirect - golang.org/x/net v0.56.0 // indirect - golang.org/x/sync v0.21.0 // indirect - golang.org/x/sys v0.46.0 // indirect - golang.org/x/text v0.39.0 // indirect + golang.org/x/crypto v0.54.0 // indirect + golang.org/x/net v0.57.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.40.0 // indirect golang.org/x/time v0.15.0 // indirect - google.golang.org/genproto v0.0.0-20260706201446-f0a921348800 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260706201446-f0a921348800 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260706201446-f0a921348800 // indirect + google.golang.org/genproto v0.0.0-20260715232425-e75dac1f907d // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260715232425-e75dac1f907d // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260715232425-e75dac1f907d // indirect ) diff --git a/go.sum b/go.sum index dcd0963..48cf2bc 100644 --- a/go.sum +++ b/go.sum @@ -1,26 +1,26 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= -cloud.google.com/go/accesscontextmanager v1.14.0 h1:50ofyZiGo2yL3Wt1gZ0j0QnD9y3YrhhcwX0N08uS6KY= -cloud.google.com/go/accesscontextmanager v1.14.0/go.mod h1:VO15iVnsM0FO9Dt8hSFPgkuHRZjq6LEYZq1szJ27U2k= -cloud.google.com/go/asset v1.27.0 h1:Lj2lg/FB7VIBAkvUTVVx7Z9HRSPyVw7WN9butFJSONg= -cloud.google.com/go/asset v1.27.0/go.mod h1:+HaDReZQAh/0syAf0uTMeUrMfXikr+KKyDtCdvf7j4M= -cloud.google.com/go/auth v0.20.0 h1:kXTssoVb4azsVDoUiF8KvxAqrsQcQtB53DcSgta74CA= -cloud.google.com/go/auth v0.20.0/go.mod h1:942/yi/itH1SsmpyrbnTMDgGfdy2BUqIKyd0cyYLc5Q= +cloud.google.com/go/accesscontextmanager v1.15.0 h1:0baVgug9IFV8y5cIoD4iN/m/QtJzk2k+j3MW+zJWCVw= +cloud.google.com/go/accesscontextmanager v1.15.0/go.mod h1:YjW9urferk8i9ALwBF3bmdcogZeQYRn2yWwR8nkhsBc= +cloud.google.com/go/asset v1.28.0 h1:M6YE1exBuZQhTi8wfIKrwjKK6b2ySfHKgrqXjPZA4EM= +cloud.google.com/go/asset v1.28.0/go.mod h1:Pnvjhay8/FgodOH9uJC8OkfJfRtSnNIIU4WSxg5JfJw= +cloud.google.com/go/auth v0.22.0 h1:Xp9wAKkLoeaYb5pYZZoQGz4E9sdPxIbzS3gywZE3ciQ= +cloud.google.com/go/auth v0.22.0/go.mod h1:M9o2Oz+YI2jAfxewJgb1vyI3vceHF+eohmxyzmrl+9s= cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= -cloud.google.com/go/iam v1.11.0 h1:KieQ9Pb+LLPak1O3Rv3GgCxhnmkYf7Xyh0P5HfF1jFM= -cloud.google.com/go/iam v1.11.0/go.mod h1:KP+nKGugNJW4LcLx1uEZcq1ok5sQHFaQehQNl4QDgV4= -cloud.google.com/go/longrunning v1.1.0 h1:qJ0R0IA8ONaRCNWTRPAS0iAmt1bj3TVgJ40z7ZGRslE= -cloud.google.com/go/longrunning v1.1.0/go.mod h1:tH+A/6UvNypiPJWAQaKCsh+xiGbB23wUO8egwUXlD2E= +cloud.google.com/go/iam v1.12.0 h1:Aki3bX9aHUDKPHfnRJfDcTdVedvy6quGBQcTqx3DRXk= +cloud.google.com/go/iam v1.12.0/go.mod h1:FEZ4lXpADAC2AIpQY7LANNjjwyQ2jK439CI2VaD+sLY= +cloud.google.com/go/longrunning v1.2.0 h1:WjYH3YHBGCxGJP9M4dWGHBfXr/cFIjMkNgWcJj7/iMM= +cloud.google.com/go/longrunning v1.2.0/go.mod h1:5KMQALFGOCtFoi2xSOA1u3H7WKlhmckgiyFw7+LGQp0= cloud.google.com/go/orgpolicy v1.20.0 h1:kpVcE/OsC5aAzHCsAiuQSg3+s6ILzgPTuPZyS7n7ejA= cloud.google.com/go/orgpolicy v1.20.0/go.mod h1:9LHqEGx5P5dhansdKTNIEXpM+QbebAIOs66+HUID4aQ= -cloud.google.com/go/osconfig v1.21.0 h1:jpq0DNmjS4FkTbNILFdp03uZUQt8D2izpUtgtmSDieQ= -cloud.google.com/go/osconfig v1.21.0/go.mod h1:BofnHqjjvu6lZQv/hqo2+rLCUiY4O6A9UYwwvVrSBjk= -cloud.google.com/go/pubsub/v2 v2.6.0 h1:8pjR0id+GTB+krKx5G6AGJoYrHog58w2Q89PCOrfM64= -cloud.google.com/go/pubsub/v2 v2.6.0/go.mod h1:4anqvV/w8Pcgu2tO0qr2XgsF3GXHowzryfQ5gOnVmWY= +cloud.google.com/go/osconfig v1.22.0 h1:r5lzneR9GNixJ96lZ6oIfJK9Cd5sKpnzZJe1LcGZf4w= +cloud.google.com/go/osconfig v1.22.0/go.mod h1:bUL0FaSR2ahPcFRRYnd6a0LyUzsQYIdUpBq8Tmxg8fE= +cloud.google.com/go/pubsub/v2 v2.6.1 h1:jX6gnC4n8BgYx6MOYICgbbaXZpr1vKeNOE3Bn17P5zg= +cloud.google.com/go/pubsub/v2 v2.6.1/go.mod h1:1y2lZnKfUFPZz0PU4YmXyk4lA11+xmYA42zbC32RkxQ= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0 h1:aokoqcHvaGjiM3VpjKDfMMnF/8epJ+Q1HLJ7CudztqE= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0/go.mod h1:/WYEx9pcM9Y+Dd/APJaNlSvVSvzl54rrMdZT5+Oi2LM= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0 h1:CU4+EJeJi3TKYWEcYuSdWsjzw0nVsK/H0MSQOiPcymU= @@ -64,8 +64,8 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/clipperhouse/uax29/v2 v2.2.0 h1:ChwIKnQN3kcZteTXMgb1wztSgaU+ZemkgWdohwgs8tY= -github.com/clipperhouse/uax29/v2 v2.2.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= +github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= +github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= @@ -73,8 +73,9 @@ github.com/coder/websocket v1.8.14 h1:9L0p0iKiNOibykf283eHkKUHHrpG7f65OE3BhhO7v9 github.com/coder/websocket v1.8.14/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 h1:5RVFMOWjMyRy8cARdy79nAmgYw3hK/4HUq48LQ6Wwqo= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -137,8 +138,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.3.18 h1:hvVi34VucdrV1IIsiWuqYM8kutw/92MxNEFxCJZEh0k= github.com/googleapis/enterprise-certificate-proxy v0.3.18/go.mod h1:rSEsBUemEBZEexP2y6jPp16LUmUbjmSbcPMQizR0o4k= -github.com/googleapis/gax-go/v2 v2.22.0 h1:PjIWBpgGIVKGoCXuiCoP64altEJCj3/Ei+kSU5vlZD4= -github.com/googleapis/gax-go/v2 v2.22.0/go.mod h1:irWBbALSr0Sk3qlqb9SyJ1h68WjgeFuiOzI4Rqw5+aY= +github.com/googleapis/gax-go/v2 v2.23.0 h1:Tchl7qkvE7Ip3y+ztvNufYFvkfqTe7NfLTYGIdJRLuE= +github.com/googleapis/gax-go/v2 v2.23.0/go.mod h1:rBQKOVJCdb8IFEzg+FCwlt1LP/xMDGuqUXhUG+XMXEg= github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -147,8 +148,8 @@ github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU= github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= -github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ= -github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk= +github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -157,14 +158,14 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lestrrat-go/blackmagic v1.0.4 h1:IwQibdnf8l2KoO+qC3uT4OaTWsW7tuRQXy9TRN9QanA= github.com/lestrrat-go/blackmagic v1.0.4/go.mod h1:6AWFyKNNj0zEXQYfTMPfZrAXUWUfTIZ5ECEUEJaijtw= -github.com/lestrrat-go/dsig v1.2.1 h1:MwxzZhE4+4fguHi+uDALKVlC3Cn+O1QU1Q/F8D7hVIc= -github.com/lestrrat-go/dsig v1.2.1/go.mod h1:RD2eOaidyPvpc7IJQoO3Qq52RWdy8ZcJs8lrOnoa1Kc= +github.com/lestrrat-go/dsig v1.3.0 h1:phjMOCXvYzhuIgn7Voe2rex8z166vGfxRxmqM25P9/Q= +github.com/lestrrat-go/dsig v1.3.0/go.mod h1:RD2eOaidyPvpc7IJQoO3Qq52RWdy8ZcJs8lrOnoa1Kc= github.com/lestrrat-go/dsig-secp256k1 v1.0.0 h1:JpDe4Aybfl0soBvoVwjqDbp+9S1Y2OM7gcrVVMFPOzY= github.com/lestrrat-go/dsig-secp256k1 v1.0.0/go.mod h1:CxUgAhssb8FToqbL8NjSPoGQlnO4w3LG1P0qPWQm/NU= github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE= github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E= -github.com/lestrrat-go/httprc/v3 v3.0.5 h1:S+Mb4L2I+bM6JGTibLmxExhyTOqnXjqx+zi9MoXw/TM= -github.com/lestrrat-go/httprc/v3 v3.0.5/go.mod h1:mSMtkZW92Z98M5YoNNztbRGxbXHql7tSitCvaxvo9l0= +github.com/lestrrat-go/httprc/v3 v3.0.6 h1:4FpLQ18KK/ypPbVU3NLWJNRvH3kcYiqKqWfKGqNWxxI= +github.com/lestrrat-go/httprc/v3 v3.0.6/go.mod h1:mSMtkZW92Z98M5YoNNztbRGxbXHql7tSitCvaxvo9l0= github.com/lestrrat-go/jwx/v3 v3.1.1 h1:yd9AdPmZ4INnQ7k42IrzXYpnEG803+SrQ6hdMvzHJzw= github.com/lestrrat-go/jwx/v3 v3.1.1/go.mod h1:uw/MN2M/Xiu4FhwcIwH11Zsh9JWx9SWzgALl7/uIEkU= github.com/lestrrat-go/option/v2 v2.0.0 h1:XxrcaJESE1fokHy3FpaQ/cXW8ZsIdWcdFzzLOcID3Ss= @@ -175,16 +176,17 @@ github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4= -github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= +github.com/mattn/go-isatty v0.0.23 h1:cYwCQTQf3HB6xUC+BtyCLZNr7IzbOmoZbmssVNzSyiQ= +github.com/mattn/go-isatty v0.0.23/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= github.com/mattn/go-runewidth v0.0.24 h1:cpokDiIn0MGnhdHwuWnJBITySJ20QyNGnY2kR/ay2DU= github.com/mattn/go-runewidth v0.0.24/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= @@ -213,8 +215,6 @@ github.com/valyala/fastjson v1.6.10 h1:/yjJg8jaVQdYR3arGxPE2X5z89xrlhS0eGXdv+ADT github.com/valyala/fastjson v1.6.10/go.mod h1:e6FubmQouUNP73jtMLmcbxS6ydWIpOfhz34TSfO3JaE= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= -go.einride.tech/aip v0.86.3 h1:jg80Ec4XBPYg1i7avzrl3MJol/dUwmMMLHtcmEMyxgM= -go.einride.tech/aip v0.86.3/go.mod h1:dZuN/0sXeoscfWqsW8QLcLrGZdvsCC1B2R2CZ4kHmao= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= @@ -238,8 +238,8 @@ go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= -golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= +golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= +golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -250,16 +250,16 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= -golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= +golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= +golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= -golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -270,12 +270,12 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= -golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus= -golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -286,26 +286,26 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/api v0.287.0 h1:CQDMqUiqZZ0U/Yge3zyjAhNQ0OSYEH0PaA7l4xtEen4= -google.golang.org/api v0.287.0/go.mod h1:pPW85yt3Iuc3unkpaMhFtMmOqnTdCwCqEOaUlnuxRlQ= +google.golang.org/api v0.289.0 h1:DmH0c6NigNFmsvsohM9bxv+MzVhag3aGHnojA5fFQjc= +google.golang.org/api v0.289.0/go.mod h1:weJZ3lldHFYI0DBFNKpJelUDNnusTt5YaOEgxvt8ci8= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20260706201446-f0a921348800 h1:NDCaohnq5LRpt5soWpH6+U2tsRCpcgD0ftByMJAAAt4= -google.golang.org/genproto v0.0.0-20260706201446-f0a921348800/go.mod h1:J1jBkXm41jiQyoU7J/Q2o2jqrZZwyFYMOKcZwWCIzDM= -google.golang.org/genproto/googleapis/api v0.0.0-20260706201446-f0a921348800 h1:admdQBe8jR3VWhBsUrAOaF2Qw6K/+p5pSm1GN8+6Fw4= -google.golang.org/genproto/googleapis/api v0.0.0-20260706201446-f0a921348800/go.mod h1:FPk7EXUKMtImne7AmknoYjT4QXqKIzzRbeQIXzLk6fQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260706201446-f0a921348800 h1:qEHAMpSaUhtD0p3NbEEI83HwNGFxEwaSJ1G9PLnCBZE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260706201446-f0a921348800/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/genproto v0.0.0-20260715232425-e75dac1f907d h1:C9v1o0/4quuhOAfmRXA2j+we0PqZIp8traLdeogF3Ms= +google.golang.org/genproto v0.0.0-20260715232425-e75dac1f907d/go.mod h1:Wz2wFJntZFmLGo7pLDXZ3wYk5hyc0Mb+SkHhDDXT+lU= +google.golang.org/genproto/googleapis/api v0.0.0-20260715232425-e75dac1f907d h1:QwnJwPte4XXAkhPu26LTDIahnsMSUV0kK8HkxbC+Pc4= +google.golang.org/genproto/googleapis/api v0.0.0-20260715232425-e75dac1f907d/go.mod h1:WRrQ7/7N19PypuT0fxLOL5Lq0waoiRri4FbtHDEKrGE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260715232425-e75dac1f907d h1:Jkpk39hlTZOIp3RbfvNX9R8Hv+Sw0X89nlU/xFOErsc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260715232425-e75dac1f907d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.82.0 h1:vguDnZUPjE26w09A63VoxZPnvPjB5Riyc0mkXPFmAIU= -google.golang.org/grpc v1.82.0/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= +google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= +google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -323,7 +323,5 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= -gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From 65b908ff0a95b0da9b7a4817f81a0f68a5d35acf Mon Sep 17 00:00:00 2001 From: apagani Date: Tue, 21 Jul 2026 11:48:45 +0200 Subject: [PATCH 10/20] chore(docs): update gitlab source docs --- docs/how-to/070_gitlab-source.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/how-to/070_gitlab-source.md b/docs/how-to/070_gitlab-source.md index fe2c9bb..e3ea8e3 100644 --- a/docs/how-to/070_gitlab-source.md +++ b/docs/how-to/070_gitlab-source.md @@ -37,7 +37,8 @@ In addition to other environment variables the GitLab source requires or accepts The first two variables are used for both sync and run modes. `GITLAB_WEBHOOK_PATH` and `GITLAB_WEBHOOK_TOKEN` are only needed when running in webhook mode -(`run`). If `GITLAB_WEBHOOK_TOKEN` is not set, the webhook endpoint will not be registered. +(`run`). `GITLAB_WEBHOOK_TOKEN` is required in that mode: if it is not set, `ibdm run gitlab` +fails to start with a `webhook token not configured` error. ## Supported Data Types From fff9b2aa144f5ed2a509c0c5c034ec857a54d019 Mon Sep 17 00:00:00 2001 From: APagani Date: Thu, 23 Jul 2026 16:59:57 +0200 Subject: [PATCH 11/20] feat: custom resource support for console source (#78) --- docs/how-to/050_console-source.md | 1 + docs/mappings/console/custom-resources.yaml | 59 +++ internal/source/console/console.go | 118 ++++-- internal/source/console/console_test.go | 382 ++++++++++++++++++++ 4 files changed, 536 insertions(+), 24 deletions(-) create mode 100644 docs/mappings/console/custom-resources.yaml diff --git a/docs/how-to/050_console-source.md b/docs/how-to/050_console-source.md index 2c51621..0e98a1b 100644 --- a/docs/how-to/050_console-source.md +++ b/docs/how-to/050_console-source.md @@ -64,5 +64,6 @@ The Console source can sync the following data types: - `project`: Information about Console projects. - `revision`: A named revision of a project. - `service`: A microservice within a project's default-branch revision (only services of type `custom` and not marked as `advanced` are emitted). +- `custom-resource`: A custom resource within a project's default-branch revision (only services of type `custom-resource` are emitted). - `cluster`: A cluster registered in the Console. Fetched via the tenant/cluster APIs; the `linkedProjects` field is stripped before mapping. - `clusterProjectRelationship`: A relationship between a cluster and a linked project. One entry is emitted per project in the cluster's `linkedProjects` list, carrying both the project and the cluster (without `linkedProjects`) as template values. diff --git a/docs/mappings/console/custom-resources.yaml b/docs/mappings/console/custom-resources.yaml new file mode 100644 index 0000000..be089f2 --- /dev/null +++ b/docs/mappings/console/custom-resources.yaml @@ -0,0 +1,59 @@ +apiVersion: console.mia-platform.eu/v1 +itemFamily: customresources +type: custom-resource +syncable: true +mappings: + identifier: |- + {{ printf "%s-%s-%s" .project._id .revision.name .customResource.name | sha256sum }} + metadata: + title: "{{ printf \"%s\" .customResource.name }}" + owner: |- + {{ $info := (get "info" .project (object)) -}} + {{- $teamContact := (get "teamContact" $info nil) -}} + {{- if $teamContact -}} + {{- $owner := object -}} + {{- $owner = set "type" "email" $owner -}} + {{- $owner = set "ref" (get "teamContact" $info "") $owner -}} + {{- $owner | toJSON -}} + {{- end }} + spec: + projectUniqueId: "{{ .project._id }}" + projectId: "{{ get \"projectId\" .project \"\" | quote }}" + tenantId: "{{ .project.tenantId }}" + revisionName: "{{ .revision.name }}" + resourceName: "{{ .customResource.name }}" + resourceKind: |- + {{ $meta := (get "meta" .customResource (object)) -}} + {{- get "kind" $meta "" }} + resourceApiVersion: |- + {{ $meta := (get "meta" .customResource (object)) -}} + {{- get "apiVersion" $meta "" }} + sourceMarketplaceItem: "{{ get \"sourceMarketplaceItem\" .customResource (object) | toJSON }}" + configuration: "{{ get \"spec\" .customResource (object) | toJSON }}" + extra: + - apiVersion: mia-platform.eu/v1 + itemFamily: relationships + deletePolicy: "cascade" + identifier: |- + {{ $src := printf "urn:mia-platform-catalog:console.mia-platform-experimental.eu:v1:CustomResource:%s" (printf "%s-%s-%s" .project._id .revision.name .customResource.name | sha256sum) -}} + {{- $type := "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:part-of.mia-platform.eu" -}} + {{- $tgt := printf "urn:mia-platform-catalog:console.mia-platform.eu:v1:Project:%s" .project._id -}} + {{- printf "%s-%s-%s" $src $type $tgt | sha256sum }} + sourceRef: |- + urn:mia-platform-catalog:console.mia-platform-experimental.eu:v1:CustomResource:{{ printf "%s-%s-%s" .project._id .revision.name .customResource.name | sha256sum }} + targetRef: |- + urn:mia-platform-catalog:console.mia-platform.eu:v1:Project:{{ .project._id }} + typeRef: "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:part-of.mia-platform.eu" + - apiVersion: mia-platform.eu/v1 + itemFamily: relationships + deletePolicy: "cascade" + identifier: |- + {{ $src := printf "urn:mia-platform-catalog:console.mia-platform-experimental.eu:v1:CustomResource:%s" (printf "%s-%s-%s" .project._id .revision.name .customResource.name | sha256sum) -}} + {{- $type := "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:part-of.mia-platform.eu" -}} + {{- $tgt := printf "urn:mia-platform-catalog:console.mia-platform.eu:v1:Revision:%s" (printf "%s-%s" .project._id .revision.name | sha256sum) -}} + {{- printf "%s-%s-%s" $src $type $tgt | sha256sum }} + sourceRef: |- + urn:mia-platform-catalog:console.mia-platform-experimental.eu:v1:CustomResource:{{ printf "%s-%s-%s" .project._id .revision.name .customResource.name | sha256sum }} + targetRef: |- + urn:mia-platform-catalog:console.mia-platform.eu:v1:Revision:{{ printf "%s-%s" .project._id .revision.name | sha256sum }} + typeRef: "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:part-of.mia-platform.eu" diff --git a/internal/source/console/console.go b/internal/source/console/console.go index 72af17f..4bacde7 100644 --- a/internal/source/console/console.go +++ b/internal/source/console/console.go @@ -29,6 +29,7 @@ const ( projectResource = "project" revisionResource = "revision" serviceResource = "service" + customResourceResource = "custom-resource" clusterResource = "cluster" clusterProjectRelationshipResource = "clusterProjectRelationship" linkedProjectsField = "linkedProjects" @@ -42,7 +43,7 @@ var ( ErrRetrievingAssets = errors.New("error retrieving assets") ErrWebhookSecretMissing = errors.New("webhook secret not configured") - configurationChainTypes = []string{projectResource, revisionResource, serviceResource} + configurationChainTypes = []string{projectResource, revisionResource, serviceResource, customResourceResource} clusterChainTypes = []string{clusterResource, clusterProjectRelationshipResource} timeSource = time.Now ) @@ -217,12 +218,17 @@ func (s *Source) listConfigurations(ctx context.Context, subtypes []string) ([]s if revName != defaultBranch { continue } - for _, svc := range configuration["services"].(map[string]any) { - svcMap := svc.(map[string]any) - if isServiceValid(svcMap) { - result = append(result, createServiceData(project, revName, svcMap, timeSource(), source.DataOperationUpsert)) - } + result = append(result, collectServicesFromConfiguration(project, revName, configuration, timeSource())...) + case customResourceResource: + defaultBranch, ok := project["defaultBranch"].(string) + if !ok { + log.Trace("nil defaultBranch for project", "_id", project["_id"], "projectId", project["projectId"], "revisionName", revName) + continue + } + if revName != defaultBranch { + continue } + result = append(result, collectCustomResourcesFromConfiguration(project, revName, configuration, timeSource())...) } } } @@ -305,6 +311,13 @@ func isServiceValid(svc map[string]any) bool { return typeFound && advancedFound && svcType.(string) == "custom" && !advanced.(bool) } +// isCustomResourceValid reports whether svc qualifies as a custom resource for +// synchronisation. A service is a custom resource when its type is "custom-resource". +func isCustomResourceValid(svc map[string]any) bool { + svcType, typeFound := svc["type"] + return typeFound && svcType.(string) == customResourceResource +} + // createProjectData assembles a [source.Data] upsert entry for a project, // embedding the normalised project maps. func createProjectData(project map[string]any, t time.Time, operation source.DataOperation) source.Data { @@ -326,12 +339,38 @@ func createRevisionData(project map[string]any, revisionName string, t time.Time Operation: operation, Time: t, Values: map[string]any{ - projectResource: buildProjectData(project), - "revision": buildRevisionData(revisionName), + projectResource: buildProjectData(project), + revisionResource: buildRevisionData(revisionName), }, } } +// collectServicesFromConfiguration returns a [source.Data] upsert entry for +// every service in configuration that passes isServiceValid. +func collectServicesFromConfiguration(project map[string]any, revName string, configuration map[string]any, t time.Time) []source.Data { + var result []source.Data + for _, svc := range configuration["services"].(map[string]any) { + svcMap := svc.(map[string]any) + if isServiceValid(svcMap) { + result = append(result, createServiceData(project, revName, svcMap, t, source.DataOperationUpsert)) + } + } + return result +} + +// collectCustomResourcesFromConfiguration returns a [source.Data] upsert entry +// for every service in configuration that passes isCustomResourceValid. +func collectCustomResourcesFromConfiguration(project map[string]any, revName string, configuration map[string]any, t time.Time) []source.Data { + var result []source.Data + for _, svc := range configuration["services"].(map[string]any) { + svcMap := svc.(map[string]any) + if isCustomResourceValid(svcMap) { + result = append(result, createCustomResourceData(project, revName, svcMap, t, source.DataOperationUpsert)) + } + } + return result +} + // createServiceData assembles a [source.Data] upsert entry for a service, // embedding the normalised project, revision and service maps. func createServiceData(project map[string]any, revisionName string, svc map[string]any, t time.Time, operation source.DataOperation) source.Data { @@ -347,6 +386,21 @@ func createServiceData(project map[string]any, revisionName string, svc map[stri } } +// createCustomResourceData assembles a [source.Data] upsert entry for a custom +// resource, embedding the normalised project, revision and raw custom resource maps. +func createCustomResourceData(project map[string]any, revisionName string, svc map[string]any, t time.Time, operation source.DataOperation) source.Data { + return source.Data{ + Type: customResourceResource, + Operation: operation, + Time: t, + Values: map[string]any{ + "project": buildProjectData(project), + "revision": buildRevisionData(revisionName), + "customResource": svc, + }, + } +} + // listClusters fetches all tenants from the Console API and, for each tenant, // retrieves its clusters. It emits [source.Data] entries for every requested // subtype (cluster, clusterProjectRelationship). @@ -440,7 +494,7 @@ func (s *Source) GetWebhook(ctx context.Context, typesToStream map[string]source switch t { case projectResource: webhookTypes = append(webhookTypes, t) - case revisionResource, serviceResource: + case revisionResource, serviceResource, customResourceResource: webhookTypes = append(webhookTypes, configurationResource) } } @@ -528,25 +582,41 @@ func (s *Source) configurationEventChain(ctx context.Context, ev event, types [] channel <- createProjectData(project, ev.UnixEventTimestamp(), ev.Operation()) case revisionResource: channel <- createRevisionData(project, revisionName, ev.UnixEventTimestamp(), ev.Operation()) - case serviceResource: - if revisionName != project["defaultBranch"].(string) { - continue - } + case serviceResource, customResourceResource: + // processed together after the loop to share one GetConfiguration call + } + } - configuration, err := s.cs.GetConfiguration(ctx, projectID, revisionName) - if err != nil { - return fmt.Errorf("%w: %w", ErrRetrievingAssets, err) - } + return s.processConfigurationServices(ctx, project, projectID, revisionName, ev, types, channel) +} - for _, svc := range configuration["services"].(map[string]any) { - svcMap := svc.(map[string]any) - if isServiceValid(svcMap) { - channel <- createServiceData(project, revisionName, svcMap, ev.UnixEventTimestamp(), ev.Operation()) - } - } - } +// processConfigurationServices fetches the project configuration once and emits +// service and custom-resource items for the given event to channel. Only the +// project's default branch is processed; non-default revisions are skipped. +func (s *Source) processConfigurationServices(ctx context.Context, project map[string]any, projectID, revisionName string, ev event, types []string, channel chan<- source.Data) error { + syncServices := slices.Contains(types, serviceResource) + syncCustomResources := slices.Contains(types, customResourceResource) + if !syncServices && !syncCustomResources { + return nil + } + if revisionName != project["defaultBranch"].(string) { + return nil + } + + configuration, err := s.cs.GetConfiguration(ctx, projectID, revisionName) + if err != nil { + return fmt.Errorf("%w: %w", ErrRetrievingAssets, err) } + for _, svc := range configuration["services"].(map[string]any) { + svcMap := svc.(map[string]any) + if syncServices && isServiceValid(svcMap) { + channel <- createServiceData(project, revisionName, svcMap, ev.UnixEventTimestamp(), ev.Operation()) + } + if syncCustomResources && isCustomResourceValid(svcMap) { + channel <- createCustomResourceData(project, revisionName, svcMap, ev.UnixEventTimestamp(), ev.Operation()) + } + } return nil } diff --git a/internal/source/console/console_test.go b/internal/source/console/console_test.go index 59fc5e1..49f9678 100644 --- a/internal/source/console/console_test.go +++ b/internal/source/console/console_test.go @@ -887,3 +887,385 @@ func Test_buildServiceData(t *testing.T) { }) } } + +func Test_isCustomResourceValid(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + input map[string]any + expected bool + }{ + "type custom-resource is valid": { + input: map[string]any{"name": "cr-1", "type": "custom-resource"}, + expected: true, + }, + "type custom is not valid": { + input: map[string]any{"name": "svc-1", "type": "custom", "advanced": false}, + expected: false, + }, + "no type field is not valid": { + input: map[string]any{"name": "cr-2"}, + expected: false, + }, + "type plugin is not valid": { + input: map[string]any{"name": "plugin-1", "type": "plugin"}, + expected: false, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + assert.Equal(t, tc.expected, isCustomResourceValid(tc.input)) + }) + } +} + +func TestSource_listAssets_customResources(t *testing.T) { + t.Run("lists custom-resources from the default branch", func(t *testing.T) { + ctx := t.Context() + + project1 := map[string]any{ + "_id": "p1", + "projectId": "project-1", + "name": "name", + "tenantId": "tenant-1", + "info": map[string]any{ + "teamContact": "contact", + }, + } + + revision1 := map[string]any{"name": "r1"} + + cr1 := map[string]any{ + "name": "example-custom-resource", + "type": "custom-resource", + "meta": map[string]any{ + "kind": "ExampleCustomKind", + "apiVersion": "custom-generator.console.mia-platform.eu/v1", + }, + "spec": map[string]any{ + "example_spec_key": float64(42), + "parsed": true, + }, + } + + // a plain "custom" service that must NOT appear in custom-resource results + svc1 := map[string]any{ + "name": "service-1", + "type": "custom", + "advanced": false, + } + + expectedData := []source.Data{ + { + Type: customResourceResource, + Operation: source.DataOperationUpsert, + Time: testTime, + Values: map[string]any{ + "project": project1, + "revision": revision1, + "customResource": cr1, + }, + }, + } + + handler := func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/backend/projects/": + projectResponse := maps.Clone(project1) + projectResponse["defaultBranch"] = "r1" + json.NewEncoder(w).Encode([]map[string]any{projectResponse}) + case "/backend/projects/p1/revisions": + json.NewEncoder(w).Encode([]map[string]any{revision1}) + case "/backend/projects/p1/revisions/r1/configuration": + json.NewEncoder(w).Encode(map[string]any{ + "services": map[string]any{ + "example-custom-resource": cr1, + "service-1": svc1, + }, + }) + default: + w.WriteHeader(http.StatusNotFound) + } + } + + server := httptest.NewServer(http.HandlerFunc(handler)) + defer server.Close() + t.Setenv("CONSOLE_ENDPOINT", server.URL) + t.Setenv("CONSOLE_WEBHOOK_PATH", "/webhook") + + s, err := NewSource() + require.NoError(t, err) + + typesToSync := map[string]source.Extra{ + customResourceResource: {}, + } + + data, err := s.listAssets(ctx, typesToSync) + require.NoError(t, err) + assert.Equal(t, expectedData, data) + }) + + t.Run("skips custom-resources from non-default branches", func(t *testing.T) { + ctx := t.Context() + + handler := func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/backend/projects/": + json.NewEncoder(w).Encode([]map[string]any{{ + "_id": "p1", + "projectId": "project-1", + "name": "name", + "tenantId": "tenant-1", + "defaultBranch": "main", + }}) + case "/backend/projects/p1/revisions": + json.NewEncoder(w).Encode([]map[string]any{{"name": "feature-branch"}}) + case "/backend/projects/p1/revisions/feature-branch/configuration": + json.NewEncoder(w).Encode(map[string]any{ + "services": map[string]any{ + "cr-1": map[string]any{"name": "cr-1", "type": "custom-resource"}, + }, + }) + default: + w.WriteHeader(http.StatusNotFound) + } + } + + server := httptest.NewServer(http.HandlerFunc(handler)) + defer server.Close() + t.Setenv("CONSOLE_ENDPOINT", server.URL) + t.Setenv("CONSOLE_WEBHOOK_PATH", "/webhook") + + s, err := NewSource() + require.NoError(t, err) + + data, err := s.listAssets(ctx, map[string]source.Extra{customResourceResource: {}}) + require.NoError(t, err) + assert.Empty(t, data) + }) +} + +func TestSource_configurationEventChain_customResource(t *testing.T) { + t.Run("emits custom-resource items on configuration event", func(t *testing.T) { + ctx := t.Context() + + cr1 := map[string]any{ + "name": "example-custom-resource", + "type": "custom-resource", + "meta": map[string]any{ + "kind": "ExampleCustomKind", + "apiVersion": "custom-generator.console.mia-platform.eu/v1", + }, + } + + handler := func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/backend/projects/p1": + json.NewEncoder(w).Encode(map[string]any{ + "_id": "p1", + "projectId": "projectId", + "name": "name", + "defaultBranch": "r1", + "tenantId": "", + "info": nil, + }) + case "/backend/projects/p1/revisions/r1/configuration": + json.NewEncoder(w).Encode(map[string]any{ + "services": map[string]any{ + "example-custom-resource": cr1, + "svc-1": map[string]any{ + "name": "svc-1", + "type": "custom", + "advanced": false, + }, + }, + }) + default: + w.WriteHeader(http.StatusNotFound) + } + } + + server := httptest.NewServer(http.HandlerFunc(handler)) + defer server.Close() + t.Setenv("CONSOLE_ENDPOINT", server.URL) + + cs, err := service.NewConsoleService() + require.NoError(t, err) + src := Source{cs: cs} + + ev := event{ + EventName: "configuration_created", + EventTimestamp: 1672531200000, + Payload: map[string]any{ + "projectId": "p1", + "revisionName": "r1", + }, + } + + ch := make(chan source.Data, 5) + typesToStream := map[string]source.Extra{ + customResourceResource: {}, + } + + err = src.handleEvent(ctx, ev, typesToStream, ch) + require.NoError(t, err) + close(ch) + + var data []source.Data + for d := range ch { + data = append(data, d) + } + + require.Len(t, data, 1) + assert.Equal(t, customResourceResource, data[0].Type) + assert.Equal(t, source.DataOperationUpsert, data[0].Operation) + assert.Equal(t, cr1, data[0].Values["customResource"]) + }) + + t.Run("custom-resource type in typesToStream subscribes to configuration webhook events", func(t *testing.T) { + ctx := t.Context() + + // Provide a server so the async goroutine can call GetProject + GetConfiguration + // without panicking on a nil ConsoleService. + handler := func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/backend/projects/p1": + json.NewEncoder(w).Encode(map[string]any{ + "_id": "p1", + "projectId": "projectId", + "name": "name", + "defaultBranch": "r1", + "tenantId": "", + "info": nil, + }) + case "/backend/projects/p1/revisions/r1/configuration": + // No custom-resource services — nothing emitted but no panic either. + json.NewEncoder(w).Encode(map[string]any{ + "services": map[string]any{}, + }) + default: + w.WriteHeader(http.StatusNotFound) + } + } + + server := httptest.NewServer(http.HandlerFunc(handler)) + defer server.Close() + t.Setenv("CONSOLE_ENDPOINT", server.URL) + + cs, err := service.NewConsoleService() + require.NoError(t, err) + + s := Source{ + cs: cs, + c: &webhookClient{ + config: webhookConfig{ + WebhookPath: "/webhook", + WebhookSecret: "secret", + }, + }, + } + + results := make(chan source.Data, 1) + typesToStream := map[string]source.Extra{customResourceResource: {}} + + webhook, err := s.GetWebhook(ctx, typesToStream, results) + require.NoError(t, err) + require.NotNil(t, webhook.Handler) + + body, err := json.Marshal(map[string]any{ + "eventName": "configuration_created", + "payload": map[string]any{ + "projectId": "p1", + "revisionName": "r1", + }, + }) + require.NoError(t, err) + + headers := signedHeaders(body, "secret", "sha256=") + // The handler dispatches async; it returns nil immediately. + err = webhook.Handler(ctx, headers, body) + require.NoError(t, err) + + // The configuration has no custom-resource services, so nothing is emitted. + // Give the goroutine a moment to run and confirm the channel stays empty. + waitCtx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + + select { + case <-results: + t.Fatal("did not expect data in channel") + case <-waitCtx.Done(): + // expected: goroutine completed, nothing emitted + } + }) +} + +func TestSource_processConfigurationServices_noRelevantTypes(t *testing.T) { + t.Run("returns early when neither service nor custom-resource is requested", func(t *testing.T) { + ctx := t.Context() + + // No server needed — the function must return before making any API call. + s := Source{} + + ev := event{ + EventName: "configuration_created", + EventTimestamp: 1672531200000, + Payload: map[string]any{}, + } + + project := map[string]any{"_id": "p1", "defaultBranch": "r1"} + ch := make(chan source.Data, 1) + + // types contains only projectResource — neither service nor custom-resource + err := s.processConfigurationServices(ctx, project, "p1", "r1", ev, []string{projectResource}, ch) + require.NoError(t, err) + assert.Empty(t, ch) + }) +} + +func TestSource_listConfigurations_customResourceNilDefaultBranch(t *testing.T) { + t.Run("skips custom-resource when project defaultBranch is not a string", func(t *testing.T) { + ctx := t.Context() + + // Project has no defaultBranch field at all — type assertion will fail. + handler := func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/backend/projects/": + json.NewEncoder(w).Encode([]map[string]any{{ + "_id": "p1", + "tenantId": "t1", + // defaultBranch intentionally absent + }}) + case "/backend/projects/p1/revisions": + json.NewEncoder(w).Encode([]map[string]any{{"name": "r1"}}) + case "/backend/projects/p1/revisions/r1/configuration": + json.NewEncoder(w).Encode(map[string]any{ + "services": map[string]any{ + "cr-1": map[string]any{"name": "cr-1", "type": "custom-resource"}, + }, + }) + default: + w.WriteHeader(http.StatusNotFound) + } + } + + server := httptest.NewServer(http.HandlerFunc(handler)) + defer server.Close() + t.Setenv("CONSOLE_ENDPOINT", server.URL) + t.Setenv("CONSOLE_WEBHOOK_PATH", "/webhook") + + s, err := NewSource() + require.NoError(t, err) + + data, err := s.listAssets(ctx, map[string]source.Extra{customResourceResource: {}}) + require.NoError(t, err) + assert.Empty(t, data) + }) +} From b2bab98e50241b3631ae51a91af1981a5a5f7ce0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2026 11:21:22 +0200 Subject: [PATCH 12/20] ci(deps): bump actions/setup-go from 6.5.0 to 7.0.0 (#83) Bumps [actions/setup-go](https://github.com/actions/setup-go) from 6.5.0 to 7.0.0. - [Release notes](https://github.com/actions/setup-go/releases) - [Commits](https://github.com/actions/setup-go/compare/924ae3a1cded613372ab5595356fb5720e22ba16...b7ad1dad31e06c5925ef5d2fc7ad053ef454303e) --- updated-dependencies: - dependency-name: actions/setup-go dependency-version: 7.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yaml | 4 ++-- .github/workflows/code-scanning.yaml | 2 +- .github/workflows/lint.yaml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 88cb751..a4a5dca 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -39,7 +39,7 @@ jobs: with: show-progress: false - name: Setup Golang - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version-file: go.mod - name: Run tests @@ -86,7 +86,7 @@ jobs: with: show-progress: false - name: Setup Golang - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version-file: go.mod - name: Docker Login to GitHub Repository diff --git a/.github/workflows/code-scanning.yaml b/.github/workflows/code-scanning.yaml index 9323910..99961d1 100644 --- a/.github/workflows/code-scanning.yaml +++ b/.github/workflows/code-scanning.yaml @@ -61,7 +61,7 @@ jobs: show-progress: false - name: Setup Golang if: matrix.language == 'go' - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version-file: go.mod - name: Initialize CodeQL diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 66a145e..8f0b6b6 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -21,7 +21,7 @@ jobs: with: show-progress: false - name: Setup Golang - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version-file: go.mod - name: golangci-lint From fb429496e320f617419621107474fed9ec0f3d45 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2026 12:46:00 +0200 Subject: [PATCH 13/20] ci(deps): bump actions/stale from 10.3.0 to 11.0.0 (#84) Bumps [actions/stale](https://github.com/actions/stale) from 10.3.0 to 11.0.0. - [Release notes](https://github.com/actions/stale/releases) - [Changelog](https://github.com/actions/stale/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/stale/compare/eb5cf3af3ac0a1aa4c9c45633dd1ae542a27a899...4391f3da665fdf50b6810c1a66712fb9ba21aa93) --- updated-dependencies: - dependency-name: actions/stale dependency-version: 11.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/inactive-issues.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/inactive-issues.yaml b/.github/workflows/inactive-issues.yaml index b616e24..483bc33 100644 --- a/.github/workflows/inactive-issues.yaml +++ b/.github/workflows/inactive-issues.yaml @@ -14,7 +14,7 @@ jobs: issues: write pull-requests: write steps: - - uses: actions/stale@eb5cf3af3ac0a1aa4c9c45633dd1ae542a27a899 # v10.3.0 + - uses: actions/stale@4391f3da665fdf50b6810c1a66712fb9ba21aa93 # v11.0.0 with: days-before-issue-stale: ${{ env.DAYS_BEFORE_INACTIVE }} days-before-issue-close: ${{ env.DAYS_BEFORE_CLOSE }} From 12fe4da90f551a15877cc8cd4813009cb352d5d9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2026 13:11:58 +0200 Subject: [PATCH 14/20] ci(deps): bump actions/checkout from 6.0.2 to 7.0.1 (#82) Bumps [actions/checkout](https://github.com/actions/checkout) from 6.0.2 to 7.0.1. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/de0fac2e4500dabe0009e67214ff5f5447ce83dd...3d3c42e5aac5ba805825da76410c181273ba90b1) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: 7.0.1 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yaml | 4 ++-- .github/workflows/code-scanning.yaml | 2 +- .github/workflows/dependencies-review.yaml | 2 +- .github/workflows/lint.yaml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a4a5dca..b1d5144 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -35,7 +35,7 @@ jobs: contents: read steps: - name: Checkout Repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: show-progress: false - name: Setup Golang @@ -82,7 +82,7 @@ jobs: if: (github.event_name == 'push') && (github.repository == 'mia-platform/ibdm') steps: - name: Checkout Repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: show-progress: false - name: Setup Golang diff --git a/.github/workflows/code-scanning.yaml b/.github/workflows/code-scanning.yaml index 99961d1..b38344d 100644 --- a/.github/workflows/code-scanning.yaml +++ b/.github/workflows/code-scanning.yaml @@ -56,7 +56,7 @@ jobs: build-mode: manual steps: - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: show-progress: false - name: Setup Golang diff --git a/.github/workflows/dependencies-review.yaml b/.github/workflows/dependencies-review.yaml index 37acf1c..7f9d2e2 100644 --- a/.github/workflows/dependencies-review.yaml +++ b/.github/workflows/dependencies-review.yaml @@ -23,7 +23,7 @@ jobs: pull-requests: write steps: - name: Checkout Repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: show-progress: false - name: Dependency Review diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 8f0b6b6..c080553 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -17,7 +17,7 @@ jobs: contents: read steps: - name: Checkout Repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: show-progress: false - name: Setup Golang From 0fca403aab47a5eb371469ad3dc73a65de7bc927 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:23:53 +0200 Subject: [PATCH 15/20] build(deps): bump ghcr.io/devcontainers/features/go from 1.3.3 to 1.3.4 (#80) Bumps ghcr.io/devcontainers/features/go from 1.3.3 to 1.3.4. --- updated-dependencies: - dependency-name: ghcr.io/devcontainers/features/go dependency-version: 1.3.4 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .devcontainer/devcontainer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index b98c8b4..4ec88d7 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -2,7 +2,7 @@ "name": "Go", "image": "mcr.microsoft.com/devcontainers/base:2.1.2-trixie", "features": { - "ghcr.io/devcontainers/features/go:1.3.3": { + "ghcr.io/devcontainers/features/go:1.3.4": { "version": "1.26.5", "golangciLintVersion": "2.11.2" } From 5814b85cb4f17d3a0e2ff982987552de831c49ba Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2026 16:41:20 +0200 Subject: [PATCH 16/20] ci(deps): bump the minor-actions-dependencies group with 4 updates (#81) Bumps the minor-actions-dependencies group with 4 updates: [docker/login-action](https://github.com/docker/login-action), [softprops/action-gh-release](https://github.com/softprops/action-gh-release), [github/codeql-action/init](https://github.com/github/codeql-action) and [github/codeql-action/analyze](https://github.com/github/codeql-action). Updates `docker/login-action` from 4.4.0 to 4.6.0 - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/af1e73f918a031802d376d3c8bbc3fe56130a9b0...dbcb813823bdd20940b903addbd779551569679f) Updates `softprops/action-gh-release` from 3.0.1 to 3.0.2 - [Release notes](https://github.com/softprops/action-gh-release/releases) - [Changelog](https://github.com/softprops/action-gh-release/blob/master/CHANGELOG.md) - [Commits](https://github.com/softprops/action-gh-release/compare/718ea10b132b3b2eba29c1007bb80653f286566b...3d0d9888cb7fd7b750713d6e236d1fcb99157228) Updates `github/codeql-action/init` from 4.36.3 to 4.37.3 - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/54f647b7e1bb85c95cddabcd46b0c578ec92bc1a...e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81) Updates `github/codeql-action/analyze` from 4.36.3 to 4.37.3 - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/54f647b7e1bb85c95cddabcd46b0c578ec92bc1a...e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81) --- updated-dependencies: - dependency-name: docker/login-action dependency-version: 4.6.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: minor-actions-dependencies - dependency-name: softprops/action-gh-release dependency-version: 3.0.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: minor-actions-dependencies - dependency-name: github/codeql-action/init dependency-version: 4.37.3 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: minor-actions-dependencies - dependency-name: github/codeql-action/analyze dependency-version: 4.37.3 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: minor-actions-dependencies ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yaml | 8 ++++---- .github/workflows/code-scanning.yaml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b1d5144..8a8c9b6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -90,18 +90,18 @@ jobs: with: go-version-file: go.mod - name: Docker Login to GitHub Repository - uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Docker Login to DockerHub - uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: username: ${{ secrets.BOT_DOCKER_USERNAME }} password: ${{ secrets.BOT_DOCKER_TOKEN }} - name: Login to Mia registry - uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: registry: nexus.mia-platform.eu username: ${{ secrets.NEXUS_USER }} @@ -209,7 +209,7 @@ jobs: COSIGN_PRIVATE_KEY: ${{ secrets.MIA_PLATFORM_KEY_KMS }} - name: Create GitHub Release if: github.ref_type == 'tag' - uses: softprops/action-gh-release@718ea10b132b3b2eba29c1007bb80653f286566b # v3.0.1 + uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3.0.2 with: preserve_order: true generate_release_notes: true diff --git a/.github/workflows/code-scanning.yaml b/.github/workflows/code-scanning.yaml index b38344d..d2256a2 100644 --- a/.github/workflows/code-scanning.yaml +++ b/.github/workflows/code-scanning.yaml @@ -65,7 +65,7 @@ jobs: with: go-version-file: go.mod - name: Initialize CodeQL - uses: github/codeql-action/init@54f647b7e1bb85c95cddabcd46b0c578ec92bc1a # v4.36.3 + uses: github/codeql-action/init@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4.37.3 with: languages: ${{ matrix.language }} build-mode: ${{ matrix.build-mode }} @@ -75,6 +75,6 @@ jobs: run: | make build - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@54f647b7e1bb85c95cddabcd46b0c578ec92bc1a # v4.36.3 + uses: github/codeql-action/analyze@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4.37.3 with: category: ${{ matrix.category }} From c196d5d4cd3d859183ac762942206ec390476505 Mon Sep 17 00:00:00 2001 From: APagani Date: Wed, 5 Aug 2026 09:55:00 +0200 Subject: [PATCH 17/20] feat: azure resources id management improvements (#85) --- docs/how-to/030_azure-source.md | 29 ++ docs/mappings/azure/cognitiveaccounts.yaml | 1 + docs/mappings/azure/containerapps.yaml | 1 + docs/mappings/azure/managedclusters.yaml | 3 +- docs/mappings/azure/postgresqldbs.yaml | 1 + docs/mappings/azure/resourcegroups.yaml | 1 + docs/mappings/azure/storageaccounts.yaml | 1 + docs/mappings/azure/subscriptions.yaml | 1 + docs/mappings/azure/virtualmachines.yaml | 1 + docs/mappings/azure/virtualnetworks.yaml | 1 + docs/mappings/azure/websites.yaml | 1 + internal/source/azure/azure.go | 40 +-- .../source/azure/azure_identifier_test.go | 129 ++++++++ internal/source/azure/azure_stream_test.go | 177 +++++++++- internal/source/azure/azure_sync_test.go | 122 ++++++- internal/source/azure/resourceid.go | 73 ++++ internal/source/azure/resourceid_test.go | 311 ++++++++++++++++++ 17 files changed, 850 insertions(+), 43 deletions(-) create mode 100644 internal/source/azure/azure_identifier_test.go create mode 100644 internal/source/azure/resourceid.go create mode 100644 internal/source/azure/resourceid_test.go diff --git a/docs/how-to/030_azure-source.md b/docs/how-to/030_azure-source.md index 82b26b6..15125ce 100644 --- a/docs/how-to/030_azure-source.md +++ b/docs/how-to/030_azure-source.md @@ -55,15 +55,44 @@ Using the `*_CONNECTION_STRING` variables is the preferred methods that will als setup the least privileges to the service account responsible to retrieve information from the REST APIs. +## Resource identifiers + +Microsoft Azure does not guarantee the letter case of the resource IDs it returns: the same +resource can arrive with `resourceGroups` from the resource graph APIs and with `resourcegroups` +from the resource provider that answers the EventHub driven read, and the provider and type +segments vary in the same way. Because the mappings hash the ID to build the Catalog identifier, +every casing difference would create a duplicate item instead of updating the existing one. + +To prevent that the source normalises every resource before handing it to the mapper: + +- `id` is lowercased in full +- `type` is set to the resource type exactly as the mapping file declares it + +The two values are therefore identical for `ibdm sync azure` and `ibdm run azure`, which makes +`{{ .id | sha256sum }}` a stable identifier and lets a delete event target the item a previous +import created. + +This is also in part suggested by Azure, since it is stated that various APIs can return names with different casing, +therefore in order to perform meaningful matches a case-insensitive comparison is recommended. +For a more in-depth explanation refer to [Naming rules and restrictions for Azure resources]. + +### Consequences for the mappings and items + +`id` is lowercase and could no longer match the casing shown in the Azure portal. +Use `.name` wherever available to display casing matters, its availability is dependant on the specific resource APIs. +The spelling Azure reported, if needed, is written to the source logs at the `Debug` level whenever it differs from the normalised value. + ## Authentication The source is using the [`DefaultAzureCredential` chain of authentication] so you can setup your preferred method of login. This authentication will be used for reading data from the REST APIs so it will need the read permissions on the resources you want to import. +Both `sync` and `run` modes use APIs to fetch the full resource, for this reason an authentication method of choice is always needed. If you choose to don’t use the `*_CONNECTION_STING` variables the same authentication will be used to receive data from the configured EventHub and to manage object inside the StorageAccount blob storage. [`DefaultAzureCredential` chain of authentication]: https://learn.microsoft.com/en-gb/azure/developer/go/sdk/authentication/credential-chains#defaultazurecredential-overview +[Naming rules and restrictions for Azure resources]: https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/resource-name-rules diff --git a/docs/mappings/azure/cognitiveaccounts.yaml b/docs/mappings/azure/cognitiveaccounts.yaml index 0ab5e14..7e7bcef 100644 --- a/docs/mappings/azure/cognitiveaccounts.yaml +++ b/docs/mappings/azure/cognitiveaccounts.yaml @@ -10,6 +10,7 @@ mappings: title: "{{ .name }}" spec: name: "{{ .name }}" + id: "{{ .id }}" location: "{{ .location | lower }}" provisioningState: "{{ .properties.provisioningState | lower }}" kind: "{{ .kind }}" diff --git a/docs/mappings/azure/containerapps.yaml b/docs/mappings/azure/containerapps.yaml index 1202e34..d32f6a9 100644 --- a/docs/mappings/azure/containerapps.yaml +++ b/docs/mappings/azure/containerapps.yaml @@ -10,6 +10,7 @@ mappings: title: "{{ .name }}" spec: name: "{{ .name }}" + id: "{{ .id }}" location: "{{ .location | lower }}" provisioningState: "{{ .properties.provisioningState | lower }}" runningStatus: "{{ .properties.runningStatus | lower }}" diff --git a/docs/mappings/azure/managedclusters.yaml b/docs/mappings/azure/managedclusters.yaml index 490ea24..11ef40f 100644 --- a/docs/mappings/azure/managedclusters.yaml +++ b/docs/mappings/azure/managedclusters.yaml @@ -10,6 +10,7 @@ mappings: title: "{{ .name }}" spec: name: "{{ .name }}" + id: "{{ .id }}" location: "{{ .location | lower }}" provisioningState: "{{ .properties.provisioningState | lower }}" currentKubernetesVersion: "{{ .properties.currentKubernetesVersion | quote }}" @@ -34,7 +35,7 @@ mappings: extra: - apiVersion: mia-platform.eu/v1 itemFamily: relationships - deletePolicy: "cascade" + deletePolicy: "none" identifier: |- {{ $src := printf "urn:mia-platform-catalog:console.mia-platform.eu:v1:Cluster:%s" (printf "%s" .properties.fqdn | sha256sum) -}} {{- $type := "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:dependency.mia-platform.eu" -}} diff --git a/docs/mappings/azure/postgresqldbs.yaml b/docs/mappings/azure/postgresqldbs.yaml index 1762c71..dfe8694 100644 --- a/docs/mappings/azure/postgresqldbs.yaml +++ b/docs/mappings/azure/postgresqldbs.yaml @@ -10,6 +10,7 @@ mappings: title: "{{ .name }}" spec: name: "{{ .name }}" + id: "{{ .id }}" location: "{{ .location | lower }}" state: "{{ .properties.state | lower }}" version: "{{ .properties.version }}" diff --git a/docs/mappings/azure/resourcegroups.yaml b/docs/mappings/azure/resourcegroups.yaml index 3e6f103..f9770a0 100644 --- a/docs/mappings/azure/resourcegroups.yaml +++ b/docs/mappings/azure/resourcegroups.yaml @@ -10,6 +10,7 @@ mappings: title: "{{ .name }}" spec: name: "{{ .name }}" + id: "{{ .id }}" location: "{{ .location | lower }}" provisioningState: "{{ .properties.provisioningState | lower }}" tags: |- diff --git a/docs/mappings/azure/storageaccounts.yaml b/docs/mappings/azure/storageaccounts.yaml index 8904d78..1274673 100644 --- a/docs/mappings/azure/storageaccounts.yaml +++ b/docs/mappings/azure/storageaccounts.yaml @@ -10,6 +10,7 @@ mappings: title: "{{ .name }}" spec: name: "{{ .name }}" + id: "{{ .id }}" provisioningState: "{{ .properties.provisioningState | lower }}" allowBlobPublicAccess: "{{ get \"allowBlobPublicAccess\" .properties false }}" publicNetworkAccess: "{{ get \"publicNetworkAccess\" .properties \"Disabled\" | lower }}" diff --git a/docs/mappings/azure/subscriptions.yaml b/docs/mappings/azure/subscriptions.yaml index 02fbd11..25dff2b 100644 --- a/docs/mappings/azure/subscriptions.yaml +++ b/docs/mappings/azure/subscriptions.yaml @@ -10,6 +10,7 @@ mappings: title: "{{ .name }}" spec: name: "{{ .name }}" + id: "{{ .id }}" displayName: "{{ get \"displayName\" . \"\" | quote }}" state: "{{ .properties.state | lower }}" tags: |- diff --git a/docs/mappings/azure/virtualmachines.yaml b/docs/mappings/azure/virtualmachines.yaml index 0e66cf7..cdb5c52 100644 --- a/docs/mappings/azure/virtualmachines.yaml +++ b/docs/mappings/azure/virtualmachines.yaml @@ -10,6 +10,7 @@ mappings: title: "{{ .name }}" spec: name: "{{ .name }}" + id: "{{ .id }}" location: "{{ .location | lower }}" provisioningState: "{{ .properties.provisioningState | lower }}" vmSize: "{{ .properties.hardwareProfile.vmSize }}" diff --git a/docs/mappings/azure/virtualnetworks.yaml b/docs/mappings/azure/virtualnetworks.yaml index 1da91a1..0432590 100644 --- a/docs/mappings/azure/virtualnetworks.yaml +++ b/docs/mappings/azure/virtualnetworks.yaml @@ -10,6 +10,7 @@ mappings: title: "{{ .name }}" spec: name: "{{ .name }}" + id: "{{ .id }}" location: "{{ .location | lower }}" provisioningState: "{{ .properties.provisioningState | lower }}" addressPrefixes: "{{ .properties.addressSpace.addressPrefixes | toJSON }}" diff --git a/docs/mappings/azure/websites.yaml b/docs/mappings/azure/websites.yaml index d9b022b..fcc0597 100644 --- a/docs/mappings/azure/websites.yaml +++ b/docs/mappings/azure/websites.yaml @@ -10,6 +10,7 @@ mappings: title: "{{ .name }}" spec: name: "{{ .name }}" + id: "{{ .id }}" location: "{{ .location | lower }}" state: "{{ .properties.state | lower }}" kind: "{{ .kind }}" diff --git a/internal/source/azure/azure.go b/internal/source/azure/azure.go index 2177c28..7b43d84 100644 --- a/internal/source/azure/azure.go +++ b/internal/source/azure/azure.go @@ -10,7 +10,6 @@ import ( "fmt" "maps" "slices" - "strings" "sync" "sync/atomic" "time" @@ -143,7 +142,7 @@ func (s *Source) StartSyncProcess(ctx context.Context, typesToFilter map[string] if data, ok := response.Data.([]any); ok { for _, item := range data { if values, ok := item.(map[string]any); ok { - values["type"] = resType // ensure type is case-normalized, and resourceGroup is normalized too + normalizeResourceValues(logger, values, resType) dataChannel <- source.Data{ Type: resType, Operation: source.DataOperationUpsert, @@ -224,18 +223,21 @@ func partitionEventHandler(client *armresources.Client, typesToFilter map[string continue } - if filterBasedOnResourceID(resID, typesSlice) { - logger.Debug("skipping event based on type", "resourceID", resID.ResourceType.String()) + // the subject can spell the resource type with any casing, so resolve the configured + // key once and use it for the apiVersion lookup and for every emitted value. + resourceType, ok := configuredResourceType(typesSlice, resID.ResourceType.String()) + if !ok { + logger.Debug("skipping event based on type", "resourceType", resID.ResourceType.String()) continue } - apiVersion, ok := typesToFilter[resID.ResourceType.String()]["apiVersion"].(string) + apiVersion, ok := typesToFilter[resourceType][apiVersionKey].(string) if !ok { - logger.Debug("skipping event with missing apiVersion", "resourceID", resID.ResourceType.String()) + logger.Debug("skipping event with missing apiVersion", "resourceType", resourceType) continue } - logger.Trace("handling resource", "resourceID", resID.ResourceType.String(), "eventType", envelope.Type, "apiVersion", apiVersion) + logger.Trace("handling resource", "resourceType", resourceType, "eventType", envelope.Type, "apiVersion", apiVersion) switch envelope.Type { case azsystemevents.TypeResourceWriteSuccess: logger.Trace("request resource data from azure", "resourceID", *envelope.Subject) @@ -255,25 +257,25 @@ func partitionEventHandler(client *armresources.Client, typesToFilter map[string continue } + normalizeResourceValues(logger, values, resourceType) dataChannel <- source.Data{ - Type: resID.ResourceType.String(), + Type: resourceType, Operation: source.DataOperationUpsert, Time: *envelope.Time, Values: values, } case azsystemevents.TypeResourceDeleteSuccess: - logger.Trace("we have to delete something", "resourceID", resID.ResourceType.String()) + logger.Trace("deleting resource", "resourceType", resourceType) + values := map[string]any{idKey: resID.String()} + normalizeResourceValues(logger, values, resourceType) dataChannel <- source.Data{ - Type: resID.ResourceType.String(), + Type: resourceType, Operation: source.DataOperationDelete, Time: *envelope.Time, - Values: map[string]any{ - "id": resID.String(), - "type": resID.ResourceType.String(), - }, + Values: values, } default: - logger.Trace("skipping resource", "resourceID", resID.ResourceType.String(), "eventType", envelope.Type, "apiVersion", apiVersion) + logger.Trace("skipping resource", "resourceType", resourceType, "eventType", envelope.Type, "apiVersion", apiVersion) } } } @@ -303,14 +305,6 @@ func resourceIDFromSubject(subject *string) (*arm.ResourceID, error) { return arm.ParseResourceID(*subject) } -// filterBasedOnResourceID checks if the resource type is in the typesToFilter slice. -func filterBasedOnResourceID(resID *arm.ResourceID, typesToFilter []string) bool { - resourceType := resID.ResourceType.String() - return !slices.ContainsFunc(typesToFilter, func(s string) bool { - return strings.EqualFold(s, resourceType) - }) -} - // Close implement source.ClosableSource. func (s *Source) Close(ctx context.Context, _ time.Duration) error { log := logger.FromContext(ctx).WithName(logName) diff --git a/internal/source/azure/azure_identifier_test.go b/internal/source/azure/azure_identifier_test.go new file mode 100644 index 0000000..601f8f6 --- /dev/null +++ b/internal/source/azure/azure_identifier_test.go @@ -0,0 +1,129 @@ +// Copyright Mia srl +// SPDX-License-Identifier: AGPL-3.0-only or Commercial + +package azure + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + fakeazcore "github.com/Azure/azure-sdk-for-go/sdk/azcore/fake" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs/v2" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources/v3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/mia-platform/ibdm/internal/source" +) + +const testSubscriptionID = "00000000-0000-0000-0000-000000000000" + +// eventDataManagedClusterCanonicalDeleteBody deletes the same managed cluster the sync and the +// stream upsert paths import. Its subject uses a lowercase resourcegroups literal that +// arm.ParseResourceID rewrites to camelCase, so the delete path builds subjectManagedClusterID. +var eventDataManagedClusterCanonicalDeleteBody = json.RawMessage(`[ +{ + "id": "00000000-0000-0000-0000-000000000000", + "source": "/subscriptions/00000000-0000-0000-0000-000000000000", + "specversion": "1.0", + "type": "Microsoft.Resources.ResourceDeleteSuccess", + "subject": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/my-rg/providers/Microsoft.ContainerService/managedClusters/my-cluster", + "time": "2020-01-01T00:00:00.0000000Z" +}]`) + +// TestAllPathsEmitTheSameIdentifier drives the three ingestion paths for one managed cluster and +// checks that they agree on the values the mappings hash into the Catalog identifier. Azure feeds +// each path a different spelling of the ID, so before the normalization the sync upsert and the +// stream upsert produced two Catalog items and the stream delete targeted neither of them. +func TestAllPathsEmitTheSameIdentifier(t *testing.T) { + t.Parallel() + + // the three spellings the fakes reproduce: Resource Graph answers with a camelCase + // resourceGroups literal, the resource provider body lowercases it, and the delete path rebuilds + // the ID from the event subject. Without this divergence the test would be vacuous. + require.NotEqual(t, graphManagedClusterID, bodyManagedClusterID, + "the sync and stream fixtures must disagree on casing") + require.Equal(t, graphManagedClusterID, subjectManagedClusterID, + "the delete path is expected to rebuild the Resource Graph spelling") + + paths := map[string]source.Data{ + "sync upsert": syncedManagedCluster(t), + "stream upsert": streamedManagedCluster(t, eventDataManagedClusterWriteBody), + "stream delete": streamedManagedCluster(t, eventDataManagedClusterCanonicalDeleteBody), + } + + assert.Equal(t, source.DataOperationUpsert, paths["sync upsert"].Operation) + assert.Equal(t, source.DataOperationUpsert, paths["stream upsert"].Operation) + assert.Equal(t, source.DataOperationDelete, paths["stream delete"].Operation) + + for pathName, data := range paths { + assert.Equal(t, normalizedManagedClusterID, data.Values[idKey], pathName) + assert.Equal(t, managedClustersType, data.Values[typeKey], pathName) + assert.Equal(t, managedClustersType, data.Type, pathName) + } +} + +// syncedManagedCluster runs a sync process against the Resource Graph fake and returns the data +// emitted for the my-cluster managed cluster. +func syncedManagedCluster(t *testing.T) source.Data { + t.Helper() + + ctx, cancel := context.WithTimeout(t.Context(), 1*time.Second) + t.Cleanup(cancel) + + azureSource := &Source{ + config: config{ + SubscriptionID: testSubscriptionID, + clientOptions: &arm.ClientOptions{ + ClientOptions: policy.ClientOptions{ + Transport: fakeResourceGraphTransport(t), + }, + }, + azureCredentials: &fakeazcore.TokenCredential{}, + }, + } + + dataChannel := make(chan source.Data, 10) + require.NoError(t, azureSource.StartSyncProcess(ctx, map[string]source.Extra{managedClustersType: nil}, dataChannel)) + close(dataChannel) + + for data := range dataChannel { + if data.Values["name"] == "my-cluster" { + return data + } + } + + require.FailNow(t, "the sync process did not emit the managed cluster") + return source.Data{} +} + +// streamedManagedCluster feeds body to the event handler and returns the single emitted data. +func streamedManagedCluster(t *testing.T, body json.RawMessage) source.Data { + t.Helper() + + ctx, cancel := context.WithTimeout(t.Context(), 1*time.Second) + t.Cleanup(cancel) + + client, err := armresources.NewClient(testSubscriptionID, &fakeazcore.TokenCredential{}, &arm.ClientOptions{ + ClientOptions: policy.ClientOptions{ + Transport: fakeClientTransport(t), + }, + }) + require.NoError(t, err) + + dataChannel := make(chan source.Data, 10) + handler := partitionEventHandler(client, map[string]source.Extra{ + managedClustersType: {apiVersionKey: managedClustersAPIVersion}, + }, dataChannel) + + handler(ctx, &azeventhubs.ReceivedEventData{EventData: azeventhubs.EventData{Body: body}}) + close(dataChannel) + + data, ok := <-dataChannel + require.True(t, ok, "the event handler did not emit any data") + return data +} diff --git a/internal/source/azure/azure_stream_test.go b/internal/source/azure/azure_stream_test.go index 33e0eba..6cdbd37 100644 --- a/internal/source/azure/azure_stream_test.go +++ b/internal/source/azure/azure_stream_test.go @@ -132,6 +132,81 @@ func TestPartitionEventHandler(t *testing.T) { }, }, }, + "resource provider returning a divergently cased id": { + contextFunc: func(tb testing.TB) (context.Context, context.CancelFunc) { + tb.Helper() + return context.WithTimeout(tb.Context(), 1*time.Second) + }, + typesToFilter: map[string]source.Extra{ + managedClustersType: {"apiVersion": managedClustersAPIVersion}, + }, + azureData: &azeventhubs.ReceivedEventData{ + EventData: azeventhubs.EventData{ + Body: eventDataManagedClusterWriteBody, + }, + }, + expectedData: []source.Data{ + { + Type: managedClustersType, + Operation: source.DataOperationUpsert, + Time: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC), + Values: map[string]any{ + "id": normalizedManagedClusterID, + "type": managedClustersType, + }, + }, + }, + }, + "non canonical subject type is not dropped": { + contextFunc: func(tb testing.TB) (context.Context, context.CancelFunc) { + tb.Helper() + return context.WithTimeout(tb.Context(), 1*time.Second) + }, + typesToFilter: map[string]source.Extra{ + managedClustersType: {"apiVersion": managedClustersAPIVersion}, + }, + azureData: &azeventhubs.ReceivedEventData{ + EventData: azeventhubs.EventData{ + Body: eventDataLowerTypeManagedClusterWriteBody, + }, + }, + expectedData: []source.Data{ + { + Type: managedClustersType, + Operation: source.DataOperationUpsert, + Time: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC), + Values: map[string]any{ + "id": normalizedManagedClusterID, + "type": managedClustersType, + }, + }, + }, + }, + "delete with non canonical subject casing": { + contextFunc: func(tb testing.TB) (context.Context, context.CancelFunc) { + tb.Helper() + return context.WithTimeout(tb.Context(), 1*time.Second) + }, + typesToFilter: map[string]source.Extra{ + managedClustersType: {"apiVersion": managedClustersAPIVersion}, + }, + azureData: &azeventhubs.ReceivedEventData{ + EventData: azeventhubs.EventData{ + Body: eventDataManagedClusterDeleteBody, + }, + }, + expectedData: []source.Data{ + { + Type: managedClustersType, + Operation: source.DataOperationDelete, + Time: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC), + Values: map[string]any{ + "id": normalizedManagedClusterID, + "type": managedClustersType, + }, + }, + }, + }, } for testName, test := range testCases { @@ -159,6 +234,9 @@ func TestPartitionEventHandler(t *testing.T) { } require.NotErrorIs(t, ctx.Err(), context.DeadlineExceeded) + // a fake branch keyed on the wrong resource ID silently emits nothing, so assert the + // received count before comparing the elements + require.Len(t, receivedData, len(test.expectedData)) assert.ElementsMatch(t, test.expectedData, receivedData) }) } @@ -185,31 +263,98 @@ func handleResourcesGetByIDRequest(tb testing.TB, resourceID, apiVersion string) tb.Helper() switch resourceID { - case "subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myRG": + case "subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/my-rg": assert.Equal(tb, "2021-04-01", apiVersion) resp = &armresources.ClientGetByIDResponse{ GenericResource: armresources.GenericResource{ - ID: to.Ptr("/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myRG"), + ID: to.Ptr("/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/my-rg"), Type: to.Ptr("Microsoft.Resources/resourceGroups"), }, } return resp, nil - case "subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myRG/providers/Microsoft.Compute/virtualMachines/myVM": + case "subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/my-rg/providers/Microsoft.Compute/virtualMachines/my-vm": assert.Equal(tb, "2021-07-01", apiVersion) return nil, assert.AnError + case managedClusterGetByIDPath: + assert.Equal(tb, managedClustersAPIVersion, apiVersion) + // Microsoft.ContainerService builds its own id form and answers with a lowercase + // resourcegroups literal even though the request carried the camelCase one + resp = &armresources.ClientGetByIDResponse{ + GenericResource: armresources.GenericResource{ + ID: to.Ptr(bodyManagedClusterID), + Type: to.Ptr("microsoft.containerservice/managedclusters"), + }, + } + return resp, nil + case lowerTypeManagedClusterGetByIDPath: + assert.Equal(tb, managedClustersAPIVersion, apiVersion) + resp = &armresources.ClientGetByIDResponse{ + GenericResource: armresources.GenericResource{ + ID: to.Ptr("/" + lowerTypeManagedClusterGetByIDPath), + Type: to.Ptr("microsoft.containerservice/managedclusters"), + }, + } + return resp, nil } return nil, nil } +const ( + managedClustersAPIVersion = "2025-10-01" + + // armresources.Client strips the leading slash before calling the server, and + // arm.ParseResourceID rewrites the resourcegroups literal of the subject to camelCase while + // leaving the provider and type segments at the casing the subject used, so these are the keys + // the fake actually receives. + managedClusterGetByIDPath = "subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/my-rg/providers/Microsoft.ContainerService/managedClusters/my-cluster" + lowerTypeManagedClusterGetByIDPath = "subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/my-rg/providers/microsoft.containerservice/managedclusters/my-cluster" +) + var ( + // eventDataManagedClusterWriteBody carries a lowercase resourcegroups literal and a canonically + // cased provider and type. + eventDataManagedClusterWriteBody = json.RawMessage(`[ + { + "id": "00000000-0000-0000-0000-000000000000", + "source": "/subscriptions/00000000-0000-0000-0000-000000000000", + "specversion": "1.0", + "type": "Microsoft.Resources.ResourceWriteSuccess", + "subject": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/my-rg/providers/Microsoft.ContainerService/managedClusters/my-cluster", + "time": "2020-01-01T00:00:00.0000000Z" + }]`) + + // eventDataLowerTypeManagedClusterWriteBody spells the provider and the type in lowercase, a + // casing the configured key does not use. + eventDataLowerTypeManagedClusterWriteBody = json.RawMessage(`[ + { + "id": "00000000-0000-0000-0000-000000000000", + "source": "/subscriptions/00000000-0000-0000-0000-000000000000", + "specversion": "1.0", + "type": "Microsoft.Resources.ResourceWriteSuccess", + "subject": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/my-rg/providers/microsoft.containerservice/managedclusters/my-cluster", + "time": "2020-01-01T00:00:00.0000000Z" + }]`) + + // eventDataManagedClusterDeleteBody spells the resource group literal, the provider and the + // type in lowercase. + eventDataManagedClusterDeleteBody = json.RawMessage(`[ + { + "id": "00000000-0000-0000-0000-000000000000", + "source": "/subscriptions/00000000-0000-0000-0000-000000000000", + "specversion": "1.0", + "type": "Microsoft.Resources.ResourceDeleteSuccess", + "subject": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/my-rg/providers/microsoft.containerservice/managedclusters/my-cluster", + "time": "2020-01-01T00:00:00.0000000Z" + }]`) + eventDataResourcesTestBody = json.RawMessage(`[ { "id": "00000000-0000-0000-0000-000000000000", "source": "/subscriptions/00000000-0000-0000-0000-000000000000", "specversion": "1.0", "type": "Microsoft.Resources.ResourceDeleteSuccess", - "subject": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/myRG", + "subject": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/my-rg", "time": "2020-01-01T00:00:00.0000000Z", "data": { "authorization": {}, @@ -217,7 +362,7 @@ var ( "correlationId": "00000000-0000-0000-0000-000000000000", "httpRequest": {}, "resourceProvider": "Microsoft.Resources", - "resourceUri": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/myRG", + "resourceUri": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/my-rg", "operationName": "Microsoft.Resources/subscriptions/resourcegroups/delete", "status": "Succeeded", "subscriptionId": "00000000-0000-0000-0000-000000000000", @@ -229,7 +374,7 @@ var ( "source": "/subscriptions/00000000-0000-0000-0000-000000000000", "specversion": "1.0", "type": "Microsoft.Resources.ResourceWriteSuccess", - "subject": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/myRG", + "subject": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/my-rg", "time": "2020-01-01T00:00:00.0000000Z", "data": { "authorization": {}, @@ -237,7 +382,7 @@ var ( "correlationId": "00000000-0000-0000-0000-000000000000", "httpRequest": {}, "resourceProvider": "Microsoft.Resources", - "resourceUri": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/myRG", + "resourceUri": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/my-rg", "operationName": "Microsoft.Resources/subscriptions/resourceGroups/write", "status": "Succeeded", "subscriptionId": "00000000-0000-0000-0000-000000000000", @@ -249,7 +394,7 @@ var ( "source": "/subscriptions/00000000-0000-0000-0000-000000000000", "specversion": "1.0", "type": "Microsoft.Resources.ResourceWriteCancel", - "subject": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/myRG", + "subject": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/my-rg", "time": "2020-01-01T00:00:00.0000000Z", "data": { "authorization": {}, @@ -257,7 +402,7 @@ var ( "correlationId": "00000000-0000-0000-0000-000000000000", "httpRequest": {}, "resourceProvider": "Microsoft.Resources", - "resourceUri": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/myRG", + "resourceUri": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/my-rg", "operationName": "Microsoft.Resources/subscriptions/resourceGroups/write", "status": "Canceled", "subscriptionId": "00000000-0000-0000-0000-000000000000", @@ -269,7 +414,7 @@ var ( "source": "/subscriptions/00000000-0000-0000-0000-000000000000", "specversion": "1.0", "type": "Microsoft.Resources.ResourceWriteSuccess", - "subject": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myRG/providers/Microsoft.Compute/virtualMachines/myVM", + "subject": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/my-rg/providers/Microsoft.Compute/virtualMachines/my-vm", "time": "2020-01-01T00:00:00.0000000Z", "data": { "authorization": {}, @@ -277,7 +422,7 @@ var ( "correlationId": "00000000-0000-0000-0000-000000000000", "httpRequest": {}, "resourceProvider": "Microsoft.Resources", - "resourceUri": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myRG/providers/Microsoft.Compute/virtualMachines/myVM", + "resourceUri": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/my-rg/providers/Microsoft.Compute/virtualMachines/my-vm", "operationName": "Microsoft.Resources/tags/write", "status": "Succeeded", "subscriptionId": "00000000-0000-0000-0000-000000000000", @@ -289,7 +434,7 @@ var ( "source": "/subscriptions/00000000-0000-0000-0000-000000000000", "specversion": "1.0", "type": "Microsoft.Resources.ResourceWriteSuccess", - "subject": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/myRG/providers/Microsoft.Storage/storageAccounts/account", + "subject": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/my-rg/providers/Microsoft.Storage/storageAccounts/account", "time": "2020-01-01T00:00:00.0000000Z", "data": { "authorization": {}, @@ -297,7 +442,7 @@ var ( "correlationId": "00000000-0000-0000-0000-000000000000", "httpRequest": {}, "resourceProvider": "Microsoft.Resources", - "resourceUri": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/myRG/providers/Microsoft.Storage/storageAccounts/account", + "resourceUri": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/my-rg/providers/Microsoft.Storage/storageAccounts/account", "operationName": "Microsoft.Resources/tags/write", "status": "Succeeded", "subscriptionId": "00000000-0000-0000-0000-000000000000", @@ -311,7 +456,8 @@ var ( Operation: source.DataOperationDelete, Time: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC), Values: map[string]any{ - "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myRG", + // the source lowercases every id so that all the ingestion paths converge + "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/my-rg", "type": "Microsoft.Resources/resourceGroups", }, }, @@ -320,7 +466,8 @@ var ( Operation: source.DataOperationUpsert, Time: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC), Values: map[string]any{ - "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myRG", + // the source lowercases every id so that all the ingestion paths converge + "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/my-rg", "type": "Microsoft.Resources/resourceGroups", }, }, diff --git a/internal/source/azure/azure_sync_test.go b/internal/source/azure/azure_sync_test.go index f8b9450..c481230 100644 --- a/internal/source/azure/azure_sync_test.go +++ b/internal/source/azure/azure_sync_test.go @@ -108,7 +108,7 @@ func TestStartSyncProcess(t *testing.T) { Time: testTime, Values: map[string]any{ "extendedLocation": nil, - "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/name", + "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/name", "identity": nil, "kind": "", "location": "region", @@ -142,7 +142,7 @@ func TestStartSyncProcess(t *testing.T) { Time: testTime, Values: map[string]any{ "extendedLocation": nil, - "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/name", + "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/name", "identity": nil, "kind": "", "location": "region", @@ -163,7 +163,7 @@ func TestStartSyncProcess(t *testing.T) { Time: testTime, Values: map[string]any{ "extendedLocation": nil, - "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/name/providers/Microsoft.Compute/virtualMachines/vm-name", + "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/name/providers/microsoft.compute/virtualmachines/vm-name", "identity": nil, "kind": "", "location": "northeurope", @@ -193,7 +193,7 @@ func TestStartSyncProcess(t *testing.T) { Time: testTime, Values: map[string]any{ "extendedLocation": nil, - "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/name/providers/Microsoft.Compute/virtualMachines/vm-name2", + "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/name/providers/microsoft.compute/virtualmachines/vm-name2", "identity": nil, "kind": "", "location": "northeurope", @@ -219,6 +219,69 @@ func TestStartSyncProcess(t *testing.T) { }, }, }, + "managed clusters with divergent casing": { + typesToFilter: map[string]source.Extra{ + managedClustersType: nil, + }, + expectedData: []source.Data{ + { + Type: managedClustersType, + Operation: source.DataOperationUpsert, + Time: testTime, + Values: map[string]any{ + "id": normalizedManagedClusterID, + "name": "my-cluster", + "type": managedClustersType, + }, + }, + { + Type: managedClustersType, + Operation: source.DataOperationUpsert, + Time: testTime, + Values: map[string]any{ + "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/my-rg/providers/microsoft.containerservice/managedclusters/my-other-cluster", + "name": "my-other-cluster", + "type": managedClustersType, + }, + }, + }, + }, + "unusable ids are emitted unchanged": { + typesToFilter: map[string]source.Extra{ + "Microsoft.Resources/malformedResources": nil, + }, + expectedData: []source.Data{ + { + Type: "Microsoft.Resources/malformedResources", + Operation: source.DataOperationUpsert, + Time: testTime, + Values: map[string]any{ + "id": "not-an-id", + "name": "no-id-resource", + "type": "Microsoft.Resources/malformedResources", + }, + }, + { + Type: "Microsoft.Resources/malformedResources", + Operation: source.DataOperationUpsert, + Time: testTime, + Values: map[string]any{ + "id": float64(42), + "name": "numeric-id-resource", + "type": "Microsoft.Resources/malformedResources", + }, + }, + { + Type: "Microsoft.Resources/malformedResources", + Operation: source.DataOperationUpsert, + Time: testTime, + Values: map[string]any{ + "name": "missing-id-resource", + "type": "Microsoft.Resources/malformedResources", + }, + }, + }, + }, "error during request": { typesToFilter: map[string]source.Extra{ "Microsoft.Resources/errorResources": nil, @@ -417,6 +480,26 @@ func handleResourceGraphQueryRequest(t *testing.T, query armresourcegraph.QueryR SkipToken: to.Ptr("skip-token-1"), }, }, nil + case fmt.Sprintf(resourceGraphQueryTemplate, managedClustersType): + return &armresourcegraph.ClientResourcesResponse{ + QueryResponse: armresourcegraph.QueryResponse{ + TotalRecords: to.Ptr(int64(2)), + Data: resourceGraphManagedClustersResponse, + ResultTruncated: to.Ptr(armresourcegraph.ResultTruncatedFalse), + Count: to.Ptr(int64(2)), + SkipToken: nil, + }, + }, nil + case fmt.Sprintf(resourceGraphQueryTemplate, "Microsoft.Resources/malformedResources"): + return &armresourcegraph.ClientResourcesResponse{ + QueryResponse: armresourcegraph.QueryResponse{ + TotalRecords: to.Ptr(int64(3)), + Data: resourceGraphMalformedResponse, + ResultTruncated: to.Ptr(armresourcegraph.ResultTruncatedFalse), + Count: to.Ptr(int64(3)), + SkipToken: nil, + }, + }, nil case fmt.Sprintf(resourceGraphQueryTemplate, "Microsoft.Resources/errorResources"): return nil, assert.AnError } @@ -425,6 +508,37 @@ func handleResourceGraphQueryRequest(t *testing.T, query armresourcegraph.QueryR } var ( + // resourceGraphManagedClustersResponse mixes the camelCase resourceGroups literal Resource Graph + // really returns with an entirely lowercase row, because the normalization must be case blind. + resourceGraphManagedClustersResponse = []any{ + map[string]any{ + "id": graphManagedClusterID, + "name": "my-cluster", + "type": "microsoft.containerservice/managedclusters", + }, + map[string]any{ + "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/my-rg/providers/microsoft.containerservice/managedclusters/my-other-cluster", + "name": "my-other-cluster", + "type": "microsoft.containerservice/managedclusters", + }, + } + + // resourceGraphMalformedResponse holds rows whose id cannot be normalized: the items must still + // be emitted with their original value. + resourceGraphMalformedResponse = []any{ + map[string]any{ + "id": "not-an-id", + "name": "no-id-resource", + }, + map[string]any{ + "id": 42, + "name": "numeric-id-resource", + }, + map[string]any{ + "name": "missing-id-resource", + }, + } + resourceGraphResourceGroupsResponse = []any{ map[string]any{ "extendedLocation": nil, diff --git a/internal/source/azure/resourceid.go b/internal/source/azure/resourceid.go new file mode 100644 index 0000000..8a81708 --- /dev/null +++ b/internal/source/azure/resourceid.go @@ -0,0 +1,73 @@ +// Copyright Mia srl +// SPDX-License-Identifier: AGPL-3.0-only or Commercial + +package azure + +import ( + "slices" + "strings" + + "github.com/mia-platform/ibdm/internal/logger" +) + +const ( + // idKey is the payload key holding the Azure resource ID. + idKey = "id" + // typeKey is the payload key holding the Azure resource type. + typeKey = "type" + // apiVersionKey is the mapping extra key holding the api-version to use for the resource type. + apiVersionKey = "apiVersion" +) + +// normalizeResourceValues rewrites the id and type entries of a raw Azure resource payload so +// that the same resource always yields the same values, whichever Azure API produced it. +// resourceType is the configured type key and is applied verbatim. +// +// Azure does not guarantee the letter case of resource IDs: different resource providers and +// different APIs return the same ID with different casing, and the mappings hash the ID to build +// the Catalog identifier, so any casing difference creates a duplicate item. Lowercasing the +// whole ID collapses every casing variant onto one value. +// +// Lowercasing is safe only while every mapped resource type has case-insensitive names, which +// Azure documents for all the types currently mapped. Before mapping a type whose names are +// case-sensitive, such as blob containers, review docs/how-to/030_azure-source.md: folding the +// case of a case-sensitive name would make two distinct resources share one Catalog identifier. +// +// The casing Azure reported is logged when it differs from the normalized value, so that the +// resource providers returning non canonical IDs stay observable. It is deliberately not added to +// the payload: mappings must always build identifiers from id. +// +// A payload whose id is missing or not a string keeps its original value so that no item is ever +// dropped or corrupted. +func normalizeResourceValues(log logger.Logger, values map[string]any, resourceType string) { + values[typeKey] = resourceType + + rawID, ok := values[idKey].(string) + if !ok || rawID == "" { + log.Warn("azure resource payload without a usable id, identifier may be unstable", + "type", resourceType) + return + } + + normalizedID := strings.ToLower(rawID) + if normalizedID != rawID { + log.Debug("azure returned a resource id with non canonical casing", + "type", resourceType, "id", normalizedID, "azureId", rawID) + } + + values[idKey] = normalizedID +} + +// configuredResourceType returns the configured type key matching resourceType ignoring case, +// reporting whether one was found. sortedTypes must be sorted so that the result is +// deterministic when two configured keys differ only by case. +func configuredResourceType(sortedTypes []string, resourceType string) (string, bool) { + idx := slices.IndexFunc(sortedTypes, func(s string) bool { + return strings.EqualFold(s, resourceType) + }) + if idx < 0 { + return "", false + } + + return sortedTypes[idx], true +} diff --git a/internal/source/azure/resourceid_test.go b/internal/source/azure/resourceid_test.go new file mode 100644 index 0000000..71c34a2 --- /dev/null +++ b/internal/source/azure/resourceid_test.go @@ -0,0 +1,311 @@ +// Copyright Mia srl +// SPDX-License-Identifier: AGPL-3.0-only or Commercial + +package azure + +import ( + "bytes" + "maps" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/mia-platform/ibdm/internal/logger" +) + +const ( + managedClustersType = "Microsoft.ContainerService/managedClusters" + + // The same managed cluster as Azure spells it on the three ingestion paths: Resource Graph and + // the event subject use camelCase resourceGroups, while the resource provider response body + // lowercases it. + graphManagedClusterID = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/my-rg/providers/Microsoft.ContainerService/managedClusters/my-cluster" + bodyManagedClusterID = "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/my-rg/providers/Microsoft.ContainerService/managedClusters/my-cluster" + subjectManagedClusterID = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/my-rg/providers/Microsoft.ContainerService/managedClusters/my-cluster" + + // normalizedManagedClusterID is what every path must converge to. + normalizedManagedClusterID = "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/my-rg/providers/microsoft.containerservice/managedclusters/my-cluster" +) + +func TestNormalizeResourceValues(t *testing.T) { + t.Parallel() + + testCases := map[string]struct { + values map[string]any + resourceType string + expectedValues map[string]any + }{ + "mixed case id is lowered and type is forced to the configured key": { + values: map[string]any{ + "id": graphManagedClusterID, + "type": "microsoft.containerservice/managedclusters", + "name": "my-cluster", + }, + resourceType: managedClustersType, + expectedValues: map[string]any{ + "id": normalizedManagedClusterID, + "type": managedClustersType, + "name": "my-cluster", + }, + }, + "already lowercase id is left unchanged": { + values: map[string]any{ + "id": normalizedManagedClusterID, + }, + resourceType: managedClustersType, + expectedValues: map[string]any{ + "id": normalizedManagedClusterID, + "type": managedClustersType, + }, + }, + "shouty id is fully lowered": { + values: map[string]any{ + "id": "/SUBSCRIPTIONS/00000000-0000-0000-0000-000000000000/RESOURCEGROUPS/MY-RG/PROVIDERS/MICROSOFT.WEB/SITES/MY-SITE", + }, + resourceType: "Microsoft.Web/sites", + expectedValues: map[string]any{ + "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/my-rg/providers/microsoft.web/sites/my-site", + "type": "Microsoft.Web/sites", + }, + }, + "malformed id is lowered without being parsed": { + values: map[string]any{ + "id": "Not-An-Azure-Id", + }, + resourceType: managedClustersType, + expectedValues: map[string]any{ + "id": "not-an-azure-id", + "type": managedClustersType, + }, + }, + "missing id leaves the other values untouched": { + values: map[string]any{ + "name": "my-rg", + "location": "westeurope", + }, + resourceType: "Microsoft.Resources/resourceGroups", + expectedValues: map[string]any{ + "name": "my-rg", + "location": "westeurope", + "type": "Microsoft.Resources/resourceGroups", + }, + }, + "non string id is preserved": { + values: map[string]any{ + "id": 42, + }, + resourceType: managedClustersType, + expectedValues: map[string]any{ + "id": 42, + "type": managedClustersType, + }, + }, + "nil id is preserved": { + values: map[string]any{ + "id": nil, + }, + resourceType: managedClustersType, + expectedValues: map[string]any{ + "id": nil, + "type": managedClustersType, + }, + }, + "empty id is preserved": { + values: map[string]any{ + "id": "", + }, + resourceType: managedClustersType, + expectedValues: map[string]any{ + "id": "", + "type": managedClustersType, + }, + }, + "unrelated keys are never touched": { + values: map[string]any{ + "id": graphManagedClusterID, + "name": "my-cluster", + "properties": map[string]any{ + "provisioningState": "Succeeded", + "fqdn": "my-cluster.example.com", + }, + "tags": map[string]any{"Env": "Prod"}, + }, + resourceType: managedClustersType, + expectedValues: map[string]any{ + "id": normalizedManagedClusterID, + "name": "my-cluster", + "properties": map[string]any{ + "provisioningState": "Succeeded", + "fqdn": "my-cluster.example.com", + }, + "tags": map[string]any{"Env": "Prod"}, + "type": managedClustersType, + }, + }, + } + + for testName, test := range testCases { + t.Run(testName, func(t *testing.T) { + t.Parallel() + + values := maps.Clone(test.values) + normalizeResourceValues(nullTestLogger(t), values, test.resourceType) + + // comparing the whole map also guards against the helper adding any field beyond id + // and type: a payload visible raw id would silently reinstate the duplicate item bug. + require.Equal(t, test.expectedValues, values) + + rawID, ok := test.values[idKey].(string) + if !ok || rawID == "" { + return + } + + normalizedID, ok := values[idKey].(string) + require.True(t, ok) + require.True(t, strings.EqualFold(rawID, normalizedID), "normalization must only change letter case") + }) + } +} + +func TestNormalizeResourceValuesConvergesDivergentCasings(t *testing.T) { + t.Parallel() + + graphValues := map[string]any{idKey: graphManagedClusterID} + bodyValues := map[string]any{idKey: bodyManagedClusterID} + subjectValues := map[string]any{idKey: subjectManagedClusterID} + + require.NotEqual(t, graphValues[idKey], bodyValues[idKey], "the fixtures must differ before normalization") + + for _, values := range []map[string]any{graphValues, bodyValues, subjectValues} { + normalizeResourceValues(nullTestLogger(t), values, managedClustersType) + } + + assert.Equal(t, normalizedManagedClusterID, graphValues[idKey]) + assert.Equal(t, normalizedManagedClusterID, bodyValues[idKey]) + assert.Equal(t, normalizedManagedClusterID, subjectValues[idKey]) +} + +func TestNormalizeResourceValuesIsIdempotent(t *testing.T) { + t.Parallel() + + values := map[string]any{ + idKey: graphManagedClusterID, + "name": "my-cluster", + } + + normalizeResourceValues(nullTestLogger(t), values, managedClustersType) + once := maps.Clone(values) + + normalizeResourceValues(nullTestLogger(t), values, managedClustersType) + assert.Equal(t, once, values) +} + +func TestNormalizeResourceValuesLogging(t *testing.T) { + t.Parallel() + + testCases := map[string]struct { + values map[string]any + expectedMessages []string + absentMessages []string + }{ + "divergent casing is reported": { + values: map[string]any{idKey: bodyManagedClusterID}, + expectedMessages: []string{"non canonical casing", bodyManagedClusterID}, + }, + "canonical casing stays silent": { + values: map[string]any{idKey: normalizedManagedClusterID}, + absentMessages: []string{"non canonical casing"}, + }, + "unusable id is reported": { + values: map[string]any{"name": "my-cluster"}, + expectedMessages: []string{"without a usable id"}, + absentMessages: []string{"non canonical casing"}, + }, + } + + for testName, test := range testCases { + t.Run(testName, func(t *testing.T) { + t.Parallel() + + buffer := new(bytes.Buffer) + log := logger.NewLogger(buffer) + log.SetLevel(logger.TRACE) + + normalizeResourceValues(log, maps.Clone(test.values), managedClustersType) + + for _, message := range test.expectedMessages { + assert.Contains(t, buffer.String(), message) + } + for _, message := range test.absentMessages { + assert.NotContains(t, buffer.String(), message) + } + }) + } +} + +func TestConfiguredResourceType(t *testing.T) { + t.Parallel() + + configuredTypes := []string{"Microsoft.ContainerService/managedClusters", "Microsoft.Web/sites"} + + testCases := map[string]struct { + sortedTypes []string + resourceType string + expectedType string + expectedFound bool + }{ + "exact match": { + sortedTypes: configuredTypes, + resourceType: managedClustersType, + expectedType: managedClustersType, + expectedFound: true, + }, + "lowercase match returns the configured key": { + sortedTypes: configuredTypes, + resourceType: "microsoft.containerservice/managedclusters", + expectedType: managedClustersType, + expectedFound: true, + }, + "shouty match returns the configured key": { + sortedTypes: configuredTypes, + resourceType: "MICROSOFT.WEB/SITES", + expectedType: "Microsoft.Web/sites", + expectedFound: true, + }, + "unconfigured type is not found": { + sortedTypes: configuredTypes, + resourceType: "Microsoft.Compute/virtualMachines", + }, + "nil slice is not found": { + resourceType: managedClustersType, + }, + "empty slice is not found": { + sortedTypes: []string{}, + resourceType: managedClustersType, + }, + "keys differing only by case resolve deterministically": { + sortedTypes: []string{"Microsoft.Web/sites", "microsoft.web/sites"}, + resourceType: "MICROSOFT.WEB/SITES", + expectedType: "Microsoft.Web/sites", + expectedFound: true, + }, + } + + for testName, test := range testCases { + t.Run(testName, func(t *testing.T) { + t.Parallel() + + resourceType, found := configuredResourceType(test.sortedTypes, test.resourceType) + assert.Equal(t, test.expectedFound, found) + assert.Equal(t, test.expectedType, resourceType) + }) + } +} + +// nullTestLogger returns a logger discarding every entry. +func nullTestLogger(tb testing.TB) logger.Logger { + tb.Helper() + return logger.FromContext(tb.Context()) +} From 4ca70f2f4f08d9e5e0109fc1a286904125d24c63 Mon Sep 17 00:00:00 2001 From: APagani Date: Mon, 10 Aug 2026 13:05:54 +0200 Subject: [PATCH 18/20] feat: azure resources sub types management and first customization (#86) --- docs/how-to/030_azure-source.md | 88 ++++ .../mappings/azure/websites_functionapps.yaml | 43 ++ internal/source/azure/azure.go | 130 +++--- internal/source/azure/azure_stream_test.go | 205 +++++++++ internal/source/azure/azure_sync_test.go | 133 ++++++ internal/source/azure/helper_test.go | 24 ++ internal/source/azure/subtypes.go | 214 ++++++++++ internal/source/azure/subtypes_test.go | 395 ++++++++++++++++++ 8 files changed, 1177 insertions(+), 55 deletions(-) create mode 100644 docs/mappings/azure/websites_functionapps.yaml create mode 100644 internal/source/azure/helper_test.go create mode 100644 internal/source/azure/subtypes.go create mode 100644 internal/source/azure/subtypes_test.go diff --git a/docs/how-to/030_azure-source.md b/docs/how-to/030_azure-source.md index 15125ce..7cc1466 100644 --- a/docs/how-to/030_azure-source.md +++ b/docs/how-to/030_azure-source.md @@ -82,6 +82,94 @@ For a more in-depth explanation refer to [Naming rules and restrictions for Azur Use `.name` wherever available to display casing matters, its availability is dependant on the specific resource APIs. The spelling Azure reported, if needed, is written to the source logs at the `Debug` level whenever it differs from the normalised value. +## Resource sub-types + +Some Azure resource types describe more than one thing: a `Microsoft.Web/sites` resource is an App +Service site, but when its `kind` carries the `functionapp` token it is also a Function App. + +For these types the Azure source produces, out of one Azure resource, both the item of the resource +itself and one or more **sub-type** items, each described by its own mapping file and related to the +item of the resource it was derived from. +A sub-type is always additive: the item of the Azure resource is produced exactly as it was before, +and the sub-type item is created next to it together with a relationship pointing at it. + +### How a sub-type mapping is dispatched + +The `type` of a sub-type mapping file is an **internal dispatch key**, not an Azure provider type: + +- it is never sent to Azure and never used to build a resource graph query +- it is never matched against the resource type of an event, because only the source can decide to + emit it +- its `extra.apiVersion` is never read, since the resource is always retrieved with the `apiVersion` + of its parent type. It is kept in the file only for symmetry with every other Azure mapping + +Which Azure type produces which sub-type is hardcoded in the source, together with the check deciding +whether a retrieved resource must produce it. Declaring a sub-type therefore takes both a new mapping +file and a change to that hardcoded dictionary: a sub-type can never be introduced by configuration +alone. + +### The sub-types shipped with ibdm + +| Azure type | Sub-type mapping | Produced when | +| --- | --- | --- | +| `Microsoft.Web/sites` | `docs/mappings/azure/websites_functionapps.yaml`, `type: functionapps` | the `kind` of the site carries the `functionapp` token | + +`kind` is a comma separated list of tokens, such as `app`, `app,linux` or `functionapp,linux`, and +the tokens are compared one by one: a site whose kind is `myfunctionapp` is not a Function App. +A site without a usable `kind` produces no sub-type and nothing fails. + +The mapping creates an `functionapps` item and, through its `extra` section, a `dependency` +relationship from that item to the `websites` item of the same site. + +Both mapping files must be loaded for the sub-type to be produced. Loading +`docs/mappings/azure/websites.yaml` alone reproduces exactly the behaviour the source had before +sub-types existed, deletion included. Loading `websites_functionapps.yaml` alone can instead never +produce anything, so the source logs a warning when it starts and carries on. + +`ibdm sync azure` and `ibdm run azure` behave identically, because the check runs on the payload the +Azure APIs returned and is indifferent to which of them retrieved it. To adopt a sub-type on an +already imported subscription load both mapping files and run `ibdm sync azure` once: every site that +already exists gets its sub-type item and its relationship. + +### Deleting a resource that has sub-types + +`Microsoft.Resources.ResourceDeleteSuccess` carries only the id of the deleted resource. Its `kind` +is gone and no API can return it any more, so at deletion time the check cannot run: the source +deletes the item of the resource **and the item of every sub-type its type can produce**, whether or +not that resource ever produced it. + +For a `Microsoft.Web/sites` resource with both mapping files loaded, three deletions reach the +catalog: + +| deleted | why | +| --- | --- | +| the `websites` item | the resource itself | +| the `functionapps` item | the only sub-type configured for its type | +| the relationship of the `functionapps` item | its `deletePolicy` is `cascade` | + +A deletion addressed to a sub-type item the resource never produced is inert: the catalog publish +reports no per item outcome, so nothing fails and nothing is left behind. The identifier of a +sub-type item also lives in its own namespace, `functionapps-` for the Function Apps, +so such a deletion can only ever name the sub-type item of that very resource. + +Removing a sub-type mapping file is not the reverse operation: the items it already published stop +being updated and stop being deleted together with their resource, so they have to be removed by +hand. + +### Authoring a sub-type mapping + +- Declare `syncable: true`. Keeping a sub-type key out of the resource graph queries is the job of + the hardcoded dictionary, not of `syncable`, and `syncable: false` would only risk confining the + sub-type to `ibdm run azure` +- Build the identifier of the item, and the identifier of every `deletePolicy: "cascade"` extra, out + of `.id` alone. A deletion payload carries only `id` and `type`, so a template reading any other + field fails to render and that deletion is lost +- Give the sub-type item its own item family, so that its identifiers can never collide with the ones + of another mapping, and make sure the item type definition for that family exists in the catalog +- Treat the payload as read only. A sub-type receives a shallow copy of the payload of its parent, so + writing into a nested value, such as `properties` or `tags`, would be seen by every other item + produced out of the same resource + ## Authentication The source is using the [`DefaultAzureCredential` chain of authentication] so you can setup diff --git a/docs/mappings/azure/websites_functionapps.yaml b/docs/mappings/azure/websites_functionapps.yaml new file mode 100644 index 0000000..5158253 --- /dev/null +++ b/docs/mappings/azure/websites_functionapps.yaml @@ -0,0 +1,43 @@ +apiVersion: azure.mia-platform.eu/v1 +itemFamily: functionapps +type: functionapps +extra: + apiVersion: "2025-03-01" +syncable: true +mappings: + identifier: |- + {{ printf "functionapps-%s" .id | sha256sum }} + metadata: + title: |- + {{ printf "Function App - %s" .name }} + spec: + name: "{{ .name }}" + id: "{{ .id }}" + location: "{{ .location | lower }}" + state: "{{ .properties.state | lower }}" + kind: "{{ .kind }}" + enabled: "{{ .properties.enabled }}" + availabilityState: "{{ .properties.availabilityState | lower }}" + defaultHostname: "{{ .properties.defaultHostName }}" + serverFarmId: "{{ .properties.serverFarmId }}" + tags: |- + {{ $tags := (get "tags" . "") -}} + {{- if $tags -}} + {{- $tags | toJSON -}} + {{- else -}} + {{- object | toJSON -}} + {{- end }} + extra: + - apiVersion: mia-platform.eu/v1 + itemFamily: relationships + deletePolicy: "cascade" + identifier: |- + {{ $src := printf "urn:mia-platform-catalog:azure.mia-platform.eu:v1:FunctionApp:%s" (printf "functionapps-%s" .id | sha256sum) -}} + {{- $type := "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:dependency.mia-platform.eu" -}} + {{- $tgt := printf "urn:mia-platform-catalog:azure.mia-platform.eu:v1:WebSite:%s" (printf "%s" .id | sha256sum) -}} + {{- printf "%s-%s-%s" $src $type $tgt | sha256sum }} + sourceRef: |- + urn:mia-platform-catalog:azure.mia-platform.eu:v1:FunctionApp:{{ printf "functionapps-%s" .id | sha256sum }} + targetRef: |- + urn:mia-platform-catalog:azure.mia-platform.eu:v1:WebSite:{{ (printf "%s" .id | sha256sum) }} + typeRef: "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:dependency.mia-platform.eu" diff --git a/internal/source/azure/azure.go b/internal/source/azure/azure.go index 7b43d84..c833133 100644 --- a/internal/source/azure/azure.go +++ b/internal/source/azure/azure.go @@ -98,6 +98,7 @@ func (s *Source) StartSyncProcess(ctx context.Context, typesToFilter map[string] if err := s.validateForSync(); err != nil { return handleError(err) } + warnOrphanSubTypes(logger, typesToFilter) client, err := s.azureGraphClient() if err != nil { @@ -111,61 +112,67 @@ func (s *Source) StartSyncProcess(ctx context.Context, typesToFilter map[string] }) for resType := range typesToFilter { - var query *string - switch resType { - case arm.ResourceGroupResourceType.String(): - graphResourceType := arm.SubscriptionResourceType.String() + "/resourceGroups" - query = to.Ptr(fmt.Sprintf(resourceContainerGraphQueryTemplate, graphResourceType)) - case arm.SubscriptionResourceType.String(): - query = to.Ptr(fmt.Sprintf(resourceContainerGraphQueryTemplate, resType)) - default: - query = to.Ptr(fmt.Sprintf(resourceGraphQueryTemplate, resType)) + // a sub-type is emitted while handling its parent resource, and its type key is an internal + // dispatch key: querying Azure for it would only ask for a type that does not exist. + if isSubTypeKey(resType) { + logger.Debug("skipping sub-type mapping, it is emitted with its parent type", "type", resType) + continue } - queryRequest := armresourcegraph.QueryRequest{ - Subscriptions: []*string{to.Ptr(s.SubscriptionID)}, - Query: query, + if err := s.syncResourceType(ctx, client, resType, typesToFilter, dataChannel); err != nil { + // handleError swallows the cancellation, so a stopped sync process is not a failure + return handleError(err) } + } - for { - timestamp := timeProvider() - response, err := client.Resources(ctx, queryRequest, nil) + s.syncContext.Swap(nil) + return nil +} - switch { - case errors.Is(err, context.Canceled): - logger.Debug("stopping sync process due to context cancellation") - return nil - case err != nil: - return handleError(err) - } +// syncResourceType pages through every resource of resType the Resource Graph returns and emits +// the item of each one of them, together with the ones of the sub-types they additionally produce. +func (s *Source) syncResourceType(ctx context.Context, client *armresourcegraph.Client, resType string, typesToFilter map[string]source.Extra, dataChannel chan<- source.Data) error { + logger := logger.FromContext(ctx).WithName(logName) + queryRequest := armresourcegraph.QueryRequest{ + Subscriptions: []*string{to.Ptr(s.SubscriptionID)}, + Query: resourceGraphQuery(resType), + } + + for { + timestamp := timeProvider() + response, err := client.Resources(ctx, queryRequest, nil) + + switch { + case errors.Is(err, context.Canceled): + logger.Debug("stopping sync process due to context cancellation") + return nil + case err != nil: + return err + } - if data, ok := response.Data.([]any); ok { - for _, item := range data { - if values, ok := item.(map[string]any); ok { - normalizeResourceValues(logger, values, resType) - dataChannel <- source.Data{ - Type: resType, - Operation: source.DataOperationUpsert, - Time: timestamp, - Values: values, - } - } else { - // something very wrong is going on, print an error and continue - logger.Debug("retrieve data item is not a valid map") + if data, ok := response.Data.([]any); ok { + for _, item := range data { + if values, ok := item.(map[string]any); ok { + normalizeResourceValues(logger, values, resType) + for _, resourceData := range resourceDataToEmit(resType, values, typesToFilter, source.DataOperationUpsert, timestamp) { + dataChannel <- resourceData } + } else { + // something very wrong is going on, print an error and continue + logger.Debug("retrieve data item is not a valid map") } - } else { - // something very wrong is going on, print an error and continue - logger.Debug("response data is not a valid type") } + } else { + // something very wrong is going on, print an error and continue + logger.Debug("response data is not a valid type") + } - if response.ResultTruncated == nil || *response.ResultTruncated == armresourcegraph.ResultTruncatedFalse { - break - } + if response.ResultTruncated == nil || *response.ResultTruncated == armresourcegraph.ResultTruncatedFalse { + break + } - queryRequest.Options = &armresourcegraph.QueryRequestOptions{ - SkipToken: response.SkipToken, - } + queryRequest.Options = &armresourcegraph.QueryRequestOptions{ + SkipToken: response.SkipToken, } } @@ -173,12 +180,27 @@ func (s *Source) StartSyncProcess(ctx context.Context, typesToFilter map[string] return nil } +// resourceGraphQuery returns the Resource Graph query retrieving every resource of resType, taken +// from the container table for the types that live in it. +func resourceGraphQuery(resType string) *string { + switch resType { + case arm.ResourceGroupResourceType.String(): + graphResourceType := arm.SubscriptionResourceType.String() + "/resourceGroups" + return to.Ptr(fmt.Sprintf(resourceContainerGraphQueryTemplate, graphResourceType)) + case arm.SubscriptionResourceType.String(): + return to.Ptr(fmt.Sprintf(resourceContainerGraphQueryTemplate, resType)) + default: + return to.Ptr(fmt.Sprintf(resourceGraphQueryTemplate, resType)) + } +} + // StartEventStream implement source.EventSource. func (s *Source) StartEventStream(ctx context.Context, typesToFilter map[string]source.Extra, dataChannel chan<- source.Data) error { logger := logger.FromContext(ctx).WithName(logName) if err := s.validateForEventStream(); err != nil { return handleError(err) } + warnOrphanSubTypes(logger, typesToFilter) client, err := s.azureClient() if err != nil { @@ -206,7 +228,9 @@ func (s *Source) StartEventStream(ctx context.Context, typesToFilter map[string] } func partitionEventHandler(client *armresources.Client, typesToFilter map[string]source.Extra, dataChannel chan<- source.Data) eventHandler { - typesSlice := slices.Sorted(maps.Keys(typesToFilter)) + // a sub-type type key is an internal dispatch key and can never be the type of an event + // subject, so it is left out of the set the subject type is resolved against. + typesSlice := slices.DeleteFunc(slices.Sorted(maps.Keys(typesToFilter)), isSubTypeKey) return func(ctx context.Context, receivedData *azeventhubs.ReceivedEventData) { logger := logger.FromContext(ctx).WithName(logName) @@ -258,21 +282,17 @@ func partitionEventHandler(client *armresources.Client, typesToFilter map[string } normalizeResourceValues(logger, values, resourceType) - dataChannel <- source.Data{ - Type: resourceType, - Operation: source.DataOperationUpsert, - Time: *envelope.Time, - Values: values, + for _, resourceData := range resourceDataToEmit(resourceType, values, typesToFilter, source.DataOperationUpsert, *envelope.Time) { + dataChannel <- resourceData } case azsystemevents.TypeResourceDeleteSuccess: logger.Trace("deleting resource", "resourceType", resourceType) + // the event carries only the resource id, so no sub-type check can run here and a + // delete is emitted for every configured sub-type of the resource type. values := map[string]any{idKey: resID.String()} normalizeResourceValues(logger, values, resourceType) - dataChannel <- source.Data{ - Type: resourceType, - Operation: source.DataOperationDelete, - Time: *envelope.Time, - Values: values, + for _, resourceData := range resourceDataToEmit(resourceType, values, typesToFilter, source.DataOperationDelete, *envelope.Time) { + dataChannel <- resourceData } default: logger.Trace("skipping resource", "resourceType", resourceType, "eventType", envelope.Type, "apiVersion", apiVersion) diff --git a/internal/source/azure/azure_stream_test.go b/internal/source/azure/azure_stream_test.go index 6cdbd37..fa8aa0c 100644 --- a/internal/source/azure/azure_stream_test.go +++ b/internal/source/azure/azure_stream_test.go @@ -182,6 +182,140 @@ func TestPartitionEventHandler(t *testing.T) { }, }, }, + "a site carrying the functionapp kind emits its item and the one of its sub-type": { + contextFunc: func(tb testing.TB) (context.Context, context.CancelFunc) { + tb.Helper() + return context.WithTimeout(tb.Context(), 1*time.Second) + }, + typesToFilter: map[string]source.Extra{ + websitesType: {apiVersionKey: websitesAPIVersion}, + functionAppsType: {apiVersionKey: websitesAPIVersion}, + }, + azureData: &azeventhubs.ReceivedEventData{ + EventData: azeventhubs.EventData{ + Body: eventDataFunctionAppWriteBody, + }, + }, + expectedData: []source.Data{ + { + Type: websitesType, + Operation: source.DataOperationUpsert, + Time: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC), + Values: streamedFunctionAppValues(), + }, + { + // the sub-type carries the payload of its parent, whose type stays the Azure + // provider type: a sub-type key is a dispatch key and never reaches the item + Type: functionAppsType, + Operation: source.DataOperationUpsert, + Time: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC), + Values: streamedFunctionAppValues(), + }, + }, + }, + "a site not carrying the functionapp kind emits its item alone": { + contextFunc: func(tb testing.TB) (context.Context, context.CancelFunc) { + tb.Helper() + return context.WithTimeout(tb.Context(), 1*time.Second) + }, + typesToFilter: map[string]source.Extra{ + websitesType: {apiVersionKey: websitesAPIVersion}, + functionAppsType: {apiVersionKey: websitesAPIVersion}, + }, + azureData: &azeventhubs.ReceivedEventData{ + EventData: azeventhubs.EventData{ + Body: eventDataWebsiteWriteBody, + }, + }, + expectedData: []source.Data{ + { + Type: websitesType, + Operation: source.DataOperationUpsert, + Time: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC), + Values: map[string]any{ + "id": normalizedWebsiteID, + "kind": webAppKindValue, + "type": websitesType, + }, + }, + }, + }, + "deleting a site broadcasts to every configured sub-type": { + contextFunc: func(tb testing.TB) (context.Context, context.CancelFunc) { + tb.Helper() + return context.WithTimeout(tb.Context(), 1*time.Second) + }, + typesToFilter: map[string]source.Extra{ + websitesType: {apiVersionKey: websitesAPIVersion}, + functionAppsType: {apiVersionKey: websitesAPIVersion}, + }, + azureData: &azeventhubs.ReceivedEventData{ + EventData: azeventhubs.EventData{ + Body: eventDataFunctionAppDeleteBody, + }, + }, + // the event carries only the resource id, so the kind check cannot run and the sub-type + // is deleted whether or not the site ever produced it + expectedData: []source.Data{ + { + Type: websitesType, + Operation: source.DataOperationDelete, + Time: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC), + Values: map[string]any{ + "id": normalizedFunctionAppID, + "type": websitesType, + }, + }, + { + Type: functionAppsType, + Operation: source.DataOperationDelete, + Time: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC), + Values: map[string]any{ + "id": normalizedFunctionAppID, + "type": websitesType, + }, + }, + }, + }, + "deleting a site without the sub-type mapping loaded keeps the previous behaviour": { + contextFunc: func(tb testing.TB) (context.Context, context.CancelFunc) { + tb.Helper() + return context.WithTimeout(tb.Context(), 1*time.Second) + }, + typesToFilter: map[string]source.Extra{ + websitesType: {apiVersionKey: websitesAPIVersion}, + }, + azureData: &azeventhubs.ReceivedEventData{ + EventData: azeventhubs.EventData{ + Body: eventDataFunctionAppDeleteBody, + }, + }, + expectedData: []source.Data{ + { + Type: websitesType, + Operation: source.DataOperationDelete, + Time: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC), + Values: map[string]any{ + "id": normalizedFunctionAppID, + "type": websitesType, + }, + }, + }, + }, + "a sub-type mapping loaded without its parent produces nothing": { + contextFunc: func(tb testing.TB) (context.Context, context.CancelFunc) { + tb.Helper() + return context.WithTimeout(tb.Context(), 1*time.Second) + }, + typesToFilter: map[string]source.Extra{ + functionAppsType: {apiVersionKey: websitesAPIVersion}, + }, + azureData: &azeventhubs.ReceivedEventData{ + EventData: azeventhubs.EventData{ + Body: eventDataFunctionAppWriteBody, + }, + }, + }, "delete with non canonical subject casing": { contextFunc: func(tb testing.TB) (context.Context, context.CancelFunc) { tb.Helper() @@ -295,6 +429,26 @@ func handleResourcesGetByIDRequest(tb testing.TB, resourceID, apiVersion string) }, } return resp, nil + case functionAppGetByIDPath: + assert.Equal(tb, websitesAPIVersion, apiVersion) + resp = &armresources.ClientGetByIDResponse{ + GenericResource: armresources.GenericResource{ + ID: to.Ptr(azureFunctionAppID), + Kind: to.Ptr(functionAppKindValue), + Type: to.Ptr(websitesType), + }, + } + return resp, nil + case websiteGetByIDPath: + assert.Equal(tb, websitesAPIVersion, apiVersion) + resp = &armresources.ClientGetByIDResponse{ + GenericResource: armresources.GenericResource{ + ID: to.Ptr(azureWebsiteID), + Kind: to.Ptr(webAppKindValue), + Type: to.Ptr(websitesType), + }, + } + return resp, nil } return nil, nil @@ -309,8 +463,25 @@ const ( // the fake actually receives. managedClusterGetByIDPath = "subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/my-rg/providers/Microsoft.ContainerService/managedClusters/my-cluster" lowerTypeManagedClusterGetByIDPath = "subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/my-rg/providers/microsoft.containerservice/managedclusters/my-cluster" + + // the same keys for the two App Service sites: their subjects already spell the resource group + // literal in camelCase, so the fake receives azureFunctionAppID and azureWebsiteID without the + // leading slash. + functionAppGetByIDPath = "subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/my-rg/providers/Microsoft.Web/sites/my-function" + websiteGetByIDPath = "subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/my-rg/providers/Microsoft.Web/sites/my-site" ) +// streamedFunctionAppValues returns the payload the event handler emits for the App Service site +// carrying the functionapp kind token. It is built on every call because a sub-type receives its +// own copy of the payload of its parent. +func streamedFunctionAppValues() map[string]any { + return map[string]any{ + "id": normalizedFunctionAppID, + "kind": functionAppKindValue, + "type": websitesType, + } +} + var ( // eventDataManagedClusterWriteBody carries a lowercase resourcegroups literal and a canonically // cased provider and type. @@ -348,6 +519,40 @@ var ( "time": "2020-01-01T00:00:00.0000000Z" }]`) + // eventDataFunctionAppWriteBody and eventDataWebsiteWriteBody import the two App Service sites, + // only the first of which carries the functionapp kind token in the body the resource provider + // answers with. + eventDataFunctionAppWriteBody = json.RawMessage(`[ + { + "id": "00000000-0000-0000-0000-000000000000", + "source": "/subscriptions/00000000-0000-0000-0000-000000000000", + "specversion": "1.0", + "type": "Microsoft.Resources.ResourceWriteSuccess", + "subject": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/my-rg/providers/Microsoft.Web/sites/my-function", + "time": "2020-01-01T00:00:00.0000000Z" + }]`) + + eventDataWebsiteWriteBody = json.RawMessage(`[ + { + "id": "00000000-0000-0000-0000-000000000000", + "source": "/subscriptions/00000000-0000-0000-0000-000000000000", + "specversion": "1.0", + "type": "Microsoft.Resources.ResourceWriteSuccess", + "subject": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/my-rg/providers/Microsoft.Web/sites/my-site", + "time": "2020-01-01T00:00:00.0000000Z" + }]`) + + // eventDataFunctionAppDeleteBody carries only the resource id, as every delete event does. + eventDataFunctionAppDeleteBody = json.RawMessage(`[ + { + "id": "00000000-0000-0000-0000-000000000000", + "source": "/subscriptions/00000000-0000-0000-0000-000000000000", + "specversion": "1.0", + "type": "Microsoft.Resources.ResourceDeleteSuccess", + "subject": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/my-rg/providers/Microsoft.Web/sites/my-function", + "time": "2020-01-01T00:00:00.0000000Z" + }]`) + eventDataResourcesTestBody = json.RawMessage(`[ { "id": "00000000-0000-0000-0000-000000000000", diff --git a/internal/source/azure/azure_sync_test.go b/internal/source/azure/azure_sync_test.go index c481230..7038942 100644 --- a/internal/source/azure/azure_sync_test.go +++ b/internal/source/azure/azure_sync_test.go @@ -246,6 +246,53 @@ func TestStartSyncProcess(t *testing.T) { }, }, }, + "websites with the sub-type mapping loaded": { + typesToFilter: map[string]source.Extra{ + websitesType: {apiVersionKey: websitesAPIVersion}, + functionAppsType: {apiVersionKey: websitesAPIVersion}, + }, + expectedData: []source.Data{ + { + Type: websitesType, + Operation: source.DataOperationUpsert, + Time: testTime, + Values: syncedFunctionAppValues(), + }, + { + // the sub-type carries the payload of its parent, whose type stays the Azure + // provider type: a sub-type key is a dispatch key and never reaches the item + Type: functionAppsType, + Operation: source.DataOperationUpsert, + Time: testTime, + Values: syncedFunctionAppValues(), + }, + { + Type: websitesType, + Operation: source.DataOperationUpsert, + Time: testTime, + Values: syncedWebsiteValues(), + }, + }, + }, + "websites without the sub-type mapping loaded keep the previous behaviour": { + typesToFilter: map[string]source.Extra{ + websitesType: {apiVersionKey: websitesAPIVersion}, + }, + expectedData: []source.Data{ + { + Type: websitesType, + Operation: source.DataOperationUpsert, + Time: testTime, + Values: syncedFunctionAppValues(), + }, + { + Type: websitesType, + Operation: source.DataOperationUpsert, + Time: testTime, + Values: syncedWebsiteValues(), + }, + }, + }, "unusable ids are emitted unchanged": { typesToFilter: map[string]source.Extra{ "Microsoft.Resources/malformedResources": nil, @@ -338,6 +385,62 @@ func TestStartSyncProcess(t *testing.T) { } } +// syncedFunctionAppValues and syncedWebsiteValues return the payloads the sync process emits for +// the two App Service sites of resourceGraphWebsitesResponse. They are built on every call because +// a sub-type receives its own copy of the payload of its parent. +func syncedFunctionAppValues() map[string]any { + return map[string]any{ + "id": normalizedFunctionAppID, + "kind": functionAppKindValue, + "name": "my-function", + "type": websitesType, + } +} + +func syncedWebsiteValues() map[string]any { + return map[string]any{ + "id": normalizedWebsiteID, + "kind": webAppKindValue, + "name": "my-site", + "type": websitesType, + } +} + +// TestStartSyncProcessEmitsSubTypesAfterTheirParent checks the emission order the table driven test +// cannot assert, because it compares the collected data as a set. +func TestStartSyncProcessEmitsSubTypesAfterTheirParent(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithTimeout(t.Context(), 1*time.Second) + defer cancel() + + azureSource := &Source{ + config: config{ + SubscriptionID: testSubscriptionID, + clientOptions: &arm.ClientOptions{ + ClientOptions: policy.ClientOptions{ + Transport: fakeResourceGraphTransport(t), + }, + }, + azureCredentials: &fakeazcore.TokenCredential{}, + }, + } + + dataChannel := make(chan source.Data, 10) + require.NoError(t, azureSource.StartSyncProcess(ctx, map[string]source.Extra{ + websitesType: {apiVersionKey: websitesAPIVersion}, + functionAppsType: {apiVersionKey: websitesAPIVersion}, + }, dataChannel)) + close(dataChannel) + + emittedTypes := make([]string, 0, 3) + for data := range dataChannel { + emittedTypes = append(emittedTypes, data.Type) + } + + assert.Equal(t, []string{websitesType, functionAppsType, websitesType}, emittedTypes) +} + func TestCancelledSyncProcess(t *testing.T) { t.Parallel() @@ -490,6 +593,18 @@ func handleResourceGraphQueryRequest(t *testing.T, query armresourcegraph.QueryR SkipToken: nil, }, }, nil + // the query is built from the Azure provider type alone: there is deliberately no branch for + // the functionapps sub-type key, so a query issued for it fails the test on the default case + case fmt.Sprintf(resourceGraphQueryTemplate, websitesType): + return &armresourcegraph.ClientResourcesResponse{ + QueryResponse: armresourcegraph.QueryResponse{ + TotalRecords: to.Ptr(int64(2)), + Data: resourceGraphWebsitesResponse, + ResultTruncated: to.Ptr(armresourcegraph.ResultTruncatedFalse), + Count: to.Ptr(int64(2)), + SkipToken: nil, + }, + }, nil case fmt.Sprintf(resourceGraphQueryTemplate, "Microsoft.Resources/malformedResources"): return &armresourcegraph.ClientResourcesResponse{ QueryResponse: armresourcegraph.QueryResponse{ @@ -523,6 +638,24 @@ var ( }, } + // resourceGraphWebsitesResponse holds one App Service site carrying the functionapp kind token + // and one that does not, so that one query exercises both the emission of a sub-type and its + // absence. + resourceGraphWebsitesResponse = []any{ + map[string]any{ + "id": azureFunctionAppID, + "kind": functionAppKindValue, + "name": "my-function", + "type": "microsoft.web/sites", + }, + map[string]any{ + "id": azureWebsiteID, + "kind": webAppKindValue, + "name": "my-site", + "type": "microsoft.web/sites", + }, + } + // resourceGraphMalformedResponse holds rows whose id cannot be normalized: the items must still // be emitted with their original value. resourceGraphMalformedResponse = []any{ diff --git a/internal/source/azure/helper_test.go b/internal/source/azure/helper_test.go new file mode 100644 index 0000000..3cacdcd --- /dev/null +++ b/internal/source/azure/helper_test.go @@ -0,0 +1,24 @@ +// Copyright Mia srl +// SPDX-License-Identifier: AGPL-3.0-only or Commercial + +package azure + +// Fixtures of the two App Service sites the sub-type tests import, shared by the sync, the stream +// and the dictionary tests because the sub-type emission runs on the retrieved payload whichever +// path retrieved it: my-function carries the functionapp kind token and my-site does not. +const ( + // websitesAPIVersion is the api-version the website mappings declare. + websitesAPIVersion = "2025-03-01" + + // functionAppKindValue and webAppKindValue are two kind values Azure returns for an App Service + // site: only the first one carries the functionapp token. + functionAppKindValue = "functionapp,linux" + webAppKindValue = "app,linux" + + // the ids as Azure spells them, with a camelCase resourceGroups literal and a canonically cased + // provider and type, and as the source normalizes them. + azureFunctionAppID = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/my-rg/providers/Microsoft.Web/sites/my-function" + normalizedFunctionAppID = "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/my-rg/providers/microsoft.web/sites/my-function" + azureWebsiteID = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/my-rg/providers/Microsoft.Web/sites/my-site" + normalizedWebsiteID = "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/my-rg/providers/microsoft.web/sites/my-site" +) diff --git a/internal/source/azure/subtypes.go b/internal/source/azure/subtypes.go new file mode 100644 index 0000000..7cf2dc6 --- /dev/null +++ b/internal/source/azure/subtypes.go @@ -0,0 +1,214 @@ +// Copyright Mia srl +// SPDX-License-Identifier: AGPL-3.0-only or Commercial + +package azure + +import ( + "maps" + "slices" + "strings" + "time" + + "github.com/mia-platform/ibdm/internal/logger" + "github.com/mia-platform/ibdm/internal/source" +) + +const ( + // websitesType is the Azure provider type of the App Service sites. + websitesType = "Microsoft.Web/sites" + + // functionAppsType is the type key of the Function App sub-type mapping. It is an internal + // dispatch key and not an Azure provider type: it is never sent to Azure and it is never used + // to build a Resource Graph query. + functionAppsType = "functionapps" + + // kindKey is the payload key holding the kind of an Azure resource. + kindKey = "kind" + + // functionAppKind is the kind token marking an App Service site as a Function App. + functionAppKind = "functionapp" + + // kindSeparator separates the tokens of a kind payload value. + kindSeparator = "," +) + +// subType couples the type key of a sub-type mapping with the predicate deciding whether a parent +// resource payload must also produce it. Keeping the two together makes it impossible to declare a +// sub-type without the check guarding it. +type subType struct { + // typeKey is the type declared by the mapping file of the sub-type. + typeKey string + + // matches reports whether values, the retrieved payload of the parent resource, must also + // produce this sub-type. It is never consulted on a delete, where the payload carries only the + // resource id. + matches func(values map[string]any) bool +} + +// subTypes maps an Azure provider type to the sub-types it can additionally produce. A sub-type is +// a specialised item, described by its own mapping file and related to the item of the resource it +// was derived from, so declaring one requires an entry here and can never be done by configuration +// alone. +// +// The order of a list is the order its sub-types are emitted in. +var subTypes = map[string][]subType{ + websitesType: { + {typeKey: functionAppsType, matches: isFunctionApp}, + }, +} + +var ( + // subTypesByParent indexes subTypes by lowercased parent type so that a lookup ignores the + // letter case the mapping file used for the Azure provider type, consistently with + // configuredResourceType. + subTypesByParent = indexSubTypesByParent() + + // subTypeKeys holds every type key declared in subTypes. Deriving it from the dictionary keeps + // the exclusion of the sub-type keys from the Azure queries in step with their emission. + subTypeKeys = collectSubTypeKeys() +) + +// indexSubTypesByParent builds the case insensitive index of subTypes. +func indexSubTypesByParent() map[string][]subType { + index := make(map[string][]subType, len(subTypes)) + for parentType, candidates := range subTypes { + index[strings.ToLower(parentType)] = candidates + } + + return index +} + +// collectSubTypeKeys builds the set of every type key declared in subTypes. +func collectSubTypeKeys() map[string]struct{} { + keys := make(map[string]struct{}, len(subTypes)) + for _, candidates := range subTypes { + for _, candidate := range candidates { + keys[candidate.typeKey] = struct{}{} + } + } + + return keys +} + +// isSubTypeKey reports whether typeKey is the type key of a sub-type mapping. Such a key is an +// internal dispatch key: it must never be used to build a Resource Graph query, nor be matched +// against the resource type carried by the subject of an event. +func isSubTypeKey(typeKey string) bool { + _, found := subTypeKeys[typeKey] + return found +} + +// subTypesFor returns the sub-types resourceType can additionally produce, ignoring its letter case. +func subTypesFor(resourceType string) []subType { + return subTypesByParent[strings.ToLower(resourceType)] +} + +// isFunctionApp reports whether the kind of a resource carries the functionapp token. +// +// Azure spells kind as a comma separated list of tokens, such as "app", "app,linux" or +// "functionapp,linux,container", so the tokens are compared one by one and a kind such as +// "myfunctionapp" does not qualify. A missing, empty or non string kind reports false, so that a +// payload without the discriminant produces the item of the resource alone instead of failing. +func isFunctionApp(values map[string]any) bool { + kind, ok := values[kindKey].(string) + if !ok { + return false + } + + for token := range strings.SplitSeq(kind, kindSeparator) { + if strings.EqualFold(strings.TrimSpace(token), functionAppKind) { + return true + } + } + + return false +} + +// subTypesToEmit returns the type keys of the sub-types a resource of resourceType must +// additionally produce, in the order the dictionary declares them. +// +// On an upsert every predicate is evaluated against values, the payload retrieved from Azure. On a +// delete the payload carries only the resource id, so no predicate can run and every configured +// sub-type is returned: deleting a sub-type item that was never created is inert, because the +// Catalog publish reports no per item outcome. +// +// A sub-type whose mapping file is not loaded is never returned, so a deployment configuring no +// sub-type mapping behaves exactly as one running against an empty dictionary. +func subTypesToEmit(resourceType string, values map[string]any, configured map[string]source.Extra, isDelete bool) []string { + candidates := subTypesFor(resourceType) + if len(candidates) == 0 { + return nil + } + + keys := make([]string, 0, len(candidates)) + for _, candidate := range candidates { + if _, found := configured[candidate.typeKey]; !found { + continue + } + + if !isDelete && !candidate.matches(values) { + continue + } + + keys = append(keys, candidate.typeKey) + } + + return keys +} + +// resourceDataToEmit builds every source.Data a retrieved resource must produce: the item of the +// resource itself first, so that it exists before the sub-types relate to it, then one item for +// every sub-type resourceType additionally produces. +// +// Every predicate runs and every payload is built before the caller sends the first value, because +// the pipeline starts reading a value as soon as it is sent: cloning after a send would race the +// mapper. +// +// The payload of a sub-type is a shallow copy of the one of its parent, because the mapping +// functions can write into the top level of the map they are handed and two items must never share +// it. The nested values stay shared, which is why the mapping templates must treat their input +// payload as read only. +func resourceDataToEmit(resourceType string, values map[string]any, configured map[string]source.Extra, operation source.DataOperation, timestamp time.Time) []source.Data { + emitted := subTypesToEmit(resourceType, values, configured, operation == source.DataOperationDelete) + + data := make([]source.Data, 0, 1+len(emitted)) + data = append(data, source.Data{ + Type: resourceType, + Operation: operation, + Time: timestamp, + Values: values, + }) + + for _, typeKey := range emitted { + data = append(data, source.Data{ + Type: typeKey, + Operation: operation, + Time: timestamp, + Values: maps.Clone(values), + }) + } + + return data +} + +// warnOrphanSubTypes logs a warning for every configured sub-type mapping whose parent type mapping +// is not configured. A sub-type is emitted only while its parent resource is handled, so such a +// mapping can never produce any item. It is a warning, and not an error, because the source always +// skips the configuration it cannot use. +func warnOrphanSubTypes(log logger.Logger, typesToFilter map[string]source.Extra) { + configuredTypes := slices.Sorted(maps.Keys(typesToFilter)) + for _, parentType := range slices.Sorted(maps.Keys(subTypes)) { + if _, configured := configuredResourceType(configuredTypes, parentType); configured { + continue + } + + for _, candidate := range subTypes[parentType] { + if _, found := typesToFilter[candidate.typeKey]; !found { + continue + } + + log.Warn("sub-type mapping configured without the mapping of its parent type, it will never produce any item", + "type", candidate.typeKey, "parentType", parentType) + } + } +} diff --git a/internal/source/azure/subtypes_test.go b/internal/source/azure/subtypes_test.go new file mode 100644 index 0000000..cb4166d --- /dev/null +++ b/internal/source/azure/subtypes_test.go @@ -0,0 +1,395 @@ +// Copyright Mia srl +// SPDX-License-Identifier: AGPL-3.0-only or Commercial + +package azure + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/mia-platform/ibdm/internal/logger" + "github.com/mia-platform/ibdm/internal/source" +) + +func TestIsFunctionApp(t *testing.T) { + t.Parallel() + + testCases := map[string]struct { + values map[string]any + expectedResult bool + }{ + "the single functionapp token matches": { + values: map[string]any{kindKey: "functionapp"}, + expectedResult: true, + }, + "the functionapp token in a list matches": { + values: map[string]any{kindKey: functionAppKindValue}, + expectedResult: true, + }, + "the functionapp token in a longer list matches": { + values: map[string]any{kindKey: "functionapp,linux,container"}, + expectedResult: true, + }, + "the token casing is ignored": { + values: map[string]any{kindKey: "FunctionApp,Linux"}, + expectedResult: true, + }, + "the token spacing is ignored": { + values: map[string]any{kindKey: "functionapp , linux"}, + expectedResult: true, + }, + "a web app does not match": { + values: map[string]any{kindKey: "app"}, + }, + "a linux web app does not match": { + values: map[string]any{kindKey: webAppKindValue}, + }, + "a token containing the term does not match": { + values: map[string]any{kindKey: "myfunctionapp"}, + }, + "a token ending with the term does not match": { + values: map[string]any{kindKey: "app,linux,functionapps"}, + }, + "an empty kind does not match": { + values: map[string]any{kindKey: ""}, + }, + "a missing kind does not match": { + values: map[string]any{idKey: normalizedWebsiteID}, + }, + "a non string kind does not match": { + values: map[string]any{kindKey: 42}, + }, + "a nil kind does not match": { + values: map[string]any{kindKey: nil}, + }, + "an empty payload does not match": { + values: map[string]any{}, + }, + } + + for testName, test := range testCases { + t.Run(testName, func(t *testing.T) { + t.Parallel() + + assert.Equal(t, test.expectedResult, isFunctionApp(test.values)) + }) + } +} + +func TestSubTypesFor(t *testing.T) { + t.Parallel() + + testCases := map[string]struct { + resourceType string + expectedTypeKeys []string + }{ + "the configured parent type returns its sub-types": { + resourceType: websitesType, + expectedTypeKeys: []string{functionAppsType}, + }, + "a lowercase parent type returns its sub-types": { + resourceType: "microsoft.web/sites", + expectedTypeKeys: []string{functionAppsType}, + }, + "a shouty parent type returns its sub-types": { + resourceType: "MICROSOFT.WEB/SITES", + expectedTypeKeys: []string{functionAppsType}, + }, + "a type without sub-types returns nothing": { + resourceType: managedClustersType, + }, + "a sub-type key is not a parent type": { + resourceType: functionAppsType, + }, + "an empty type returns nothing": {}, + } + + for testName, test := range testCases { + t.Run(testName, func(t *testing.T) { + t.Parallel() + + candidates := subTypesFor(test.resourceType) + typeKeys := make([]string, 0, len(candidates)) + for _, candidate := range candidates { + typeKeys = append(typeKeys, candidate.typeKey) + assert.NotNil(t, candidate.matches, "every sub-type must carry the check guarding it") + } + + assert.ElementsMatch(t, test.expectedTypeKeys, typeKeys) + }) + } +} + +func TestIsSubTypeKey(t *testing.T) { + t.Parallel() + + testCases := map[string]struct { + typeKey string + expectedResult bool + }{ + "a declared sub-type key is one": { + typeKey: functionAppsType, + expectedResult: true, + }, + "an Azure provider type is not one": { + typeKey: websitesType, + }, + "an unmapped type is not one": { + typeKey: managedClustersType, + }, + "an empty type is not one": {}, + } + + for testName, test := range testCases { + t.Run(testName, func(t *testing.T) { + t.Parallel() + + assert.Equal(t, test.expectedResult, isSubTypeKey(test.typeKey)) + }) + } +} + +func TestSubTypesToEmit(t *testing.T) { + t.Parallel() + + bothMappings := map[string]source.Extra{ + websitesType: {apiVersionKey: websitesAPIVersion}, + functionAppsType: {apiVersionKey: websitesAPIVersion}, + } + parentMappingOnly := map[string]source.Extra{ + websitesType: {apiVersionKey: websitesAPIVersion}, + } + + testCases := map[string]struct { + resourceType string + values map[string]any + configured map[string]source.Extra + isDelete bool + expectedKeys []string + }{ + "an upsert of a matching payload returns the sub-type": { + resourceType: websitesType, + values: map[string]any{idKey: normalizedWebsiteID, kindKey: functionAppKindValue}, + configured: bothMappings, + expectedKeys: []string{functionAppsType}, + }, + "an upsert of a non matching payload returns nothing": { + resourceType: websitesType, + values: map[string]any{idKey: normalizedWebsiteID, kindKey: webAppKindValue}, + configured: bothMappings, + }, + "an upsert of a matching payload of a differently cased type returns the sub-type": { + resourceType: "microsoft.web/sites", + values: map[string]any{idKey: normalizedWebsiteID, kindKey: functionAppKindValue}, + configured: bothMappings, + expectedKeys: []string{functionAppsType}, + }, + "a delete returns every configured sub-type whatever the payload": { + resourceType: websitesType, + values: map[string]any{idKey: normalizedWebsiteID}, + configured: bothMappings, + isDelete: true, + expectedKeys: []string{functionAppsType}, + }, + "an upsert returns nothing when the sub-type mapping is not loaded": { + resourceType: websitesType, + values: map[string]any{idKey: normalizedWebsiteID, kindKey: functionAppKindValue}, + configured: parentMappingOnly, + }, + "a delete returns nothing when the sub-type mapping is not loaded": { + resourceType: websitesType, + values: map[string]any{idKey: normalizedWebsiteID}, + configured: parentMappingOnly, + isDelete: true, + }, + "a type without sub-types returns nothing": { + resourceType: managedClustersType, + values: map[string]any{idKey: normalizedManagedClusterID, kindKey: "functionapp"}, + configured: bothMappings, + }, + "a type without sub-types returns nothing on a delete too": { + resourceType: managedClustersType, + values: map[string]any{idKey: normalizedManagedClusterID}, + configured: bothMappings, + isDelete: true, + }, + } + + for testName, test := range testCases { + t.Run(testName, func(t *testing.T) { + t.Parallel() + + assert.ElementsMatch(t, test.expectedKeys, subTypesToEmit(test.resourceType, test.values, test.configured, test.isDelete)) + }) + } +} + +func TestResourceDataToEmit(t *testing.T) { + t.Parallel() + + bothMappings := map[string]source.Extra{ + websitesType: {apiVersionKey: websitesAPIVersion}, + functionAppsType: {apiVersionKey: websitesAPIVersion}, + } + functionAppValues := map[string]any{ + idKey: normalizedWebsiteID, + typeKey: websitesType, + kindKey: functionAppKindValue, + } + webAppValues := map[string]any{ + idKey: normalizedWebsiteID, + typeKey: websitesType, + kindKey: webAppKindValue, + } + deleteValues := map[string]any{ + idKey: normalizedWebsiteID, + typeKey: websitesType, + } + + testCases := map[string]struct { + resourceType string + values map[string]any + configured map[string]source.Extra + operation source.DataOperation + expectedData []source.Data + }{ + "a matching resource emits its item and the one of its sub-type": { + resourceType: websitesType, + values: functionAppValues, + configured: bothMappings, + operation: source.DataOperationUpsert, + expectedData: []source.Data{ + {Type: websitesType, Operation: source.DataOperationUpsert, Time: testTime, Values: functionAppValues}, + {Type: functionAppsType, Operation: source.DataOperationUpsert, Time: testTime, Values: functionAppValues}, + }, + }, + "a non matching resource emits its item alone": { + resourceType: websitesType, + values: webAppValues, + configured: bothMappings, + operation: source.DataOperationUpsert, + expectedData: []source.Data{ + {Type: websitesType, Operation: source.DataOperationUpsert, Time: testTime, Values: webAppValues}, + }, + }, + "a delete broadcasts to every configured sub-type": { + resourceType: websitesType, + values: deleteValues, + configured: bothMappings, + operation: source.DataOperationDelete, + expectedData: []source.Data{ + {Type: websitesType, Operation: source.DataOperationDelete, Time: testTime, Values: deleteValues}, + {Type: functionAppsType, Operation: source.DataOperationDelete, Time: testTime, Values: deleteValues}, + }, + }, + "a type without sub-types emits its item alone": { + resourceType: managedClustersType, + values: map[string]any{idKey: normalizedManagedClusterID, typeKey: managedClustersType}, + configured: map[string]source.Extra{managedClustersType: nil}, + operation: source.DataOperationUpsert, + expectedData: []source.Data{ + { + Type: managedClustersType, + Operation: source.DataOperationUpsert, + Time: testTime, + Values: map[string]any{idKey: normalizedManagedClusterID, typeKey: managedClustersType}, + }, + }, + }, + } + + for testName, test := range testCases { + t.Run(testName, func(t *testing.T) { + t.Parallel() + + data := resourceDataToEmit(test.resourceType, test.values, test.configured, test.operation, testTime) + + // the item of the resource is always emitted first, so that it exists before the + // relationship of a sub-type references it + require.Equal(t, test.expectedData, data) + }) + } +} + +// TestResourceDataToEmitIsolatesThePayloads checks the property the emission relies on: the +// mapping functions can write into the top level of the payload they are handed, so two items must +// never share one map. +func TestResourceDataToEmitIsolatesThePayloads(t *testing.T) { + t.Parallel() + + values := map[string]any{ + idKey: normalizedWebsiteID, + typeKey: websitesType, + kindKey: functionAppKindValue, + } + + data := resourceDataToEmit(websitesType, values, map[string]source.Extra{ + websitesType: {apiVersionKey: websitesAPIVersion}, + functionAppsType: {apiVersionKey: websitesAPIVersion}, + }, source.DataOperationUpsert, testTime) + require.Len(t, data, 2) + + data[0].Values["writtenByTheMapper"] = true + assert.NotContains(t, data[1].Values, "writtenByTheMapper") + + data[1].Values["writtenByTheOtherMapper"] = true + assert.NotContains(t, data[0].Values, "writtenByTheOtherMapper") +} + +func TestWarnOrphanSubTypes(t *testing.T) { + t.Parallel() + + const orphanMessage = "without the mapping of its parent type" + + testCases := map[string]struct { + typesToFilter map[string]source.Extra + expectedMessages []string + absentMessages []string + }{ + "a sub-type mapping loaded alone is reported": { + typesToFilter: map[string]source.Extra{functionAppsType: nil}, + expectedMessages: []string{orphanMessage, functionAppsType, websitesType}, + }, + "a sub-type mapping loaded with its parent stays silent": { + typesToFilter: map[string]source.Extra{websitesType: nil, functionAppsType: nil}, + absentMessages: []string{orphanMessage}, + }, + "a sub-type mapping loaded with a differently cased parent stays silent": { + typesToFilter: map[string]source.Extra{"microsoft.web/sites": nil, functionAppsType: nil}, + absentMessages: []string{orphanMessage}, + }, + "a parent mapping loaded alone stays silent": { + typesToFilter: map[string]source.Extra{websitesType: nil}, + absentMessages: []string{orphanMessage}, + }, + "an unrelated mapping stays silent": { + typesToFilter: map[string]source.Extra{managedClustersType: nil}, + absentMessages: []string{orphanMessage}, + }, + "no mapping stays silent": { + absentMessages: []string{orphanMessage}, + }, + } + + for testName, test := range testCases { + t.Run(testName, func(t *testing.T) { + t.Parallel() + + buffer := new(bytes.Buffer) + log := logger.NewLogger(buffer) + log.SetLevel(logger.TRACE) + + warnOrphanSubTypes(log, test.typesToFilter) + + for _, message := range test.expectedMessages { + assert.Contains(t, buffer.String(), message) + } + for _, message := range test.absentMessages { + assert.NotContains(t, buffer.String(), message) + } + }) + } +} From bef79ec47bcb4a9fc8bcff81b88a33d245f925e6 Mon Sep 17 00:00:00 2001 From: APagani Date: Wed, 12 Aug 2026 12:13:38 +0200 Subject: [PATCH 19/20] feat: add new azure mappings (#87) --- docs/mappings/azure/apim_services.yaml | 46 +++++++++++ docs/mappings/azure/containerregistries.yaml | 82 +++++++++++++++++++ docs/mappings/azure/keyvaults.yaml | 45 ++++++++++ .../azure/notificationhubs_namespaces.yaml | 37 +++++++++ .../notificationhubs_notificationhubs.yaml | 48 +++++++++++ 5 files changed, 258 insertions(+) create mode 100644 docs/mappings/azure/apim_services.yaml create mode 100644 docs/mappings/azure/containerregistries.yaml create mode 100644 docs/mappings/azure/keyvaults.yaml create mode 100644 docs/mappings/azure/notificationhubs_namespaces.yaml create mode 100644 docs/mappings/azure/notificationhubs_notificationhubs.yaml diff --git a/docs/mappings/azure/apim_services.yaml b/docs/mappings/azure/apim_services.yaml new file mode 100644 index 0000000..befddea --- /dev/null +++ b/docs/mappings/azure/apim_services.yaml @@ -0,0 +1,46 @@ +apiVersion: azure.mia-platform.eu/v1 +itemFamily: apimservices +type: Microsoft.ApiManagement/Service +extra: + apiVersion: "2025-03-01" +syncable: true +mappings: + identifier: "{{ .id | sha256sum }}" + metadata: + title: "{{ .name }}" + spec: + name: "{{ .name }}" + id: "{{ .id }}" + kind: "{{ .kind }}" + location: "{{ .location }}" + managedBy: "{{ .managedBy }}" + plan: "{{ .plan }}" + apiVersionConstraintMinApiVersion: "{{ .properties.apiVersionConstraint.minApiVersion | toJSON }}" + certificates: "{{ .properties.certificates | toJSON }}" + configurationApi: "{{ .properties.configurationApi | toJSON }}" + createdAtUtc: "{{ .properties.createdAtUtc }}" + customProperties: "{{ .properties.customProperties | toJSON }}" + dataApiUrl: "{{ .properties.dataApiUrl }}" + developerPortalStatus: "{{ .properties.developerPortalStatus }}" + developerPortalUrl: "{{ .properties.developerPortalUrl }}" + disableGateway: "{{ .properties.disableGateway }}" + gatewayRegionalUrl: "{{ .properties.gatewayRegionalUrl }}" + gatewayUrl: "{{ .properties.gatewayUrl }}" + legacyPortalStatus: "{{ .properties.legacyPortalStatus }}" + managementApiUrl: "{{ .properties.managementApiUrl }}" + notificationSenderEmail: "{{ .properties.notificationSenderEmail }}" + platformVersion: "{{ .properties.platformVersion }}" + portalUrl: "{{ .properties.portalUrl }}" + provisioningState: "{{ .properties.provisioningState }}" + publicIPAddresses: "{{ .properties.publicIPAddresses | toJSON }}" + publicNetworkAccess: "{{ .properties.publicNetworkAccess }}" + publisherEmail: "{{ .properties.publisherEmail }}" + publisherName: "{{ .properties.publisherName }}" + scmUrl: "{{ .properties.scmUrl }}" + tags: |- + {{ $tags := (get "tags" . "") -}} + {{- if $tags -}} + {{- $tags | toJSON -}} + {{- else -}} + {{- object | toJSON -}} + {{- end }} diff --git a/docs/mappings/azure/containerregistries.yaml b/docs/mappings/azure/containerregistries.yaml new file mode 100644 index 0000000..d50d1a1 --- /dev/null +++ b/docs/mappings/azure/containerregistries.yaml @@ -0,0 +1,82 @@ +apiVersion: azure.mia-platform.eu/v1 +itemFamily: containerregistries +type: Microsoft.ContainerRegistry/registries +extra: + apiVersion: "2026-01-01" +syncable: true +mappings: + identifier: "{{ .id | sha256sum }}" + metadata: + title: "{{ .name }}" + spec: + name: "{{ .name }}" + id: "{{ .id }}" + identity: |- + {{ $identity := (get "identity" . "") -}} + {{- if $identity -}} + {{- .identity -}} + {{- else -}} + {{- "" | quote -}} + {{- end }} + kind: "{{ .kind }}" + location: "{{ .location }}" + managedBy: "{{ .managedBy }}" + plan: |- + {{ $plan := (get "plan" . "") -}} + {{- if $plan -}} + {{- .plan -}} + {{- else -}} + {{- "" | quote -}} + {{- end }} + adminUserEnabled: "{{ .properties.adminUserEnabled }}" + anonymousPullEnabled: "{{ .properties.anonymousPullEnabled }}" + creationDate: "{{ .properties.creationDate }}" + dataEndpointEnabled: "{{ .properties.dataEndpointEnabled }}" + dataEndpointHostNames: "{{ .properties.dataEndpointHostNames | toJSON }}" + encryptionStatus: |- + {{ $encryption := get "encryption" .properties (object) -}} + {{- get "status" $encryption "" | quote }} + endpointProtocol: "{{ .properties.endpointProtocol }}" + loginServer: "{{ .properties.loginServer }}" + networkRuleBypassAllowedForTasks: "{{ .properties.networkRuleBypassAllowedForTasks }}" + networkRuleBypassOptions: "{{ .properties.networkRuleBypassOptions }}" + azureADAuthenticationAsArmPolicyStatus: |- + {{ $policies := get "policies" .properties (object) -}} + {{- $policy := get "azureADAuthenticationAsArmPolicy" $policies (object) -}} + {{- get "status" $policy "" | quote }} + exportPolicyStatus: |- + {{ $policies := get "policies" .properties (object) -}} + {{- $policy := get "exportPolicy" $policies (object) -}} + {{- get "status" $policy "" | quote }} + quarantinePolicyStatus: |- + {{ $policies := get "policies" .properties (object) -}} + {{- $policy := get "quarantinePolicy" $policies (object) -}} + {{- get "status" $policy "" | quote }} + retentionPolicyDays: |- + {{ $policies := get "policies" .properties (object) -}} + {{- $policy := get "retentionPolicy" $policies (object) -}} + {{- $days := get "days" $policy -1 }} + {{- $days }} + retentionPolicyStatus: |- + {{ $policies := get "policies" .properties (object) -}} + {{- $policy := get "retentionPolicy" $policies (object) -}} + {{- get "status" $policy "" | quote }} + softDeletePolicyRetentionDays: |- + {{ $policies := get "policies" .properties (object) -}} + {{- $policy := get "softDeletePolicy" $policies (object) -}} + {{- $days := get "retentionDays" $policy -1 }} + {{- $days }} + softDeletePolicyStatus: |- + {{ $policies := get "policies" .properties (object) -}} + {{- $policy := get "softDeletePolicy" $policies (object) -}} + {{- get "status" $policy "" | quote }} + provisioningState: "{{ .properties.provisioningState }}" + publicNetworkAccess: "{{ .properties.publicNetworkAccess }}" + zoneRedundancy: "{{ .properties.zoneRedundancy }}" + tags: |- + {{ $tags := (get "tags" . "") -}} + {{- if $tags -}} + {{- $tags | toJSON -}} + {{- else -}} + {{- object | toJSON -}} + {{- end }} diff --git a/docs/mappings/azure/keyvaults.yaml b/docs/mappings/azure/keyvaults.yaml new file mode 100644 index 0000000..a11b9d6 --- /dev/null +++ b/docs/mappings/azure/keyvaults.yaml @@ -0,0 +1,45 @@ +apiVersion: azure.mia-platform.eu/v1 +itemFamily: keyvaults +type: Microsoft.KeyVault/vaults +extra: + apiVersion: "2026-01-01" +syncable: true +mappings: + identifier: "{{ .id | sha256sum }}" + metadata: + title: "{{ .name }}" + spec: + name: "{{ .name }}" + id: "{{ .id }}" + identity: |- + {{ $identity := (get "identity" . "") -}} + {{- if $identity -}} + {{- .identity -}} + {{- else -}} + {{- "" | quote -}} + {{- end }} + kind: "{{ .kind }}" + location: "{{ .location }}" + managedBy: "{{ .managedBy }}" + plan: |- + {{ $plan := (get "plan" . "") -}} + {{- if $plan -}} + {{- .plan -}} + {{- else -}} + {{- "" | quote -}} + {{- end }} + enableRbacAuthorization: "{{ .properties.enableRbacAuthorization }}" + enableSoftDelete: "{{ .properties.enableSoftDelete }}" + enabledForDeployment: "{{ .properties.enabledForDeployment }}" + enabledForDiskEncryption: "{{ .properties.enabledForDiskEncryption }}" + provisioningState: "{{ .properties.provisioningState }}" + publicNetworkAccess: "{{ .properties.publicNetworkAccess }}" + tenantId: "{{ .properties.tenantId }}" + vaultUri: "{{ .properties.vaultUri }}" + tags: |- + {{ $tags := (get "tags" . "") -}} + {{- if $tags -}} + {{- $tags | toJSON -}} + {{- else -}} + {{- object | toJSON -}} + {{- end }} diff --git a/docs/mappings/azure/notificationhubs_namespaces.yaml b/docs/mappings/azure/notificationhubs_namespaces.yaml new file mode 100644 index 0000000..adcc200 --- /dev/null +++ b/docs/mappings/azure/notificationhubs_namespaces.yaml @@ -0,0 +1,37 @@ +apiVersion: azure.mia-platform.eu/v1 +itemFamily: notificationhubsnamespaces +type: Microsoft.NotificationHubs/namespaces +extra: + apiVersion: "2026-01-01" +syncable: true +mappings: + identifier: "{{ .id | sha256sum }}" + metadata: + title: "{{ .name }}" + spec: + name: "{{ .name }}" + id: "{{ .id }}" + kind: "{{ .kind }}" + location: "{{ .location }}" + managedBy: "{{ .managedBy }}" + plan: |- + {{ $plan := (get "plan" . "") -}} + {{- if $plan -}} + {{- .plan -}} + {{- else -}} + {{- "" | quote -}} + {{- end }} + createdAt: "{{ .properties.createdAt }}" + critical: "{{ .properties.critical }}" + enabled: "{{ .properties.enabled }}" + provisioningState: "{{ .properties.provisioningState }}" + serviceBusEndpoint: "{{ .properties.serviceBusEndpoint }}" + status: "{{ .properties.status }}" + updatedAt: "{{ .properties.updatedAt }}" + tags: |- + {{ $tags := (get "tags" . "") -}} + {{- if $tags -}} + {{- $tags | toJSON -}} + {{- else -}} + {{- object | toJSON -}} + {{- end }} diff --git a/docs/mappings/azure/notificationhubs_notificationhubs.yaml b/docs/mappings/azure/notificationhubs_notificationhubs.yaml new file mode 100644 index 0000000..4d66ec1 --- /dev/null +++ b/docs/mappings/azure/notificationhubs_notificationhubs.yaml @@ -0,0 +1,48 @@ +apiVersion: azure.mia-platform.eu/v1 +itemFamily: notificationhubs +type: Microsoft.NotificationHubs/namespaces/notificationhubs +extra: + apiVersion: "2026-01-01" +syncable: true +mappings: + identifier: "{{ .id | sha256sum }}" + metadata: + title: "{{ .name }}" + spec: + name: "{{ .name }}" + id: "{{ .id }}" + kind: "{{ .kind }}" + location: "{{ .location }}" + managedBy: "{{ .managedBy }}" + plan: |- + {{ $plan := (get "plan" . "") -}} + {{- if $plan -}} + {{- .plan -}} + {{- else -}} + {{- "" | quote -}} + {{- end }} + dailyMaxActiveDevices: "{{ .properties.dailyMaxActiveDevices }}" + hubName: "{{ .properties.name }}" + registrationTtl: "{{ .properties.registrationTtl }}" + tags: |- + {{ $tags := (get "tags" . "") -}} + {{- if $tags -}} + {{- $tags | toJSON -}} + {{- else -}} + {{- object | toJSON -}} + {{- end }} + extra: + - apiVersion: mia-platform.eu/v1 + itemFamily: relationships + deletePolicy: "cascade" + identifier: |- + {{ $namespaceId := printf "%s" (first (split "/notificationhubs/" .id)) -}} + {{- $src := printf "urn:mia-platform-catalog:azure.mia-platform.eu:v1:NotificationHub:%s" (printf "%s" .id | sha256sum) -}} + {{- $type := "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:part-of.mia-platform.eu" -}} + {{- $tgt := printf "urn:mia-platform-catalog:azure.mia-platform.eu:v1:NotificationHubsNamespace:%s" ($namespaceId | sha256sum) -}} + {{- printf "%s-%s-%s" $src $type $tgt | sha256sum }} + sourceRef: |- + urn:mia-platform-catalog:azure.mia-platform.eu:v1:NotificationHub:{{ printf "%s" .id | sha256sum }} + targetRef: |- + urn:mia-platform-catalog:azure.mia-platform.eu:v1:NotificationHubsNamespace:{{ printf "%s" (first (split "/notificationhubs/" .id)) | sha256sum }} + typeRef: "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:part-of.mia-platform.eu" From a4a6dadbfc0559f9f3009b407d4ad556bbf5da3b Mon Sep 17 00:00:00 2001 From: APagani Date: Thu, 13 Aug 2026 11:59:50 +0200 Subject: [PATCH 20/20] fix: remove missing fields from API in mappings and fix nullable fields (#88) --- docs/mappings/azure/apim_services.yaml | 46 +++++++++++++++---- docs/mappings/azure/containerregistries.yaml | 20 +------- docs/mappings/azure/keyvaults.yaml | 27 ++++------- .../azure/notificationhubs_namespaces.yaml | 12 +---- .../notificationhubs_notificationhubs.yaml | 16 +++---- 5 files changed, 52 insertions(+), 69 deletions(-) diff --git a/docs/mappings/azure/apim_services.yaml b/docs/mappings/azure/apim_services.yaml index befddea..207a664 100644 --- a/docs/mappings/azure/apim_services.yaml +++ b/docs/mappings/azure/apim_services.yaml @@ -2,7 +2,7 @@ apiVersion: azure.mia-platform.eu/v1 itemFamily: apimservices type: Microsoft.ApiManagement/Service extra: - apiVersion: "2025-03-01" + apiVersion: "2024-05-01" syncable: true mappings: identifier: "{{ .id | sha256sum }}" @@ -11,16 +11,36 @@ mappings: spec: name: "{{ .name }}" id: "{{ .id }}" - kind: "{{ .kind }}" - location: "{{ .location }}" - managedBy: "{{ .managedBy }}" - plan: "{{ .plan }}" - apiVersionConstraintMinApiVersion: "{{ .properties.apiVersionConstraint.minApiVersion | toJSON }}" - certificates: "{{ .properties.certificates | toJSON }}" - configurationApi: "{{ .properties.configurationApi | toJSON }}" + apiVersionConstraintMinApiVersion: |- + {{ $apiVersionConstraintMinApiVersion := (get "minApiVersion" .properties.apiVersionConstraint "") -}} + {{- if $apiVersionConstraintMinApiVersion -}} + {{- $apiVersionConstraintMinApiVersion -}} + {{- else -}} + {{- "" | quote -}} + {{- end }} + certificates: |- + {{ $certificates := (get "certificates" .properties "") -}} + {{- if $certificates -}} + {{- .properties.certificates | toJSON -}} + {{- else -}} + {{- list | toJSON -}} + {{- end }} + configurationApi: |- + {{ $configurationApi := (get "configurationApi" .properties "") -}} + {{- if $configurationApi -}} + {{- .properties.configurationApi | toJSON -}} + {{- else -}} + {{- object | toJSON -}} + {{- end }} createdAtUtc: "{{ .properties.createdAtUtc }}" customProperties: "{{ .properties.customProperties | toJSON }}" - dataApiUrl: "{{ .properties.dataApiUrl }}" + dataApiUrl: |- + {{ $dataApiUrl := (get "dataApiUrl" .properties "") -}} + {{- if $dataApiUrl -}} + {{- .properties.dataApiUrl -}} + {{- else -}} + {{- "" | quote -}} + {{- end }} developerPortalStatus: "{{ .properties.developerPortalStatus }}" developerPortalUrl: "{{ .properties.developerPortalUrl }}" disableGateway: "{{ .properties.disableGateway }}" @@ -30,7 +50,13 @@ mappings: managementApiUrl: "{{ .properties.managementApiUrl }}" notificationSenderEmail: "{{ .properties.notificationSenderEmail }}" platformVersion: "{{ .properties.platformVersion }}" - portalUrl: "{{ .properties.portalUrl }}" + portalUrl: |- + {{ $portalUrl := (get "portalUrl" .properties "") -}} + {{- if $portalUrl -}} + {{- .properties.portalUrl -}} + {{- else -}} + {{- "" | quote -}} + {{- end }} provisioningState: "{{ .properties.provisioningState }}" publicIPAddresses: "{{ .properties.publicIPAddresses | toJSON }}" publicNetworkAccess: "{{ .properties.publicNetworkAccess }}" diff --git a/docs/mappings/azure/containerregistries.yaml b/docs/mappings/azure/containerregistries.yaml index d50d1a1..9031752 100644 --- a/docs/mappings/azure/containerregistries.yaml +++ b/docs/mappings/azure/containerregistries.yaml @@ -2,7 +2,7 @@ apiVersion: azure.mia-platform.eu/v1 itemFamily: containerregistries type: Microsoft.ContainerRegistry/registries extra: - apiVersion: "2026-01-01" + apiVersion: "2025-11-01" syncable: true mappings: identifier: "{{ .id | sha256sum }}" @@ -11,23 +11,6 @@ mappings: spec: name: "{{ .name }}" id: "{{ .id }}" - identity: |- - {{ $identity := (get "identity" . "") -}} - {{- if $identity -}} - {{- .identity -}} - {{- else -}} - {{- "" | quote -}} - {{- end }} - kind: "{{ .kind }}" - location: "{{ .location }}" - managedBy: "{{ .managedBy }}" - plan: |- - {{ $plan := (get "plan" . "") -}} - {{- if $plan -}} - {{- .plan -}} - {{- else -}} - {{- "" | quote -}} - {{- end }} adminUserEnabled: "{{ .properties.adminUserEnabled }}" anonymousPullEnabled: "{{ .properties.anonymousPullEnabled }}" creationDate: "{{ .properties.creationDate }}" @@ -36,7 +19,6 @@ mappings: encryptionStatus: |- {{ $encryption := get "encryption" .properties (object) -}} {{- get "status" $encryption "" | quote }} - endpointProtocol: "{{ .properties.endpointProtocol }}" loginServer: "{{ .properties.loginServer }}" networkRuleBypassAllowedForTasks: "{{ .properties.networkRuleBypassAllowedForTasks }}" networkRuleBypassOptions: "{{ .properties.networkRuleBypassOptions }}" diff --git a/docs/mappings/azure/keyvaults.yaml b/docs/mappings/azure/keyvaults.yaml index a11b9d6..d047336 100644 --- a/docs/mappings/azure/keyvaults.yaml +++ b/docs/mappings/azure/keyvaults.yaml @@ -2,7 +2,7 @@ apiVersion: azure.mia-platform.eu/v1 itemFamily: keyvaults type: Microsoft.KeyVault/vaults extra: - apiVersion: "2026-01-01" + apiVersion: "2026-02-01" syncable: true mappings: identifier: "{{ .id | sha256sum }}" @@ -11,27 +11,16 @@ mappings: spec: name: "{{ .name }}" id: "{{ .id }}" - identity: |- - {{ $identity := (get "identity" . "") -}} - {{- if $identity -}} - {{- .identity -}} - {{- else -}} - {{- "" | quote -}} - {{- end }} - kind: "{{ .kind }}" - location: "{{ .location }}" - managedBy: "{{ .managedBy }}" - plan: |- - {{ $plan := (get "plan" . "") -}} - {{- if $plan -}} - {{- .plan -}} - {{- else -}} - {{- "" | quote -}} - {{- end }} enableRbacAuthorization: "{{ .properties.enableRbacAuthorization }}" enableSoftDelete: "{{ .properties.enableSoftDelete }}" enabledForDeployment: "{{ .properties.enabledForDeployment }}" - enabledForDiskEncryption: "{{ .properties.enabledForDiskEncryption }}" + enabledForDiskEncryption: |- + {{ $enabledForDiskEncryption := (get "enabledForDiskEncryption" .properties "") -}} + {{- if $enabledForDiskEncryption -}} + {{- .properties.enabledForDiskEncryption -}} + {{- else -}} + {{- false -}} + {{- end }} provisioningState: "{{ .properties.provisioningState }}" publicNetworkAccess: "{{ .properties.publicNetworkAccess }}" tenantId: "{{ .properties.tenantId }}" diff --git a/docs/mappings/azure/notificationhubs_namespaces.yaml b/docs/mappings/azure/notificationhubs_namespaces.yaml index adcc200..45b7b6f 100644 --- a/docs/mappings/azure/notificationhubs_namespaces.yaml +++ b/docs/mappings/azure/notificationhubs_namespaces.yaml @@ -2,7 +2,7 @@ apiVersion: azure.mia-platform.eu/v1 itemFamily: notificationhubsnamespaces type: Microsoft.NotificationHubs/namespaces extra: - apiVersion: "2026-01-01" + apiVersion: "2023-09-01" syncable: true mappings: identifier: "{{ .id | sha256sum }}" @@ -11,16 +11,6 @@ mappings: spec: name: "{{ .name }}" id: "{{ .id }}" - kind: "{{ .kind }}" - location: "{{ .location }}" - managedBy: "{{ .managedBy }}" - plan: |- - {{ $plan := (get "plan" . "") -}} - {{- if $plan -}} - {{- .plan -}} - {{- else -}} - {{- "" | quote -}} - {{- end }} createdAt: "{{ .properties.createdAt }}" critical: "{{ .properties.critical }}" enabled: "{{ .properties.enabled }}" diff --git a/docs/mappings/azure/notificationhubs_notificationhubs.yaml b/docs/mappings/azure/notificationhubs_notificationhubs.yaml index 4d66ec1..8fa5f8a 100644 --- a/docs/mappings/azure/notificationhubs_notificationhubs.yaml +++ b/docs/mappings/azure/notificationhubs_notificationhubs.yaml @@ -2,7 +2,7 @@ apiVersion: azure.mia-platform.eu/v1 itemFamily: notificationhubs type: Microsoft.NotificationHubs/namespaces/notificationhubs extra: - apiVersion: "2026-01-01" + apiVersion: "2017-04-01" syncable: true mappings: identifier: "{{ .id | sha256sum }}" @@ -11,18 +11,14 @@ mappings: spec: name: "{{ .name }}" id: "{{ .id }}" - kind: "{{ .kind }}" - location: "{{ .location }}" - managedBy: "{{ .managedBy }}" - plan: |- - {{ $plan := (get "plan" . "") -}} - {{- if $plan -}} - {{- .plan -}} + dailyMaxActiveDevices: "{{ .properties.dailyMaxActiveDevices }}" + hubName: |- + {{ $name := (get "name" .properties "") -}} + {{- if $name -}} + {{- .properties.name -}} {{- else -}} {{- "" | quote -}} {{- end }} - dailyMaxActiveDevices: "{{ .properties.dailyMaxActiveDevices }}" - hubName: "{{ .properties.name }}" registrationTtl: "{{ .properties.registrationTtl }}" tags: |- {{ $tags := (get "tags" . "") -}}