From 8aca78c47d0dbe1dd6a94ff02203764cab1e7595 Mon Sep 17 00:00:00 2001 From: apagani Date: Fri, 31 Jul 2026 15:54:15 +0200 Subject: [PATCH 1/4] add id field to mappings --- docs/mappings/azure/cognitiveaccounts.yaml | 1 + docs/mappings/azure/containerapps.yaml | 1 + docs/mappings/azure/managedclusters.yaml | 1 + docs/mappings/azure/postgresqldbs.yaml | 1 + docs/mappings/azure/resourcegroups.yaml | 1 + docs/mappings/azure/storageaccounts.yaml | 1 + docs/mappings/azure/subscriptions.yaml | 1 + docs/mappings/azure/virtualmachines.yaml | 1 + docs/mappings/azure/virtualnetworks.yaml | 1 + docs/mappings/azure/websites.yaml | 1 + 10 files changed, 10 insertions(+) diff --git a/docs/mappings/azure/cognitiveaccounts.yaml b/docs/mappings/azure/cognitiveaccounts.yaml index 0ab5e14..7e7bcef 100644 --- a/docs/mappings/azure/cognitiveaccounts.yaml +++ b/docs/mappings/azure/cognitiveaccounts.yaml @@ -10,6 +10,7 @@ mappings: title: "{{ .name }}" spec: name: "{{ .name }}" + id: "{{ .id }}" location: "{{ .location | lower }}" provisioningState: "{{ .properties.provisioningState | lower }}" kind: "{{ .kind }}" diff --git a/docs/mappings/azure/containerapps.yaml b/docs/mappings/azure/containerapps.yaml index 1202e34..d32f6a9 100644 --- a/docs/mappings/azure/containerapps.yaml +++ b/docs/mappings/azure/containerapps.yaml @@ -10,6 +10,7 @@ mappings: title: "{{ .name }}" spec: name: "{{ .name }}" + id: "{{ .id }}" location: "{{ .location | lower }}" provisioningState: "{{ .properties.provisioningState | lower }}" runningStatus: "{{ .properties.runningStatus | lower }}" diff --git a/docs/mappings/azure/managedclusters.yaml b/docs/mappings/azure/managedclusters.yaml index 490ea24..0bc4c8a 100644 --- a/docs/mappings/azure/managedclusters.yaml +++ b/docs/mappings/azure/managedclusters.yaml @@ -10,6 +10,7 @@ mappings: title: "{{ .name }}" spec: name: "{{ .name }}" + id: "{{ .id }}" location: "{{ .location | lower }}" provisioningState: "{{ .properties.provisioningState | lower }}" currentKubernetesVersion: "{{ .properties.currentKubernetesVersion | quote }}" diff --git a/docs/mappings/azure/postgresqldbs.yaml b/docs/mappings/azure/postgresqldbs.yaml index 1762c71..dfe8694 100644 --- a/docs/mappings/azure/postgresqldbs.yaml +++ b/docs/mappings/azure/postgresqldbs.yaml @@ -10,6 +10,7 @@ mappings: title: "{{ .name }}" spec: name: "{{ .name }}" + id: "{{ .id }}" location: "{{ .location | lower }}" state: "{{ .properties.state | lower }}" version: "{{ .properties.version }}" diff --git a/docs/mappings/azure/resourcegroups.yaml b/docs/mappings/azure/resourcegroups.yaml index 3e6f103..f9770a0 100644 --- a/docs/mappings/azure/resourcegroups.yaml +++ b/docs/mappings/azure/resourcegroups.yaml @@ -10,6 +10,7 @@ mappings: title: "{{ .name }}" spec: name: "{{ .name }}" + id: "{{ .id }}" location: "{{ .location | lower }}" provisioningState: "{{ .properties.provisioningState | lower }}" tags: |- diff --git a/docs/mappings/azure/storageaccounts.yaml b/docs/mappings/azure/storageaccounts.yaml index 8904d78..1274673 100644 --- a/docs/mappings/azure/storageaccounts.yaml +++ b/docs/mappings/azure/storageaccounts.yaml @@ -10,6 +10,7 @@ mappings: title: "{{ .name }}" spec: name: "{{ .name }}" + id: "{{ .id }}" provisioningState: "{{ .properties.provisioningState | lower }}" allowBlobPublicAccess: "{{ get \"allowBlobPublicAccess\" .properties false }}" publicNetworkAccess: "{{ get \"publicNetworkAccess\" .properties \"Disabled\" | lower }}" diff --git a/docs/mappings/azure/subscriptions.yaml b/docs/mappings/azure/subscriptions.yaml index 02fbd11..25dff2b 100644 --- a/docs/mappings/azure/subscriptions.yaml +++ b/docs/mappings/azure/subscriptions.yaml @@ -10,6 +10,7 @@ mappings: title: "{{ .name }}" spec: name: "{{ .name }}" + id: "{{ .id }}" displayName: "{{ get \"displayName\" . \"\" | quote }}" state: "{{ .properties.state | lower }}" tags: |- diff --git a/docs/mappings/azure/virtualmachines.yaml b/docs/mappings/azure/virtualmachines.yaml index 0e66cf7..cdb5c52 100644 --- a/docs/mappings/azure/virtualmachines.yaml +++ b/docs/mappings/azure/virtualmachines.yaml @@ -10,6 +10,7 @@ mappings: title: "{{ .name }}" spec: name: "{{ .name }}" + id: "{{ .id }}" location: "{{ .location | lower }}" provisioningState: "{{ .properties.provisioningState | lower }}" vmSize: "{{ .properties.hardwareProfile.vmSize }}" diff --git a/docs/mappings/azure/virtualnetworks.yaml b/docs/mappings/azure/virtualnetworks.yaml index 1da91a1..0432590 100644 --- a/docs/mappings/azure/virtualnetworks.yaml +++ b/docs/mappings/azure/virtualnetworks.yaml @@ -10,6 +10,7 @@ mappings: title: "{{ .name }}" spec: name: "{{ .name }}" + id: "{{ .id }}" location: "{{ .location | lower }}" provisioningState: "{{ .properties.provisioningState | lower }}" addressPrefixes: "{{ .properties.addressSpace.addressPrefixes | toJSON }}" diff --git a/docs/mappings/azure/websites.yaml b/docs/mappings/azure/websites.yaml index d9b022b..fcc0597 100644 --- a/docs/mappings/azure/websites.yaml +++ b/docs/mappings/azure/websites.yaml @@ -10,6 +10,7 @@ mappings: title: "{{ .name }}" spec: name: "{{ .name }}" + id: "{{ .id }}" location: "{{ .location | lower }}" state: "{{ .properties.state | lower }}" kind: "{{ .kind }}" From 78ac6f082325f4b4f1aadd4aeea5b2c678cee78d Mon Sep 17 00:00:00 2001 From: apagani Date: Fri, 31 Jul 2026 18:49:34 +0200 Subject: [PATCH 2/4] fix: azure resource ID normalization (lowercase fold) --- docs/how-to/030_azure-source.md | 50 +++ internal/source/azure/azure.go | 40 +-- .../source/azure/azure_identifier_test.go | 129 ++++++++ internal/source/azure/azure_stream_test.go | 177 +++++++++- internal/source/azure/azure_sync_test.go | 122 ++++++- internal/source/azure/resourceid.go | 73 ++++ internal/source/azure/resourceid_test.go | 311 ++++++++++++++++++ 7 files changed, 860 insertions(+), 42 deletions(-) create mode 100644 internal/source/azure/azure_identifier_test.go create mode 100644 internal/source/azure/resourceid.go create mode 100644 internal/source/azure/resourceid_test.go diff --git a/docs/how-to/030_azure-source.md b/docs/how-to/030_azure-source.md index 82b26b6..a1646a5 100644 --- a/docs/how-to/030_azure-source.md +++ b/docs/how-to/030_azure-source.md @@ -55,6 +55,56 @@ Using the `*_CONNECTION_STRING` variables is the preferred methods that will als setup the least privileges to the service account responsible to retrieve information from the REST APIs. +## Resource identifiers + +Microsoft Azure does not guarantee the letter case of the resource IDs it returns: the same +resource can arrive with `resourceGroups` from the resource graph APIs and with `resourcegroups` +from the resource provider that answers the EventHub driven read, and the provider and type +segments vary in the same way. Because the mappings hash the ID to build the Catalog identifier, +every casing difference would create a duplicate item instead of updating the existing one. + +To prevent that the source normalises every resource before handing it to the mapper: + +- `id` is lowercased in full +- `type` is set to the resource type exactly as the mapping file declares it + +The two values are therefore identical for `ibdm sync azure` and `ibdm run azure`, which makes +`{{ .id | sha256sum }}` a stable identifier and lets a delete event target the item a previous +import created. + +### Mapping a resource type with case-sensitive names + +The normalisation is safe only while every mapped resource type has **case-insensitive** names, +which Microsoft Azure documents for all the types shipped in `docs/mappings/azure`. Blob +containers and Log Analytics solutions are the two documented exceptions: their names *are* +case-sensitive, so Azure can hold two distinct resources whose IDs differ only by case. Lowercasing +would collapse them onto a single Catalog `metadata.name` and the second import would silently +overwrite the first. Review this behaviour before adding a mapping for such a type — nothing +detects the situation at runtime. + +### Consequences for the mappings + +`id` is lowercase and no longer matches the casing shown in the Azure portal. Use `.name` wherever +display casing matters. The spelling Azure reported is deliberately not exposed to the mappings: it +is written to the source logs at the `Debug` level whenever it differs from the normalised value, +so the resource providers returning non canonical IDs stay observable without any template being +able to hash the unstable value by mistake. + +### Upgrading from a release without the normalisation + +Canonical Azure IDs contain uppercase characters, so lowercasing changes `sha256sum` for **every** +Azure resource, not only the ones affected by the divergence. Each item therefore receives a new +identifier, the destination inserts it as a new Catalog item and the previous one is left behind as +an orphan, together with the relationship items that cascade from it. + +1. Capture the current list of Azure items **before** upgrading: identifiers are opaque hashes, so + afterwards a stale item is indistinguishable from a fresh one. +1. Run `ibdm sync azure` right after deploying so that the new items exist. +1. Delete the items captured at step 1 and their cascaded relationships. This cleanup is manual. + +Expect the item count to grow slightly as well: resources whose events spelled the resource type +with a casing different from the mapping file were previously dropped and now get imported. + ## Authentication The source is using the [`DefaultAzureCredential` chain of authentication] so you can setup diff --git a/internal/source/azure/azure.go b/internal/source/azure/azure.go index 2177c28..7b43d84 100644 --- a/internal/source/azure/azure.go +++ b/internal/source/azure/azure.go @@ -10,7 +10,6 @@ import ( "fmt" "maps" "slices" - "strings" "sync" "sync/atomic" "time" @@ -143,7 +142,7 @@ func (s *Source) StartSyncProcess(ctx context.Context, typesToFilter map[string] if data, ok := response.Data.([]any); ok { for _, item := range data { if values, ok := item.(map[string]any); ok { - values["type"] = resType // ensure type is case-normalized, and resourceGroup is normalized too + normalizeResourceValues(logger, values, resType) dataChannel <- source.Data{ Type: resType, Operation: source.DataOperationUpsert, @@ -224,18 +223,21 @@ func partitionEventHandler(client *armresources.Client, typesToFilter map[string continue } - if filterBasedOnResourceID(resID, typesSlice) { - logger.Debug("skipping event based on type", "resourceID", resID.ResourceType.String()) + // the subject can spell the resource type with any casing, so resolve the configured + // key once and use it for the apiVersion lookup and for every emitted value. + resourceType, ok := configuredResourceType(typesSlice, resID.ResourceType.String()) + if !ok { + logger.Debug("skipping event based on type", "resourceType", resID.ResourceType.String()) continue } - apiVersion, ok := typesToFilter[resID.ResourceType.String()]["apiVersion"].(string) + apiVersion, ok := typesToFilter[resourceType][apiVersionKey].(string) if !ok { - logger.Debug("skipping event with missing apiVersion", "resourceID", resID.ResourceType.String()) + logger.Debug("skipping event with missing apiVersion", "resourceType", resourceType) continue } - logger.Trace("handling resource", "resourceID", resID.ResourceType.String(), "eventType", envelope.Type, "apiVersion", apiVersion) + logger.Trace("handling resource", "resourceType", resourceType, "eventType", envelope.Type, "apiVersion", apiVersion) switch envelope.Type { case azsystemevents.TypeResourceWriteSuccess: logger.Trace("request resource data from azure", "resourceID", *envelope.Subject) @@ -255,25 +257,25 @@ func partitionEventHandler(client *armresources.Client, typesToFilter map[string continue } + normalizeResourceValues(logger, values, resourceType) dataChannel <- source.Data{ - Type: resID.ResourceType.String(), + Type: resourceType, Operation: source.DataOperationUpsert, Time: *envelope.Time, Values: values, } case azsystemevents.TypeResourceDeleteSuccess: - logger.Trace("we have to delete something", "resourceID", resID.ResourceType.String()) + logger.Trace("deleting resource", "resourceType", resourceType) + values := map[string]any{idKey: resID.String()} + normalizeResourceValues(logger, values, resourceType) dataChannel <- source.Data{ - Type: resID.ResourceType.String(), + Type: resourceType, Operation: source.DataOperationDelete, Time: *envelope.Time, - Values: map[string]any{ - "id": resID.String(), - "type": resID.ResourceType.String(), - }, + Values: values, } default: - logger.Trace("skipping resource", "resourceID", resID.ResourceType.String(), "eventType", envelope.Type, "apiVersion", apiVersion) + logger.Trace("skipping resource", "resourceType", resourceType, "eventType", envelope.Type, "apiVersion", apiVersion) } } } @@ -303,14 +305,6 @@ func resourceIDFromSubject(subject *string) (*arm.ResourceID, error) { return arm.ParseResourceID(*subject) } -// filterBasedOnResourceID checks if the resource type is in the typesToFilter slice. -func filterBasedOnResourceID(resID *arm.ResourceID, typesToFilter []string) bool { - resourceType := resID.ResourceType.String() - return !slices.ContainsFunc(typesToFilter, func(s string) bool { - return strings.EqualFold(s, resourceType) - }) -} - // Close implement source.ClosableSource. func (s *Source) Close(ctx context.Context, _ time.Duration) error { log := logger.FromContext(ctx).WithName(logName) diff --git a/internal/source/azure/azure_identifier_test.go b/internal/source/azure/azure_identifier_test.go new file mode 100644 index 0000000..601f8f6 --- /dev/null +++ b/internal/source/azure/azure_identifier_test.go @@ -0,0 +1,129 @@ +// Copyright Mia srl +// SPDX-License-Identifier: AGPL-3.0-only or Commercial + +package azure + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + fakeazcore "github.com/Azure/azure-sdk-for-go/sdk/azcore/fake" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs/v2" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources/v3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/mia-platform/ibdm/internal/source" +) + +const testSubscriptionID = "00000000-0000-0000-0000-000000000000" + +// eventDataManagedClusterCanonicalDeleteBody deletes the same managed cluster the sync and the +// stream upsert paths import. Its subject uses a lowercase resourcegroups literal that +// arm.ParseResourceID rewrites to camelCase, so the delete path builds subjectManagedClusterID. +var eventDataManagedClusterCanonicalDeleteBody = json.RawMessage(`[ +{ + "id": "00000000-0000-0000-0000-000000000000", + "source": "/subscriptions/00000000-0000-0000-0000-000000000000", + "specversion": "1.0", + "type": "Microsoft.Resources.ResourceDeleteSuccess", + "subject": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/my-rg/providers/Microsoft.ContainerService/managedClusters/my-cluster", + "time": "2020-01-01T00:00:00.0000000Z" +}]`) + +// TestAllPathsEmitTheSameIdentifier drives the three ingestion paths for one managed cluster and +// checks that they agree on the values the mappings hash into the Catalog identifier. Azure feeds +// each path a different spelling of the ID, so before the normalization the sync upsert and the +// stream upsert produced two Catalog items and the stream delete targeted neither of them. +func TestAllPathsEmitTheSameIdentifier(t *testing.T) { + t.Parallel() + + // the three spellings the fakes reproduce: Resource Graph answers with a camelCase + // resourceGroups literal, the resource provider body lowercases it, and the delete path rebuilds + // the ID from the event subject. Without this divergence the test would be vacuous. + require.NotEqual(t, graphManagedClusterID, bodyManagedClusterID, + "the sync and stream fixtures must disagree on casing") + require.Equal(t, graphManagedClusterID, subjectManagedClusterID, + "the delete path is expected to rebuild the Resource Graph spelling") + + paths := map[string]source.Data{ + "sync upsert": syncedManagedCluster(t), + "stream upsert": streamedManagedCluster(t, eventDataManagedClusterWriteBody), + "stream delete": streamedManagedCluster(t, eventDataManagedClusterCanonicalDeleteBody), + } + + assert.Equal(t, source.DataOperationUpsert, paths["sync upsert"].Operation) + assert.Equal(t, source.DataOperationUpsert, paths["stream upsert"].Operation) + assert.Equal(t, source.DataOperationDelete, paths["stream delete"].Operation) + + for pathName, data := range paths { + assert.Equal(t, normalizedManagedClusterID, data.Values[idKey], pathName) + assert.Equal(t, managedClustersType, data.Values[typeKey], pathName) + assert.Equal(t, managedClustersType, data.Type, pathName) + } +} + +// syncedManagedCluster runs a sync process against the Resource Graph fake and returns the data +// emitted for the my-cluster managed cluster. +func syncedManagedCluster(t *testing.T) source.Data { + t.Helper() + + ctx, cancel := context.WithTimeout(t.Context(), 1*time.Second) + t.Cleanup(cancel) + + azureSource := &Source{ + config: config{ + SubscriptionID: testSubscriptionID, + clientOptions: &arm.ClientOptions{ + ClientOptions: policy.ClientOptions{ + Transport: fakeResourceGraphTransport(t), + }, + }, + azureCredentials: &fakeazcore.TokenCredential{}, + }, + } + + dataChannel := make(chan source.Data, 10) + require.NoError(t, azureSource.StartSyncProcess(ctx, map[string]source.Extra{managedClustersType: nil}, dataChannel)) + close(dataChannel) + + for data := range dataChannel { + if data.Values["name"] == "my-cluster" { + return data + } + } + + require.FailNow(t, "the sync process did not emit the managed cluster") + return source.Data{} +} + +// streamedManagedCluster feeds body to the event handler and returns the single emitted data. +func streamedManagedCluster(t *testing.T, body json.RawMessage) source.Data { + t.Helper() + + ctx, cancel := context.WithTimeout(t.Context(), 1*time.Second) + t.Cleanup(cancel) + + client, err := armresources.NewClient(testSubscriptionID, &fakeazcore.TokenCredential{}, &arm.ClientOptions{ + ClientOptions: policy.ClientOptions{ + Transport: fakeClientTransport(t), + }, + }) + require.NoError(t, err) + + dataChannel := make(chan source.Data, 10) + handler := partitionEventHandler(client, map[string]source.Extra{ + managedClustersType: {apiVersionKey: managedClustersAPIVersion}, + }, dataChannel) + + handler(ctx, &azeventhubs.ReceivedEventData{EventData: azeventhubs.EventData{Body: body}}) + close(dataChannel) + + data, ok := <-dataChannel + require.True(t, ok, "the event handler did not emit any data") + return data +} diff --git a/internal/source/azure/azure_stream_test.go b/internal/source/azure/azure_stream_test.go index 33e0eba..6cdbd37 100644 --- a/internal/source/azure/azure_stream_test.go +++ b/internal/source/azure/azure_stream_test.go @@ -132,6 +132,81 @@ func TestPartitionEventHandler(t *testing.T) { }, }, }, + "resource provider returning a divergently cased id": { + contextFunc: func(tb testing.TB) (context.Context, context.CancelFunc) { + tb.Helper() + return context.WithTimeout(tb.Context(), 1*time.Second) + }, + typesToFilter: map[string]source.Extra{ + managedClustersType: {"apiVersion": managedClustersAPIVersion}, + }, + azureData: &azeventhubs.ReceivedEventData{ + EventData: azeventhubs.EventData{ + Body: eventDataManagedClusterWriteBody, + }, + }, + expectedData: []source.Data{ + { + Type: managedClustersType, + Operation: source.DataOperationUpsert, + Time: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC), + Values: map[string]any{ + "id": normalizedManagedClusterID, + "type": managedClustersType, + }, + }, + }, + }, + "non canonical subject type is not dropped": { + contextFunc: func(tb testing.TB) (context.Context, context.CancelFunc) { + tb.Helper() + return context.WithTimeout(tb.Context(), 1*time.Second) + }, + typesToFilter: map[string]source.Extra{ + managedClustersType: {"apiVersion": managedClustersAPIVersion}, + }, + azureData: &azeventhubs.ReceivedEventData{ + EventData: azeventhubs.EventData{ + Body: eventDataLowerTypeManagedClusterWriteBody, + }, + }, + expectedData: []source.Data{ + { + Type: managedClustersType, + Operation: source.DataOperationUpsert, + Time: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC), + Values: map[string]any{ + "id": normalizedManagedClusterID, + "type": managedClustersType, + }, + }, + }, + }, + "delete with non canonical subject casing": { + contextFunc: func(tb testing.TB) (context.Context, context.CancelFunc) { + tb.Helper() + return context.WithTimeout(tb.Context(), 1*time.Second) + }, + typesToFilter: map[string]source.Extra{ + managedClustersType: {"apiVersion": managedClustersAPIVersion}, + }, + azureData: &azeventhubs.ReceivedEventData{ + EventData: azeventhubs.EventData{ + Body: eventDataManagedClusterDeleteBody, + }, + }, + expectedData: []source.Data{ + { + Type: managedClustersType, + Operation: source.DataOperationDelete, + Time: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC), + Values: map[string]any{ + "id": normalizedManagedClusterID, + "type": managedClustersType, + }, + }, + }, + }, } for testName, test := range testCases { @@ -159,6 +234,9 @@ func TestPartitionEventHandler(t *testing.T) { } require.NotErrorIs(t, ctx.Err(), context.DeadlineExceeded) + // a fake branch keyed on the wrong resource ID silently emits nothing, so assert the + // received count before comparing the elements + require.Len(t, receivedData, len(test.expectedData)) assert.ElementsMatch(t, test.expectedData, receivedData) }) } @@ -185,31 +263,98 @@ func handleResourcesGetByIDRequest(tb testing.TB, resourceID, apiVersion string) tb.Helper() switch resourceID { - case "subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myRG": + case "subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/my-rg": assert.Equal(tb, "2021-04-01", apiVersion) resp = &armresources.ClientGetByIDResponse{ GenericResource: armresources.GenericResource{ - ID: to.Ptr("/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myRG"), + ID: to.Ptr("/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/my-rg"), Type: to.Ptr("Microsoft.Resources/resourceGroups"), }, } return resp, nil - case "subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myRG/providers/Microsoft.Compute/virtualMachines/myVM": + case "subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/my-rg/providers/Microsoft.Compute/virtualMachines/my-vm": assert.Equal(tb, "2021-07-01", apiVersion) return nil, assert.AnError + case managedClusterGetByIDPath: + assert.Equal(tb, managedClustersAPIVersion, apiVersion) + // Microsoft.ContainerService builds its own id form and answers with a lowercase + // resourcegroups literal even though the request carried the camelCase one + resp = &armresources.ClientGetByIDResponse{ + GenericResource: armresources.GenericResource{ + ID: to.Ptr(bodyManagedClusterID), + Type: to.Ptr("microsoft.containerservice/managedclusters"), + }, + } + return resp, nil + case lowerTypeManagedClusterGetByIDPath: + assert.Equal(tb, managedClustersAPIVersion, apiVersion) + resp = &armresources.ClientGetByIDResponse{ + GenericResource: armresources.GenericResource{ + ID: to.Ptr("/" + lowerTypeManagedClusterGetByIDPath), + Type: to.Ptr("microsoft.containerservice/managedclusters"), + }, + } + return resp, nil } return nil, nil } +const ( + managedClustersAPIVersion = "2025-10-01" + + // armresources.Client strips the leading slash before calling the server, and + // arm.ParseResourceID rewrites the resourcegroups literal of the subject to camelCase while + // leaving the provider and type segments at the casing the subject used, so these are the keys + // the fake actually receives. + managedClusterGetByIDPath = "subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/my-rg/providers/Microsoft.ContainerService/managedClusters/my-cluster" + lowerTypeManagedClusterGetByIDPath = "subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/my-rg/providers/microsoft.containerservice/managedclusters/my-cluster" +) + var ( + // eventDataManagedClusterWriteBody carries a lowercase resourcegroups literal and a canonically + // cased provider and type. + eventDataManagedClusterWriteBody = json.RawMessage(`[ + { + "id": "00000000-0000-0000-0000-000000000000", + "source": "/subscriptions/00000000-0000-0000-0000-000000000000", + "specversion": "1.0", + "type": "Microsoft.Resources.ResourceWriteSuccess", + "subject": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/my-rg/providers/Microsoft.ContainerService/managedClusters/my-cluster", + "time": "2020-01-01T00:00:00.0000000Z" + }]`) + + // eventDataLowerTypeManagedClusterWriteBody spells the provider and the type in lowercase, a + // casing the configured key does not use. + eventDataLowerTypeManagedClusterWriteBody = json.RawMessage(`[ + { + "id": "00000000-0000-0000-0000-000000000000", + "source": "/subscriptions/00000000-0000-0000-0000-000000000000", + "specversion": "1.0", + "type": "Microsoft.Resources.ResourceWriteSuccess", + "subject": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/my-rg/providers/microsoft.containerservice/managedclusters/my-cluster", + "time": "2020-01-01T00:00:00.0000000Z" + }]`) + + // eventDataManagedClusterDeleteBody spells the resource group literal, the provider and the + // type in lowercase. + eventDataManagedClusterDeleteBody = json.RawMessage(`[ + { + "id": "00000000-0000-0000-0000-000000000000", + "source": "/subscriptions/00000000-0000-0000-0000-000000000000", + "specversion": "1.0", + "type": "Microsoft.Resources.ResourceDeleteSuccess", + "subject": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/my-rg/providers/microsoft.containerservice/managedclusters/my-cluster", + "time": "2020-01-01T00:00:00.0000000Z" + }]`) + eventDataResourcesTestBody = json.RawMessage(`[ { "id": "00000000-0000-0000-0000-000000000000", "source": "/subscriptions/00000000-0000-0000-0000-000000000000", "specversion": "1.0", "type": "Microsoft.Resources.ResourceDeleteSuccess", - "subject": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/myRG", + "subject": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/my-rg", "time": "2020-01-01T00:00:00.0000000Z", "data": { "authorization": {}, @@ -217,7 +362,7 @@ var ( "correlationId": "00000000-0000-0000-0000-000000000000", "httpRequest": {}, "resourceProvider": "Microsoft.Resources", - "resourceUri": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/myRG", + "resourceUri": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/my-rg", "operationName": "Microsoft.Resources/subscriptions/resourcegroups/delete", "status": "Succeeded", "subscriptionId": "00000000-0000-0000-0000-000000000000", @@ -229,7 +374,7 @@ var ( "source": "/subscriptions/00000000-0000-0000-0000-000000000000", "specversion": "1.0", "type": "Microsoft.Resources.ResourceWriteSuccess", - "subject": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/myRG", + "subject": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/my-rg", "time": "2020-01-01T00:00:00.0000000Z", "data": { "authorization": {}, @@ -237,7 +382,7 @@ var ( "correlationId": "00000000-0000-0000-0000-000000000000", "httpRequest": {}, "resourceProvider": "Microsoft.Resources", - "resourceUri": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/myRG", + "resourceUri": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/my-rg", "operationName": "Microsoft.Resources/subscriptions/resourceGroups/write", "status": "Succeeded", "subscriptionId": "00000000-0000-0000-0000-000000000000", @@ -249,7 +394,7 @@ var ( "source": "/subscriptions/00000000-0000-0000-0000-000000000000", "specversion": "1.0", "type": "Microsoft.Resources.ResourceWriteCancel", - "subject": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/myRG", + "subject": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/my-rg", "time": "2020-01-01T00:00:00.0000000Z", "data": { "authorization": {}, @@ -257,7 +402,7 @@ var ( "correlationId": "00000000-0000-0000-0000-000000000000", "httpRequest": {}, "resourceProvider": "Microsoft.Resources", - "resourceUri": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/myRG", + "resourceUri": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/my-rg", "operationName": "Microsoft.Resources/subscriptions/resourceGroups/write", "status": "Canceled", "subscriptionId": "00000000-0000-0000-0000-000000000000", @@ -269,7 +414,7 @@ var ( "source": "/subscriptions/00000000-0000-0000-0000-000000000000", "specversion": "1.0", "type": "Microsoft.Resources.ResourceWriteSuccess", - "subject": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myRG/providers/Microsoft.Compute/virtualMachines/myVM", + "subject": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/my-rg/providers/Microsoft.Compute/virtualMachines/my-vm", "time": "2020-01-01T00:00:00.0000000Z", "data": { "authorization": {}, @@ -277,7 +422,7 @@ var ( "correlationId": "00000000-0000-0000-0000-000000000000", "httpRequest": {}, "resourceProvider": "Microsoft.Resources", - "resourceUri": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myRG/providers/Microsoft.Compute/virtualMachines/myVM", + "resourceUri": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/my-rg/providers/Microsoft.Compute/virtualMachines/my-vm", "operationName": "Microsoft.Resources/tags/write", "status": "Succeeded", "subscriptionId": "00000000-0000-0000-0000-000000000000", @@ -289,7 +434,7 @@ var ( "source": "/subscriptions/00000000-0000-0000-0000-000000000000", "specversion": "1.0", "type": "Microsoft.Resources.ResourceWriteSuccess", - "subject": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/myRG/providers/Microsoft.Storage/storageAccounts/account", + "subject": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/my-rg/providers/Microsoft.Storage/storageAccounts/account", "time": "2020-01-01T00:00:00.0000000Z", "data": { "authorization": {}, @@ -297,7 +442,7 @@ var ( "correlationId": "00000000-0000-0000-0000-000000000000", "httpRequest": {}, "resourceProvider": "Microsoft.Resources", - "resourceUri": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/myRG/providers/Microsoft.Storage/storageAccounts/account", + "resourceUri": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/my-rg/providers/Microsoft.Storage/storageAccounts/account", "operationName": "Microsoft.Resources/tags/write", "status": "Succeeded", "subscriptionId": "00000000-0000-0000-0000-000000000000", @@ -311,7 +456,8 @@ var ( Operation: source.DataOperationDelete, Time: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC), Values: map[string]any{ - "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myRG", + // the source lowercases every id so that all the ingestion paths converge + "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/my-rg", "type": "Microsoft.Resources/resourceGroups", }, }, @@ -320,7 +466,8 @@ var ( Operation: source.DataOperationUpsert, Time: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC), Values: map[string]any{ - "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myRG", + // the source lowercases every id so that all the ingestion paths converge + "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/my-rg", "type": "Microsoft.Resources/resourceGroups", }, }, diff --git a/internal/source/azure/azure_sync_test.go b/internal/source/azure/azure_sync_test.go index f8b9450..c481230 100644 --- a/internal/source/azure/azure_sync_test.go +++ b/internal/source/azure/azure_sync_test.go @@ -108,7 +108,7 @@ func TestStartSyncProcess(t *testing.T) { Time: testTime, Values: map[string]any{ "extendedLocation": nil, - "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/name", + "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/name", "identity": nil, "kind": "", "location": "region", @@ -142,7 +142,7 @@ func TestStartSyncProcess(t *testing.T) { Time: testTime, Values: map[string]any{ "extendedLocation": nil, - "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/name", + "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/name", "identity": nil, "kind": "", "location": "region", @@ -163,7 +163,7 @@ func TestStartSyncProcess(t *testing.T) { Time: testTime, Values: map[string]any{ "extendedLocation": nil, - "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/name/providers/Microsoft.Compute/virtualMachines/vm-name", + "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/name/providers/microsoft.compute/virtualmachines/vm-name", "identity": nil, "kind": "", "location": "northeurope", @@ -193,7 +193,7 @@ func TestStartSyncProcess(t *testing.T) { Time: testTime, Values: map[string]any{ "extendedLocation": nil, - "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/name/providers/Microsoft.Compute/virtualMachines/vm-name2", + "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/name/providers/microsoft.compute/virtualmachines/vm-name2", "identity": nil, "kind": "", "location": "northeurope", @@ -219,6 +219,69 @@ func TestStartSyncProcess(t *testing.T) { }, }, }, + "managed clusters with divergent casing": { + typesToFilter: map[string]source.Extra{ + managedClustersType: nil, + }, + expectedData: []source.Data{ + { + Type: managedClustersType, + Operation: source.DataOperationUpsert, + Time: testTime, + Values: map[string]any{ + "id": normalizedManagedClusterID, + "name": "my-cluster", + "type": managedClustersType, + }, + }, + { + Type: managedClustersType, + Operation: source.DataOperationUpsert, + Time: testTime, + Values: map[string]any{ + "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/my-rg/providers/microsoft.containerservice/managedclusters/my-other-cluster", + "name": "my-other-cluster", + "type": managedClustersType, + }, + }, + }, + }, + "unusable ids are emitted unchanged": { + typesToFilter: map[string]source.Extra{ + "Microsoft.Resources/malformedResources": nil, + }, + expectedData: []source.Data{ + { + Type: "Microsoft.Resources/malformedResources", + Operation: source.DataOperationUpsert, + Time: testTime, + Values: map[string]any{ + "id": "not-an-id", + "name": "no-id-resource", + "type": "Microsoft.Resources/malformedResources", + }, + }, + { + Type: "Microsoft.Resources/malformedResources", + Operation: source.DataOperationUpsert, + Time: testTime, + Values: map[string]any{ + "id": float64(42), + "name": "numeric-id-resource", + "type": "Microsoft.Resources/malformedResources", + }, + }, + { + Type: "Microsoft.Resources/malformedResources", + Operation: source.DataOperationUpsert, + Time: testTime, + Values: map[string]any{ + "name": "missing-id-resource", + "type": "Microsoft.Resources/malformedResources", + }, + }, + }, + }, "error during request": { typesToFilter: map[string]source.Extra{ "Microsoft.Resources/errorResources": nil, @@ -417,6 +480,26 @@ func handleResourceGraphQueryRequest(t *testing.T, query armresourcegraph.QueryR SkipToken: to.Ptr("skip-token-1"), }, }, nil + case fmt.Sprintf(resourceGraphQueryTemplate, managedClustersType): + return &armresourcegraph.ClientResourcesResponse{ + QueryResponse: armresourcegraph.QueryResponse{ + TotalRecords: to.Ptr(int64(2)), + Data: resourceGraphManagedClustersResponse, + ResultTruncated: to.Ptr(armresourcegraph.ResultTruncatedFalse), + Count: to.Ptr(int64(2)), + SkipToken: nil, + }, + }, nil + case fmt.Sprintf(resourceGraphQueryTemplate, "Microsoft.Resources/malformedResources"): + return &armresourcegraph.ClientResourcesResponse{ + QueryResponse: armresourcegraph.QueryResponse{ + TotalRecords: to.Ptr(int64(3)), + Data: resourceGraphMalformedResponse, + ResultTruncated: to.Ptr(armresourcegraph.ResultTruncatedFalse), + Count: to.Ptr(int64(3)), + SkipToken: nil, + }, + }, nil case fmt.Sprintf(resourceGraphQueryTemplate, "Microsoft.Resources/errorResources"): return nil, assert.AnError } @@ -425,6 +508,37 @@ func handleResourceGraphQueryRequest(t *testing.T, query armresourcegraph.QueryR } var ( + // resourceGraphManagedClustersResponse mixes the camelCase resourceGroups literal Resource Graph + // really returns with an entirely lowercase row, because the normalization must be case blind. + resourceGraphManagedClustersResponse = []any{ + map[string]any{ + "id": graphManagedClusterID, + "name": "my-cluster", + "type": "microsoft.containerservice/managedclusters", + }, + map[string]any{ + "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/my-rg/providers/microsoft.containerservice/managedclusters/my-other-cluster", + "name": "my-other-cluster", + "type": "microsoft.containerservice/managedclusters", + }, + } + + // resourceGraphMalformedResponse holds rows whose id cannot be normalized: the items must still + // be emitted with their original value. + resourceGraphMalformedResponse = []any{ + map[string]any{ + "id": "not-an-id", + "name": "no-id-resource", + }, + map[string]any{ + "id": 42, + "name": "numeric-id-resource", + }, + map[string]any{ + "name": "missing-id-resource", + }, + } + resourceGraphResourceGroupsResponse = []any{ map[string]any{ "extendedLocation": nil, diff --git a/internal/source/azure/resourceid.go b/internal/source/azure/resourceid.go new file mode 100644 index 0000000..8a81708 --- /dev/null +++ b/internal/source/azure/resourceid.go @@ -0,0 +1,73 @@ +// Copyright Mia srl +// SPDX-License-Identifier: AGPL-3.0-only or Commercial + +package azure + +import ( + "slices" + "strings" + + "github.com/mia-platform/ibdm/internal/logger" +) + +const ( + // idKey is the payload key holding the Azure resource ID. + idKey = "id" + // typeKey is the payload key holding the Azure resource type. + typeKey = "type" + // apiVersionKey is the mapping extra key holding the api-version to use for the resource type. + apiVersionKey = "apiVersion" +) + +// normalizeResourceValues rewrites the id and type entries of a raw Azure resource payload so +// that the same resource always yields the same values, whichever Azure API produced it. +// resourceType is the configured type key and is applied verbatim. +// +// Azure does not guarantee the letter case of resource IDs: different resource providers and +// different APIs return the same ID with different casing, and the mappings hash the ID to build +// the Catalog identifier, so any casing difference creates a duplicate item. Lowercasing the +// whole ID collapses every casing variant onto one value. +// +// Lowercasing is safe only while every mapped resource type has case-insensitive names, which +// Azure documents for all the types currently mapped. Before mapping a type whose names are +// case-sensitive, such as blob containers, review docs/how-to/030_azure-source.md: folding the +// case of a case-sensitive name would make two distinct resources share one Catalog identifier. +// +// The casing Azure reported is logged when it differs from the normalized value, so that the +// resource providers returning non canonical IDs stay observable. It is deliberately not added to +// the payload: mappings must always build identifiers from id. +// +// A payload whose id is missing or not a string keeps its original value so that no item is ever +// dropped or corrupted. +func normalizeResourceValues(log logger.Logger, values map[string]any, resourceType string) { + values[typeKey] = resourceType + + rawID, ok := values[idKey].(string) + if !ok || rawID == "" { + log.Warn("azure resource payload without a usable id, identifier may be unstable", + "type", resourceType) + return + } + + normalizedID := strings.ToLower(rawID) + if normalizedID != rawID { + log.Debug("azure returned a resource id with non canonical casing", + "type", resourceType, "id", normalizedID, "azureId", rawID) + } + + values[idKey] = normalizedID +} + +// configuredResourceType returns the configured type key matching resourceType ignoring case, +// reporting whether one was found. sortedTypes must be sorted so that the result is +// deterministic when two configured keys differ only by case. +func configuredResourceType(sortedTypes []string, resourceType string) (string, bool) { + idx := slices.IndexFunc(sortedTypes, func(s string) bool { + return strings.EqualFold(s, resourceType) + }) + if idx < 0 { + return "", false + } + + return sortedTypes[idx], true +} diff --git a/internal/source/azure/resourceid_test.go b/internal/source/azure/resourceid_test.go new file mode 100644 index 0000000..71c34a2 --- /dev/null +++ b/internal/source/azure/resourceid_test.go @@ -0,0 +1,311 @@ +// Copyright Mia srl +// SPDX-License-Identifier: AGPL-3.0-only or Commercial + +package azure + +import ( + "bytes" + "maps" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/mia-platform/ibdm/internal/logger" +) + +const ( + managedClustersType = "Microsoft.ContainerService/managedClusters" + + // The same managed cluster as Azure spells it on the three ingestion paths: Resource Graph and + // the event subject use camelCase resourceGroups, while the resource provider response body + // lowercases it. + graphManagedClusterID = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/my-rg/providers/Microsoft.ContainerService/managedClusters/my-cluster" + bodyManagedClusterID = "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/my-rg/providers/Microsoft.ContainerService/managedClusters/my-cluster" + subjectManagedClusterID = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/my-rg/providers/Microsoft.ContainerService/managedClusters/my-cluster" + + // normalizedManagedClusterID is what every path must converge to. + normalizedManagedClusterID = "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/my-rg/providers/microsoft.containerservice/managedclusters/my-cluster" +) + +func TestNormalizeResourceValues(t *testing.T) { + t.Parallel() + + testCases := map[string]struct { + values map[string]any + resourceType string + expectedValues map[string]any + }{ + "mixed case id is lowered and type is forced to the configured key": { + values: map[string]any{ + "id": graphManagedClusterID, + "type": "microsoft.containerservice/managedclusters", + "name": "my-cluster", + }, + resourceType: managedClustersType, + expectedValues: map[string]any{ + "id": normalizedManagedClusterID, + "type": managedClustersType, + "name": "my-cluster", + }, + }, + "already lowercase id is left unchanged": { + values: map[string]any{ + "id": normalizedManagedClusterID, + }, + resourceType: managedClustersType, + expectedValues: map[string]any{ + "id": normalizedManagedClusterID, + "type": managedClustersType, + }, + }, + "shouty id is fully lowered": { + values: map[string]any{ + "id": "/SUBSCRIPTIONS/00000000-0000-0000-0000-000000000000/RESOURCEGROUPS/MY-RG/PROVIDERS/MICROSOFT.WEB/SITES/MY-SITE", + }, + resourceType: "Microsoft.Web/sites", + expectedValues: map[string]any{ + "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/my-rg/providers/microsoft.web/sites/my-site", + "type": "Microsoft.Web/sites", + }, + }, + "malformed id is lowered without being parsed": { + values: map[string]any{ + "id": "Not-An-Azure-Id", + }, + resourceType: managedClustersType, + expectedValues: map[string]any{ + "id": "not-an-azure-id", + "type": managedClustersType, + }, + }, + "missing id leaves the other values untouched": { + values: map[string]any{ + "name": "my-rg", + "location": "westeurope", + }, + resourceType: "Microsoft.Resources/resourceGroups", + expectedValues: map[string]any{ + "name": "my-rg", + "location": "westeurope", + "type": "Microsoft.Resources/resourceGroups", + }, + }, + "non string id is preserved": { + values: map[string]any{ + "id": 42, + }, + resourceType: managedClustersType, + expectedValues: map[string]any{ + "id": 42, + "type": managedClustersType, + }, + }, + "nil id is preserved": { + values: map[string]any{ + "id": nil, + }, + resourceType: managedClustersType, + expectedValues: map[string]any{ + "id": nil, + "type": managedClustersType, + }, + }, + "empty id is preserved": { + values: map[string]any{ + "id": "", + }, + resourceType: managedClustersType, + expectedValues: map[string]any{ + "id": "", + "type": managedClustersType, + }, + }, + "unrelated keys are never touched": { + values: map[string]any{ + "id": graphManagedClusterID, + "name": "my-cluster", + "properties": map[string]any{ + "provisioningState": "Succeeded", + "fqdn": "my-cluster.example.com", + }, + "tags": map[string]any{"Env": "Prod"}, + }, + resourceType: managedClustersType, + expectedValues: map[string]any{ + "id": normalizedManagedClusterID, + "name": "my-cluster", + "properties": map[string]any{ + "provisioningState": "Succeeded", + "fqdn": "my-cluster.example.com", + }, + "tags": map[string]any{"Env": "Prod"}, + "type": managedClustersType, + }, + }, + } + + for testName, test := range testCases { + t.Run(testName, func(t *testing.T) { + t.Parallel() + + values := maps.Clone(test.values) + normalizeResourceValues(nullTestLogger(t), values, test.resourceType) + + // comparing the whole map also guards against the helper adding any field beyond id + // and type: a payload visible raw id would silently reinstate the duplicate item bug. + require.Equal(t, test.expectedValues, values) + + rawID, ok := test.values[idKey].(string) + if !ok || rawID == "" { + return + } + + normalizedID, ok := values[idKey].(string) + require.True(t, ok) + require.True(t, strings.EqualFold(rawID, normalizedID), "normalization must only change letter case") + }) + } +} + +func TestNormalizeResourceValuesConvergesDivergentCasings(t *testing.T) { + t.Parallel() + + graphValues := map[string]any{idKey: graphManagedClusterID} + bodyValues := map[string]any{idKey: bodyManagedClusterID} + subjectValues := map[string]any{idKey: subjectManagedClusterID} + + require.NotEqual(t, graphValues[idKey], bodyValues[idKey], "the fixtures must differ before normalization") + + for _, values := range []map[string]any{graphValues, bodyValues, subjectValues} { + normalizeResourceValues(nullTestLogger(t), values, managedClustersType) + } + + assert.Equal(t, normalizedManagedClusterID, graphValues[idKey]) + assert.Equal(t, normalizedManagedClusterID, bodyValues[idKey]) + assert.Equal(t, normalizedManagedClusterID, subjectValues[idKey]) +} + +func TestNormalizeResourceValuesIsIdempotent(t *testing.T) { + t.Parallel() + + values := map[string]any{ + idKey: graphManagedClusterID, + "name": "my-cluster", + } + + normalizeResourceValues(nullTestLogger(t), values, managedClustersType) + once := maps.Clone(values) + + normalizeResourceValues(nullTestLogger(t), values, managedClustersType) + assert.Equal(t, once, values) +} + +func TestNormalizeResourceValuesLogging(t *testing.T) { + t.Parallel() + + testCases := map[string]struct { + values map[string]any + expectedMessages []string + absentMessages []string + }{ + "divergent casing is reported": { + values: map[string]any{idKey: bodyManagedClusterID}, + expectedMessages: []string{"non canonical casing", bodyManagedClusterID}, + }, + "canonical casing stays silent": { + values: map[string]any{idKey: normalizedManagedClusterID}, + absentMessages: []string{"non canonical casing"}, + }, + "unusable id is reported": { + values: map[string]any{"name": "my-cluster"}, + expectedMessages: []string{"without a usable id"}, + absentMessages: []string{"non canonical casing"}, + }, + } + + for testName, test := range testCases { + t.Run(testName, func(t *testing.T) { + t.Parallel() + + buffer := new(bytes.Buffer) + log := logger.NewLogger(buffer) + log.SetLevel(logger.TRACE) + + normalizeResourceValues(log, maps.Clone(test.values), managedClustersType) + + for _, message := range test.expectedMessages { + assert.Contains(t, buffer.String(), message) + } + for _, message := range test.absentMessages { + assert.NotContains(t, buffer.String(), message) + } + }) + } +} + +func TestConfiguredResourceType(t *testing.T) { + t.Parallel() + + configuredTypes := []string{"Microsoft.ContainerService/managedClusters", "Microsoft.Web/sites"} + + testCases := map[string]struct { + sortedTypes []string + resourceType string + expectedType string + expectedFound bool + }{ + "exact match": { + sortedTypes: configuredTypes, + resourceType: managedClustersType, + expectedType: managedClustersType, + expectedFound: true, + }, + "lowercase match returns the configured key": { + sortedTypes: configuredTypes, + resourceType: "microsoft.containerservice/managedclusters", + expectedType: managedClustersType, + expectedFound: true, + }, + "shouty match returns the configured key": { + sortedTypes: configuredTypes, + resourceType: "MICROSOFT.WEB/SITES", + expectedType: "Microsoft.Web/sites", + expectedFound: true, + }, + "unconfigured type is not found": { + sortedTypes: configuredTypes, + resourceType: "Microsoft.Compute/virtualMachines", + }, + "nil slice is not found": { + resourceType: managedClustersType, + }, + "empty slice is not found": { + sortedTypes: []string{}, + resourceType: managedClustersType, + }, + "keys differing only by case resolve deterministically": { + sortedTypes: []string{"Microsoft.Web/sites", "microsoft.web/sites"}, + resourceType: "MICROSOFT.WEB/SITES", + expectedType: "Microsoft.Web/sites", + expectedFound: true, + }, + } + + for testName, test := range testCases { + t.Run(testName, func(t *testing.T) { + t.Parallel() + + resourceType, found := configuredResourceType(test.sortedTypes, test.resourceType) + assert.Equal(t, test.expectedFound, found) + assert.Equal(t, test.expectedType, resourceType) + }) + } +} + +// nullTestLogger returns a logger discarding every entry. +func nullTestLogger(tb testing.TB) logger.Logger { + tb.Helper() + return logger.FromContext(tb.Context()) +} From b08b154edbb568922810c4cbec45b9b2e7d9e6f6 Mon Sep 17 00:00:00 2001 From: apagani Date: Tue, 4 Aug 2026 16:35:54 +0200 Subject: [PATCH 3/4] fix: set managedcluster relationship deletePolicy to none since in deletion fqdn is not available --- docs/mappings/azure/managedclusters.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/mappings/azure/managedclusters.yaml b/docs/mappings/azure/managedclusters.yaml index 0bc4c8a..11ef40f 100644 --- a/docs/mappings/azure/managedclusters.yaml +++ b/docs/mappings/azure/managedclusters.yaml @@ -35,7 +35,7 @@ mappings: extra: - apiVersion: mia-platform.eu/v1 itemFamily: relationships - deletePolicy: "cascade" + deletePolicy: "none" identifier: |- {{ $src := printf "urn:mia-platform-catalog:console.mia-platform.eu:v1:Cluster:%s" (printf "%s" .properties.fqdn | sha256sum) -}} {{- $type := "urn:mia-platform-catalog:mia-platform.eu:v1:RelationshipType:dependency.mia-platform.eu" -}} From e799f3d527d81d89ca1f62e9d50a8c3f0975adfd Mon Sep 17 00:00:00 2001 From: apagani Date: Tue, 4 Aug 2026 18:35:42 +0200 Subject: [PATCH 4/4] fix docs --- docs/how-to/030_azure-source.md | 39 ++++++++------------------------- 1 file changed, 9 insertions(+), 30 deletions(-) diff --git a/docs/how-to/030_azure-source.md b/docs/how-to/030_azure-source.md index a1646a5..15125ce 100644 --- a/docs/how-to/030_azure-source.md +++ b/docs/how-to/030_azure-source.md @@ -72,38 +72,15 @@ The two values are therefore identical for `ibdm sync azure` and `ibdm run azure `{{ .id | sha256sum }}` a stable identifier and lets a delete event target the item a previous import created. -### Mapping a resource type with case-sensitive names +This is also in part suggested by Azure, since it is stated that various APIs can return names with different casing, +therefore in order to perform meaningful matches a case-insensitive comparison is recommended. +For a more in-depth explanation refer to [Naming rules and restrictions for Azure resources]. -The normalisation is safe only while every mapped resource type has **case-insensitive** names, -which Microsoft Azure documents for all the types shipped in `docs/mappings/azure`. Blob -containers and Log Analytics solutions are the two documented exceptions: their names *are* -case-sensitive, so Azure can hold two distinct resources whose IDs differ only by case. Lowercasing -would collapse them onto a single Catalog `metadata.name` and the second import would silently -overwrite the first. Review this behaviour before adding a mapping for such a type — nothing -detects the situation at runtime. +### Consequences for the mappings and items -### Consequences for the mappings - -`id` is lowercase and no longer matches the casing shown in the Azure portal. Use `.name` wherever -display casing matters. The spelling Azure reported is deliberately not exposed to the mappings: it -is written to the source logs at the `Debug` level whenever it differs from the normalised value, -so the resource providers returning non canonical IDs stay observable without any template being -able to hash the unstable value by mistake. - -### Upgrading from a release without the normalisation - -Canonical Azure IDs contain uppercase characters, so lowercasing changes `sha256sum` for **every** -Azure resource, not only the ones affected by the divergence. Each item therefore receives a new -identifier, the destination inserts it as a new Catalog item and the previous one is left behind as -an orphan, together with the relationship items that cascade from it. - -1. Capture the current list of Azure items **before** upgrading: identifiers are opaque hashes, so - afterwards a stale item is indistinguishable from a fresh one. -1. Run `ibdm sync azure` right after deploying so that the new items exist. -1. Delete the items captured at step 1 and their cascaded relationships. This cleanup is manual. - -Expect the item count to grow slightly as well: resources whose events spelled the resource type -with a casing different from the mapping file were previously dropped and now get imported. +`id` is lowercase and could no longer match the casing shown in the Azure portal. +Use `.name` wherever available to display casing matters, its availability is dependant on the specific resource APIs. +The spelling Azure reported, if needed, is written to the source logs at the `Debug` level whenever it differs from the normalised value. ## Authentication @@ -111,9 +88,11 @@ The source is using the [`DefaultAzureCredential` chain of authentication] so yo your preferred method of login. This authentication will be used for reading data from the REST APIs so it will need the read permissions on the resources you want to import. +Both `sync` and `run` modes use APIs to fetch the full resource, for this reason an authentication method of choice is always needed. If you choose to don’t use the `*_CONNECTION_STING` variables the same authentication will be used to receive data from the configured EventHub and to manage object inside the StorageAccount blob storage. [`DefaultAzureCredential` chain of authentication]: https://learn.microsoft.com/en-gb/azure/developer/go/sdk/authentication/credential-chains#defaultazurecredential-overview +[Naming rules and restrictions for Azure resources]: https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/resource-name-rules