ACM-39327: Fix non-admin SSE OOM under large inventory (release-2.13)

backend/src/routes/aggregators/applications.ts

@@ -10,13 +10,13 @@ import { addArgoQueryInputs, cacheArgoApplications } from './applicationsArgo'
 import { getGiganticApps } from '../../lib/gigantic'
 
 export enum AppColumns {
-  'name' = 0,
-  'type',
-  'namespace',
-  'clusters',
-  'repo',
-  'timeWindow',
-  'created',
+  name = 0,
+  type,
+  namespace,
+  clusters,
+  repo,
+  timeWindow,
+  created,
 }
 export interface IArgoApplication extends IResource {
   cluster?: string

backend/src/routes/events.ts

@@ -8,13 +8,27 @@ import { Http2ServerRequest, Http2ServerResponse } from 'node:http2'
 import pluralize from 'pluralize'
 import { Stream } from 'stream'
 import { promisify } from 'util'
-import { jsonPost } from '../lib/json-request'
 import { logger } from '../lib/logger'
 import type { ITransformedResource } from '../lib/pagination'
 import { type ServerSideEvent, ServerSideEvents } from '../lib/server-side-events'
 import { getCACertificate, getServiceAccountToken } from '../lib/serviceAccountToken'
 import { getAuthenticatedToken } from '../lib/token'
 import type { IResource } from '../resources/resource'
+import { canAccess, canGetResource, canListClusterScopedKind, canListNamespacedScopedKind } from './eventsAccess'
+import { definitions, type IWatchOptions } from './eventsDefinitions'
+import { startAccessCacheCleanup, stopAccessCacheCleanup } from './eventsCache'
+
+export {
+  ACCESS_CACHE_TTL,
+  ACCESS_CACHE_CLEANUP_INTERVAL,
+  ACCESS_CACHE_MAX_TOKENS,
+  ACCESS_CACHE_MAX_ENTRIES_PER_TOKEN,
+  cleanupAccessCache,
+  getAccessCache,
+  hashAccessToken,
+  resetAccessCache,
+} from './eventsCache'
+export { canAccess, canGetResource } from './eventsAccess'
 
 const { map, split } = eventStream
 const pipeline = promisify(Stream.pipeline)
@@ -131,111 +145,15 @@ export function initResourceCache(cache: ResourceCache) {
 
 export let resourceCache: ResourceCache = {}
 
-const accessCache: Record<string, Record<string, { time: number; promise: Promise<boolean> }>> = {}
-
-const definitions: IWatchOptions[] = [
-  { kind: 'ClusterManagementAddOn', apiVersion: 'addon.open-cluster-management.io/v1alpha1' },
-  { kind: 'ManagedClusterAddOn', apiVersion: 'addon.open-cluster-management.io/v1alpha1' },
-  { kind: 'Agent', apiVersion: 'agent-install.openshift.io/v1beta1' },
-  { kind: 'AgentServiceConfig', apiVersion: 'agent-install.openshift.io/v1beta1' },
-  { kind: 'InfraEnv', apiVersion: 'agent-install.openshift.io/v1beta1' },
-  { kind: 'NMStateConfig', apiVersion: 'agent-install.openshift.io/v1beta1' },
-  { kind: 'Application', apiVersion: 'app.k8s.io/v1beta1' },
-  { kind: 'Channel', apiVersion: 'apps.open-cluster-management.io/v1' },
-  { kind: 'GitOpsCluster', apiVersion: 'apps.open-cluster-management.io/v1beta1' },
-  { kind: 'HelmRelease', apiVersion: 'apps.open-cluster-management.io/v1' },
-  { kind: 'PlacementRule', apiVersion: 'apps.open-cluster-management.io/v1' },
-  { kind: 'Subscription', apiVersion: 'apps.open-cluster-management.io/v1' },
-  { kind: 'SubscriptionReport', apiVersion: 'apps.open-cluster-management.io/v1alpha1' },
-  { kind: 'Application', apiVersion: 'argoproj.io/v1alpha1' },
-  { kind: 'ApplicationSet', apiVersion: 'argoproj.io/v1alpha1' },
-  { kind: 'ArgoCD', apiVersion: 'argoproj.io/v1alpha1' },
-  { kind: 'MulticlusterApplicationSetReport', apiVersion: 'apps.open-cluster-management.io/v1alpha1' },
-  { kind: 'Infrastructure', apiVersion: 'config.openshift.io/v1' },
-  {
-    kind: 'CertificateSigningRequest',
-    apiVersion: 'certificates.k8s.io/v1',
-    labelSelector: { 'open-cluster-management.io/cluster-name': '' },
-  },
-  { kind: 'ManagedCluster', apiVersion: 'cluster.open-cluster-management.io/v1' },
-  { kind: 'Placement', apiVersion: 'cluster.open-cluster-management.io/v1beta1' },
-  { kind: 'Placement', apiVersion: 'cluster.open-cluster-management.io/v1alpha1' },
-  { kind: 'PlacementDecision', apiVersion: 'cluster.open-cluster-management.io/v1alpha1' },
-  { kind: 'PlacementDecision', apiVersion: 'cluster.open-cluster-management.io/v1beta1' },
-  { kind: 'ManagedClusterSetBinding', apiVersion: 'cluster.open-cluster-management.io/v1beta2' },
-  { kind: 'ManagedClusterSet', apiVersion: 'cluster.open-cluster-management.io/v1beta2' },
-  { kind: 'ClusterCurator', apiVersion: 'cluster.open-cluster-management.io/v1beta1' },
-  { kind: 'Subscription', apiVersion: 'operators.coreos.com/v1alpha1' },
-  { kind: 'DiscoveredCluster', apiVersion: 'discovery.open-cluster-management.io/v1' },
-  { kind: 'DiscoveryConfig', apiVersion: 'discovery.open-cluster-management.io/v1' },
-  { kind: 'AgentClusterInstall', apiVersion: 'extensions.hive.openshift.io/v1beta1' },
-  { kind: 'ClusterClaim', apiVersion: 'hive.openshift.io/v1' },
-  { kind: 'ClusterDeployment', apiVersion: 'hive.openshift.io/v1' },
-  { kind: 'ClusterImageSet', apiVersion: 'hive.openshift.io/v1' },
-  { kind: 'ClusterPool', apiVersion: 'hive.openshift.io/v1' },
-  { kind: 'ClusterProvision', apiVersion: 'hive.openshift.io/v1' },
-  { kind: 'MachinePool', apiVersion: 'hive.openshift.io/v1' },
-  { kind: 'ManagedClusterInfo', apiVersion: 'internal.open-cluster-management.io/v1beta1' },
-  { kind: 'BareMetalHost', apiVersion: 'metal3.io/v1alpha1' },
-  { kind: 'MultiClusterEngine', apiVersion: 'multicluster.openshift.io/v1' },
-  { kind: 'ClusterVersion', apiVersion: 'config.openshift.io/v1' },
-  { kind: 'StorageClass', apiVersion: 'storage.k8s.io/v1' },
-  { kind: 'PlacementBinding', apiVersion: 'policy.open-cluster-management.io/v1' },
-  { kind: 'Policy', apiVersion: 'policy.open-cluster-management.io/v1' },
-  { kind: 'PolicyAutomation', apiVersion: 'policy.open-cluster-management.io/v1beta1' },
-  { kind: 'PolicySet', apiVersion: 'policy.open-cluster-management.io/v1beta1' },
-  { kind: 'SubmarinerConfig', apiVersion: 'submarineraddon.open-cluster-management.io/v1alpha1' },
-  { kind: 'AnsibleJob', apiVersion: 'tower.ansible.com/v1alpha1' },
-  {
-    kind: 'ConfigMap',
-    apiVersion: 'v1',
-    fieldSelector: { 'metadata.name': 'assisted-service' },
-  },
-  {
-    kind: 'ConfigMap',
-    apiVersion: 'v1',
-    fieldSelector: { 'metadata.namespace': 'openshift-config-managed', 'metadata.name': 'console-public' },
-  },
-  { kind: 'ConfigMap', apiVersion: 'v1', fieldSelector: { 'metadata.name': 'console-search-config' } },
-  { kind: 'Namespace', apiVersion: 'v1' },
-  { kind: 'Secret', apiVersion: 'v1', labelSelector: { 'cluster.open-cluster-management.io/credentials': '' } },
-  // **Need to look for creds with: 'cluster.open-cluster-management.io/type': 'ans', for edit scenarios
-  { kind: 'Secret', apiVersion: 'v1', labelSelector: { 'cluster.open-cluster-management.io/type': 'ans' } },
-  { kind: 'Secret', apiVersion: 'v1', fieldSelector: { 'metadata.name': 'auto-import-secret' } },
-  { kind: 'PolicyReport', apiVersion: 'wgpolicyk8s.io/v1alpha2' },
-  { kind: 'HostedCluster', apiVersion: 'hypershift.openshift.io/v1beta1' },
-  { kind: 'NodePool', apiVersion: 'hypershift.openshift.io/v1beta1' },
-  { kind: 'AgentMachine', apiVersion: 'capi-provider.agent-install.openshift.io/v1alpha1' },
-  { kind: 'ConfigMap', apiVersion: 'v1', labelSelector: { 'hypershift.openshift.io/supported-versions': 'true' } },
-  { kind: 'Search', apiVersion: 'search.open-cluster-management.io/v1alpha1' },
-  // Configmaps that contain Grafana dashboard IDs
-  {
-    kind: 'ConfigMap',
-    apiVersion: 'v1',
-    fieldSelector: { 'metadata.name': 'grafana-dashboard-acm-openshift-virtualization-clusters-overview' },
-  },
-  {
-    kind: 'ConfigMap',
-    apiVersion: 'v1',
-    fieldSelector: { 'metadata.name': 'grafana-dashboard-acm-openshift-virtualization-single-vm-view' },
-  },
-]
-
 export function startWatching(): void {
   ServerSideEvents.eventFilter = eventFilter
+  startAccessCacheCleanup()
 
   for (const definition of definitions) {
     void listAndWatch(definition)
   }
 }
 
-interface IWatchOptions {
-  apiVersion: string
-  kind: string
-  labelSelector?: Record<string, string>
-  fieldSelector?: Record<string, string>
-}
-
 // https://kubernetes.io/docs/reference/using-api/api-concepts/
 async function listAndWatch(options: IWatchOptions) {
   while (!stopping) {
@@ -626,14 +544,15 @@ function eventFilter(token: string, serverSideEvent: ServerSideEvent<ServerSideE
       return Promise.resolve(true)
     case 'ADDED':
     case 'MODIFIED': {
-      const watchEvent = serverSideEvent.data
-      const resource = watchEvent.object
+      const resource = serverSideEvent.data.object
+      if (!resource?.kind || !resource?.apiVersion) {
+        return Promise.resolve(false)
+      }
+      // Fast path: cluster-scoped list (admins / broad ClusterRoles).
       return canListClusterScopedKind(resource, token).then((allowed) => {
         if (allowed) return true
-        return canListNamespacedScopedKind(resource, token).then((allowed) => {
-          if (allowed) return true
-          return canGetResource(resource, token)
-        })
+        // After cluster list is denied, use SelfSubjectRulesReview instead of O(N) SSARs.
+        return canGetResource(resource, token)
       })
     }
     default:
@@ -642,83 +561,10 @@ function eventFilter(token: string, serverSideEvent: ServerSideEvent<ServerSideE
   }
 }
 
-function canListClusterScopedKind(resource: IResource, token: string): Promise<boolean> {
-  return canAccess({ kind: resource.kind, apiVersion: resource.apiVersion }, 'list', token)
-}
-
-function canListNamespacedScopedKind(resource: IResource, token: string): Promise<boolean> {
-  if (!resource.metadata?.namespace) return Promise.resolve(false)
-  return canAccess(
-    {
-      kind: resource.kind,
-      apiVersion: resource.apiVersion,
-      metadata: { namespace: resource.metadata.namespace },
-    },
-    'list',
-    token
-  )
-}
-
-function canGetResource(resource: IResource, token: string): Promise<boolean> {
-  return canAccess(resource, 'get', token)
-}
-
-export function canAccess(
-  resource: { kind: string; apiVersion: string; metadata?: { name?: string; namespace?: string } },
-  verb: 'get' | 'list' | 'create',
-  token: string
-): Promise<boolean> {
-  // TODO make sure old cache items get cleaned up
-
-  const key = `${resource.kind}:${resource.metadata?.namespace}:${resource.metadata?.name}`
-  if (!accessCache[token]) accessCache[token] = {}
-  const existing = accessCache[token][key]
-  if (existing && existing.time > Date.now() - 60 * 1000) {
-    return existing.promise
-  }
-
-  const promise = jsonPost<{ status: { allowed: boolean } }>(
-    process.env.CLUSTER_API_URL + '/apis/authorization.k8s.io/v1/selfsubjectaccessreviews',
-    {
-      apiVersion: 'authorization.k8s.io/v1',
-      kind: 'SelfSubjectAccessReview',
-      metadata: {},
-      spec: {
-        resourceAttributes: {
-          group: resource.apiVersion.includes('/') ? resource.apiVersion.split('/')[0] : '',
-          name: resource.metadata?.name,
-          namespace:
-            resource.metadata?.namespace ?? (resource.kind === 'Namespace' ? resource.metadata?.name : undefined),
-          resource: pluralize(resource.kind.toLowerCase()),
-          verb,
-        },
-      },
-    },
-    token
-  ).then((result) => {
-    if (process.env.LOG_ACCESS === 'true') {
-      logger.debug({
-        msg: 'access',
-        allowed: result.body.status.allowed,
-        verb,
-        resource: pluralize(resource.kind.toLowerCase()),
-        name: resource.metadata?.name,
-        namespace: resource.metadata?.namespace,
-      })
-    }
-    return result.body.status.allowed
-  })
-
-  accessCache[token][key] = {
-    time: Date.now(),
-    promise,
-  }
-  return promise
-}
-
 let stopping = false
 export function stopWatching(): void {
   stopping = true
+  stopAccessCacheCleanup()
   for (const request of requests) {
     request.cancel()
   }