Integrating Metrics Forwarder with Log forwarder

.github/workflows/release-template-files.yaml

@@ -4,8 +4,6 @@ on:
   push:
     branches:
       - main
-    paths:
-      - '*.yaml'
 
 jobs:
   build-and-deploy:
@@ -43,6 +41,17 @@ jobs:
       - name: Verify the changes
         run: cat lambda-template.yaml
 
+      # Update the LambdaInstrumentationVersion in lambda-template.yaml
+      - name: Update LambdaInstrumentationVersion in lambda-template.yaml
+        run: |
+          # Extract the version from version.go
+          version=$(grep 'const InstrumentationVersion' src/common/version.go | awk -F '"' '{print $2}')
+          echo "Extracted version: $version"
+
+          # Replace the version in lambda-template.yaml
+          sed -i "s/LambdaInstrumentationVersion: [^ ]*/LambdaInstrumentationVersion: $version/" lambda-template.yaml
+          cat lambda-template.yaml
+
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v3
         with:

lambda-template.yaml

@@ -1,7 +1,10 @@
 AWSTemplateFormatVersion: "2010-09-09"
 Transform: AWS::Serverless-2016-10-31
 Description: New Relic Template to forward logs using S3 bucket and/or cloudwatch using lambda
-
+Mappings:
+  VersionMap:
+    MetricsForwarder:
+      LambdaInstrumentationVersion: 1.0.0   #This will be dynamially updated by release workflow
 Metadata:
   AWS::ServerlessRepo::Application:
     Name: NewRelic-log-forwarding
@@ -30,7 +33,9 @@ Metadata:
         default: 'Common Attributes to be added to the log events'
       StoreNRLicenseKeyInSecretManager: 
         default: 'Store New Relic License Key in AWS Secrets Manager'
-
+      MonitoringTier:
+        default: 'Metrics Monitoring Collection Tier'
+
 Parameters:
   LicenseKey:
     Type: String
@@ -68,13 +73,27 @@ Parameters:
     Default: "true"
     AllowedValues:
       - "true"
-      - "false" 
+      - "false"
+  MonitoringTier:
+    Type: String
+    Description: |
+      Determines the metrics collection frequency and detail level for Lambda Log Forwarder
+      - None: Send No metrics
+      - Basic: Limited metrics every 15 minutes
+      - Advanced: Full metrics every 5 minutes
+    Default: "Basic"
+    AllowedValues:
+      - "None"
+      - "Basic"
+      - "Advanced"
+
 
 Conditions:
   ShouldCreateSecret: !Equals [ !Ref StoreNRLicenseKeyInSecretManager, "true" ]
   AddS3Trigger: !Not [ !Equals [!Ref S3BucketNames , ""]]
   AddCloudwatchTrigger: !Not [ !Equals [!Ref LogGroupConfig , ""]]
   IsCommonAttributesNotBlank: !Not [!Equals [!Ref CommonAttributes, ""]]
+  EnableMetricsForwarder: !Not [ !Equals [!Ref MonitoringTier, "None"]]
 
 Resources:
   NewRelicLogsLicenseKeySecret:
@@ -453,6 +472,33 @@ Resources:
         LambdaFunctionArn: !GetAtt NewRelicLogsServerlessLogForwarder.Arn
         InvalidLogGroups: !GetAtt NewRelicLogsResourceForLogGroupArnConstruction.InvalidLogGroups
 
+  NewRelicMetricsForwarderTrigger:
+    Type: 'AWS::CloudFormation::Stack'
+    Condition: EnableMetricsForwarder
+    Properties:
+      TemplateURL:  https://s3.us-east-2.amazonaws.com/unified-lambda-serverless/logging-metrics-template.yaml
+      Parameters:
+        LicenseKey: !Ref LicenseKey
+        LicenseKeySecretArn: !If 
+          - ShouldCreateSecret
+          - !Ref NewRelicLogsLicenseKeySecret
+          - ""
+        NewRelicRegion: !Ref NewRelicRegion
+        NewRelicAccountId: !Ref NewRelicAccountId
+        StoreNRLicenseKeyInSecretManager: !Ref StoreNRLicenseKeyInSecretManager
+        MonitoringTier: !Ref MonitoringTier
+        LogForwarderLambdaInputJSON: 
+          Fn::Sub:
+            - |
+              [
+                {
+                  "Arn": "${NewRelicLogsServerlessLogForwarder.Arn}",
+                  "InstrumentationVersion": "${lambda_instrumentation_version}"
+                }
+              ]
+            - lambda_instrumentation_version: !FindInMap [VersionMap, MetricsForwarder, LambdaInstrumentationVersion]
+        LogForwarderFirehoseInputJSON: ""
+
 Outputs:
   NewRelicLogsLambdaFunctionArn:
     Description: The ARN of the Lambda function.

logging-firehose-metric-polling.yaml

@@ -40,6 +40,7 @@ Metadata:
           - LoggingS3BackupBucketName
           - EnableCloudWatchLoggingForFirehose
           - StoreNRLicenseKeyInSecretManager
+          - MonitoringTier
 
     ParameterLabels:
       IntegrationName:
@@ -65,7 +66,9 @@ Metadata:
       CommonAttributes: 
         default: 'Common Attributes to be added to the log events'
       StoreNRLicenseKeyInSecretManager:
-        default: 'Store New Relic License Key in AWS Secrets Manager'  
+        default: 'Store New Relic License Key in AWS Secrets Manager'
+      MonitoringTier:
+        default: 'Metrics Monitoring Collection Tier'  
 
 Parameters:
   IAMRoleName:
@@ -144,6 +147,18 @@ Parameters:
     AllowedValues:
       - "true"
       - "false"      
+  MonitoringTier:
+    Type: String
+    Description: |
+      Determines the metrics collection frequency and detail level for Firehose Log Forwarder
+      - None: Send No metrics
+      - Basic: Limited metrics every 15 minutes
+      - Advanced: Full metrics every 5 minutes
+    Default: "Basic"
+    AllowedValues:
+      - "None"
+      - "Basic"
+      - "Advanced"
 
 Resources: 
   NewRelicLogsFirehose:
@@ -160,6 +175,7 @@ Resources:
         EnableCloudWatchLoggingForFirehose: !Ref EnableCloudWatchLoggingForFirehose
         CommonAttributes: !Ref CommonAttributes
         StoreNRLicenseKeyInSecretManager: !Ref StoreNRLicenseKeyInSecretManager
+        MonitoringTier: !Ref MonitoringTier
 
   # New Relic Metrics Polling Stack
   NewrelicMetricsPollingStack:

logging-firehose-metric-stream.yaml

@@ -44,6 +44,7 @@ Metadata:
           - LoggingS3BackupBucketName
           - EnableCloudWatchLoggingForFirehose
           - StoreNRLicenseKeyInSecretManager
+          - MonitoringTier
       -
         Label:
           default: 'AWS Config Service Configuration (Optional)'
@@ -88,6 +89,8 @@ Metadata:
         default: 'Common Attributes to be added to the log events'
       StoreNRLicenseKeyInSecretManager:
         default: 'Store New Relic License Key in AWS Secrets Manager'  
+      MonitoringTier:
+        default: 'Metrics Monitoring Collection Tier' 
 
 Parameters:
   IAMRoleName:
@@ -206,6 +209,18 @@ Parameters:
     AllowedValues:
       - "true"
       - "false"     
+  MonitoringTier:
+    Type: String
+    Description: |
+      Determines the metrics collection frequency and detail level for Firehose Log Forwarder
+      - None: Send No metrics
+      - Basic: Limited metrics every 15 minutes
+      - Advanced: Full metrics every 5 minutes
+    Default: "Basic"
+    AllowedValues:
+      - "None"
+      - "Basic"
+      - "Advanced"
 
 Resources: 
   NewRelicLogsFirehose:
@@ -222,6 +237,7 @@ Resources:
         EnableCloudWatchLoggingForFirehose: !Ref EnableCloudWatchLoggingForFirehose
         CommonAttributes: !Ref CommonAttributes
         StoreNRLicenseKeyInSecretManager: !Ref StoreNRLicenseKeyInSecretManager
+        MonitoringTier: !Ref MonitoringTier
 
   # New Relic Metrics Stack
   NewrelicMetricsStack:

logging-lambda-firehose-metric-polling.yaml

@@ -40,7 +40,8 @@ Metadata:
           - LoggingFirehoseStreamName
           - LoggingS3BackupBucketName
           - EnableCloudWatchLoggingForFirehose
-          - StoreNRLicenseKeyInSecretManager          
+          - StoreNRLicenseKeyInSecretManager 
+          - MonitoringTier         
 
     ParameterLabels:
       IntegrationName:
@@ -69,6 +70,8 @@ Metadata:
         default: 'Enables CloudWatch logging for Logging Firehose stream'
       StoreNRLicenseKeyInSecretManager:
         default: 'Store New Relic License Key in AWS Secrets Manager'
+      MonitoringTier:
+        default: 'Metrics Monitoring Collection Tier'
 
 Parameters:
   IAMRoleName:
@@ -148,7 +151,19 @@ Parameters:
     Default: "true"
     AllowedValues:
       - "true"
-      - "false"       
+      - "false"   
+  MonitoringTier:
+    Type: String
+    Description: |
+      Determines the metrics collection frequency and detail level for Log Forwarder:
+      - None: Send No metrics
+      - Basic: Limited metrics every 15 minutes
+      - Advanced: Full metrics every 5 minutes
+    Default: "Basic"
+    AllowedValues:
+      - "None"
+      - "Basic"
+      - "Advanced"   
 
 Resources: 
   NewRelicLogsLoggingLambdaFirehose:
@@ -166,6 +181,7 @@ Resources:
         LoggingS3BackupBucketName: !Ref LoggingS3BackupBucketName
         EnableCloudWatchLoggingForFirehose: !Ref EnableCloudWatchLoggingForFirehose
         StoreNRLicenseKeyInSecretManager: !Ref StoreNRLicenseKeyInSecretManager
+        MonitoringTier: !Ref MonitoringTier
 
   # New Relic Metrics Polling Stack
   NewrelicMetricsPollingStack:

logging-lambda-firehose-metric-stream.yaml

@@ -46,6 +46,7 @@ Metadata:
           - LoggingS3BackupBucketName
           - EnableCloudWatchLoggingForFirehose
           - StoreNRLicenseKeyInSecretManager
+          - MonitoringTier
       -
         Label:
           default: 'AWS Config Service Configuration (Optional)'
@@ -92,6 +93,8 @@ Metadata:
         default: 'Enables CloudWatch logging for Logging Firehose stream'
       StoreNRLicenseKeyInSecretManager:
         default: 'Store New Relic License Key in AWS Secrets Manager'
+      MonitoringTier:
+        default: 'Metrics Monitoring Collection Tier'
 
 Parameters:
   IAMRoleName:
@@ -212,6 +215,18 @@ Parameters:
     AllowedValues:
       - "true"
       - "false" 
+  MonitoringTier:
+    Type: String
+    Description: |
+      Determines the metrics collection frequency and detail level for Log Forwarder:
+      - None: Send No metrics
+      - Basic: Limited metrics every 15 minutes
+      - Advanced: Full metrics every 5 minutes
+    Default: "Basic"
+    AllowedValues:
+      - "None"
+      - "Basic"
+      - "Advanced"
 
 Resources: 
   NewRelicLogsLoggingLambdaFirehose:
@@ -229,6 +244,7 @@ Resources:
         LoggingS3BackupBucketName: !Ref LoggingS3BackupBucketName
         EnableCloudWatchLoggingForFirehose: !Ref EnableCloudWatchLoggingForFirehose
         StoreNRLicenseKeyInSecretManager: !Ref StoreNRLicenseKeyInSecretManager
+        MonitoringTier: !Ref MonitoringTier
 
   # New Relic Metrics Stack
   NewrelicMetricsStack:

logging-lambda-firehose-template.yaml

@@ -36,6 +36,8 @@ Metadata:
         default: 'Enables CloudWatch logging for Logging Firehose stream'
       StoreNRLicenseKeyInSecretManager:
         default: 'Store New Relic License Key in AWS Secrets Manager'
+      MonitoringTier:
+        default: 'Metrics Monitoring Collection Tier'
 
 Parameters:
   LicenseKey:
@@ -96,7 +98,20 @@ Parameters:
     Default: "true"
     AllowedValues:
       - "true"
-      - "false" 
+      - "false"
+  MonitoringTier:
+    Type: String
+    Description: |
+      Determines the metrics collection frequency and detail level for Log Forwarder:
+      - None: Send No metrics
+      - Basic: Limited metrics every 15 minutes
+      - Advanced: Full metrics every 5 minutes
+    Default: "Basic"
+    AllowedValues:
+      - "None"
+      - "Basic"
+      - "Advanced"
+
 
 Resources:
 
@@ -112,7 +127,8 @@ Resources:
         LogGroupConfig: "" 
         CommonAttributes: !Ref CommonAttributes
         StoreNRLicenseKeyInSecretManager: !Ref StoreNRLicenseKeyInSecretManager
-
+        MonitoringTier: !Ref MonitoringTier
+
   NewRelicLogsFirehose:
     Type: 'AWS::CloudFormation::Stack'
     Properties:
@@ -127,4 +143,4 @@ Resources:
         EnableCloudWatchLoggingForFirehose: !Ref EnableCloudWatchLoggingForFirehose
         CommonAttributes: !Ref CommonAttributes
         StoreNRLicenseKeyInSecretManager: !Ref StoreNRLicenseKeyInSecretManager
-
+        MonitoringTier: !Ref MonitoringTier

logging-lambda-metric-polling.yaml

@@ -39,6 +39,7 @@ Metadata:
           - LogGroupConfig
           - CommonAttributes
           - StoreNRLicenseKeyInSecretManager
+          - MonitoringTier
 
     ParameterLabels:
       IntegrationName:
@@ -61,6 +62,8 @@ Metadata:
         default: 'Common Attributes to be added to the log events'
       StoreNRLicenseKeyInSecretManager:
         default: 'Store New Relic License Key in AWS Secrets Manager'
+      MonitoringTier:
+        default: 'Metrics Monitoring Collection Tier'
 
 Parameters:
   IAMRoleName:
@@ -118,7 +121,19 @@ Parameters:
     Default: "true"
     AllowedValues:
       - "true"
-      - "false"     
+      - "false"
+  MonitoringTier:
+    Type: String
+    Description: |
+      Determines the metrics collection frequency and detail level for Lambda Log Forwarder:
+      - None: Send No metrics
+      - Basic: Limited metrics every 15 minutes
+      - Advanced: Full metrics every 5 minutes
+    Default: "Basic"
+    AllowedValues:
+      - "None"
+      - "Basic"
+      - "Advanced"
 
 Resources: 
   NewRelicLogsLoggingLambda:
@@ -133,6 +148,7 @@ Resources:
         LogGroupConfig : !Ref LogGroupConfig
         CommonAttributes : !Ref CommonAttributes
         StoreNRLicenseKeyInSecretManager: !Ref StoreNRLicenseKeyInSecretManager
+        MonitoringTier: !Ref MonitoringTier
 
   # New Relic Metrics Polling Stack
   NewrelicMetricsPollingStack: