Telemetry API Extension

.gitignore

@@ -6,4 +6,5 @@ extensions/
 response.out
 .aws-sam/
 coverage.txt
-preview-extensions-ggqizro707
+extenstion.zip
+.DS_Store

Makefile

@@ -1,25 +1,32 @@
 build: clean
 	go build -o ./extensions/newrelic-lambda-extension
 
+build-arm64: clean
+	mkdir extensions
+	env GOARCH=arm64 GOOS=linux go build -ldflags="-s -w" -o ./extensions/newrelic-lambda-extension
+	chmod +x ./extensions/newrelic-lambda-extension
+	zip -r ./extensions/extension.zip ./extensions/
+
+build-x86_64: clean
+	mkdir extensions
+	env GOARCH=amd64 GOOS=linux go build -ldflags="-s -w" -o ./extensions/newrelic-lambda-extension
+	chmod +x ./extensions/newrelic-lambda-extension
+	zip -r ./extensions/extension.zip ./extensions/
+
 clean:
 	rm -rf extensions
-	rm -f preview-extensions-ggqizro707
-	rm -f /tmp/newrelic-lambda-extension.x86_64.zip
-	rm -f /tmp/newrelic-lambda-extension.arm64.zip
 
-dist-x86_64: clean
+ci-build-x86_64: clean
 	env GOARCH=amd64 GOOS=linux go build -ldflags="-s -w" -o ./extensions/newrelic-lambda-extension
-	touch preview-extensions-ggqizro707
 
-dist-arm64: clean
+ci-build-arm64: clean
 	env GOARCH=arm64 GOOS=linux go build -ldflags="-s -w" -o ./extensions/newrelic-lambda-extension
-	touch preview-extensions-ggqizro707
 
-zip-x86_64: dist-x86_64
-	zip -r /tmp/newrelic-lambda-extension.x86_64.zip preview-extensions-ggqizro707 extensions
+zip-x86_64: ci-build-x86_64
+	zip -r /tmp/newrelic-lambda-extension.x86_64.zip extensions
 
 zip-arm64: dist-arm64
-	zip -r /tmp/newrelic-lambda-extension.arm64.zip preview-extensions-ggqizro707 extensions
+	zip -r /tmp/newrelic-lambda-extension.arm64.zip extensions
 
 test:
 	@echo "Normal tests"
@@ -29,6 +36,3 @@ test:
 
 coverage:
 	./coverage.sh
-
-publish: zip-x86_64
-	aws lambda publish-layer-version --no-cli-pager --layer-name newrelic-lambda-extension-x86_64 --zip-file fileb:///tmp/newrelic-lambda-extension.x86_64.zip

README.md

@@ -1,11 +1,18 @@
-[![Community Project header](https://github.com/newrelic/opensource-website/raw/master/src/images/categories/Community_Project.png)](https://opensource.newrelic.com/oss-category/#community-project)
-
-# newrelic-lambda-extension [![Build Status](https://circleci.com/gh/newrelic/newrelic-lambda-extension.svg?style=svg)](https://circleci.com/gh/newrelic/newrelic-lambda-extension) [![Coverage](https://codecov.io/gh/newrelic/newrelic-lambda-extension/branch/main/graph/badge.svg?token=T73UEDVA5K)](https://codecov.io/gh/newrelic/newrelic-lambda-extension)
-
-An AWS Lambda extension to collect, enhance, and transport telemetry data from your AWS Lambda functions to New Relic without requiring an external transport such as CloudWatch Logs or Kinesis.
-
-This lightweight AWS Lambda Extension runs alongside your AWS Lambda functions and automatically handles the collection and transport of telemetry data from
-supported New Relic serverless agents.
+# New Relic Telemetry API Extension
+
+The New Relic Telemetry API Extension collects telemetry data from both AWS Lambda Telemetry API and New Relic Agents and sends it to New Relic. Please note that Telemetry API collects all logs for an invocation, so when using an agent do not forward logs or they will be duplicated. If you are using a New Relic Agent, be sure to follow its [lambda/serverless installation documentation](https://docs.newrelic.com/docs/serverless-function-monitoring/aws-lambda-monitoring/enable-lambda-monitoring/instrument-example/). The use of this layer does not require a New Relic agent to be present, and can be configured to send telemetry without one. The following environment variables are available to configure this extension:
+
+| Environment Variable | Required | Description |
+| --- | --- | --- |
+| NEW_RELIC_ACCOUNT_ID | **always** | The account ID of the New Relic account you want to send data to |
+| NEW_RELIC_LICENSE_KEY | *conditional* | A plaintext New Relic license key. Either this or the value retrieved from NEW_RELIC_LICENSE_KEY_SECRET must contain a valid New Relic license key or the application will exit. If a plaintext license key is provided, it will override the license key retrieved from an AWS Secret. |
+| NEW_RELIC_LICENSE_KEY_SECRET | *conditional* | The name of an AWS Secrets Manager Secret containing a New Relic license key. If no plaintext license key is provided, the value of this variable must be set to the name of an AWS Secret containing a valid New Relic license key. |
+| NEW_RELIC_EXTENSION_AGENT_DATA_COLLECTION_ENABLED | optional | Setting this to "false" will prevent the extension from collecting data from New Relic Agents. This will not prevent the agent from running. |
+| NEW_RELIC_EXTENSION_AGENT_DATA_BATCH_SIZE | optional | The number of invocations to store before sending Agent Data to New Relic. If your lamba function gets invoked at a high frequency, increasing this number will improve the performance of the extension and avoid dropped data and improve performance. Default: 1 |
+| NEW_RELIC_EXTENSION_TELEMETRY_API_BATCH_SIZE | optional | The number of Telemetry API events and logs to batch before sending them to New Relic. If your application invokes frequently, increase this number to avoid data getting dropped and to improve performance. Default: 1 | 
+| NEW_RELIC_EXTENSION_DATA_COLLECTION_TIMEOUT |  optional | A valid time.Duration string for how long the extension should wait to attempt to send agent data to New Relic in the event of a timeout/retry loop scenario. Example: 1s, 1500ms; Default: 10s |
+| NEW_RELIC_EXTENSION_COLLECTOR_OVERRIDE | optional | An override for the New Relic collection endpoint you want to send data to. By default, this will be detected based on the region of your New Relic license key. |
+| NEW_RELIC_EXTENSION_LOG_LEVEL | optional | The log level of the New Relic Telemetry API Extension. For more verbose logs, set to "debug". To log only warnings and errors, set it to "warn". For Advanced troubleshooting, set to "trace". Note that debug, and especially trace, log levels will cause a significant increase in log lines that are printed and saved to cloudwatch. |
 
 ## Installation
 
@@ -16,118 +23,9 @@ Lambda function. The current layer ARN can be found [here][3].
 
 **Note:** This extension is included with all New Relic AWS Lambda layers going forward.
 
-You'll also need to make the New Relic license key available to the extension. Use the [New Relic Lambda CLI][4]
-to install the managed secret, and then add the permission for the secret to your Lambda execution role.
-
-[4]: https://github.com/newrelic/newrelic-lambda-cli
-
-    newrelic-lambda integrations install \
-        --nr-account-id <account id> \
-        --nr-api-key <api key> \
-        --linked-account-name <linked account name> \
-        --enable-license-key-secret
-
-Each of the example functions in the `examples` directory has the appropriate license key secret permission. 
-
-After deploying your AWS Lambda function with one of the layer ARNs from the
-link above you should begin seeing telemetry data in New Relic.
-
-See below for details on supported New Relic agents.
-
-## Supported Configurations
-
-AWS's [Extension API supports](https://docs.aws.amazon.com/lambda/latest/dg/runtimes-extensions-api.html) only a subset 
-of all their runtimes. Notably absent as of this writing are Node JS before 10, Python before 3.7, Go (all versions), 
-Dotnet before 3.1, and the older "java8" runtime, though "java8.al2" is supported.
-
-For Go lambdas, we suggest using "provided" or "provided.al2". The Go example's deploy script contains compiler flags
-that produce a suitable self-hosting Go executable. See the [Custom runtime](https://docs.aws.amazon.com/lambda/latest/dg/runtimes-custom.html)
-docs for more details on this feature. 
-
-All of our layers include the extension, and the latest Agent version for the Layer's runtime. The latest 
-layer version ARNs for your runtime and region are available [here](https://layers.newrelic-external.com/). The 
-`NewRelicLambdaExtension` layer is suitable for Go, Java and Dotnet.
-
-## Building
-
-Use the included `Makefile` to compile the extension. 
-
-```sh
-make dist
-```
-
-This creates the extension binary in `./extensions/newrelic-lambda-extension`. The binary is compiled for Amazon Linux, which is likely different from the platform you're working on.
-
-## Deploying
-
-To publish the extension to your AWS account, run the following command:
-
-```sh
-    make publish
-```
-
-This packages the extension, and publishes a new layer version in your AWS account. Be sure that the AWS CLI is configured correctly. You can use the usual [AWS CLI environment variables](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html) to control the account and region for the CLI.
-
-## Startup Checks
-
-This Lambda Extension will perform a series of checks on initialization. Should any of
-these checks fail, the extension wil attempt to output troubleshooting recommendations to both
-CloudWatch Logs and New Relic Logs. If you have any issues using this extension, be sure
-to check your logs for messages starting with `Startup check failed:` for
-troubleshooting recommendations.
-
-Startup checks include:
-
-* New Relic agent version checks
-* Lambda handler configuration checks
-* Lambda environment variable checks
-* Vendored New Relic agent checks
-
-## Disabling Extension
-
-The New Relic Lambda Extension is enabled by default. To disable it, after adding or
-updating the Lambda layer, set the `NEW_RELIC_LAMBDA_EXTENSION_ENABLED` environment
-variable to `false`.
-
-## Testing
-
-To test locally, acquire the AWS extension test harness first. Then:
-
->TODO: Link to the AWS SDK that has the test harness, assuming it gets published.
-
-1. (Optional) Use the `newrelic-lambda` CLI to create the license key managed secret in your AWS account and region.
-2. Build the docker container for sample function code. Give it the tag `lambda_ext`.
-   - Be sure to include your lambda function in the container.
-3. Start up your container.
-
-   - Using AWS Secret Manager
-
-            export AWS_ACCESS_KEY_ID=$(aws configure get aws_access_key_id --profile default)
-            export AWS_SECRET_ACCESS_KEY=$(aws configure get aws_secret_access_key --profile default)
-            export AWS_SESSION_TOKEN=$(aws configure get aws_session_token --profile default)
-
-            docker run --rm -v $(pwd)/extensions:/opt/extensions -p 9001:8080 \
-                -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_SESSION_TOKEN \
-                lambda_ext:latest \
-                -h function.handler -c '{}' -t 60000
-
-   - Or, setting the license key directly
-
-            docker run --rm \
-                -v $(pwd)/extensions:/opt/extensions \
-                -p 9001:8080 \
-                lambda_ext:latest \
-                -h function.handler -c '{"NEW_RELIC_LICENSE_KEY": "your-license-key-here"}' -t 60000
-
-4. To invoke the sample lambda run: 
-
-       curl -XPOST 'http://localhost:9001/2015-03-31/functions/function.handler/invocations' \
-           -d 'invoke-payload'
-
-5. Finally, you can exercise the container shutdown lifecycle event with:
+## Building Locally
 
-        curl -XPOST 'http://localhost:9001/test/shutdown' \
-            -d '{"timeoutMs": 5000 }'
+Use the `make build-arm64` or `make build-amd64` commands to build local version of this binary. Make sure that you have docker enabled and conifgured properly. This runs the build command in a linux docker container to prevent errors caused by MacOS filesystem artifacts from occuring.
 
 ## Support
 

agentTelemetry/batch.go

@@ -0,0 +1,121 @@
+package agentTelemetry
+
+import (
+	"time"
+
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	MustHarvestError = "telemetry can not be added until batch is harvested"
+)
+
+var l = log.WithFields(log.Fields{"pkg": "agentTelemetry"})
+
+// Batch represents the unsent invocations and their telemetry, along with timing data.
+type Batch struct {
+	extractTraceID bool
+	harvestSize    int
+	invocations    map[string]*Invocation
+}
+
+// NewBatch constructs a new batch.
+func NewBatch(size int, extractTraceID bool, logLevel log.Level) *Batch {
+	log.SetLevel(logLevel)
+	return &Batch{
+		invocations:    make(map[string]*Invocation),
+		harvestSize:    size,
+		extractTraceID: extractTraceID,
+	}
+}
+
+func (b *Batch) ReadyToHarvest() bool {
+	l.Debugf("[agentTelemetry] Have %d / %d invocations needed to do a harvest", len(b.invocations), b.harvestSize)
+	return len(b.invocations) >= b.harvestSize
+}
+
+// AddInvocation should be called just after the next API response. It creates the Invocation record so that we can attach telemetry later.
+func (b *Batch) AddInvocation(requestID string, start time.Time) {
+	invocation := NewInvocation(requestID, start)
+	b.invocations[requestID] = &invocation
+}
+
+// HasInvocation checks if an invocation has been created for this request ID
+func (b *Batch) HasInvocation(requestID string) bool {
+	_, ok := b.invocations[requestID]
+	return ok
+}
+
+// AddTelemetry attaches telemetry to an existing Invocation, identified by requestId
+func (b *Batch) AddTelemetry(requestId string, telemetry []byte) *Invocation {
+	inv, ok := b.invocations[requestId]
+	if ok {
+		inv.Telemetry = append(inv.Telemetry, telemetry)
+		if b.extractTraceID {
+			traceId, err := ExtractTraceID(telemetry)
+			if err != nil {
+				l.Errorf("[agent telemtry: ExtractTraceID] %v", err)
+			}
+			// We don't want to unset a previously set trace ID
+			if traceId != "" {
+				inv.TraceId = traceId
+			}
+		}
+		return inv
+	}
+	return nil
+}
+
+// Close aggressively harvests all telemetry from the Batch. The Batch is no longer valid.
+func (b *Batch) Close() []*Invocation {
+	return b.Harvest(true)
+}
+
+// Harvest all ready invocations. It removes harvested invocations from the batch and updates the lastHarvest timestamp.
+func (b *Batch) Harvest(force bool) []*Invocation {
+	ret := make([]*Invocation, 0, len(b.invocations))
+	for k, v := range b.invocations {
+		if force {
+			if !v.IsEmpty() {
+				ret = append(ret, v)
+				delete(b.invocations, k)
+			}
+		} else {
+			if !v.IsEmpty() && v.IsRipe() {
+				ret = append(ret, v)
+				delete(b.invocations, k)
+			}
+		}
+	}
+
+	l.Debugf("[agentTelemetry] harvesting %d invocations\n", len(ret))
+	return ret
+}
+
+// An Invocation holds telemetry for a request, and knows when the request began.
+// Invocations are parts of a Batch, and should only be used by the batch object.
+type Invocation struct {
+	Start     time.Time
+	RequestId string
+	TraceId   string
+	Telemetry [][]byte
+}
+
+// NewInvocation creates an Invocation, which can hold telemetry
+func NewInvocation(requestId string, start time.Time) Invocation {
+	return Invocation{
+		Start:     start,
+		RequestId: requestId,
+		Telemetry: make([][]byte, 0, 2),
+	}
+}
+
+// IsRipe indicates that an Invocation has all the telemetry it's likely to get. Sending a ripe invocation won't omit data.
+func (inv *Invocation) IsRipe() bool {
+	return len(inv.Telemetry) >= 1
+}
+
+// IsEmpty is true when the invocation has no telemetry. The invocation has begun, but has received no agent payload, nor platform logs.
+func (inv *Invocation) IsEmpty() bool {
+	return len(inv.Telemetry) == 0
+}