diff --git a/.github/workflows/nexmark-runner-ci.yml b/.github/workflows/nexmark-runner-ci.yml new file mode 100644 index 0000000..5d58247 --- /dev/null +++ b/.github/workflows/nexmark-runner-ci.yml @@ -0,0 +1,60 @@ +name: Nexmark Runner CI + +on: + push: + branches: [main] + paths: + - 'nexmark-runner/**' + - 'nexmark/**' + - '.github/workflows/nexmark-runner-ci.yml' + pull_request: + branches: [main] + paths: + - 'nexmark-runner/**' + - 'nexmark/**' + - '.github/workflows/nexmark-runner-ci.yml' + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-hudi-gcs: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + strategy: + matrix: + flink_version: ["1.18.1"] + hudi_version: ["1.1.1"] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push Docker image + uses: docker/build-push-action@v6 + with: + context: . + file: nexmark-runner/docker/Dockerfile + push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }}-flink${{ matrix.flink_version }}-hudi${{ matrix.hudi_version }}-gcs + build-args: | + FLINK_VERSION=${{ matrix.flink_version }} + NEXMARK_CONFIG_FILE=nexmark-hudi-gcs.yaml + HUDI_VERSION=${{ matrix.hudi_version }} + cache-from: type=gha,scope=hudi-gcs-flink${{ matrix.flink_version }}-hudi${{ matrix.hudi_version }} + cache-to: type=gha,mode=max,scope=hudi-gcs-flink${{ matrix.flink_version }}-hudi${{ matrix.hudi_version }} diff --git a/.github/workflows/nexmark.yml b/.github/workflows/nexmark.yml new file mode 100644 index 0000000..49b4ed3 --- /dev/null +++ b/.github/workflows/nexmark.yml @@ -0,0 +1,33 @@ +name: Nexmark CI + +on: + push: + branches: [main] + paths: + - 'nexmark/**' + - '.github/workflows/nexmark.yml' + pull_request: + branches: [main] + paths: + - 'nexmark/**' + - '.github/workflows/nexmark.yml' + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + java-version: '11' + distribution: 'temurin' + cache: maven + + - name: Build and test + run: | + cd nexmark + mvn -B verify diff --git a/nexmark-runner/Makefile b/nexmark-runner/Makefile new file mode 100644 index 0000000..5d8de90 --- /dev/null +++ b/nexmark-runner/Makefile @@ -0,0 +1,319 @@ +# Nexmark Docker Build Makefile +# Usage: +# make build CONFIG=hudi-gcs +# make build CONFIG=hudi-local +# make build CONFIG=blackhole +# make push CONFIG=hudi-gcs PROJECT_ID=your-project + +.PHONY: help pre-setup build push info build-amd64 setup-k8s teardown-k8s run-k8s redeploy-k8s restart-k8s setup-local teardown-local run-local redeploy-local restart-local + +# Configuration +IMAGE_NAME ?= nexmark-runner +HUDI_VERSION ?= 1.1.1 +FLINK_VERSION ?= 1.18 +PROJECT_ID ?= +REGION ?= us-central1 + +# Derived paths +CONFIGS_DIR = configs + +# Derived from CONFIG +# For blackhole, no config file needed (uses nexmark defaults) +NEXMARK_CONFIG_FILE = $(if $(filter blackhole,$(CONFIG)),,nexmark-$(CONFIG).yaml) +# Image tag format: flink-[-] +# blackhole doesn't use Hudi, so skip Hudi version suffix for it +IMAGE_TAG = $(if $(filter blackhole,$(CONFIG)),flink$(FLINK_VERSION)-$(CONFIG),flink$(FLINK_VERSION)-$(CONFIG)-$(HUDI_VERSION)) + +# Default target +help: + @echo "Nexmark Docker Build" + @echo "" + @echo "Usage:" + @echo " make build CONFIG= Build Docker image" + @echo " make push CONFIG= PROJECT_ID= Build and push to GCR" + @echo "" + @echo "Available configs:" + @ls -1 $(CONFIGS_DIR)/*.yaml 2>/dev/null | xargs -I{} basename {} .yaml | sed 's/^nexmark-/ /' + @echo "" + @echo "Examples:" + @echo " make build CONFIG=hudi-gcs" + @echo " make build CONFIG=hudi-local" + @echo " make build CONFIG=blackhole" + @echo " make push CONFIG=hudi-gcs PROJECT_ID=my-project" + @echo "" + @echo "Other targets:" + @echo " make pre-setup Check prerequisites and show install commands" + @echo " make info CONFIG= Show build configuration" + @echo "" + @echo "K8s targets (GKE):" + @echo " make setup-k8s Setup GKE cluster (terraform + k8s)" + @echo " make teardown-k8s Teardown GKE cluster" + @echo " make run-k8s QUERY=q0 [SINK=hudi] [BUCKET=name]" + @echo " make restart-k8s Restart Flink pods (after Ctrl+C)" + @echo " make redeploy-k8s CONFIG=hudi-gcs Update config + restart" + @echo " make redeploy-k8s CONFIG=hudi-gcs REBUILD=1 PROJECT_ID= Rebuild + push + restart" + @echo "" + @echo "Local targets (Kind + MinIO):" + @echo " make setup-local Setup local Kind cluster" + @echo " make teardown-local Teardown local cluster" + @echo " make run-local QUERY=q0 [ARGS='--sink hudi']" + @echo " make restart-local Restart Flink pods (after Ctrl+C)" + @echo " make redeploy-local Update config + restart Flink" + @echo " make redeploy-local REBUILD=1 Rebuild image + update + restart" + @echo "" + @echo "Configuration variables:" + @echo " IMAGE_NAME Docker image name (default: nexmark-runner)" + @echo " HUDI_VERSION Hudi version (default: 1.1.1)" + @echo " FLINK_VERSION Flink version for Hudi bundle (default: 1.18)" + @echo " PROJECT_ID GCP project ID (required for push)" + @echo " REGION GCP region (default: us-central1)" + +# Check prerequisites and show install commands +pre-setup: + @echo "=== Checking Prerequisites ===" + @echo "" + @echo "Common (required for all):" + @printf " docker: "; command -v docker >/dev/null 2>&1 && echo "OK" || echo "MISSING - https://docs.docker.com/get-docker/" + @printf " kubectl: "; command -v kubectl >/dev/null 2>&1 && echo "OK" || echo "MISSING - brew install kubectl" + @echo "" + @echo "Local development (Kind + MinIO):" + @printf " kind: "; command -v kind >/dev/null 2>&1 && echo "OK" || echo "MISSING - brew install kind" + @printf " helm: "; command -v helm >/dev/null 2>&1 && echo "OK" || echo "MISSING - brew install helm" + @echo "" + @echo "GKE deployment:" + @printf " gcloud: "; command -v gcloud >/dev/null 2>&1 && echo "OK" || echo "MISSING - https://cloud.google.com/sdk/docs/install" + @printf " terraform: "; command -v terraform >/dev/null 2>&1 && echo "OK" || echo "MISSING - brew install terraform" + +# Show configuration +info: +ifndef CONFIG + @echo "Usage: make info CONFIG=hudi-gcs" + @exit 1 +endif + @echo "Configuration:" + @echo " CONFIG: $(CONFIG)" + @echo " IMAGE_NAME: $(IMAGE_NAME)" + @echo " IMAGE_TAG: $(IMAGE_TAG)" + @echo " NEXMARK_CONFIG_FILE: $(NEXMARK_CONFIG_FILE)" + @echo " HUDI_VERSION: $(HUDI_VERSION)" + @echo " PROJECT_ID: $(PROJECT_ID)" + @echo " REGION: $(REGION)" + +# Build target +# Usage: make build CONFIG=hudi-local [NO_CACHE=1] +build: +ifndef CONFIG + @echo "Error: CONFIG not specified" + @echo "Usage: make build CONFIG=hudi-gcs|hudi-local|blackhole" + @exit 1 +endif + @echo "=== Building Nexmark Docker Image ===" + @echo "Config: $(CONFIG)" + @echo "Image: $(IMAGE_NAME):$(IMAGE_TAG)" + @echo "" + docker build \ + $(if $(NO_CACHE),--no-cache,) \ + --build-arg NEXMARK_CONFIG_FILE=$(NEXMARK_CONFIG_FILE) \ + --build-arg HUDI_VERSION=$(HUDI_VERSION) \ + -t "$(IMAGE_NAME):$(IMAGE_TAG)" \ + -f docker/Dockerfile . + @echo "" + @echo "=== Build Complete ===" + @echo "Image: $(IMAGE_NAME):$(IMAGE_TAG)" + @echo "" + @echo "To load into Kind cluster:" + @echo " kind load docker-image $(IMAGE_NAME):$(IMAGE_TAG) --name nexmark" + +# Push target +push: build +ifndef PROJECT_ID + @echo "Error: PROJECT_ID not specified" + @echo "Usage: make push CONFIG=hudi-gcs PROJECT_ID=your-project" + @exit 1 +endif + @REGISTRY_URL="$(REGION)-docker.pkg.dev/$(PROJECT_ID)/flink"; \ + IMAGE_URL="$$REGISTRY_URL/$(IMAGE_NAME):$(IMAGE_TAG)"; \ + echo "=== Pushing to Artifact Registry ==="; \ + echo "Configuring Docker authentication..."; \ + gcloud auth configure-docker "$(REGION)-docker.pkg.dev" --quiet; \ + echo "Tagging image..."; \ + docker tag "$(IMAGE_NAME):$(IMAGE_TAG)" "$$IMAGE_URL"; \ + echo "Pushing to $$IMAGE_URL..."; \ + docker push "$$IMAGE_URL"; \ + echo ""; \ + echo "=== Push Complete ==="; \ + echo "Image: $$IMAGE_URL" + +# Build AMD64 image for GKE (from Mac ARM) +# Usage: make build-amd64 CONFIG=hudi-gcs [NO_CACHE=1] +build-amd64: +ifndef CONFIG + @echo "Error: CONFIG not specified" + @echo "Usage: make build-amd64 CONFIG=hudi-gcs" + @exit 1 +endif + @echo "=== Building AMD64 Image for GKE ===" + docker build \ + --platform linux/amd64 \ + $(if $(NO_CACHE),--no-cache,) \ + --build-arg NEXMARK_CONFIG_FILE=$(NEXMARK_CONFIG_FILE) \ + --build-arg HUDI_VERSION=$(HUDI_VERSION) \ + -t "$(IMAGE_NAME):$(IMAGE_TAG)" \ + -f docker/Dockerfile . + @echo "" + @echo "=== Build Complete ===" + @echo "Image: $(IMAGE_NAME):$(IMAGE_TAG) (linux/amd64)" + +# K8s targets +setup-k8s: + @echo "=== Setting up GKE environment ===" + ./k8s/scripts/setup.sh + +teardown-k8s: + @echo "=== Tearing down GKE environment ===" + ./k8s/scripts/teardown.sh + +run-k8s: +ifndef QUERY + @echo "Error: QUERY not specified" + @echo "Usage: make run-k8s QUERY=q0 [SINK=hudi] [BUCKET=my-bucket]" + @exit 1 +endif + @ARGS="$(QUERY)"; \ + if [ -n "$(SINK)" ]; then ARGS="$$ARGS --sink $(SINK)"; fi; \ + if [ -n "$(BUCKET)" ]; then ARGS="$$ARGS --bucket $(BUCKET)"; fi; \ + ./k8s/scripts/run.sh $$ARGS + +# Local targets (Kind + MinIO) +setup-local: + @echo "=== Setting up local Kind environment ===" + ./local/setup.sh + +teardown-local: + @echo "=== Tearing down local environment ===" + ./local/teardown.sh + +run-local: +ifndef QUERY + @echo "Error: QUERY not specified" + @echo "Usage: make run-local QUERY=q0 [ARGS='--sink hudi --storage minio']" + @exit 1 +endif + ./local/run.sh $(QUERY) $(ARGS) + +# Restart Flink pods (use after Ctrl+C or stuck run) +restart-local: + @echo "=== Restarting FlinkDeployment ===" + kubectl delete flinkdeployment flink-session -n flink-nexmark --ignore-not-found + sleep 2 + kubectl apply -f local/flink-session.yaml + @echo "=== Waiting for Flink pods ===" + @for i in $$(seq 1 30); do \ + kubectl get pod -n flink-nexmark -l component=jobmanager 2>/dev/null | grep -q flink-session && break; \ + echo " Waiting for pod creation... ($$i/30)"; \ + sleep 2; \ + done + kubectl wait --for=condition=Ready pod -l component=jobmanager -n flink-nexmark --timeout=300s + @echo "=== Done. Run: make run-local QUERY=q0 ===" + +# Redeploy: update config + restart Flink (or rebuild image if REBUILD=1) +# Usage: +# make redeploy-local # Update config + restart +# make redeploy-local REBUILD=1 # Rebuild image + update config + restart +# make redeploy-local REBUILD=1 NO_CACHE=1 # Force full rebuild +redeploy-local: + $(eval CONFIG := $(or $(CONFIG),hudi-local)) +ifdef REBUILD + @$(MAKE) build CONFIG=$(CONFIG) $(if $(NO_CACHE),NO_CACHE=1,) + @echo "=== Tagging and loading image into Kind ===" + docker tag "$(IMAGE_NAME):$(IMAGE_TAG)" nexmark-k8s:local + kind load docker-image nexmark-k8s:local --name nexmark +endif + @echo "=== Updating ConfigMap ===" + kubectl create configmap nexmark-config \ + --from-file=nexmark.yaml="configs/$(NEXMARK_CONFIG_FILE)" \ + -n flink-nexmark --dry-run=client -o yaml | kubectl apply -f - + @echo "=== Applying metrics service ===" + kubectl apply -f local/nexmark-metrics-service.yaml + @echo "=== Restarting FlinkDeployment ===" + kubectl delete flinkdeployment flink-session -n flink-nexmark --ignore-not-found + sleep 2 + kubectl apply -f local/flink-session.yaml + @echo "=== Waiting for Flink pods ===" + @for i in $$(seq 1 30); do \ + kubectl get pod -n flink-nexmark -l component=jobmanager 2>/dev/null | grep -q flink-session && break; \ + echo " Waiting for pod creation... ($$i/30)"; \ + sleep 2; \ + done + kubectl wait --for=condition=Ready pod -l component=jobmanager -n flink-nexmark --timeout=300s + kubectl wait --for=condition=Ready pod -l component=taskmanager -n flink-nexmark --timeout=120s 2>/dev/null || true + @echo "=== Done. Run: make run-local QUERY=q0 ===" + +# GKE: Redeploy - update config + restart Flink (or rebuild image if REBUILD=1) +# Usage: +# make redeploy-k8s CONFIG=hudi-gcs PROJECT_ID= # Update config + restart +# make redeploy-k8s CONFIG=hudi-gcs PROJECT_ID= REBUILD=1 # Rebuild + push + restart +# make redeploy-k8s CONFIG=hudi-gcs PROJECT_ID= REBUILD=1 NO_CACHE=1 # Force full rebuild +redeploy-k8s: +ifndef CONFIG + @echo "Error: CONFIG not specified" + @echo "Usage: make redeploy-k8s CONFIG=hudi-gcs PROJECT_ID= [REBUILD=1] [NO_CACHE=1]" + @exit 1 +endif +ifndef PROJECT_ID + @echo "Error: PROJECT_ID not specified" + @echo "Usage: make redeploy-k8s CONFIG=hudi-gcs PROJECT_ID=your-project [REBUILD=1] [NO_CACHE=1]" + @exit 1 +endif +ifdef REBUILD + @$(MAKE) push CONFIG=$(CONFIG) PROJECT_ID=$(PROJECT_ID) $(if $(NO_CACHE),NO_CACHE=1,) +endif + @echo "=== Updating ConfigMap ===" + kubectl create configmap nexmark-config \ + --from-file=nexmark.yaml="configs/$(NEXMARK_CONFIG_FILE)" \ + -n flink-nexmark --dry-run=client -o yaml | kubectl apply -f - + @echo "=== Applying metrics service ===" + kubectl apply -f k8s/nexmark-metrics-service.yaml + @echo "=== Restarting FlinkDeployment ===" + kubectl delete flinkdeployment flink-session -n flink-nexmark --ignore-not-found + sleep 2 + @IMAGE_URL="$(REGION)-docker.pkg.dev/$(PROJECT_ID)/flink/$(IMAGE_NAME):$(IMAGE_TAG)"; \ + sed "s|image: nexmark-runner:.*|image: $$IMAGE_URL|g" k8s/flink-session.yaml | kubectl apply -f - + @echo "=== Waiting for Flink pods ===" + @for i in $$(seq 1 30); do \ + kubectl get pod -n flink-nexmark -l component=jobmanager 2>/dev/null | grep -q flink-session && break; \ + echo " Waiting for pod creation... ($$i/30)"; \ + sleep 2; \ + done + kubectl wait --for=condition=Ready pod -l component=jobmanager -n flink-nexmark --timeout=300s + kubectl wait --for=condition=Ready pod -l component=taskmanager -n flink-nexmark --timeout=120s 2>/dev/null || true + @echo "=== Done. Run: make run-k8s QUERY=q0 ===" + +# GKE: Restart Flink pods (use after Ctrl+C or stuck run) +# Usage: make restart-k8s CONFIG=hudi-gcs PROJECT_ID= +restart-k8s: +ifndef CONFIG + @echo "Error: CONFIG not specified" + @echo "Usage: make restart-k8s CONFIG=hudi-gcs PROJECT_ID=your-project" + @exit 1 +endif +ifndef PROJECT_ID + @echo "Error: PROJECT_ID not specified" + @echo "Usage: make restart-k8s CONFIG=hudi-gcs PROJECT_ID=your-project" + @exit 1 +endif + @echo "=== Applying metrics service ===" + kubectl apply -f k8s/nexmark-metrics-service.yaml + @echo "=== Restarting FlinkDeployment ===" + kubectl delete flinkdeployment flink-session -n flink-nexmark --ignore-not-found + sleep 2 + @IMAGE_URL="$(REGION)-docker.pkg.dev/$(PROJECT_ID)/flink/$(IMAGE_NAME):$(IMAGE_TAG)"; \ + sed "s|image: nexmark-runner:.*|image: $$IMAGE_URL|g" k8s/flink-session.yaml | kubectl apply -f - + @echo "=== Waiting for Flink pods ===" + @for i in $$(seq 1 30); do \ + kubectl get pod -n flink-nexmark -l component=jobmanager 2>/dev/null | grep -q flink-session && break; \ + echo " Waiting for pod creation... ($$i/30)"; \ + sleep 2; \ + done + kubectl wait --for=condition=Ready pod -l component=jobmanager -n flink-nexmark --timeout=300s + @echo "=== Done. Run: make run-k8s QUERY=q0 ===" diff --git a/nexmark-runner/README.md b/nexmark-runner/README.md new file mode 100644 index 0000000..4f8d0fd --- /dev/null +++ b/nexmark-runner/README.md @@ -0,0 +1,99 @@ +# nexmark-runner + +Run [Nexmark](https://github.com/nexmark/nexmark) streaming benchmarks on Apache Flink with pluggable sinks (Hudi, blackhole). + +## Prerequisites + +```bash +# Check prerequisites and see install commands +make pre-setup +``` + +## Local Development (Kind + MinIO) + +```bash +# Setup cluster +make setup-local + +# Run benchmark +make run-local QUERY=q0 + +# Update config + restart Flink +make redeploy-local + +# Rebuild image + update config + restart +make redeploy-local REBUILD=1 + +# Force full rebuild (no Docker cache) +make redeploy-local REBUILD=1 NO_CACHE=1 + +# Restart Flink pods only +make restart-local + +# View MinIO console (minioadmin/minioadmin) +kubectl port-forward svc/minio 9001:9001 -n flink-nexmark + +# Teardown +make teardown-local +``` + +## GCP GKE Deployment + +```bash +# 1. Configure +cd k8s/gcp && cp terraform.tfvars.example terraform.tfvars +# Edit terraform.tfvars with your project settings + +# 2. Setup cluster +make setup-k8s + +# 3. Run benchmark +make run-k8s QUERY=q0 SINK=hudi BUCKET=your-bucket + +# Update config + restart Flink +make redeploy-k8s CONFIG=hudi-gcs PROJECT_ID=your-project + +# Rebuild image + push + restart +make redeploy-k8s CONFIG=hudi-gcs PROJECT_ID=your-project REBUILD=1 + +# Force full rebuild (no Docker cache) +make redeploy-k8s CONFIG=hudi-gcs PROJECT_ID=your-project REBUILD=1 NO_CACHE=1 + +# Restart Flink pods only +make restart-k8s CONFIG=hudi-gcs PROJECT_ID=your-project + +# 4. Teardown +make teardown-k8s +``` + +## Build Commands + +```bash +# Build for different configs +make build CONFIG=blackhole # flink1.18-blackhole +make build CONFIG=hudi-local # flink1.18-hudi-local-1.1.1 +make build CONFIG=hudi-gcs # flink1.18-hudi-gcs-1.1.1 + +# Build without Docker cache +make build CONFIG=hudi-local NO_CACHE=1 + +# Push to GCR +make push CONFIG=hudi-gcs PROJECT_ID=your-project + +# Show config info +make info CONFIG=hudi-gcs +``` + +## Queries + +All Nexmark queries (q0-q22) are supported. Common ones: + +| Query | Description | +|-------|-------------| +| q0 | Pass-through | +| q5 | Hot items (HOP window) | +| q7 | Highest bid (TUMBLE window) | + +## License + +Apache License 2.0 diff --git a/nexmark-runner/configs/nexmark-hudi-gcs.yaml b/nexmark-runner/configs/nexmark-hudi-gcs.yaml new file mode 100644 index 0000000..163b82a --- /dev/null +++ b/nexmark-runner/configs/nexmark-hudi-gcs.yaml @@ -0,0 +1,159 @@ +# Nexmark benchmark configuration - Hudi sink with GCS storage +# For cloud deployment on GCP GKE with Google Cloud Storage. + +# JARs to include (Maven coordinates downloaded during build) +include_jars: + - org.apache.hudi:hudi-flink${FLINK_VERSION_SHORT}-bundle:${HUDI_VERSION} + +# Metric reporter (sender connects to this address) +nexmark.metric.reporter.host: nexmark-metrics.flink-nexmark.svc.cluster.local +nexmark.metric.reporter.port: 9098 +# Metric receiver (binds to this address, falls back to reporter.host if not set) +nexmark.metric.reporter.receiving.host: 0.0.0.0 +nexmark.metric.monitor.delay: 0s + +# Workload configuration +nexmark.workload.suite.100m.events.num: 100000000 +nexmark.workload.suite.100m.tps: 10000000 +nexmark.workload.suite.100m.queries: "q0,q1,q2,q3,q4,q5,q7,q8,q9,q10,q11,q12,q13,q14,q15,q16,q17,q18,q19,q20,q21,q22" +nexmark.workload.suite.100m.warmup.duration: 120s +nexmark.workload.suite.100m.warmup.events.num: 100000000 +nexmark.workload.suite.100m.warmup.tps: 10000000 + +# Flink REST +flink.rest.address: localhost +flink.rest.port: 8081 + +# Base Hudi config for GCS (anchor - filtered out by SinkDdlProvider) +_base_hudi_gcs: &base + connector: hudi + table.type: MERGE_ON_READ + write.tasks: "8" + index.type: FLINK_STATE + write.operation: insert + hoodie.metadata.enable: "false" + hive_sync.enabled: "false" + hoodie.parquet.compression.codec: snappy + compaction.schedule.enabled: "false" + compaction.async.enabled: "false" + write.parquet.max.file.size: "120" + write.parquet.block.size: "120" + write.parquet.page.size: "1" + hoodie.parquet.small.file.limit: "0" + hoodie.fs.atomic_creation.support: gs + hadoop.fs.gs.impl: com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem + hadoop.fs.AbstractFileSystem.gs.impl: com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS + hadoop.fs.gs.auth.type: APPLICATION_DEFAULT + +# Hudi sink DDL configuration per query +# Format follows nexmark's SinkDdlProvider expectations + +nexmark.sink.ddl.q0: + <<: *base + record.key.field: auction,bidder,dateTime + ordering.fields: dateTime + +nexmark.sink.ddl.q1: + <<: *base + record.key.field: auction,bidder,dateTime + ordering.fields: dateTime + +nexmark.sink.ddl.q2: + <<: *base + record.key.field: auction + ordering.fields: auction + +nexmark.sink.ddl.q3: + <<: *base + record.key.field: id + ordering.fields: id + +nexmark.sink.ddl.q4: + <<: *base + record.key.field: id + ordering.fields: id + +nexmark.sink.ddl.q5: + <<: *base + record.key.field: auction + ordering.fields: auction + +nexmark.sink.ddl.q7: + <<: *base + record.key.field: auction,bidder,dateTime + ordering.fields: dateTime + +nexmark.sink.ddl.q8: + <<: *base + record.key.field: id + ordering.fields: stime + +nexmark.sink.ddl.q9: + <<: *base + record.key.field: id + ordering.fields: dateTime + +nexmark.sink.ddl.q10: + <<: *base + record.key.field: auction,bidder,dateTime + ordering.fields: dateTime + +nexmark.sink.ddl.q11: + <<: *base + record.key.field: bidder,starttime + ordering.fields: starttime + +nexmark.sink.ddl.q12: + <<: *base + record.key.field: bidder,starttime + ordering.fields: starttime + +nexmark.sink.ddl.q13: + <<: *base + record.key.field: auction,bidder,dateTime + ordering.fields: dateTime + +nexmark.sink.ddl.q14: + <<: *base + record.key.field: auction,bidder,dateTime + ordering.fields: dateTime + +nexmark.sink.ddl.q15: + <<: *base + record.key.field: day + ordering.fields: day + +nexmark.sink.ddl.q16: + <<: *base + record.key.field: channel,day,minute + ordering.fields: day + +nexmark.sink.ddl.q17: + <<: *base + record.key.field: auction,day + ordering.fields: day + +nexmark.sink.ddl.q18: + <<: *base + record.key.field: bidder,auction + ordering.fields: dateTime + +nexmark.sink.ddl.q19: + <<: *base + record.key.field: auction,rank_number + ordering.fields: dateTime + +nexmark.sink.ddl.q20: + <<: *base + record.key.field: auction,bidder + ordering.fields: bid_dateTime + +nexmark.sink.ddl.q21: + <<: *base + record.key.field: auction,bidder,dateTime + ordering.fields: dateTime + +nexmark.sink.ddl.q22: + <<: *base + record.key.field: auction,bidder,dateTime + ordering.fields: dateTime diff --git a/nexmark-runner/configs/nexmark-hudi-local.yaml b/nexmark-runner/configs/nexmark-hudi-local.yaml new file mode 100644 index 0000000..57211a1 --- /dev/null +++ b/nexmark-runner/configs/nexmark-hudi-local.yaml @@ -0,0 +1,185 @@ +# Nexmark benchmark configuration - Hudi sink for local development +# For local development with Kind cluster. + +# JARs to include (Maven coordinates downloaded during build) +include_jars: + - org.apache.hudi:hudi-flink${FLINK_VERSION_SHORT}-bundle:${HUDI_VERSION} + +# Metric reporter (sender connects to this address) +nexmark.metric.reporter.host: nexmark-metrics.flink-nexmark.svc.cluster.local +nexmark.metric.reporter.port: 9098 +# Metric receiver (binds to this address, falls back to reporter.host if not set) +nexmark.metric.reporter.receiving.host: 0.0.0.0 +nexmark.metric.monitor.delay: 0s + +# Workload configuration +nexmark.workload.suite.100m.events.num: 100000000 +nexmark.workload.suite.100m.tps: 10000000 +nexmark.workload.suite.100m.queries: "q0,q1,q2,q3,q4,q5,q7,q8,q9,q10,q11,q12,q13,q14,q15,q16,q17,q18,q19,q20,q21,q22" +nexmark.workload.suite.100m.warmup.duration: 5s +nexmark.workload.suite.100m.warmup.events.num: 100000 +nexmark.workload.suite.100m.warmup.tps: 100000 + +# Flink REST +flink.rest.address: localhost +flink.rest.port: 8081 + +# Base Hudi config for MinIO (anchor - filtered out by SinkDdlProvider) +_base_hudi_minio: &base + connector: hudi + table.type: MERGE_ON_READ + write.tasks: "8" + index.type: FLINK_STATE + write.operation: insert + hoodie.metadata.enable: "false" + hive_sync.enabled: "false" + hoodie.parquet.compression.codec: snappy + compaction.schedule.enabled: "false" + compaction.async.enabled: "false" + write.parquet.max.file.size: "120" + write.parquet.block.size: "120" + write.parquet.page.size: "1" + hoodie.parquet.small.file.limit: "0" + hoodie.fs.atomic_creation.support: s3a + hadoop.fs.s3a.endpoint: http://minio:9000 + hadoop.fs.s3a.access.key: minioadmin + hadoop.fs.s3a.secret.key: minioadmin + hadoop.fs.s3a.path.style.access: "true" + hadoop.fs.s3a.connection.ssl.enabled: "false" + hadoop.fs.s3a.impl: org.apache.hadoop.fs.s3a.S3AFileSystem + hadoop.fs.s3a.aws.credentials.provider: org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider + +# Hudi sink DDL configuration per query +# Format follows nexmark's SinkDdlProvider expectations + +nexmark.sink.ddl.q0: + <<: *base + path: s3a://nexmark/q0 + record.key.field: auction,bidder,dateTime + ordering.fields: dateTime + +nexmark.sink.ddl.q1: + <<: *base + path: s3a://nexmark/q1 + record.key.field: auction,bidder,dateTime + ordering.fields: dateTime + +nexmark.sink.ddl.q2: + <<: *base + path: s3a://nexmark/q2 + record.key.field: auction + ordering.fields: auction + +nexmark.sink.ddl.q3: + <<: *base + path: s3a://nexmark/q3 + record.key.field: id + ordering.fields: id + +nexmark.sink.ddl.q4: + <<: *base + path: s3a://nexmark/q4 + record.key.field: id + ordering.fields: id + +nexmark.sink.ddl.q5: + <<: *base + path: s3a://nexmark/q5 + record.key.field: auction + ordering.fields: auction + +nexmark.sink.ddl.q7: + <<: *base + path: s3a://nexmark/q7 + record.key.field: auction,bidder,dateTime + ordering.fields: dateTime + +nexmark.sink.ddl.q8: + <<: *base + path: s3a://nexmark/q8 + record.key.field: id + ordering.fields: stime + +nexmark.sink.ddl.q9: + <<: *base + path: s3a://nexmark/q9 + record.key.field: id + ordering.fields: dateTime + +nexmark.sink.ddl.q10: + <<: *base + path: s3a://nexmark/q10 + record.key.field: auction,bidder,dateTime + ordering.fields: dateTime + +nexmark.sink.ddl.q11: + <<: *base + path: s3a://nexmark/q11 + record.key.field: bidder,starttime + ordering.fields: starttime + +nexmark.sink.ddl.q12: + <<: *base + path: s3a://nexmark/q12 + record.key.field: bidder,starttime + ordering.fields: starttime + +nexmark.sink.ddl.q13: + <<: *base + path: s3a://nexmark/q13 + record.key.field: auction,bidder,dateTime + ordering.fields: dateTime + +nexmark.sink.ddl.q14: + <<: *base + path: s3a://nexmark/q14 + record.key.field: auction,bidder,dateTime + ordering.fields: dateTime + +nexmark.sink.ddl.q15: + <<: *base + path: s3a://nexmark/q15 + record.key.field: day + ordering.fields: day + +nexmark.sink.ddl.q16: + <<: *base + path: s3a://nexmark/q16 + record.key.field: channel,day,minute + ordering.fields: day + +nexmark.sink.ddl.q17: + <<: *base + path: s3a://nexmark/q17 + record.key.field: auction,day + ordering.fields: day + +nexmark.sink.ddl.q18: + <<: *base + path: s3a://nexmark/q18 + record.key.field: bidder,auction + ordering.fields: dateTime + +nexmark.sink.ddl.q19: + <<: *base + path: s3a://nexmark/q19 + record.key.field: auction,rank_number + ordering.fields: dateTime + +nexmark.sink.ddl.q20: + <<: *base + path: s3a://nexmark/q20 + record.key.field: auction,bidder + ordering.fields: bid_dateTime + +nexmark.sink.ddl.q21: + <<: *base + path: s3a://nexmark/q21 + record.key.field: auction,bidder,dateTime + ordering.fields: dateTime + +nexmark.sink.ddl.q22: + <<: *base + path: s3a://nexmark/q22 + record.key.field: auction,bidder,dateTime + ordering.fields: dateTime diff --git a/nexmark-runner/docker/Dockerfile b/nexmark-runner/docker/Dockerfile new file mode 100644 index 0000000..0b734ac --- /dev/null +++ b/nexmark-runner/docker/Dockerfile @@ -0,0 +1,151 @@ +ARG FLINK_VERSION=1.18.1 +FROM flink:${FLINK_VERSION}-java11 + +# Build arguments +# Note: ARGs before FROM go out of scope after FROM, so FLINK_VERSION must be re-declared +ARG FLINK_VERSION=1.18.1 +# NEXMARK_CONFIG_FILE: Config for JAR selection and runtime sink DDL +# Empty/unset = no additional JARs (blackhole sink uses nexmark defaults) +ARG NEXMARK_CONFIG_FILE= +ARG HUDI_VERSION=1.1.1 +ARG SIDE_INPUT_NUM=10000 + +ENV FLINK_VERSION=${FLINK_VERSION} +ENV HUDI_VERSION=${HUDI_VERSION} + +USER root + +# Install JDK, git, Maven, gettext (for envsubst), yq (base image only has JRE) +RUN apt-get update && \ + apt-get install -y openjdk-11-jdk-headless git maven gettext-base && \ + rm -rf /var/lib/apt/lists/* && \ + ln -sf /usr/lib/jvm/java-11-openjdk-* /usr/lib/jvm/java-11-openjdk && \ + wget -q -O /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/v4.40.5/yq_linux_amd64 && \ + chmod +x /usr/local/bin/yq + +ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk + +# Download Flink plugins and connectors (single layer) +# - GCS/S3 filesystem plugins for Flink +# - Hadoop bundle + AWS SDK for Hudi S3/MinIO support +# - GCS connector for Hudi GCS support +RUN mkdir -p /opt/flink/plugins/gs-fs-hadoop /opt/flink/plugins/s3-fs-hadoop && \ + wget -q -P /opt/flink/plugins/gs-fs-hadoop \ + https://repo1.maven.org/maven2/org/apache/flink/flink-gs-fs-hadoop/${FLINK_VERSION}/flink-gs-fs-hadoop-${FLINK_VERSION}.jar && \ + wget -q -P /opt/flink/plugins/s3-fs-hadoop \ + https://repo1.maven.org/maven2/org/apache/flink/flink-s3-fs-hadoop/${FLINK_VERSION}/flink-s3-fs-hadoop-${FLINK_VERSION}.jar && \ + wget -q -P /opt/flink/lib \ + https://repo1.maven.org/maven2/org/apache/flink/flink-shaded-hadoop-2-uber/2.8.3-10.0/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar && \ + wget -q -P /opt/flink/lib \ + https://repo1.maven.org/maven2/com/google/cloud/bigdataoss/gcs-connector/hadoop2-2.2.18/gcs-connector-hadoop2-2.2.18-shaded.jar && \ + wget -q -P /opt/flink/lib \ + https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/2.8.5/hadoop-aws-2.8.5.jar && \ + wget -q -P /opt/flink/lib \ + https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/1.11.375/aws-java-sdk-bundle-1.11.375.jar && \ + wget -q -P /opt/flink/lib \ + https://repo1.maven.org/maven2/com/google/guava/guava/27.0-jre/guava-27.0-jre.jar && \ + echo "Downloaded all Flink plugins and connectors" + +# Replace planner-loader with full flink-table-planner +# Removes loader to avoid "Multiple factories for identifier 'default'" conflict +RUN rm -f /opt/flink/lib/flink-table-planner-loader-*.jar && \ + cp /opt/flink/opt/flink-table-planner_2.12-${FLINK_VERSION}.jar /opt/flink/lib/ + +# Copy and build Nexmark from local directory (with pluggable sink support) +COPY ../nexmark /tmp/nexmark +RUN cd /tmp/nexmark && \ + mvn clean package -pl nexmark-flink -am -Dmaven.test.skip=true && \ + cp -r nexmark-flink/target/nexmark-flink-bin/nexmark-flink /opt/nexmark && \ + cp /opt/nexmark/lib/nexmark-flink*.jar /opt/flink/lib/ && \ + rm -rf /tmp/nexmark ~/.m2 + +# Copy JARs specified in config's include_jars list +# Supports two formats: +# 1. JAR filename (ending with .jar): copied from docker/jars/ +# 2. Maven coordinates (groupId:artifactId:version): downloaded from Maven Central +# Variables like ${FLINK_VERSION_SHORT}, ${HUDI_VERSION} are resolved against env vars +# If NEXMARK_CONFIG_FILE is empty, no additional JARs are included (blackhole mode) +COPY nexmark-runner/docker/jars/ /tmp/build-jars/ +COPY nexmark-runner/configs/ /tmp/build-configs/ +RUN set -e && \ + FLINK_VERSION_SHORT=$(echo "$FLINK_VERSION" | cut -d. -f1,2) && \ + export FLINK_VERSION_SHORT FLINK_VERSION HUDI_VERSION && \ + if [ -z "$NEXMARK_CONFIG_FILE" ]; then \ + echo "No NEXMARK_CONFIG_FILE specified (blackhole mode - no additional JARs)"; \ + elif [ -f "/tmp/build-configs/$NEXMARK_CONFIG_FILE" ]; then \ + CONFIG_PATH="/tmp/build-configs/$NEXMARK_CONFIG_FILE"; \ + if yq -e '.include_jars' "$CONFIG_PATH" > /dev/null 2>&1; then \ + echo "=== Processing include_jars from $NEXMARK_CONFIG_FILE ==="; \ + yq -r '.include_jars[]' "$CONFIG_PATH" > /tmp/jar_entries.txt; \ + while IFS= read -r entry || [ -n "$entry" ]; do \ + resolved=$(echo "$entry" | envsubst); \ + echo " Processing: $entry -> $resolved"; \ + if echo "$resolved" | grep -q '\.jar$'; then \ + if [ -f "/tmp/build-jars/$resolved" ]; then \ + cp "/tmp/build-jars/$resolved" /opt/flink/lib/; \ + echo " Copied: $resolved"; \ + else \ + echo "ERROR: JAR not found: $resolved"; \ + echo "Available JARs in docker/jars/:"; \ + ls -la /tmp/build-jars/ || echo " (empty)"; \ + exit 1; \ + fi; \ + else \ + GROUP_ID=$(echo "$resolved" | cut -d: -f1); \ + ARTIFACT_ID=$(echo "$resolved" | cut -d: -f2); \ + VERSION=$(echo "$resolved" | cut -d: -f3); \ + if [ -z "$GROUP_ID" ] || [ -z "$ARTIFACT_ID" ] || [ -z "$VERSION" ]; then \ + echo "ERROR: Invalid Maven coordinate: $resolved"; \ + echo "Expected format: groupId:artifactId:version"; \ + exit 1; \ + fi; \ + GROUP_PATH=$(echo "$GROUP_ID" | tr '.' '/'); \ + JAR_NAME="${ARTIFACT_ID}-${VERSION}.jar"; \ + MAVEN_URL="https://repo1.maven.org/maven2/${GROUP_PATH}/${ARTIFACT_ID}/${VERSION}/${JAR_NAME}"; \ + echo " Downloading from Maven Central: $MAVEN_URL"; \ + wget -q -P /opt/flink/lib "$MAVEN_URL"; \ + echo " Downloaded: $JAR_NAME"; \ + fi; \ + done < /tmp/jar_entries.txt; \ + rm -f /tmp/jar_entries.txt; \ + else \ + echo "No include_jars in $NEXMARK_CONFIG_FILE"; \ + fi; \ + else \ + echo "ERROR: Config file not found: $NEXMARK_CONFIG_FILE"; \ + exit 1; \ + fi && \ + rm -rf /tmp/build-jars /tmp/build-configs + +# Create nexmark config directory (nexmark.yaml provided at runtime) +# K8s: ConfigMap mounts nexmark.yaml directly to /opt/nexmark/conf/ +# YARN: ship nexmark.yaml via -files, set NEXMARK_CONF_DIR to point to it +RUN mkdir -p /opt/nexmark/conf + +# Generate side input data for q13 (bounded side input join) +ARG SIDE_INPUT_NUM +RUN mkdir -p /opt/flink/data && \ + java -cp "/opt/nexmark/lib/*:/opt/flink/lib/*" \ + com.github.nexmark.flink.generator.SideInputGenerator \ + --num ${SIDE_INPUT_NUM} --path /opt/flink/data/side_input.txt + +# Set environment variables +ENV FLINK_HOME=/opt/flink +ENV NEXMARK_HOME=/opt/nexmark +ENV NEXMARK_CONF_DIR=/opt/nexmark/conf + +# Create directories and set permissions +RUN mkdir -p /opt/flink/usrlib && \ + chown -R flink:flink /opt/flink /opt/nexmark + +# Verify setup +RUN echo "=== Nexmark Package ===" && \ + ls -la /opt/nexmark/ && \ + echo "=== Nexmark Config Dir (runtime mount) ===" && \ + ls -la /opt/nexmark/conf/ && \ + echo "=== Flink Lib JARs ===" && \ + ls -la /opt/flink/lib/*.jar && \ + echo "=== Flink Filesystem Plugins ===" && \ + ls -la /opt/flink/plugins/*/ + +USER flink diff --git a/nexmark-runner/docker/jars/.gitkeep b/nexmark-runner/docker/jars/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/nexmark-runner/k8s/flink-session.yaml b/nexmark-runner/k8s/flink-session.yaml new file mode 100644 index 0000000..a0e2e31 --- /dev/null +++ b/nexmark-runner/k8s/flink-session.yaml @@ -0,0 +1,125 @@ +# Flink 1.18 Session Cluster for GKE +# Config is loaded from ConfigMap mounted at /opt/nexmark/conf/ (NEXMARK_CONF_DIR) +# GCS credentials via Workload Identity (configured in Terraform) +apiVersion: flink.apache.org/v1beta1 +kind: FlinkDeployment +metadata: + name: flink-session + namespace: flink-nexmark +spec: + image: nexmark-runner:flink1.18-hudi-gcs-1.1.1 + imagePullPolicy: IfNotPresent + flinkVersion: v1_18 + serviceAccount: flink + + flinkConfiguration: + taskmanager.numberOfTaskSlots: "4" + parallelism.default: "8" + + # Checkpointing (Nexmark recommended: 3 min interval, exactly-once, incremental) + execution.checkpointing.interval: "180000" + execution.checkpointing.mode: EXACTLY_ONCE + execution.checkpointing.timeout: "300000" + state.backend: rocksdb + state.backend.incremental: "true" + state.backend.local-recovery: "true" + # Note: checkpoints dir should be overridden via helm or CLI for GCS bucket + + # RocksDB tuning for high throughput + state.backend.rocksdb.memory.managed: "true" + state.backend.rocksdb.block.cache-size: "256mb" + + # MiniBatch optimization (Nexmark V1 recommended) + table.exec.mini-batch.enabled: "true" + table.exec.mini-batch.allow-latency: "2s" + table.exec.mini-batch.size: "50000" + + # Distinct aggregation optimization + table.optimizer.distinct-agg.split.enabled: "true" + + # Disable final checkpoint for faster job completion + execution.checkpointing.checkpoints-after-tasks-finish.enabled: "false" + + # Memory tuning + taskmanager.memory.process.size: "8192m" + taskmanager.memory.managed.fraction: "0.4" + jobmanager.memory.process.size: "4096m" + taskmanager.network.memory.fraction: "0.15" + + # REST API + rest.address: "0.0.0.0" + rest.port: "8081" + rest.bind-address: "0.0.0.0" + jobmanager.rpc.address: "localhost" + + # Force sql-client to use remote execution target + execution.target: "remote" + + # GCS authentication via Workload Identity + gs.auth.type: APPLICATION_DEFAULT + + jobManager: + resource: + memory: "4096m" + cpu: 2 + podTemplate: + spec: + containers: + - name: flink-main-container + ports: + - containerPort: 9098 + name: metrics + protocol: TCP + + taskManager: + resource: + memory: "8192m" + cpu: 4 + replicas: 2 + podTemplate: + spec: + shareProcessNamespace: true + containers: + - name: flink-main-container + volumeMounts: + - name: shared-tmp + mountPath: /tmp + - name: metrics-sender + image: nexmark-runner:flink1.18-hudi-gcs-1.1.1 + imagePullPolicy: IfNotPresent + command: ["/bin/bash", "-c"] + args: + - | + sleep 10 + java -cp "/opt/nexmark/lib/*:/opt/flink/lib/*" \ + com.github.nexmark.flink.metric.process.ProcessMetricSender + volumeMounts: + - name: nexmark-config + mountPath: /opt/nexmark/conf/nexmark.yaml + subPath: nexmark.yaml + - name: shared-tmp + mountPath: /tmp + resources: + requests: + memory: "256Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "200m" + + podTemplate: + spec: + containers: + - name: flink-main-container + volumeMounts: + - name: nexmark-config + mountPath: /opt/nexmark/conf/nexmark.yaml + subPath: nexmark.yaml + volumes: + - name: nexmark-config + configMap: + name: nexmark-config + - name: shared-tmp + emptyDir: {} + + mode: standalone diff --git a/nexmark-runner/k8s/gcp/gcs.tf b/nexmark-runner/k8s/gcp/gcs.tf new file mode 100644 index 0000000..d1af428 --- /dev/null +++ b/nexmark-runner/k8s/gcp/gcs.tf @@ -0,0 +1,5 @@ +# Reference existing GCS bucket for Hudi tables and Flink checkpoints +# Bucket is managed externally (default: hudi-benchmark) +data "google_storage_bucket" "hudi_tables" { + name = var.gcs_bucket_name +} diff --git a/nexmark-runner/k8s/gcp/helm.tf b/nexmark-runner/k8s/gcp/helm.tf new file mode 100644 index 0000000..2aed400 --- /dev/null +++ b/nexmark-runner/k8s/gcp/helm.tf @@ -0,0 +1,113 @@ +# Create Flink namespace +resource "kubernetes_namespace" "flink" { + metadata { + name = var.flink_namespace + } + + depends_on = [google_container_node_pool.jobmanager_nodes, google_container_node_pool.taskmanager_nodes] +} + +# cert-manager namespace +resource "kubernetes_namespace" "cert_manager" { + metadata { + name = "cert-manager" + } + + depends_on = [google_container_node_pool.jobmanager_nodes, google_container_node_pool.taskmanager_nodes] +} + +# Install cert-manager (required for Flink Operator webhooks) +resource "helm_release" "cert_manager" { + name = "cert-manager" + repository = "https://charts.jetstack.io" + chart = "cert-manager" + version = var.cert_manager_version + namespace = kubernetes_namespace.cert_manager.metadata[0].name + + set { + name = "installCRDs" + value = "true" + } + + # Wait for cert-manager to be fully ready + wait = true + timeout = 600 +} + +# Install Flink Kubernetes Operator +resource "helm_release" "flink_operator" { + name = "flink-kubernetes-operator" + repository = "https://downloads.apache.org/flink/flink-kubernetes-operator-${var.flink_operator_version}/" + chart = "flink-kubernetes-operator" + version = var.flink_operator_version + namespace = kubernetes_namespace.flink.metadata[0].name + + # Webhook configuration + set { + name = "webhook.create" + value = "true" + } + + # Watch the Flink namespace + set { + name = "watchNamespaces[0]" + value = var.flink_namespace + } + + # Default Flink configuration for all deployments + values = [ + yamlencode({ + defaultConfiguration = { + create = true + append = true + "flink-conf.yaml" = <<-EOT + # Checkpointing + execution.checkpointing.interval: 60000 + execution.checkpointing.mode: EXACTLY_ONCE + state.backend: rocksdb + state.checkpoints.dir: gs://${data.google_storage_bucket.hudi_tables.name}/checkpoints + state.savepoints.dir: gs://${data.google_storage_bucket.hudi_tables.name}/savepoints + + # GCS authentication via Workload Identity + gs.auth.type: APPLICATION_DEFAULT + + # Memory configuration defaults + taskmanager.memory.process.size: 8192m + jobmanager.memory.process.size: 2048m + taskmanager.numberOfTaskSlots: 4 + + # High availability (optional, disabled for research) + # high-availability.type: kubernetes + # high-availability.storageDir: gs://${data.google_storage_bucket.hudi_tables.name}/ha + EOT + } + }) + ] + + depends_on = [helm_release.cert_manager] + + wait = true + timeout = 600 +} + +# Annotate the Flink job service account for Workload Identity +# (The 'flink' SA is created by Helm; we add the GCP SA annotation for GCS access) +resource "kubernetes_annotations" "flink_sa_workload_identity" { + api_version = "v1" + kind = "ServiceAccount" + + metadata { + name = "flink" + namespace = var.flink_namespace + } + + annotations = { + "iam.gke.io/gcp-service-account" = google_service_account.flink_sa.email + } + + depends_on = [helm_release.flink_operator] +} + +# Note: RBAC for Flink jobs is managed by the Helm chart (creates 'flink' role +# and 'flink-role-binding'). Do not create separate RBAC resources here as they +# conflict with Helm-managed ones. diff --git a/nexmark-runner/k8s/gcp/iam.tf b/nexmark-runner/k8s/gcp/iam.tf new file mode 100644 index 0000000..4906bc9 --- /dev/null +++ b/nexmark-runner/k8s/gcp/iam.tf @@ -0,0 +1,29 @@ +# Service account for Flink workloads to access GCS +resource "google_service_account" "flink_sa" { + account_id = "flink-nexmark-sa" + display_name = "Flink Nexmark Service Account" + project = var.project_id + + depends_on = [google_project_service.required_apis] +} + +# Grant Storage Object Admin on the existing Hudi bucket +resource "google_storage_bucket_iam_member" "flink_bucket_access" { + bucket = data.google_storage_bucket.hudi_tables.name + role = "roles/storage.objectAdmin" + member = "serviceAccount:${google_service_account.flink_sa.email}" +} + +# Workload Identity binding: allow Kubernetes SA to impersonate GCP SA +resource "google_service_account_iam_member" "workload_identity_binding" { + service_account_id = google_service_account.flink_sa.name + role = "roles/iam.workloadIdentityUser" + member = "serviceAccount:${var.project_id}.svc.id.goog[${var.flink_namespace}/flink]" +} + +# Grant Artifact Registry Reader for pulling images +resource "google_project_iam_member" "flink_artifact_reader" { + project = var.project_id + role = "roles/artifactregistry.reader" + member = "serviceAccount:${google_service_account.flink_sa.email}" +} diff --git a/nexmark-runner/k8s/gcp/main.tf b/nexmark-runner/k8s/gcp/main.tf new file mode 100644 index 0000000..7280b81 --- /dev/null +++ b/nexmark-runner/k8s/gcp/main.tf @@ -0,0 +1,175 @@ +# Provider configuration +provider "google" { + project = var.project_id + region = var.region +} + +# Enable required APIs +resource "google_project_service" "required_apis" { + for_each = toset([ + "container.googleapis.com", + "artifactregistry.googleapis.com", + "iam.googleapis.com", + ]) + + project = var.project_id + service = each.value + disable_on_destroy = false +} + +# VPC Network +resource "google_compute_network" "vpc" { + name = "${var.cluster_name}-vpc" + auto_create_subnetworks = false + project = var.project_id + + depends_on = [google_project_service.required_apis] +} + +# Subnet +resource "google_compute_subnetwork" "subnet" { + name = "${var.cluster_name}-subnet" + ip_cidr_range = "10.0.0.0/24" + region = var.region + network = google_compute_network.vpc.id + project = var.project_id + + secondary_ip_range { + range_name = "pods" + ip_cidr_range = "10.1.0.0/16" + } + + secondary_ip_range { + range_name = "services" + ip_cidr_range = "10.2.0.0/20" + } +} + +# GKE Cluster +resource "google_container_cluster" "flink_cluster" { + name = var.cluster_name + location = var.zone + project = var.project_id + + # Use separately managed node pool + remove_default_node_pool = true + initial_node_count = 1 + + # Network configuration + network = google_compute_network.vpc.name + subnetwork = google_compute_subnetwork.subnet.name + + ip_allocation_policy { + cluster_secondary_range_name = "pods" + services_secondary_range_name = "services" + } + + # Workload Identity for secure GCS access + workload_identity_config { + workload_pool = "${var.project_id}.svc.id.goog" + } + + # Release channel for automatic upgrades + release_channel { + channel = "REGULAR" + } + + depends_on = [google_project_service.required_apis] +} + +# Node pool for Flink JobManager (fixed size, no autoscaling) +resource "google_container_node_pool" "jobmanager_nodes" { + name = "jobmanager-pool" + cluster = google_container_cluster.flink_cluster.name + location = var.zone + project = var.project_id + + node_count = var.jm_node_count + + node_config { + machine_type = var.jm_machine_type + disk_size_gb = var.jm_disk_size_gb + disk_type = "pd-ssd" + + # Workload Identity + workload_metadata_config { + mode = "GKE_METADATA" + } + + oauth_scopes = [ + "https://www.googleapis.com/auth/cloud-platform" + ] + + labels = { + "flink-role" = "jobmanager" + } + } + + management { + auto_repair = true + auto_upgrade = true + } +} + +# Node pool for Flink TaskManagers (fixed size, no autoscaling) +resource "google_container_node_pool" "taskmanager_nodes" { + name = "taskmanager-pool" + cluster = google_container_cluster.flink_cluster.name + location = var.zone + project = var.project_id + + node_count = var.tm_node_count + + node_config { + machine_type = var.tm_machine_type + disk_size_gb = var.tm_disk_size_gb + disk_type = "pd-ssd" + + # Workload Identity + workload_metadata_config { + mode = "GKE_METADATA" + } + + oauth_scopes = [ + "https://www.googleapis.com/auth/cloud-platform" + ] + + labels = { + "flink-role" = "taskmanager" + } + } + + management { + auto_repair = true + auto_upgrade = true + } +} + +# Artifact Registry for Docker images +resource "google_artifact_registry_repository" "flink_repo" { + location = var.region + repository_id = "flink" + description = "Docker repository for Flink images" + format = "DOCKER" + project = var.project_id + + depends_on = [google_project_service.required_apis] +} + +# Configure Kubernetes provider +data "google_client_config" "default" {} + +provider "kubernetes" { + host = "https://${google_container_cluster.flink_cluster.endpoint}" + token = data.google_client_config.default.access_token + cluster_ca_certificate = base64decode(google_container_cluster.flink_cluster.master_auth[0].cluster_ca_certificate) +} + +# Configure Helm provider +provider "helm" { + kubernetes { + host = "https://${google_container_cluster.flink_cluster.endpoint}" + token = data.google_client_config.default.access_token + cluster_ca_certificate = base64decode(google_container_cluster.flink_cluster.master_auth[0].cluster_ca_certificate) + } +} diff --git a/nexmark-runner/k8s/gcp/outputs.tf b/nexmark-runner/k8s/gcp/outputs.tf new file mode 100644 index 0000000..dfddc8f --- /dev/null +++ b/nexmark-runner/k8s/gcp/outputs.tf @@ -0,0 +1,55 @@ +output "project_id" { + description = "GCP Project ID" + value = var.project_id +} + +output "region" { + description = "GCP Region" + value = var.region +} + +output "zone" { + description = "GCP Zone" + value = var.zone +} + +output "cluster_name" { + description = "GKE cluster name" + value = google_container_cluster.flink_cluster.name +} + +output "cluster_endpoint" { + description = "GKE cluster endpoint" + value = google_container_cluster.flink_cluster.endpoint + sensitive = true +} + +output "gcs_bucket_name" { + description = "GCS bucket for Hudi tables" + value = data.google_storage_bucket.hudi_tables.name +} + +output "gcs_bucket_url" { + description = "GCS bucket URL" + value = "gs://${data.google_storage_bucket.hudi_tables.name}" +} + +output "flink_service_account_email" { + description = "Flink GCP service account email" + value = google_service_account.flink_sa.email +} + +output "artifact_registry_url" { + description = "Artifact Registry URL for Docker images" + value = "${var.region}-docker.pkg.dev/${var.project_id}/${google_artifact_registry_repository.flink_repo.repository_id}" +} + +output "flink_image_url" { + description = "Full URL for the Hudi Nexmark image" + value = "${var.region}-docker.pkg.dev/${var.project_id}/${google_artifact_registry_repository.flink_repo.repository_id}/hudi-nexmark:latest" +} + +output "kubectl_config_command" { + description = "Command to configure kubectl" + value = "gcloud container clusters get-credentials ${google_container_cluster.flink_cluster.name} --zone ${var.zone} --project ${var.project_id}" +} diff --git a/nexmark-runner/k8s/gcp/terraform.tfvars.example b/nexmark-runner/k8s/gcp/terraform.tfvars.example new file mode 100644 index 0000000..6aee7e9 --- /dev/null +++ b/nexmark-runner/k8s/gcp/terraform.tfvars.example @@ -0,0 +1,27 @@ +# GCP Project Configuration +project_id = "your-gcp-project-id" + +# Region and Zone +region = "us-central1" +zone = "us-central1-a" + +# GKE Cluster Configuration +cluster_name = "flink-nexmark-cluster" + +# JobManager node pool (1 x e2-standard-2, fixed size) +jm_node_count = 1 +jm_machine_type = "e2-standard-2" # 2 vCPU, 8GB RAM +jm_disk_size_gb = 20 + +# TaskManager node pool (2 x e2-standard-8, fixed size) +tm_node_count = 2 +tm_machine_type = "e2-standard-8" # 8 vCPU, 32GB RAM +tm_disk_size_gb = 40 # More disk for checkpoints + +# GCS Bucket (must exist - not created by Terraform) +gcs_bucket_name = "hudi-benchmark" + +# Flink Configuration +flink_namespace = "flink-nexmark" +flink_operator_version = "1.10.0" +cert_manager_version = "v1.18.2" diff --git a/nexmark-runner/k8s/gcp/variables.tf b/nexmark-runner/k8s/gcp/variables.tf new file mode 100644 index 0000000..0638438 --- /dev/null +++ b/nexmark-runner/k8s/gcp/variables.tf @@ -0,0 +1,84 @@ +variable "project_id" { + description = "GCP Project ID" + type = string +} + +variable "region" { + description = "GCP region for resources" + type = string + default = "us-central1" +} + +variable "zone" { + description = "GCP zone for GKE cluster" + type = string + default = "us-central1-a" +} + +variable "cluster_name" { + description = "Name of the GKE cluster" + type = string + default = "flink-nexmark-cluster" +} + +# JobManager node pool configuration (fixed size, no autoscaling) +variable "jm_node_count" { + description = "Number of nodes in the JobManager pool" + type = number + default = 1 +} + +variable "jm_machine_type" { + description = "Machine type for JobManager nodes" + type = string + default = "e2-standard-8" # 8 vCPU, 32GB RAM +} + +# TaskManager node pool configuration (fixed size, no autoscaling) +variable "tm_node_count" { + description = "Number of nodes in the TaskManager pool" + type = number + default = 2 +} + +variable "tm_machine_type" { + description = "Machine type for TaskManager nodes" + type = string + default = "e2-standard-8" # 8 vCPU, 32GB RAM +} + +variable "jm_disk_size_gb" { + description = "Disk size in GB for JobManager nodes" + type = number + default = 20 +} + +variable "tm_disk_size_gb" { + description = "Disk size in GB for TaskManager nodes (needs more for checkpoints)" + type = number + default = 40 +} + +variable "gcs_bucket_name" { + description = "Name of the existing GCS bucket for Hudi tables" + type = string + default = "hudi-benchmark" +} + +variable "flink_namespace" { + description = "Kubernetes namespace for Flink workloads" + type = string + default = "flink-nexmark" +} + +variable "flink_operator_version" { + description = "Flink Kubernetes Operator Helm chart version" + type = string + default = "1.10.0" +} + +variable "cert_manager_version" { + description = "cert-manager version" + type = string + default = "v1.18.2" +} diff --git a/nexmark-runner/k8s/gcp/versions.tf b/nexmark-runner/k8s/gcp/versions.tf new file mode 100644 index 0000000..10aeedb --- /dev/null +++ b/nexmark-runner/k8s/gcp/versions.tf @@ -0,0 +1,18 @@ +terraform { + required_version = ">= 1.0" + + required_providers { + google = { + source = "hashicorp/google" + version = "~> 5.0" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = "~> 2.25" + } + helm = { + source = "hashicorp/helm" + version = "~> 2.12" + } + } +} diff --git a/nexmark-runner/k8s/nexmark-metrics-service.yaml b/nexmark-runner/k8s/nexmark-metrics-service.yaml new file mode 100644 index 0000000..a31803e --- /dev/null +++ b/nexmark-runner/k8s/nexmark-metrics-service.yaml @@ -0,0 +1,17 @@ +# Service to expose ProcessMetricReceiver port on JobManager +# Allows sidecar ProcessMetricSender containers on TaskManagers to send metrics +apiVersion: v1 +kind: Service +metadata: + name: nexmark-metrics + namespace: flink-nexmark +spec: + selector: + app: flink-session + component: jobmanager + ports: + - port: 9098 + targetPort: 9098 + protocol: TCP + name: metrics + type: ClusterIP diff --git a/nexmark-runner/k8s/scripts/run.sh b/nexmark-runner/k8s/scripts/run.sh new file mode 100755 index 0000000..e34429d --- /dev/null +++ b/nexmark-runner/k8s/scripts/run.sh @@ -0,0 +1,114 @@ +#!/bin/bash +# Run Nexmark benchmark on GCP GKE Flink session cluster +# Uses Nexmark's native pluggable sink system via CLI arguments +# +# Usage: run.sh [queries] [--sink TYPE] [--bucket NAME] [-D key=value] +# +# Examples: +# ./run.sh all # All queries (blackhole sink) +# ./run.sh q0 # Single query (blackhole sink) +# ./run.sh q0 --sink hudi --bucket my-bucket # Hudi sink to GCS +# ./run.sh q0 --sink hudi --bucket b -D nexmark.sink.hudi.write.tasks=4 +set -e + +NAMESPACE="flink-nexmark" +QUERIES="" +SINK="" +BUCKET="" +EXTRA_ARGS=() + +# Parse arguments +while [[ $# -gt 0 ]]; do + case $1 in + --sink) + SINK="$2" + shift 2 + ;; + --bucket) + BUCKET="$2" + shift 2 + ;; + -D) + EXTRA_ARGS+=("-D" "$2") + shift 2 + ;; + -D*) + # Handle -Dkey=value (no space) + EXTRA_ARGS+=("$1") + shift + ;; + *) + if [ -z "$QUERIES" ]; then + QUERIES="$1" + else + # Pass through unknown args + EXTRA_ARGS+=("$1") + fi + shift + ;; + esac +done + +QUERIES="${QUERIES:-all}" + +echo "=== Nexmark Benchmark (GCP GKE) ===" +echo "" + +# Check if session cluster exists +if ! kubectl get flinkdeployment flink-session -n "$NAMESPACE" &>/dev/null; then + echo "Error: Flink session cluster not found." + echo "Run setup.sh first to deploy the cluster." + exit 1 +fi + +# Get JobManager pod name +POD=$(kubectl get pod -l app=flink-session,component=jobmanager -n "$NAMESPACE" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) +if [ -z "$POD" ]; then + echo "Waiting for Flink JobManager pod..." + kubectl wait --for=condition=Ready pod -l app=flink-session,component=jobmanager -n "$NAMESPACE" --timeout=120s + POD=$(kubectl get pod -l app=flink-session,component=jobmanager -n "$NAMESPACE" -o jsonpath='{.items[0].metadata.name}') +fi + +echo "Using pod: $POD" +echo "" + +# Build Nexmark CLI arguments +NEXMARK_ARGS=() + +if [ -n "$SINK" ]; then + NEXMARK_ARGS+=("--sink" "$SINK") + + # Determine storage type and path prefix based on sink + if [ "$SINK" = "hudi" ]; then + if [ -z "$BUCKET" ]; then + echo "Error: --bucket is required when using --sink hudi" + echo "Usage: ./run.sh q0 --sink hudi --bucket my-bucket" + exit 1 + fi + NEXMARK_ARGS+=("--storage" "gcs") + NEXMARK_ARGS+=("--path-prefix" "gs://${BUCKET}") + echo "Sink: hudi (gs://${BUCKET}/hudi/)" + else + echo "Sink: $SINK" + fi +else + echo "Sink: blackhole (default)" +fi + +echo "Queries: $QUERIES" +if [ ${#EXTRA_ARGS[@]} -gt 0 ]; then + echo "Extra args: ${EXTRA_ARGS[*]}" +fi +echo "" + +# Run Nexmark benchmark using native CLI +kubectl exec -t -n "$NAMESPACE" "$POD" -- \ + /opt/nexmark/bin/run_query.sh "$QUERIES" "${NEXMARK_ARGS[@]}" "${EXTRA_ARGS[@]}" + +echo "" +if [ -n "$BUCKET" ]; then + echo "Output location: gs://${BUCKET}/hudi/" + echo " gsutil ls gs://${BUCKET}/hudi/" + echo "" +fi +echo "Flink UI: kubectl port-forward svc/flink-session-rest 8081:8081 -n $NAMESPACE" diff --git a/nexmark-runner/k8s/scripts/setup.sh b/nexmark-runner/k8s/scripts/setup.sh new file mode 100755 index 0000000..ff299e0 --- /dev/null +++ b/nexmark-runner/k8s/scripts/setup.sh @@ -0,0 +1,104 @@ +#!/bin/bash +# Setup Nexmark benchmark environment on GCP GKE +# Usage: ./setup.sh +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(dirname "$(dirname "$SCRIPT_DIR")")" + +INFRA_DIR="$PROJECT_ROOT/k8s/gcp" + +echo "=== Nexmark Benchmark Setup (GCP GKE) ===" +echo "" + +# Check prerequisites +echo "Checking prerequisites..." +command -v terraform >/dev/null 2>&1 || { echo "Error: terraform not found"; exit 1; } +command -v kubectl >/dev/null 2>&1 || { echo "Error: kubectl not found"; exit 1; } +command -v docker >/dev/null 2>&1 || { echo "Error: docker not found"; exit 1; } +command -v gcloud >/dev/null 2>&1 || { echo "Error: gcloud CLI not found"; exit 1; } + +# Check for terraform.tfvars +if [ ! -f "$INFRA_DIR/terraform.tfvars" ]; then + echo "Error: $INFRA_DIR/terraform.tfvars not found" + echo "Please copy terraform.tfvars.example to terraform.tfvars and configure it" + exit 1 +fi + +# Step 1: Apply Terraform +echo "" +echo "=== Step 1: Provisioning GKE cluster and GCS bucket ===" +cd "$INFRA_DIR" +terraform init +terraform apply + +# Get GKE outputs +PROJECT_ID=$(terraform output -raw project_id) +REGION=$(terraform output -raw region) +ZONE=$(terraform output -raw zone) +CLUSTER_NAME=$(terraform output -raw cluster_name) +GCS_BUCKET=$(terraform output -raw gcs_bucket_name) +IMAGE_URL=$(terraform output -raw flink_image_url) + +echo "" +echo "Project ID: $PROJECT_ID" +echo "GCS Bucket: $GCS_BUCKET" +echo "Image URL: $IMAGE_URL" + +# Step 2: Configure kubectl +echo "" +echo "=== Step 2: Configuring kubectl ===" +gcloud container clusters get-credentials "$CLUSTER_NAME" --zone "$ZONE" --project "$PROJECT_ID" + +# Wait for Flink operator to be ready +echo "Waiting for Flink Kubernetes Operator to be ready..." +kubectl wait --for=condition=available --timeout=300s deployment/flink-kubernetes-operator -n flink-nexmark || { + echo "Warning: Flink operator not ready yet. Check with: kubectl get pods -n flink-nexmark" +} + +# Step 3: Build and push Docker image +echo "" +echo "=== Step 3: Building and pushing Docker image ===" +cd "$PROJECT_ROOT" +make push CONFIG=hudi-gcs PROJECT_ID="$PROJECT_ID" REGION="$REGION" + +# Step 4: Create ConfigMap from GCS config +echo "" +echo "=== Step 4: Creating ConfigMap ===" +kubectl create configmap nexmark-config \ + --from-file=nexmark.yaml="$PROJECT_ROOT/configs/nexmark-hudi-gcs.yaml" \ + -n flink-nexmark \ + --dry-run=client -o yaml | kubectl apply -f - + +# Step 5: Deploy Flink session cluster and metrics service +echo "" +echo "=== Step 5: Deploying Flink session cluster ===" +kubectl apply -f "$PROJECT_ROOT/k8s/nexmark-metrics-service.yaml" +kubectl delete flinkdeployment flink-session -n flink-nexmark --ignore-not-found +sleep 2 +# Patch FlinkDeployment with registry image URL +sed "s|image: nexmark-runner:.*|image: $IMAGE_URL|g" "$PROJECT_ROOT/k8s/flink-session.yaml" | kubectl apply -f - + +echo "" +echo "Waiting for Flink pods to be ready..." +for i in $(seq 1 30); do + if kubectl get pod -n flink-nexmark -l component=jobmanager 2>/dev/null | grep -q flink-session; then + break + fi + echo " Waiting for pod creation... ($i/30)" + sleep 2 +done +kubectl wait --for=condition=Ready pod -l component=jobmanager -n flink-nexmark --timeout=300s + +echo "" +echo "=== Setup Complete ===" +echo "" +kubectl get pods -n flink-nexmark +echo "" +echo "GCS Bucket: gs://$GCS_BUCKET" +echo "Image URL: $IMAGE_URL" +echo "" +echo "Run benchmark:" +echo " ./k8s/scripts/run.sh q0 --sink hudi --bucket $GCS_BUCKET" +echo "" +echo "Flink UI: kubectl port-forward svc/flink-session-rest 8081:8081 -n flink-nexmark" diff --git a/nexmark-runner/k8s/scripts/teardown.sh b/nexmark-runner/k8s/scripts/teardown.sh new file mode 100755 index 0000000..11c4284 --- /dev/null +++ b/nexmark-runner/k8s/scripts/teardown.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# Teardown Nexmark benchmark environment on GCP GKE +# Usage: ./teardown.sh +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(dirname "$(dirname "$SCRIPT_DIR")")" + +INFRA_DIR="$PROJECT_ROOT/k8s/gcp" + +echo "=== Nexmark Benchmark Teardown (GCP GKE) ===" +echo "" +echo "This will destroy all resources:" +echo " - GKE cluster" +echo " - GCS bucket (and all data)" +echo " - Service accounts" +echo " - VPC and networking" +echo " - Artifact Registry repository" +echo "" + +read -p "Are you sure you want to continue? (yes/no): " confirm +if [ "$confirm" != "yes" ]; then + echo "Teardown cancelled." + exit 0 +fi + +# Delete Kubernetes resources first (optional, Terraform will handle this) +echo "" +echo "=== Deleting Kubernetes resources ===" +kubectl delete flinkdeployment --all -n flink-nexmark 2>/dev/null || true +kubectl delete flinksessionjob --all -n flink-nexmark 2>/dev/null || true + +# Run Terraform destroy +echo "" +echo "=== Running Terraform destroy ===" +cd "$INFRA_DIR" +terraform destroy + +echo "" +echo "=== Teardown Complete ===" +echo "" +echo "All resources have been destroyed." diff --git a/nexmark-runner/local/flink-session.yaml b/nexmark-runner/local/flink-session.yaml new file mode 100644 index 0000000..3a36607 --- /dev/null +++ b/nexmark-runner/local/flink-session.yaml @@ -0,0 +1,136 @@ +# Flink 1.18 Session Cluster for local Kind testing +# Config is loaded from ConfigMap mounted at /opt/nexmark/conf/ (NEXMARK_CONF_DIR) +apiVersion: flink.apache.org/v1beta1 +kind: FlinkDeployment +metadata: + name: flink-session + namespace: flink-nexmark +spec: + image: nexmark-k8s:local + imagePullPolicy: Never + flinkVersion: v1_18 + serviceAccount: flink + + flinkConfiguration: + taskmanager.numberOfTaskSlots: "4" + parallelism.default: "8" + + # Checkpointing (Nexmark recommended: 3 min interval, exactly-once, incremental) + execution.checkpointing.interval: "180000" + execution.checkpointing.mode: EXACTLY_ONCE + execution.checkpointing.timeout: "300000" + state.backend: rocksdb + state.backend.incremental: "true" + state.backend.local-recovery: "true" + state.checkpoints.dir: file:///tmp/flink-checkpoints + state.savepoints.dir: file:///tmp/flink-savepoints + + # RocksDB tuning for high throughput + state.backend.rocksdb.memory.managed: "true" + state.backend.rocksdb.block.cache-size: "256mb" + + # MiniBatch optimization (Nexmark V1 recommended) + table.exec.mini-batch.enabled: "true" + table.exec.mini-batch.allow-latency: "2s" + table.exec.mini-batch.size: "50000" + + # Distinct aggregation optimization + table.optimizer.distinct-agg.split.enabled: "true" + + # Disable final checkpoint for faster job completion + execution.checkpointing.checkpoints-after-tasks-finish.enabled: "false" + + # Memory tuning + taskmanager.memory.process.size: "8192m" + taskmanager.memory.managed.fraction: "0.4" + jobmanager.memory.process.size: "4096m" + taskmanager.network.memory.fraction: "0.15" + + # REST API + rest.address: "0.0.0.0" + rest.port: "8081" + rest.bind-address: "0.0.0.0" + jobmanager.rpc.address: "localhost" + + # Force sql-client to use remote execution target + execution.target: "remote" + + # S3/MinIO configuration for Flink filesystem plugin + s3.endpoint: http://minio:9000 + s3.access-key: minioadmin + s3.secret-key: minioadmin + s3.path-style-access: "true" + + # Hadoop S3A configuration for Hudi (uses Hadoop FileSystem directly) + fs.s3a.endpoint: http://minio:9000 + fs.s3a.access.key: minioadmin + fs.s3a.secret.key: minioadmin + fs.s3a.path.style.access: "true" + fs.s3a.connection.ssl.enabled: "false" + fs.s3a.impl: org.apache.hadoop.fs.s3a.S3AFileSystem + + jobManager: + resource: + memory: "4096m" + cpu: 2 + podTemplate: + spec: + containers: + - name: flink-main-container + ports: + - containerPort: 9098 + name: metrics + protocol: TCP + + taskManager: + resource: + memory: "8192m" + cpu: 4 + replicas: 2 + podTemplate: + spec: + shareProcessNamespace: true + containers: + - name: flink-main-container + volumeMounts: + - name: shared-tmp + mountPath: /tmp + - name: metrics-sender + image: nexmark-k8s:local + imagePullPolicy: Never + command: ["/bin/bash", "-c"] + args: + - | + sleep 10 + java -cp "/opt/nexmark/lib/*:/opt/flink/lib/*" \ + com.github.nexmark.flink.metric.process.ProcessMetricSender + volumeMounts: + - name: nexmark-config + mountPath: /opt/nexmark/conf/nexmark.yaml + subPath: nexmark.yaml + - name: shared-tmp + mountPath: /tmp + resources: + requests: + memory: "256Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "200m" + + podTemplate: + spec: + containers: + - name: flink-main-container + volumeMounts: + - name: nexmark-config + mountPath: /opt/nexmark/conf/nexmark.yaml + subPath: nexmark.yaml + volumes: + - name: nexmark-config + configMap: + name: nexmark-config + - name: shared-tmp + emptyDir: {} + + mode: standalone diff --git a/nexmark-runner/local/kind-config.yaml b/nexmark-runner/local/kind-config.yaml new file mode 100644 index 0000000..0d32e8f --- /dev/null +++ b/nexmark-runner/local/kind-config.yaml @@ -0,0 +1,13 @@ +# Kind cluster configuration for Nexmark benchmarking +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +name: nexmark +nodes: + - role: control-plane + # Expose port for Flink UI + extraPortMappings: + - containerPort: 30081 + hostPort: 8081 + protocol: TCP + - role: worker + - role: worker diff --git a/nexmark-runner/local/minio.yaml b/nexmark-runner/local/minio.yaml new file mode 100644 index 0000000..4b71287 --- /dev/null +++ b/nexmark-runner/local/minio.yaml @@ -0,0 +1,111 @@ +# MinIO for local S3-compatible storage +# Default credentials: minioadmin / minioadmin +# Bucket: nexmark (created by init job) +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: minio-pvc + namespace: flink-nexmark +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: minio + namespace: flink-nexmark +spec: + replicas: 1 + selector: + matchLabels: + app: minio + template: + metadata: + labels: + app: minio + spec: + containers: + - name: minio + image: minio/minio:latest + args: + - server + - /data + - --console-address + - ":9001" + env: + - name: MINIO_ROOT_USER + value: "minioadmin" + - name: MINIO_ROOT_PASSWORD + value: "minioadmin" + ports: + - containerPort: 9000 + name: api + - containerPort: 9001 + name: console + volumeMounts: + - name: data + mountPath: /data + readinessProbe: + httpGet: + path: /minio/health/ready + port: 9000 + initialDelaySeconds: 5 + periodSeconds: 5 + livenessProbe: + httpGet: + path: /minio/health/live + port: 9000 + initialDelaySeconds: 5 + periodSeconds: 10 + volumes: + - name: data + persistentVolumeClaim: + claimName: minio-pvc +--- +apiVersion: v1 +kind: Service +metadata: + name: minio + namespace: flink-nexmark +spec: + selector: + app: minio + ports: + - name: api + port: 9000 + targetPort: 9000 + - name: console + port: 9001 + targetPort: 9001 +--- +# Job to create the nexmark bucket +apiVersion: batch/v1 +kind: Job +metadata: + name: minio-create-bucket + namespace: flink-nexmark +spec: + ttlSecondsAfterFinished: 60 + template: + spec: + restartPolicy: OnFailure + containers: + - name: mc + image: minio/mc:latest + command: + - /bin/sh + - -c + - | + # Wait for MinIO to be ready + until mc alias set myminio http://minio:9000 minioadmin minioadmin; do + echo "Waiting for MinIO..." + sleep 2 + done + # Create bucket (ignore if exists) + mc mb myminio/nexmark --ignore-existing + echo "Bucket 'nexmark' ready" diff --git a/nexmark-runner/local/nexmark-metrics-service.yaml b/nexmark-runner/local/nexmark-metrics-service.yaml new file mode 100644 index 0000000..36e692a --- /dev/null +++ b/nexmark-runner/local/nexmark-metrics-service.yaml @@ -0,0 +1,17 @@ +# Service to expose metrics receiver port on JobManager +# Used by ProcessMetricSender sidecar on TaskManagers to send CPU metrics +apiVersion: v1 +kind: Service +metadata: + name: nexmark-metrics + namespace: flink-nexmark +spec: + selector: + app: flink-session + component: jobmanager + ports: + - port: 9098 + targetPort: 9098 + protocol: TCP + name: metrics + type: ClusterIP diff --git a/nexmark-runner/local/run.sh b/nexmark-runner/local/run.sh new file mode 100755 index 0000000..4100629 --- /dev/null +++ b/nexmark-runner/local/run.sh @@ -0,0 +1,73 @@ +#!/bin/bash +# Run Nexmark benchmark locally +# Usage: ./run.sh [queries] [--sink TYPE] [--storage PROFILE] [--path-prefix PATH] [-D key=value] +# +# Examples: +# ./run.sh q0 # Use config from ConfigMap +# ./run.sh q0 --sink blackhole # Override to blackhole +# ./run.sh q0 --sink hudi --storage minio --path-prefix s3a://nexmark +# ./run.sh q0 --sink hudi -Dnexmark.sink.hudi.write.tasks=16 +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +QUERIES="" +EXTRA_ARGS=() + +# Parse arguments - collect query and pass through everything else +while [[ $# -gt 0 ]]; do + case $1 in + --sink|--storage|--path-prefix|-D*) + EXTRA_ARGS+=("$1") + if [[ "$1" != -D* ]]; then + shift + EXTRA_ARGS+=("$1") + fi + shift + ;; + *) + if [ -z "$QUERIES" ]; then + QUERIES="$1" + else + EXTRA_ARGS+=("$1") + fi + shift + ;; + esac +done + +QUERIES="${QUERIES:-q0}" + +echo "=== Nexmark Benchmark (Local) ===" +echo "" + +# Check if cluster is ready +kubectl get flinkdeployment flink-session -n flink-nexmark >/dev/null 2>&1 || { + echo "Error: Flink session cluster not found. Run ./setup.sh first." + exit 1 +} + +# Get JobManager pod name +POD=$(kubectl get pod -l component=jobmanager -n flink-nexmark -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) +if [ -z "$POD" ]; then + echo "Waiting for Flink JobManager pod..." + kubectl wait --for=condition=Ready pod -l component=jobmanager -n flink-nexmark --timeout=120s + POD=$(kubectl get pod -l component=jobmanager -n flink-nexmark -o jsonpath='{.items[0].metadata.name}') +fi + +echo "Using pod: $POD" +echo "Query: $QUERIES" +if [ ${#EXTRA_ARGS[@]} -gt 0 ]; then + echo "Extra args: ${EXTRA_ARGS[*]}" +fi +echo "" + +# Run Nexmark benchmark using native runner +# Config is loaded from /opt/nexmark/conf/nexmark.yaml (mounted via ConfigMap) +# CLI args (--sink, --storage, --path-prefix, -D) override ConfigMap values +kubectl exec -t -n flink-nexmark "$POD" -- \ + /opt/nexmark/bin/run_query.sh "$QUERIES" "${EXTRA_ARGS[@]}" + +echo "" +echo "Flink UI: kubectl port-forward svc/flink-session-rest 8081:8081 -n flink-nexmark" +echo "MinIO: kubectl port-forward svc/minio 9001:9001 -n flink-nexmark" diff --git a/nexmark-runner/local/setup.sh b/nexmark-runner/local/setup.sh new file mode 100755 index 0000000..1d05185 --- /dev/null +++ b/nexmark-runner/local/setup.sh @@ -0,0 +1,269 @@ +#!/bin/bash +# Setup local Nexmark environment using Kind +# Usage: ./setup.sh [--nuke] [--quick] [--config CONFIG] +# +# Options: +# --nuke Delete existing cluster first (full rebuild) +# --quick Rebuild image + restart Flink only (skip cluster/operator setup) +# --config CONFIG Config file to use (default: nexmark-hudi-local.yaml) +# +# Examples: +# ./setup.sh # Create/update cluster with Hudi config +# ./setup.sh --config nexmark-blackhole.yaml # Use blackhole sink config +# ./setup.sh --nuke # Delete and recreate from scratch +# ./setup.sh --quick # Rebuild image and restart Flink +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +cd "$SCRIPT_DIR" + +NUKE=false +QUICK=false +CONFIG="nexmark-hudi-local.yaml" +CLUSTER_NAME="nexmark" +IMAGE_NAME="nexmark-k8s:local" + +# Parse arguments +while [[ $# -gt 0 ]]; do + case $1 in + --nuke) + NUKE=true + shift + ;; + --quick) + QUICK=true + shift + ;; + --config) + CONFIG="$2" + shift 2 + ;; + *) + echo "Unknown option: $1" + echo "Usage: ./setup.sh [--nuke] [--quick] [--config CONFIG]" + exit 1 + ;; + esac +done + +CONFIG_PATH="$REPO_ROOT/configs/$CONFIG" + +# Check prerequisites +command -v docker >/dev/null 2>&1 || { echo "Error: docker not found"; exit 1; } +command -v kind >/dev/null 2>&1 || { echo "Error: kind not found. Install: brew install kind"; exit 1; } +command -v kubectl >/dev/null 2>&1 || { echo "Error: kubectl not found"; exit 1; } +command -v helm >/dev/null 2>&1 || { echo "Error: helm not found. Install: brew install helm"; exit 1; } + +# Check for config file +if [ ! -f "$CONFIG_PATH" ]; then + echo "Error: Config file not found: $CONFIG_PATH" + echo "Available configs:" + ls "$REPO_ROOT/configs/"*.yaml + exit 1 +fi + +# Function to wait for Flink pods +wait_for_flink() { + echo "" + echo ">>> Waiting for Flink to be ready..." + for i in {1..30}; do + if kubectl get pod -n flink-nexmark -l component=jobmanager 2>/dev/null | grep -q flink-session; then + break + fi + echo " Waiting for pod creation... ($i/30)" + sleep 2 + done + kubectl wait --for=condition=Ready pod -l component=jobmanager -n flink-nexmark --timeout=300s + kubectl wait --for=condition=Ready pod -l component=taskmanager -n flink-nexmark --timeout=120s 2>/dev/null || true +} + +# Quick mode: rebuild image + restart Flink only +if [ "$QUICK" = true ]; then + echo "=== Quick Redeploy (image + Flink restart) ===" + echo "" + + # Build Docker image + echo ">>> Building Docker image..." + cd "$REPO_ROOT" + docker build -t "$IMAGE_NAME" -f docker/Dockerfile \ + --build-arg NEXMARK_CONFIG_FILE="$CONFIG" . + cd "$SCRIPT_DIR" + + # Load image into Kind + echo "" + echo ">>> Loading image into Kind cluster..." + kind load docker-image "$IMAGE_NAME" --name "$CLUSTER_NAME" + + # Update ConfigMap + echo "" + echo ">>> Updating ConfigMap from $CONFIG..." + kubectl create configmap nexmark-config \ + --from-file=nexmark.yaml="$CONFIG_PATH" \ + -n flink-nexmark \ + --dry-run=client -o yaml | kubectl apply -f - + + # Ensure metrics service exists and restart Flink deployment + echo "" + echo ">>> Applying metrics service..." + kubectl apply -f nexmark-metrics-service.yaml + + echo "" + echo ">>> Reapplying FlinkDeployment..." + kubectl delete flinkdeployment flink-session -n flink-nexmark --ignore-not-found + sleep 2 + kubectl apply -f flink-session.yaml + + wait_for_flink + + echo "" + echo "=== Quick Redeploy Complete ===" + echo "" + kubectl get pods -n flink-nexmark + echo "" + echo "Run benchmark:" + echo " ./run.sh q0 # Uses config from ConfigMap" + echo " ./run.sh q0 --sink hudi --storage minio --path-prefix s3a://nexmark" + exit 0 +fi + +# Full setup mode +echo "=== Local Nexmark Setup ===" +echo "Config: $CONFIG" +echo "" + +# Step 1: Handle existing cluster +if kind get clusters 2>/dev/null | grep -q "$CLUSTER_NAME"; then + if [ "$NUKE" = true ]; then + echo ">>> Deleting existing cluster..." + kind delete cluster --name "$CLUSTER_NAME" + else + echo ">>> Cluster '$CLUSTER_NAME' already exists (use --nuke to recreate)" + fi +fi + +# Step 2: Create cluster if needed +if ! kind get clusters 2>/dev/null | grep -q "$CLUSTER_NAME"; then + echo ">>> Creating Kind cluster..." + kind create cluster --config kind-config.yaml +fi + +# Step 3: Build Docker image +echo "" +echo ">>> Building Docker image..." +cd "$REPO_ROOT" +docker build -t "$IMAGE_NAME" -f docker/Dockerfile \ + --build-arg NEXMARK_CONFIG_FILE="$CONFIG" . +cd "$SCRIPT_DIR" + +# Step 4: Load image into Kind +echo "" +echo ">>> Loading image into Kind cluster..." +kind load docker-image "$IMAGE_NAME" --name "$CLUSTER_NAME" + +# Step 5: Install Metrics Server (for kubectl top / resource monitoring) +if ! kubectl get deployment metrics-server -n kube-system &>/dev/null; then + echo "" + echo ">>> Installing Metrics Server..." + kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml + kubectl patch deployment metrics-server -n kube-system --type='json' -p='[ + {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--kubelet-insecure-tls"}, + {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--kubelet-preferred-address-types=InternalIP"} + ]' 2>/dev/null || true + echo " Waiting for metrics-server to be ready..." + kubectl wait --for=condition=Available deployment metrics-server -n kube-system --timeout=120s 2>/dev/null || echo " (metrics-server may take a minute to start)" +else + echo "" + echo ">>> Metrics Server already installed" +fi + +# Step 6: Install cert-manager (if not present) +if ! kubectl get namespace cert-manager &>/dev/null; then + echo "" + echo ">>> Installing cert-manager..." + kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.14.0/cert-manager.yaml + kubectl wait --for=condition=Available deployment --all -n cert-manager --timeout=300s +else + echo "" + echo ">>> cert-manager already installed" +fi + +# Step 7: Install Flink Operator (if not present) +echo "" +echo ">>> Installing Flink Kubernetes Operator..." +kubectl create namespace flink-nexmark --dry-run=client -o yaml | kubectl apply -f - +helm repo add flink-operator https://downloads.apache.org/flink/flink-kubernetes-operator-1.10.0/ 2>/dev/null || true +helm repo update +helm upgrade --install flink-operator flink-operator/flink-kubernetes-operator \ + -n flink-nexmark --wait --timeout 5m + +# Step 8: Deploy MinIO +echo "" +echo ">>> Deploying MinIO..." +kubectl apply -f minio.yaml +kubectl wait --for=condition=Ready pod -l app=minio -n flink-nexmark --timeout=120s +kubectl wait --for=condition=Complete job/minio-create-bucket -n flink-nexmark --timeout=60s 2>/dev/null || true + +# Step 9: Create ConfigMap from config file +echo "" +echo ">>> Creating ConfigMap from $CONFIG..." +kubectl create configmap nexmark-config \ + --from-file=nexmark.yaml="$CONFIG_PATH" \ + -n flink-nexmark \ + --dry-run=client -o yaml | kubectl apply -f - + +# Step 10: Setup RBAC +echo "" +echo ">>> Setting up RBAC..." +kubectl delete rolebinding flink-role-binding -n flink-nexmark 2>/dev/null || true +kubectl delete role flink-role -n flink-nexmark 2>/dev/null || true +kubectl apply -f - <>> Deploying metrics service..." +kubectl apply -f nexmark-metrics-service.yaml + +echo "" +echo ">>> Deploying Flink session cluster..." +kubectl delete flinkdeployment flink-session -n flink-nexmark --ignore-not-found +sleep 2 +kubectl apply -f flink-session.yaml + +wait_for_flink + +echo "" +echo "=== Setup Complete ===" +echo "" +kubectl get pods -n flink-nexmark +echo "" +echo "Run benchmark:" +echo " ./run.sh q0 # Uses config from ConfigMap" +echo " ./run.sh q0 --sink hudi --storage minio --path-prefix s3a://nexmark" +echo "" +echo "Flink UI: kubectl port-forward svc/flink-session-rest 8081:8081 -n flink-nexmark" +echo "MinIO: kubectl port-forward svc/minio 9001:9001 -n flink-nexmark" diff --git a/nexmark-runner/local/teardown.sh b/nexmark-runner/local/teardown.sh new file mode 100755 index 0000000..899f8a1 --- /dev/null +++ b/nexmark-runner/local/teardown.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Teardown local Nexmark environment +# Usage: ./teardown.sh +set -e + +CLUSTER_NAME="nexmark" + +echo "=== Tearing Down Local Nexmark Environment ===" +echo "" + +# Delete Kind cluster +if kind get clusters 2>/dev/null | grep -q "$CLUSTER_NAME"; then + echo ">>> Deleting Kind cluster '$CLUSTER_NAME'..." + kind delete cluster --name "$CLUSTER_NAME" + echo "Cluster deleted." +else + echo ">>> Cluster '$CLUSTER_NAME' not found (already deleted?)" +fi + +echo "" +echo "=== Teardown Complete ===" diff --git a/nexmark-runner/yarn/.gitkeep b/nexmark-runner/yarn/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/nexmark/.gitignore b/nexmark/.gitignore new file mode 100644 index 0000000..cffce1d --- /dev/null +++ b/nexmark/.gitignore @@ -0,0 +1,47 @@ +# Compiled class file +*.class + +# Log file +*.log + +# BlueJ files +*.ctxt + +# Mobile Tools for Java (J2ME) +.mtj.tmp/ + +# Package Files # +*.jar +*.war +*.nar +*.ear +*.zip +*.tar.gz +*.rar +*.tgz + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* + +.cache +scalastyle-output.xml +.classpath +.idea +.metadata +.settings +.project +.version.properties +filter.properties +logs.zip +target +tmp +*.iml +*.swp +*.pyc +.DS_Store +atlassian-ide-plugin.xml +out/ +*.ipr +*.iws +load_*_*.mk +.java-version \ No newline at end of file diff --git a/nexmark/GUIDE_V1.md b/nexmark/GUIDE_V1.md new file mode 100644 index 0000000..6b4a9c7 --- /dev/null +++ b/nexmark/GUIDE_V1.md @@ -0,0 +1,128 @@ +## Nexmark Benchmark Guideline (For V1 runner) + +### Requirements + +The Nexmark benchmark framework runs Flink queries on [standalone cluster](https://ci.apache.org/projects/flink/flink-docs-release-1.13/ops/deployment/cluster_setup.html), see the Flink documentation for more detailed requirements and how to setup it. + +#### Software Requirements + +The cluster should consist of **one master node** and **one or more worker nodes**. All of them should be **Linux** environment (the CPU monitor script requries to run on Linux). Please make sure you have the following software installed **on each node**: + +- **JDK 1.8.x** or higher (Nexmark scripts uses some tools of JDK), +- **ssh** (sshd must be running to use the Flink and Nexmark scripts that manage + remote components) + +If your cluster does not fulfill these software requirements you will need to install/upgrade it. + +Having [**passwordless SSH**](https://linuxize.com/post/how-to-setup-passwordless-ssh-login/) and __the same directory structure__ on all your cluster nodes will allow you to use our scripts to control +everything. + +#### Environment Variables + +The following environment variable should be set on every node for the Flink and Nexmark scripts. + + - `JAVA_HOME`: point to the directory of your JDK installation. + - `FLINK_HOME`: point to the directory of your Flink installation. + +### Build Nexmark + +Before start to run the benchmark, you should build the Nexmark benchmark first to have a benchmark package. Please make sure you have installed `maven` in your build machine. And run the `./build.sh` command under `nexmark-flink` directoy. Then you will get the `nexmark-flink.tgz` archive under the directory. + +### Setup Cluster + +- Step 1: Download the latest Flink package from the [download page](https://flink.apache.org/downloads.html). Say `flink--bin-scala_2.11.tgz`. +- Step2: Copy the archives (`flink--bin-scala_2.11.tgz`, `nexmark-flink.tgz`) to your master node and extract it. + ``` + tar xzf flink--bin-scala_2.11.tgz; tar xzf nexmark-flink.tgz + mv flink- flink; mv nexmark-flink nexmark + ``` +- Step3: Copy the jars under `nexmark/lib` to `flink/lib` which contains the Nexmark source generator. +- Step4: Configure Flink. + - Edit `flink/conf/workers` and enter the IP address of each worker node. Recommand to set 8 entries. + - Replace `flink/conf/config.yaml` by `nexmark/conf/config.yaml`. Remember to update the following configurations: + - Set `jobmanager.rpc.address` to you master IP address + - Set `state.checkpoints.dir` to your local file path (recommend to use SSD), e.g. `file:///home/username/checkpoint`. + - Set `state.backend.rocksdb.localdir` to your local file path (recommend to use SSD), e.g. `/home/username/rocksdb`. +- Step5: Configure Nexmark benchmark. + - Edit `nexmark/conf/nexmark.yaml` and set `nexmark.metric.reporter.host` to your master IP address. +- Step6: Copy `flink` and `nexmark` to your worker nodes using `scp`. +- Step7: Start Flink Cluster by running `flink/bin/start-cluster.sh` on the master node. +- Step8: Setup the benchmark cluster by running `nexmark/bin/setup_cluster.sh` on the master node. +- (If you want to use kafka source instead of datagen source) Step9: Prepare Kafka + - (Please make sure Flink Kafka Jar is ready in flink/lib/ [download page](https://ci.apache.org/projects/flink/flink-docs-release-1.13/docs/connectors/table/kafka/#dependencies)) + - Start your kafka cluster. (recommend to use SSD) + - Create kafka topic: `bin/kafka-topics.sh --create --topic nexmark --bootstrap-server localhost:9092 --partitions 8`. + - Edit `nexmark/conf/nexmark.yaml`, set `kafka.bootstrap.servers`. + - Prepare source data: `nexmark/bin/run_query.sh insert_kafka`. + - NOTE: Kafka source is endless, only supports tps mode (unlimited events.num) now. + +### Run Nexmark + +You can run the Nexmark benchmark by running `nexmark/bin/run_query.sh all` on the master node. It will run all the queries one by one, and collect benchmark metrics automatically. It will take 50 minutes to finish the benchmark by default. At last, it will print the benchmark summary result (Cores * Time(s) for each query) on the console. + +You can also run specific queries by running `nexmark/bin/run_query.sh q1,q2`. + +You can also tune the workload of the queries by editing `nexmark/conf/nexmark.yaml` with the `nexmark.workload.*` prefix options. + +## Nexmark Benchmark Result + +### Machines + +Minimum requirements: +- 3 worker node +- Each machine has 8 cores and 32 GB RAM +- 800 GB SSD local disk + +### Flink Configuration + +Use the default configuration file `config.yaml` defined in `nexmark-flink/src/main/resources/conf/`. + +Some notable configurations including: + +- 8 TaskManagers, each has only 1 slot +- 8 GB for each TaskManager and JobManager +- Job parallelism: 8 +- Checkpoint enabled with exactly once mode and 3 minutes interval +- Use RocksDB state backend with incremental checkpoint enabled +- MiniBatch optimization enabled with 2 seconds interval and 50000 rows +- Splitting distinct aggregation optimization is enabled + +Flink version: 1.13 ~ 2.0. + +### Workloads + +Source total events number is 100 million. Source generates 10M records per seconds. The percentage of 3 stream is Bid: 92%, Auction: 6%, Person: 2%. + +### Benchmark Results + +An example of result table is as follows: + +``` ++-------------------+-------------------+-------------------+-------------------+-------------------+-------------------+ +| Nexmark Query | Events Num | Cores | Time(s) | Cores * Time(s) | Throughput/Cores | ++-------------------+-------------------+-------------------+-------------------+-------------------+-------------------+ +|q0 |100,000,000 |8.45 |76.323 |645.087 |155.02 K/s | +|q1 |100,000,000 |8.26 |76.643 |633.165 |157.94 K/s | +|q2 |100,000,000 |8.23 |69.309 |570.736 |175.21 K/s | +|q3 |100,000,000 |8.59 |76.531 |657.384 |152.12 K/s | +|q4 |100,000,000 |12.85 |226.605 |2912.841 |34.33 K/s | +|q5 |100,000,000 |10.8 |418.242 |4516.930 |22.14 K/s | +|q7 |100,000,000 |14.21 |570.983 |8112.884 |12.33 K/s | +|q8 |100,000,000 |9.42 |72.673 |684.288 |146.14 K/s | +|q9 |100,000,000 |16.11 |435.882 |7022.197 |14.24 K/s | +|q10 |100,000,000 |8.09 |213.795 |1729.775 |57.81 K/s | +|q11 |100,000,000 |10.6 |237.599 |2518.946 |39.7 K/s | +|q12 |100,000,000 |13.69 |96.559 |1321.536 |75.67 K/s | +|q13 |100,000,000 |8.24 |92.839 |764.952 |130.73 K/s | +|q14 |100,000,000 |8.28 |74.861 |620.220 |161.23 K/s | +|q15 |100,000,000 |8.73 |158.224 |1380.927 |72.42 K/s | +|q16 |100,000,000 |11.51 |466.008 |5362.602 |18.65 K/s | +|q17 |100,000,000 |9.24 |92.666 |856.162 |116.8 K/s | +|q18 |100,000,000 |12.49 |149.076 |1862.171 |53.7 K/s | +|q19 |100,000,000 |21.38 |106.190 |2270.551 |44.04 K/s | +|q20 |100,000,000 |17.27 |305.099 |5267.805 |18.98 K/s | +|q21 |100,000,000 |8.33 |121.845 |1015.293 |98.49 K/s | +|q22 |100,000,000 |8.25 |93.244 |769.471 |129.96 K/s | +|Total |2,200,000,000 |243.029 |4231.196 |51495.920 |1.89 M/s | ++-------------------+-------------------+-------------------+-------------------+-------------------+-------------------+ +``` \ No newline at end of file diff --git a/nexmark/GUIDE_V2.md b/nexmark/GUIDE_V2.md new file mode 100644 index 0000000..a4d2848 --- /dev/null +++ b/nexmark/GUIDE_V2.md @@ -0,0 +1,126 @@ +## Nexmark Benchmark Guideline (For V2 runner) + +### Requirements + +The Nexmark benchmark framework runs Flink queries on [standalone cluster](https://ci.apache.org/projects/flink/flink-docs-release-1.13/ops/deployment/cluster_setup.html), see the Flink documentation for more detailed requirements and how to setup it. + +#### Software Requirements + +The cluster should consist of **one master node** and **one or more worker nodes**. All of them should be **Linux** environment (the CPU monitor script requries to run on Linux). Please make sure you have the following software installed **on each node**: + +- **JDK 1.11.x** or higher (Nexmark scripts uses some tools of JDK), +- **ssh** (sshd must be running to use the Flink and Nexmark scripts that manage + remote components) +- **Flink 2.0.0** or above. +- **DFS environment** (HDFS, S3, etc.) is necessary for the checkpointing and state backend. + +If your cluster does not fulfill these software requirements you will need to install/upgrade it. + +Having [**passwordless SSH**](https://linuxize.com/post/how-to-setup-passwordless-ssh-login/) and __the same directory structure__ on all your cluster nodes will allow you to use our scripts to control +everything. + +#### Environment Variables + +The following environment variable should be set on every node for the Flink and Nexmark scripts. + + - `JAVA_HOME`: point to the directory of your JDK installation. + - `FLINK_HOME`: point to the directory of your Flink installation. + +### Build Nexmark + +Before start to run the benchmark, you should build the Nexmark benchmark first to have a benchmark package. Please make sure you have installed `maven` in your build machine. And run the `./build.sh` command under `nexmark-flink` directoy. Then you will get the `nexmark-flink.tgz` archive under the directory. + +### Setup Cluster + +- Step 1: Download the latest Flink package from the [download page](https://flink.apache.org/downloads.html). Say `flink--bin-scala_2.11.tgz`. +- Step2: Copy the archives (`flink--bin-scala_2.11.tgz`, `nexmark-flink.tgz`) to your master node and extract it. + ``` + tar xzf flink--bin-scala_2.11.tgz; tar xzf nexmark-flink.tgz + mv flink- flink; mv nexmark-flink nexmark + ``` +- Step3: Copy the jars under `nexmark/lib` to `flink/lib` which contains the Nexmark source generator. +- Step4: Configure Flink. + - Edit `flink/conf/workers` and enter the IP address of each worker node. Recommand to set 8 entries. + - Replace `flink/conf/config.yaml` by `nexmark/conf/config.yaml`. Remember to update the following configurations: + - Set `jobmanager.rpc.address` to you master IP address + - Set `execution.checkpointing.dir` to DFS path, e.g. `hdfs:///checkpoints/`. + - Set a proper state backend via `state.backend.type`, e.g. `forst` or `rocksdb`. + - Set `io.tmp.dirs` to your local file path (recommend to use SSD), e.g. `/mnt/disk1/tmp`. +- Step5: Configure Nexmark benchmark. + - Replace `nexmark/conf/nexmark.yaml` by `nexmark/conf/nexmark_v2.yaml`. + - Edit `nexmark/conf/nexmark.yaml` and set `nexmark.metric.reporter.host` to your master IP address. +- Step6: Copy `flink` and `nexmark` to your worker nodes using `scp`. +- Step7: Start Flink Cluster by running `flink/bin/start-cluster.sh` on the master node. +- Step8: Setup the benchmark cluster by running `nexmark/bin/setup_cluster.sh` on the master node. + +### Run Nexmark + +You can run the Nexmark benchmark by running `nexmark/bin/run_query.sh all` on the master node. It will run all the queries one by one, and collect benchmark metrics automatically. It will take 50 minutes to finish the benchmark by default. At last, it will print the benchmark summary result (Cores * Time(s) for each query) on the console. + +You can also run specific queries by running `nexmark/bin/run_query.sh q1,q2`. + +You can also tune the workload of the queries by editing `nexmark/conf/nexmark.yaml` with the `nexmark.workload.*` prefix options. + +## Nexmark Benchmark Result + +### Machines + +Minimum requirements: +- 4 worker node +- Each machine has 16 cores and 32 GB RAM +- 100 GB SSD local disk (or less for disaggregated state) +- HDFS with 4 data nodes with SSD disks on 1Gbps LAN + +### Flink Configuration + +Use the default configuration file `config_v2.yaml` defined in `nexmark-flink/src/main/resources/conf/`. + +Some notable configurations including: + +- 8 TaskManagers, each has only 1 slot +- 8 GB for each TaskManager and JobManager +- Job parallelism: 8 +- Checkpoint enabled with exactly once mode and 30 seconds interval +- Use RocksDB state backend with incremental checkpoint enabled +- MiniBatch optimization enabled with 2 seconds interval and 50000 rows +- Splitting distinct aggregation optimization is enabled + +Flink version: 2.0 or above. + +### Workloads + +Source total events number is 200 million (150M for warmup and 50M for evaluation). Source generates 100K records per seconds. The percentage of 3 stream is Bid: 92%, Auction: 6%, Person: 2%. + +### Benchmark Results + +An example of result table is as follows: + +``` ++------+-----------------+--------+----------+--------------+-------------------+ +| Query| Events Num | Cores | Time(s) | Throughput | Recovery Time(s) | ++------+-----------------+--------+----------+--------------+-------------------+ +|q0 |50,000,000 |7 |18.875 |2.65 M/s |0.004 | +|q1 |50,000,000 |7.24 |17.462 |2.86 M/s |0.002 | +|q2 |50,000,000 |7.13 |16.927 |2.95 M/s |0.002 | +|q3 |50,000,000 |8.69 |26.250 |1.9 M/s |0.002 | +|q4 |50,000,000 |12.33 |143.745 |347.84 K/s |0.002 | +|q5 |50,000,000 |12.35 |54.324 |920.4 K/s |0.001 | +|q7 |50,000,000 |11.45 |190.508 |262.46 K/s |3.001 | +|q8 |50,000,000 |12.83 |23.994 |2.08 M/s |0.004 | +|q9 |50,000,000 |10.86 |334.395 |149.52 K/s |9.009 | +|q10 |50,000,000 |6.81 |55.290 |904.32 K/s |0.002 | +|q11 |50,000,000 |9.29 |116.856 |427.88 K/s |0.002 | +|q12 |50,000,000 |10.03 |25.656 |1.95 M/s |0.001 | +|q13 |50,000,000 |7.97 |33.326 |1.5 M/s |0.003 | +|q14 |50,000,000 |7.83 |24.660 |2.03 M/s |0.001 | +|q15 |50,000,000 |8.16 |36.419 |1.37 M/s |0.002 | +|q16 |50,000,000 |10.77 |102.608 |487.29 K/s |0.002 | +|q17 |50,000,000 |10.15 |30.599 |1.63 M/s |0.003 | +|q18 |50,000,000 |11.77 |64.954 |769.77 K/s |3.017 | +|q19 |50,000,000 |13.09 |89.471 |558.84 K/s |3.003 | +|q20 |50,000,000 |10.98 |256.461 |194.96 K/s |6.035 | +|q21 |50,000,000 |7.81 |43.492 |1.15 M/s |0.001 | +|q22 |50,000,000 |7.7 |30.725 |1.63 M/s |0.001 | +|Total |1,100,000,000 |212.24 |1736.997 |28.7 M/s |24.1 | ++------+-----------------+--------+----------+--------------+-------------------+ +``` \ No newline at end of file diff --git a/nexmark/README.md b/nexmark/README.md new file mode 100644 index 0000000..0bbdd1c --- /dev/null +++ b/nexmark/README.md @@ -0,0 +1,78 @@ +# Nexmark Benchmark + +Inspired by and modified based on https://github.com/nexmark/nexmark + +## What is Nexmark + +Nexmark is a benchmark suite for queries over continuous data streams. This project is inspired by the [NEXMark research paper](https://web.archive.org/web/20100620010601/http://datalab.cs.pdx.edu/niagaraST/NEXMark/) and [Apache Beam Nexmark](https://beam.apache.org/documentation/sdks/java/testing/nexmark/). + +## Nexmark Benchmark Suite + +### Schemas + +These are multiple queries over a three entities model representing on online auction system: + +- **Person** represents a person submitting an item for auction and/or making a bid on an auction. +- **Auction** represents an item under auction. +- **Bid** represents a bid for an item under auction. + +### Queries + +| Query | Name | Summary | Flink | +| -------- | -------- | -------- | ------ | +| q0 | Pass Through | Measures the monitoring overhead including the source generator. | ✅ | +| q1 | Currency Conversion | Convert each bid value from dollars to euros. | ✅ | +| q2 | Selection | Find bids with specific auction ids and show their bid price. | ✅ | +| q3 | Local Item Suggestion | Who is selling in OR, ID or CA in category 10, and for what auction ids? | ✅ | +| q4 | Average Price for a Category | Select the average of the wining bid prices for all auctions in each category. | ✅ | +| q5 | Hot Items | Which auctions have seen the most bids in the last period? | ✅ | +| q6 | Average Selling Price by Seller | What is the average selling price per seller for their last 10 closed auctions. | [FLINK-19059](https://issues.apache.org/jira/browse/FLINK-19059) | +| q7 | Highest Bid | Select the bids with the highest bid price in the last period. | ✅ | +| q8 | Monitor New Users | Select people who have entered the system and created auctions in the last period. | ✅ | +| q9 | Winning Bids | Find the winning bid for each auction. | ✅ | +| q10 | Log to File System | Log all events to file system. Illustrates windows streaming data into partitioned file system. | ✅ | +| q11 | User Sessions | How many bids did a user make in each session they were active? Illustrates session windows. | ✅ | +| q12 | Processing Time Windows | How many bids does a user make within a fixed processing time limit? Illustrates working in processing time window. | ✅ | +| q13 | Bounded Side Input Join | Joins a stream to a bounded side input, modeling basic stream enrichment. | ✅ | +| q14 | Calculation | Convert bid timestamp into types and find bids with specific price. Illustrates more complex projection and filter. | ✅ | +| q15 | Bidding Statistics Report | How many distinct users join the bidding for different level of price? Illustrates multiple distinct aggregations with filters. | ✅ | +| q16 | Channel Statistics Report | How many distinct users join the bidding for different level of price for a channel? Illustrates multiple distinct aggregations with filters for multiple keys. | ✅ | +| q17 | Auction Statistics Report | How many bids on an auction made a day and what is the price? Illustrates an unbounded group aggregation. | ✅ | +| q18 | Find last bid | What's a's last bid for bidder to auction? Illustrates a Deduplicate query. | ✅ | +| q19 | Auction TOP-10 Price | What's the top price 10 bids of an auction? Illustrates a TOP-N query. | ✅ | +| q20 | Expand bid with auction | Get bids with the corresponding auction information where category is 10. Illustrates a filter join. | ✅ | +| q21 | Add channel id | Add a channel_id column to the bid table. Illustrates a 'CASE WHEN' + 'REGEXP_EXTRACT' SQL. | ✅ | +| q22 | Get URL Directories | What is the directory structure of the URL? Illustrates a SPLIT_INDEX SQL. | ✅ | + +*Note: q1 ~ q8 are from original [NEXMark queries](https://web.archive.org/web/20100620010601/http://datalab.cs.pdx.edu/niagaraST/NEXMark/), q0 and q9 ~ q13 are from [Apache Beam](https://beam.apache.org/documentation/sdks/java/testing/nexmark), others are extended to cover more scenarios.* + +### Metrics + +For evaluating the performance, there are two performance measurement terms used in Nexmark that are **cores** and **time**. + +Cores is the CPU usage used by the stream processing system. Usually CPU allows preemption, not like memory can be limited. Therefore, how the stream processing system effectively use CPU resources, how much throughput is contributed per core, they are important aspect for streaming performance benchmark. +For Flink, we deploy a CPU usage collector on every worker node and send the usage metric to the benchmark runner for summarizing. We don't use the `Status.JVM.CPU.Load` metric provided by Flink, because it is not accurate. + +Time is the cost time for specified number of events executed by the stream processing system. With Cores * Time, we can know how many resources the stream processing system uses to process specified number of events. + +## Nexmark Benchmark Guideline + +For the Nexmark benchmark, we provide a guideline to help you run the benchmark on Flink. +Currently, we have two versions of the Nexmark benchmark runner, named V1 and V2 respectively. +The V1 runner is the original version of the Nexmark benchmark runner, where each query contains one dataset. The framework can evaluate in either throughput or throughput per core in heavy load or daily load. +The V2 runner is a new introduced two-phase benchmark runner, the main difference is that the V2 runner only evaluates the performance of latter phase of each query, which is more accurate to evaluate the performance of the long-running queries. Only heavy load with data generator source supported. +For more details of V2, please read [the proposal of New Runner](https://github.com/nexmark/nexmark/issues/65). + +Select one of the versions to run the benchmark according to your requirements. + - [Nexmark Benchmark V1 Guideline](./GUIDE_V1.md) + - [Nexmark Benchmark V2 Guideline](./GUIDE_V2.md) + +## Roadmap + +1. Run Nexmark benchmark for more stream processing systems, such as Spark, KSQL. However, they don't have complete streaming SQL features. Therefore, not all of the queries can be ran in these systems. But we can implement the queries in programing way using Spark Streaming, Kafka Streams. +2. Support Latency metric for the benchmark. Latency measures the required time from a record entering the system to some results produced after some actions performed on the record. However, this is not easy to support for SQL queries unless we modify the queries. + +## References + +- Pete Tucker, Kristin Tufte, Vassilis Papadimos, David Maier. NEXMark – A Benchmark for Queries over Data Streams. June 2010. + diff --git a/nexmark/nexmark-core/pom.xml b/nexmark/nexmark-core/pom.xml new file mode 100644 index 0000000..a094fb2 --- /dev/null +++ b/nexmark/nexmark-core/pom.xml @@ -0,0 +1,50 @@ + + + + 4.0.0 + + + com.github.nexmark + nexmark + 0.3-SNAPSHOT + + + nexmark-core + jar + + + + org.yaml + snakeyaml + 2.5 + + + + + org.junit.jupiter + junit-jupiter + 5.14.2 + test + + + + diff --git a/nexmark/nexmark-core/src/main/java/com/github/nexmark/core/SinkDdlProvider.java b/nexmark/nexmark-core/src/main/java/com/github/nexmark/core/SinkDdlProvider.java new file mode 100644 index 0000000..c8994d0 --- /dev/null +++ b/nexmark/nexmark-core/src/main/java/com/github/nexmark/core/SinkDdlProvider.java @@ -0,0 +1,263 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.core; + +import org.yaml.snakeyaml.Yaml; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Provides sink DDL configuration for Nexmark queries. Reads from a YAML file using SnakeYAML. Can + * be used by any engine (Flink, Spark, etc.). + * + *

YAML format: + * + * 

+ * nexmark.sink.ddl.q0:
+ *   connector: blackhole
+ *
+ * nexmark.sink.ddl.q10:
+ *   connector: filesystem
+ *   path: file://...
+ *   format: csv
+ *
+ * + *

Config resolution: user config overrides defaults from nexmark-sink-defaults.yaml + */ +public class SinkDdlProvider { + + /** Config key prefix for sink DDL. */ + public static final String SINK_DDL_KEY_PREFIX = "nexmark.sink.ddl"; + + /** Resource path for default sink configurations. */ + public static final String DEFAULTS_RESOURCE = "nexmark-sink-defaults.yaml"; + + /** Map from query key (e.g., "nexmark.sink.ddl.q0") to DDL string */ + private final Map queryDdlMap; + + private SinkDdlProvider(Map queryDdlMap) { + this.queryDdlMap = queryDdlMap; + } + + /** + * Get the config key for a specific query. + * + * @param queryName the query name (e.g., "q0", "q10") + * @return the config key (e.g., "nexmark.sink.ddl.q0") + */ + public static String getConfigKey(String queryName) { + return SINK_DDL_KEY_PREFIX + "." + queryName; + } + + /** + * Create provider from a YAML config file, merged with defaults. + * + *

Resolves ${NEXMARK_DIR} and ${SUBMIT_TIME} placeholders in default configurations. + * User-provided configurations are used as-is. + * + * @param yamlPath path to nexmark.yaml + * @param nexmarkDir value to substitute for ${NEXMARK_DIR} + * @param submitTime value to substitute for ${SUBMIT_TIME} + * @return configured provider + */ + public static SinkDdlProvider fromYaml(Path yamlPath, String nexmarkDir, String submitTime) + throws IOException { + Map> defaults = loadDefaults(); + Map> userConfig = parseYamlFile(yamlPath); + + // Merge: user config overrides defaults + Map merged = new HashMap<>(); + for (Map.Entry> entry : defaults.entrySet()) { + merged.put( + entry.getKey(), + resolveVariables(toDdlString(entry.getValue()), nexmarkDir, submitTime)); + } + for (Map.Entry> entry : userConfig.entrySet()) { + merged.put(entry.getKey(), toDdlString(entry.getValue())); + } + + return new SinkDdlProvider(merged); + } + + /** + * Create provider from YAML file at NEXMARK_CONF_DIR/nexmark.yaml. Falls back to defaults only + * if env var not set or file not found. + * + *

Resolves ${NEXMARK_DIR} and ${SUBMIT_TIME} placeholders in default configurations. + * User-provided configurations are used as-is. + * + * TODO: consolidate with NexmarkGlobalConfiguration + * + * @param nexmarkDir value to substitute for ${NEXMARK_DIR} + * @param submitTime value to substitute for ${SUBMIT_TIME} + */ + public static SinkDdlProvider fromEnv(String nexmarkDir, String submitTime) { + String confDir = System.getenv("NEXMARK_CONF_DIR"); + if (confDir != null) { + Path yamlPath = Path.of(confDir, "nexmark.yaml"); + if (Files.exists(yamlPath)) { + try { + return fromYaml(yamlPath, nexmarkDir, submitTime); + } catch (IOException e) { + // Fall through to defaults only + } + } + } + // Use defaults only + Map> defaults = loadDefaults(); + Map ddlMap = new HashMap<>(); + for (Map.Entry> entry : defaults.entrySet()) { + ddlMap.put( + entry.getKey(), + resolveVariables(toDdlString(entry.getValue()), nexmarkDir, submitTime)); + } + return new SinkDdlProvider(ddlMap); + } + + /** + * Create provider from a pre-built config map (for Flink integration). Keys like + * "nexmark.sink.ddl.q0.connector" -> nested structure. + * + *

Resolves ${NEXMARK_DIR} and ${SUBMIT_TIME} placeholders in default configurations. + * User-provided configurations are used as-is. + * + * @param flatConfig flat config map from Flink Configuration + * @param nexmarkDir value to substitute for ${NEXMARK_DIR} + * @param submitTime value to substitute for ${SUBMIT_TIME} + */ + public static SinkDdlProvider fromMap( + Map flatConfig, String nexmarkDir, String submitTime) { + Map> defaults = loadDefaults(); + Map> userConfig = parseFlatConfig(flatConfig); + + Map merged = new HashMap<>(); + for (Map.Entry> entry : defaults.entrySet()) { + merged.put( + entry.getKey(), + resolveVariables(toDdlString(entry.getValue()), nexmarkDir, submitTime)); + } + for (Map.Entry> entry : userConfig.entrySet()) { + merged.put(entry.getKey(), toDdlString(entry.getValue())); + } + + return new SinkDdlProvider(merged); + } + + /** + * Get sink DDL for a query. + * + * @param queryName the query name (e.g., "q0", "q10") + * @return the DDL string for the WITH clause + */ + public String getSinkDdl(String queryName) { + String queryKey = getConfigKey(queryName); + return queryDdlMap.getOrDefault(queryKey, "'connector' = 'blackhole'"); + } + + /** + * Convert key-value map to DDL string format. E.g., {connector: filesystem, path: /foo} -> + * "'connector' = 'filesystem', 'path' = '/foo'" + */ + private static String toDdlString(Map properties) { + return properties.entrySet().stream() + .map(e -> "'" + e.getKey() + "' = '" + e.getValue() + "'") + .collect(Collectors.joining(",\n ")); + } + + /** Resolve ${NEXMARK_DIR} and ${SUBMIT_TIME} placeholders in a DDL string. */ + private static String resolveVariables(String ddl, String nexmarkDir, String submitTime) { + return ddl.replace("${NEXMARK_DIR}", nexmarkDir).replace("${SUBMIT_TIME}", submitTime); + } + + /** Load default sink configurations from classpath resource. */ + private static Map> loadDefaults() { + try (InputStream is = + SinkDdlProvider.class.getClassLoader().getResourceAsStream(DEFAULTS_RESOURCE)) { + if (is != null) { + return parseYamlStream(is); + } + } catch (IOException e) { + // Use empty defaults + } + return new HashMap<>(); + } + + private static Map> parseYamlFile(Path yamlPath) + throws IOException { + try (InputStream is = Files.newInputStream(yamlPath)) { + return parseYamlStream(is); + } + } + + @SuppressWarnings("unchecked") + private static Map> parseYamlStream(InputStream is) { + Yaml yaml = new Yaml(); + Map rawConfig = yaml.load(is); + if (rawConfig == null) { + return new HashMap<>(); + } + + // Filter and convert to expected structure + Map> result = new HashMap<>(); + for (Map.Entry entry : rawConfig.entrySet()) { + String key = entry.getKey(); + if (key.startsWith(SINK_DDL_KEY_PREFIX + ".") && entry.getValue() instanceof Map) { + result.put(key, (Map) entry.getValue()); + } + } + return result; + } + + /** + * Parse flat config map (from Flink Configuration.toMap()). Keys like + * "nexmark.sink.ddl.q0.connector" -> nested structure. + */ + private static Map> parseFlatConfig( + Map flatConfig) { + Map> result = new HashMap<>(); + + for (Map.Entry entry : flatConfig.entrySet()) { + String key = entry.getKey(); + if (!key.startsWith(SINK_DDL_KEY_PREFIX + ".")) { + continue; + } + + // Parse: nexmark.sink.ddl.q0.connector -> queryKey=nexmark.sink.ddl.q0, prop=connector + String afterPrefix = key.substring(SINK_DDL_KEY_PREFIX.length() + 1); + int dotIdx = afterPrefix.indexOf("."); + if (dotIdx > 0) { + String queryName = afterPrefix.substring(0, dotIdx); + String propName = afterPrefix.substring(dotIdx + 1); + String queryKey = SINK_DDL_KEY_PREFIX + "." + queryName; + + result.computeIfAbsent(queryKey, k -> new LinkedHashMap<>()) + .put(propName, entry.getValue()); + } + } + + return result; + } +} diff --git a/nexmark/nexmark-core/src/main/resources/nexmark-sink-defaults.yaml b/nexmark/nexmark-core/src/main/resources/nexmark-sink-defaults.yaml new file mode 100644 index 0000000..7360585 --- /dev/null +++ b/nexmark/nexmark-core/src/main/resources/nexmark-sink-defaults.yaml @@ -0,0 +1,96 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default sink DDL configurations for Nexmark queries. +# Each query has its own section with key-value pairs for connector options. +# Override these in your project's nexmark.yaml. + +nexmark.sink.ddl.q0: + connector: blackhole + +nexmark.sink.ddl.q1: + connector: blackhole + +nexmark.sink.ddl.q2: + connector: blackhole + +nexmark.sink.ddl.q3: + connector: blackhole + +nexmark.sink.ddl.q4: + connector: blackhole + +nexmark.sink.ddl.q5: + connector: blackhole + +nexmark.sink.ddl.q7: + connector: blackhole + +nexmark.sink.ddl.q8: + connector: blackhole + +nexmark.sink.ddl.q9: + connector: blackhole + +nexmark.sink.ddl.q10: + connector: filesystem + path: file://${NEXMARK_DIR}/data/output/${SUBMIT_TIME}/bid/ + format: csv + sink.partition-commit.trigger: partition-time + sink.partition-commit.delay: 1 min + sink.partition-commit.policy.kind: success-file + partition.time-extractor.timestamp-pattern: $dt $hm:00 + sink.rolling-policy.rollover-interval: 1min + sink.rolling-policy.check-interval: 1min + +nexmark.sink.ddl.q11: + connector: blackhole + +nexmark.sink.ddl.q12: + connector: blackhole + +nexmark.sink.ddl.q13: + connector: blackhole + +nexmark.sink.ddl.q14: + connector: blackhole + +nexmark.sink.ddl.q15: + connector: blackhole + +nexmark.sink.ddl.q16: + connector: blackhole + +nexmark.sink.ddl.q17: + connector: blackhole + +nexmark.sink.ddl.q18: + connector: blackhole + +nexmark.sink.ddl.q19: + connector: blackhole + +nexmark.sink.ddl.q20: + connector: blackhole + +nexmark.sink.ddl.q21: + connector: blackhole + +nexmark.sink.ddl.q22: + connector: blackhole + +nexmark.sink.ddl.q23: + connector: blackhole diff --git a/nexmark/nexmark-core/src/test/java/com/github/nexmark/core/SinkDdlProviderTest.java b/nexmark/nexmark-core/src/test/java/com/github/nexmark/core/SinkDdlProviderTest.java new file mode 100644 index 0000000..fa46587 --- /dev/null +++ b/nexmark/nexmark-core/src/test/java/com/github/nexmark/core/SinkDdlProviderTest.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.core; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** Unit tests for {@link SinkDdlProvider}. */ +class SinkDdlProviderTest { + + private static final String TEST_NEXMARK_DIR = "/opt/nexmark"; + private static final String TEST_SUBMIT_TIME = "2024-01-15T10:30:00"; + private static final List BLACKHOLE_QUERIES = + Arrays.asList( + "q0", "q1", "q2", "q3", "q4", "q5", "q7", "q8", "q9", "q11", "q12", "q13", + "q14", "q15", "q16", "q17", "q18", "q19", "q20", "q21", "q22", "q23"); + + @Test + void testFromEnvLoadsDefaults() { + SinkDdlProvider provider = SinkDdlProvider.fromEnv(TEST_NEXMARK_DIR, TEST_SUBMIT_TIME); + + // Verify blackhole queries + for (String query : BLACKHOLE_QUERIES) { + String ddl = provider.getSinkDdl(query); + assertEquals( + "'connector' = 'blackhole'", + ddl, + "Query " + query + " should have blackhole connector"); + } + + // Verify q10 filesystem with resolved variables + String q10Ddl = provider.getSinkDdl("q10"); + assertTrue(q10Ddl.contains("'connector' = 'filesystem'")); + assertTrue( + q10Ddl.contains( + "'path' = 'file://" + + TEST_NEXMARK_DIR + + "/data/output/" + + TEST_SUBMIT_TIME + + "/bid/'")); + assertFalse(q10Ddl.contains("${NEXMARK_DIR}")); + assertFalse(q10Ddl.contains("${SUBMIT_TIME}")); + + // Verify unknown query fallback + assertEquals("'connector' = 'blackhole'", provider.getSinkDdl("unknown_query")); + } + + @Test + void testFromYamlOverridesDefaults() throws IOException { + Path tempYaml = Files.createTempFile("nexmark-test", ".yaml"); + try { + String yamlContent = + "nexmark.sink.ddl.q0:\n" + + " connector: kafka\n" + + " topic: nexmark-q0\n" + + "\n" + + "nexmark.sink.ddl.q10:\n" + + " connector: hudi\n" + + " path: s3a://bucket/q10\n"; + Files.writeString(tempYaml, yamlContent); + + SinkDdlProvider provider = + SinkDdlProvider.fromYaml(tempYaml, TEST_NEXMARK_DIR, TEST_SUBMIT_TIME); + + // q0 overridden + assertTrue(provider.getSinkDdl("q0").contains("'connector' = 'kafka'")); + + // q10 overridden (no longer filesystem) + String q10Ddl = provider.getSinkDdl("q10"); + assertTrue(q10Ddl.contains("'connector' = 'hudi'")); + assertFalse(q10Ddl.contains("'connector' = 'filesystem'")); + + // q1 still default + assertEquals("'connector' = 'blackhole'", provider.getSinkDdl("q1")); + } finally { + Files.deleteIfExists(tempYaml); + } + } + + @Test + void testFromMapOverridesDefaults() { + Map flatConfig = new HashMap<>(); + flatConfig.put("nexmark.sink.ddl.q0.connector", "iceberg"); + flatConfig.put("nexmark.sink.ddl.q0.catalog-name", "hive_catalog"); + flatConfig.put("some.other.config", "ignored"); + + SinkDdlProvider provider = + SinkDdlProvider.fromMap(flatConfig, TEST_NEXMARK_DIR, TEST_SUBMIT_TIME); + + // q0 overridden + String q0Ddl = provider.getSinkDdl("q0"); + assertTrue(q0Ddl.contains("'connector' = 'iceberg'")); + assertTrue(q0Ddl.contains("'catalog-name' = 'hive_catalog'")); + + // q1 still default + assertEquals("'connector' = 'blackhole'", provider.getSinkDdl("q1")); + } + + @Test + void testGetConfigKey() { + assertEquals("nexmark.sink.ddl.q0", SinkDdlProvider.getConfigKey("q0")); + assertEquals("nexmark.sink.ddl.q10", SinkDdlProvider.getConfigKey("q10")); + } + + @Test + void testEmptyConfigUsesDefaults() throws IOException { + // Empty YAML + Path tempYaml = Files.createTempFile("nexmark-test", ".yaml"); + try { + Files.writeString(tempYaml, "# Empty\n"); + SinkDdlProvider yamlProvider = + SinkDdlProvider.fromYaml(tempYaml, TEST_NEXMARK_DIR, TEST_SUBMIT_TIME); + assertEquals("'connector' = 'blackhole'", yamlProvider.getSinkDdl("q0")); + } finally { + Files.deleteIfExists(tempYaml); + } + + // Empty Map + SinkDdlProvider mapProvider = + SinkDdlProvider.fromMap(new HashMap<>(), TEST_NEXMARK_DIR, TEST_SUBMIT_TIME); + assertEquals("'connector' = 'blackhole'", mapProvider.getSinkDdl("q0")); + assertTrue(mapProvider.getSinkDdl("q10").contains("'connector' = 'filesystem'")); + } +} diff --git a/nexmark/nexmark-flink/build.sh b/nexmark/nexmark-flink/build.sh new file mode 100755 index 0000000..819b3b4 --- /dev/null +++ b/nexmark/nexmark-flink/build.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +MVN=${MVN:-mvn} +DIR=`pwd` + +$MVN clean package -DskipTests +cd target/nexmark-flink-bin/ +tar czf "nexmark-flink.tgz" nexmark-flink +cp nexmark-flink.tgz ${DIR} +cd ${DIR} \ No newline at end of file diff --git a/nexmark/nexmark-flink/pom.xml b/nexmark/nexmark-flink/pom.xml new file mode 100644 index 0000000..3ab3ee1 --- /dev/null +++ b/nexmark/nexmark-flink/pom.xml @@ -0,0 +1,155 @@ + + + + + nexmark + com.github.nexmark + 0.3-SNAPSHOT + + 4.0.0 + + nexmark-flink + + + + com.github.nexmark + nexmark-core + ${project.version} + + + commons-io + commons-io + 2.15.1 + + + commons-cli + commons-cli + 1.3.1 + + + org.apache.httpcomponents + httpclient + 4.5.3 + + + + + + org.apache.flink + flink-table-api-java + ${flink.version} + provided + + + org.apache.flink + flink-table-api-java-bridge + ${flink.version} + provided + + + org.apache.flink + flink-core + ${flink.version} + provided + + + org.apache.flink + flink-streaming-java + ${flink.version} + provided + + + + + org.apache.flink + flink-table-planner_2.12 + ${flink.version} + test + + + org.apache.flink + flink-test-utils + ${flink.version} + test + + + org.mockito + mockito-core + 2.21.0 + jar + test + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + shade-flink + package + + shade + + + false + false + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + maven-assembly-plugin + + + bin + package + + single + + + + src/main/assemblies/bin.xml + + nexmark-flink-bin + false + + + + + + + \ No newline at end of file diff --git a/nexmark/nexmark-flink/src/main/assemblies/bin.xml b/nexmark/nexmark-flink/src/main/assemblies/bin.xml new file mode 100644 index 0000000..7681a9e --- /dev/null +++ b/nexmark/nexmark-flink/src/main/assemblies/bin.xml @@ -0,0 +1,86 @@ + + + bin + + dir + + + true + nexmark-flink + + + + + target/nexmark-flink-${project.version}.jar + lib/ + 0644 + + + + + + + src/main/resources/bin + bin + 0755 + + + + src/main/resources/conf + conf + 0755 + + + + src/main/resources/queries + queries + 0755 + + + + src/main/resources/queries-cep + queries-cep + 0755 + + + + ./ + + README.md + + ./ + 0755 + + + + + src/main/resources/bin/ + log + 0644 + + **/* + + + + + diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/Benchmark.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/Benchmark.java new file mode 100644 index 0000000..4a51431 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/Benchmark.java @@ -0,0 +1,447 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink; + +import org.apache.flink.configuration.Configuration; + +import com.github.nexmark.core.SinkDdlProvider; +import com.github.nexmark.flink.metric.FlinkRestClient; +import com.github.nexmark.flink.metric.JobBenchmarkMetric; +import com.github.nexmark.flink.metric.MetricReporter; +import com.github.nexmark.flink.metric.process.ProcessMetricReceiver; +import com.github.nexmark.flink.utils.NexmarkGlobalConfiguration; +import com.github.nexmark.flink.workload.Workload; +import com.github.nexmark.flink.workload.WorkloadSuite; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; + +import java.io.File; +import java.nio.file.Path; +import java.time.Duration; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static com.github.nexmark.flink.metric.BenchmarkMetric.NUMBER_FORMAT; +import static com.github.nexmark.flink.metric.BenchmarkMetric.formatDoubleValue; +import static com.github.nexmark.flink.metric.BenchmarkMetric.formatLongValue; +import static com.github.nexmark.flink.metric.BenchmarkMetric.formatLongValuePerSecond; + +/** + * The entry point to run benchmark for nexmark queries. + */ +public class Benchmark { + + // TODO: remove this once q6 is supported + private static final Set UNSUPPORTED_QUERIES = Collections.singleton("q6"); + + private static final Option LOCATION = new Option("l", "location", true, + "Nexmark directory."); + private static final Option QUERIES = new Option("q", "queries", true, + "Query to run. If the value is 'all', all queries will be run."); + private static final Option CATEGORY = new Option("c", "category", true, + "Query category."); + + public static final String CATEGORY_OA = "oa"; + + public static void main(String[] args) throws ParseException { + if (args == null || args.length == 0) { + throw new RuntimeException("Usage: --queries q1,q3 --category oa --location /path/to/nexmark"); + } + Options options = getOptions(); + DefaultParser parser = new DefaultParser(); + CommandLine line = parser.parse(options, args, true); + Path location = new File(line.getOptionValue(LOCATION.getOpt())).toPath(); + String category = CATEGORY.getValue(CATEGORY_OA).toLowerCase(); + boolean isQueryOa = CATEGORY_OA.equals(category); + Path queryLocation = isQueryOa ? location.resolve("queries") : location.resolve("queries-" + category); + List queries = getQueries(queryLocation, line.getOptionValue(QUERIES.getOpt()), isQueryOa); + System.out.println("Benchmark Queries: " + queries); + runQueries(queries, location, category); + } + + private static void runQueries(List queries, Path location, String category) { + String flinkHome = System.getenv("FLINK_HOME"); + if (flinkHome == null) { + throw new IllegalArgumentException("FLINK_HOME environment variable is not set."); + } + Path flinkDist = new File(flinkHome).toPath(); + + // start metric servers + Configuration nexmarkConf = NexmarkGlobalConfiguration.loadConfiguration(); + String jmAddress = nexmarkConf.get(FlinkNexmarkOptions.FLINK_REST_ADDRESS); + int jmPort = nexmarkConf.get(FlinkNexmarkOptions.FLINK_REST_PORT); + // Use receiving-specific host/port if configured, otherwise fall back to reporter host/port + String reporterAddress = nexmarkConf.getOptional(FlinkNexmarkOptions.METRIC_REPORTER_RECEIVING_HOST) + .filter(s -> !s.isEmpty()) + .orElseGet(() -> nexmarkConf.get(FlinkNexmarkOptions.METRIC_REPORTER_HOST)); + int reporterPort = nexmarkConf.getOptional(FlinkNexmarkOptions.METRIC_REPORTER_RECEIVING_PORT) + .orElseGet(() -> nexmarkConf.get(FlinkNexmarkOptions.METRIC_REPORTER_PORT)); + FlinkRestClient flinkRestClient = new FlinkRestClient(jmAddress, jmPort); + ProcessMetricReceiver processMetricReceiver = new ProcessMetricReceiver(reporterAddress, reporterPort); + processMetricReceiver.runServer(); + + String runnerVersion = nexmarkConf.get(FlinkNexmarkOptions.RUNNER_VERSION); + + Duration monitorDelay = nexmarkConf.get(FlinkNexmarkOptions.METRIC_MONITOR_DELAY); + Duration monitorInterval = nexmarkConf.get(FlinkNexmarkOptions.METRIC_MONITOR_INTERVAL); + Duration monitorDuration = nexmarkConf.get(FlinkNexmarkOptions.METRIC_MONITOR_DURATION); + + WorkloadSuite workloadSuite = WorkloadSuite.fromConf(nexmarkConf, category); + + // create sink DDL provider from environment or defaults + LocalDateTime now = LocalDateTime.now(); + String submitTime = now.minusNanos(now.getNano()).toString(); + SinkDdlProvider sinkDdlProvider = SinkDdlProvider.fromEnv( + location.toFile().getAbsolutePath(), submitTime); + + // start to run queries + LinkedHashMap totalMetrics = new LinkedHashMap<>(); + + if (runnerVersion.equalsIgnoreCase("V2")) { + executeQueriesV2( + queries, + workloadSuite, + flinkRestClient, + processMetricReceiver, + monitorDelay, + monitorInterval, + monitorDuration, + location, + flinkDist, + totalMetrics, + category, + sinkDdlProvider); + } else { + executeQueries( + queries, + workloadSuite, + flinkRestClient, + processMetricReceiver, + monitorDelay, + monitorInterval, + monitorDuration, + location, + flinkDist, + totalMetrics, + category, + sinkDdlProvider); + } + + // print benchmark summary + if (runnerVersion.equalsIgnoreCase("V2")) { + printSummaryV2(totalMetrics); + } else { + printSummary(totalMetrics); + } + + flinkRestClient.close(); + processMetricReceiver.close(); + } + + /** + * Returns the mapping from query name to query file path. + */ + private static List getQueries(Path queryLocation, String queries, boolean isQueryOa) { + List queryList = new ArrayList<>(); + if (queries.equals("all")) { + File queriesDir = queryLocation.toFile(); + if (!queriesDir.exists()) { + throw new IllegalArgumentException( + String.format("The queries dir \"%s\" does not exist.", queryLocation)); + } + for (int i = 0; i < 100; i++) { + String queryName = "q" + i; + if (isQueryOa && UNSUPPORTED_QUERIES.contains(queryName)) { + continue; + } + File queryFile = new File(queryLocation.toFile(), queryName + ".sql"); + if (queryFile.exists()) { + queryList.add(queryName); + } + } + } else { + for (String queryName : queries.split(",")) { + if (isQueryOa && UNSUPPORTED_QUERIES.contains(queryName)) { + continue; + } + File queryFile = new File(queryLocation.toFile(), queryName + ".sql"); + if (!queryFile.exists()) { + throw new IllegalArgumentException( + String.format("The query path \"%s\" does not exist.", queryFile.getAbsolutePath())); + } + queryList.add(queryName); + } + } + return queryList; + } + + private static void executeQueries( + List queries, + WorkloadSuite workloadSuite, + FlinkRestClient flinkRestClient, + ProcessMetricReceiver processMetricReceiver, + Duration monitorDelay, + Duration monitorInterval, + Duration monitorDuration, + Path location, + Path flinkDist, + LinkedHashMap totalMetrics, + String category, + SinkDdlProvider sinkDdlProvider) { + for (String queryName : queries) { + Workload workload = workloadSuite.getQueryWorkload(queryName); + if (workload == null) { + throw new IllegalArgumentException( + String.format("The workload of query %s is not defined.", queryName)); + } + workload.validateWorkload(monitorDuration); + + MetricReporter reporter = + new MetricReporter( + flinkRestClient, + processMetricReceiver, + monitorDelay, + monitorInterval, + monitorDuration); + QueryRunner runner = + new QueryRunner( + queryName, + workload, + location, + flinkDist, + reporter, + flinkRestClient, + category, + sinkDdlProvider); + JobBenchmarkMetric metric = runner.run(); + totalMetrics.put(queryName, metric); + } + } + + private static void executeQueriesV2( + List queries, + WorkloadSuite workloadSuite, + FlinkRestClient flinkRestClient, + ProcessMetricReceiver processMetricReceiver, + Duration monitorDelay, + Duration monitorInterval, + Duration monitorDuration, + Path location, + Path flinkDist, + LinkedHashMap totalMetrics, + String category, + SinkDdlProvider sinkDdlProvider) { + for (String queryName : queries) { + Workload workload = workloadSuite.getQueryWorkload(queryName); + if (workload == null) { + throw new IllegalArgumentException( + String.format("The workload of query %s is not defined.", queryName)); + } + workload.validateWorkload(monitorDuration); + + MetricReporter reporter = + new MetricReporter( + flinkRestClient, + processMetricReceiver, + monitorDelay, + monitorInterval, + monitorDuration); + QueryRunnerV2 runner = + new QueryRunnerV2( + queryName, + workload, + location, + flinkDist, + reporter, + flinkRestClient, + category, + sinkDdlProvider); + JobBenchmarkMetric metric = runner.run(); + totalMetrics.put(queryName, metric); + } + } + + public static void printSummary(LinkedHashMap totalMetrics) { + if (totalMetrics.isEmpty()) { + return; + } + System.err.println("-------------------------------- Nexmark Results --------------------------------"); + System.err.println(); + if (totalMetrics.values().iterator().next().getEventsNum() != 0) { + printEventNumSummary(totalMetrics); + } else { + printTPSSummary(totalMetrics); + } + System.err.println(); + } + + private static void printEventNumSummary(LinkedHashMap totalMetrics) { + int[] itemMaxLength = {7, 18, 9, 11, 18, 15, 18}; + printLine('-', "+", itemMaxLength, "", "", "", "", "", "", ""); + printLine(' ', "|", itemMaxLength, " Query", " Events Num", " Cores", " Time(s)", " Cores * Time(s)", " Throughput ", " Throughput/Cores"); + printLine('-', "+", itemMaxLength, "", "", "", "", "", "", ""); + + long totalEventsNum = 0; + double totalCpus = 0; + double totalTimeSeconds = 0; + double totalCoresMultiplyTimeSeconds = 0; + double totalThroughput = 0; + double totalThroughputPerCore = 0; + for (Map.Entry entry : totalMetrics.entrySet()) { + JobBenchmarkMetric metric = entry.getValue(); + double throughput = metric.getEventsNum() / metric.getTimeSeconds(); + double throughputPerCore = metric.getEventsNum() / metric.getCoresMultiplyTimeSeconds(); + printLine(' ', "|", itemMaxLength, + entry.getKey(), + NUMBER_FORMAT.format(metric.getEventsNum()), + NUMBER_FORMAT.format(metric.getCpu()), + formatDoubleValue(metric.getTimeSeconds()), + formatDoubleValue(metric.getCoresMultiplyTimeSeconds()), + formatLongValuePerSecond((long) throughput), + formatLongValuePerSecond((long) throughputPerCore)); + totalEventsNum += metric.getEventsNum(); + totalCpus += metric.getCpu(); + totalTimeSeconds += metric.getTimeSeconds(); + totalCoresMultiplyTimeSeconds += metric.getCoresMultiplyTimeSeconds(); + totalThroughput += throughput; + totalThroughputPerCore += throughputPerCore; + } + printLine(' ', "|", itemMaxLength, + "Total", + NUMBER_FORMAT.format(totalEventsNum), + formatDoubleValue(totalCpus), + formatDoubleValue(totalTimeSeconds), + formatDoubleValue(totalCoresMultiplyTimeSeconds), + formatLongValuePerSecond((long) totalThroughput), + formatLongValuePerSecond((long) totalThroughputPerCore)); + printLine('-', "+", itemMaxLength, "", "", "", "", "", "", ""); + } + + private static void printTPSSummary(LinkedHashMap totalMetrics) { + int[] itemMaxLength = {7, 18, 10, 18}; + printLine('-', "+", itemMaxLength, "", "", "", ""); + printLine(' ', "|", itemMaxLength, " Query", " Throughput (r/s)", " Cores", " Throughput/Cores"); + printLine('-', "+", itemMaxLength, "", "", "", ""); + + long totalTpsPerCore = 0; + for (Map.Entry entry : totalMetrics.entrySet()) { + JobBenchmarkMetric metric = entry.getValue(); + printLine(' ', "|", itemMaxLength, + entry.getKey(), + metric.getPrettyTps(), + metric.getPrettyCpu(), + metric.getPrettyTpsPerCore()); + totalTpsPerCore += metric.getTpsPerCore(); + } + printLine(' ', "|", itemMaxLength, + "Total", + "-", + "-", + formatLongValue(totalTpsPerCore)); + printLine('-', "+", itemMaxLength, "", "", "", ""); + } + + public static void printSummaryV2(LinkedHashMap totalMetrics) { + if (totalMetrics.isEmpty()) { + return; + } + System.err.println("-------------------------------- Nexmark Results --------------------------------"); + System.err.println(); + printEventNumSummaryV2(totalMetrics); + System.err.println(); + } + + private static void printEventNumSummaryV2(LinkedHashMap totalMetrics) { + int[] itemMaxLength = {7, 18, 9, 11, 15, 20}; + printLine('-', "+", itemMaxLength, "", "", "", "", "", ""); + printLine(' ', "|", itemMaxLength, " Query", " Events Num", " Cores", " Time(s)", " Throughput ", " Recovery Time(s)"); + printLine('-', "+", itemMaxLength, "", "", "", "", "", ""); + + long totalEventsNum = 0; + double totalCpus = 0; + double totalTimeSeconds = 0; + double totalThroughput = 0; + double totalRecoverySeconds = 0; + for (Map.Entry entry : totalMetrics.entrySet()) { + JobBenchmarkMetric metric = entry.getValue(); + double throughput = metric.getEventsNum() / metric.getTimeSeconds(); + printLine(' ', "|", itemMaxLength, + entry.getKey(), + NUMBER_FORMAT.format(metric.getEventsNum()), + NUMBER_FORMAT.format(metric.getCpu()), + formatDoubleValue(metric.getTimeSeconds()), + formatLongValuePerSecond((long) throughput), + formatDoubleValue(metric.getJobInitializedTimeSeconds())); + totalEventsNum += metric.getEventsNum(); + totalCpus += metric.getCpu(); + totalTimeSeconds += metric.getTimeSeconds(); + totalThroughput += throughput; + totalRecoverySeconds += metric.getJobInitializedTimeSeconds(); + } + printLine(' ', "|", itemMaxLength, + "Total", + NUMBER_FORMAT.format(totalEventsNum), + formatDoubleValue(totalCpus), + formatDoubleValue(totalTimeSeconds), + formatLongValuePerSecond((long) totalThroughput), + formatDoubleValue(totalRecoverySeconds)); + printLine('-', "+", itemMaxLength, "", "", "", "", "", ""); + } + + private static void printLine( + char charToFill, + String separator, + int[] itemMaxLength, + String... items) { + StringBuilder builder = new StringBuilder(); + Iterator lengthIterator = Arrays.stream(itemMaxLength).iterator(); + int lineLength = 0; + for (String item : items) { + if (lengthIterator.hasNext()) { + lineLength = lengthIterator.next(); + } + builder.append(separator); + builder.append(item); + int left = lineLength - item.length() - separator.length(); + for (int i = 0; i < left; i++) { + builder.append(charToFill); + } + } + builder.append(separator); + System.err.println(builder.toString()); + } + + private static Options getOptions() { + Options options = new Options(); + options.addOption(QUERIES); + options.addOption(CATEGORY); + options.addOption(LOCATION); + return options; + } +} diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/FlinkNexmarkOptions.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/FlinkNexmarkOptions.java new file mode 100644 index 0000000..def8bf1 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/FlinkNexmarkOptions.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink; + +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.configuration.ConfigOptions; + +import java.time.Duration; + +/** + * Options to control the nexmark-flink benchmark behaviors. + */ +public class FlinkNexmarkOptions { + + + public static final ConfigOption RUNNER_VERSION = ConfigOptions + .key("nexmark.runner.version") + .stringType() + .defaultValue("v1") + .withDescription("The version of the runner, v1 or v2."); + + public static final ConfigOption METRIC_MONITOR_DELAY = ConfigOptions + .key("nexmark.metric.monitor.delay") + .durationType() + .defaultValue(Duration.ofSeconds(10)) + .withDescription("When to monitor the metrics, default 10secs after job is started"); + + public static final ConfigOption METRIC_MONITOR_DURATION = ConfigOptions + .key("nexmark.metric.monitor.duration") + .durationType() + .defaultValue(Duration.ofNanos(Long.MAX_VALUE)) + .withDescription("How long to monitor the metrics, default never end, " + + "monitor until job is finished."); + + public static final ConfigOption METRIC_MONITOR_INTERVAL = ConfigOptions + .key("nexmark.metric.monitor.interval") + .durationType() + .defaultValue(Duration.ofSeconds(5)) + .withDescription("The interval to request the metrics."); + + + public static final ConfigOption METRIC_REPORTER_HOST = ConfigOptions + .key("nexmark.metric.reporter.host") + .stringType() + .defaultValue("localhost") + .withDescription("The metric reporter host that sender connects to."); + + public static final ConfigOption METRIC_REPORTER_PORT = ConfigOptions + .key("nexmark.metric.reporter.port") + .intType() + .defaultValue(9098) + .withDescription("The metric reporter port that sender connects to."); + + public static final ConfigOption METRIC_REPORTER_RECEIVING_HOST = ConfigOptions + .key("nexmark.metric.reporter.receiving.host") + .stringType() + .noDefaultValue() + .withDescription("The host for metric receiver to bind to. " + + "If not set, falls back to nexmark.metric.reporter.host. " + + "Useful in K8s where receiver binds to 0.0.0.0 but sender connects via service DNS."); + + public static final ConfigOption METRIC_REPORTER_RECEIVING_PORT = ConfigOptions + .key("nexmark.metric.reporter.receiving.port") + .intType() + .noDefaultValue() + .withDescription("The port for metric receiver to bind to. " + + "If not set, falls back to nexmark.metric.reporter.port."); + + public static final ConfigOption FLINK_REST_ADDRESS = ConfigOptions + .key("flink.rest.address") + .stringType() + .defaultValue("localhost") + .withDescription("Flink REST address."); + + public static final ConfigOption FLINK_REST_PORT = ConfigOptions + .key("flink.rest.port") + .intType() + .defaultValue(8081) + .withDescription("Flink REST port."); + +} diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/NexmarkConfiguration.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/NexmarkConfiguration.java new file mode 100644 index 0000000..69c1c09 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/NexmarkConfiguration.java @@ -0,0 +1,237 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink; + +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonProcessingException; + +import com.github.nexmark.flink.utils.NexmarkUtils; + +import java.io.Serializable; +import java.util.Objects; + +public class NexmarkConfiguration implements Serializable { + + /** + * Number of events to generate. If zero, generate as many as possible without overflowing + * internal counters etc. + */ + @JsonProperty + public long numEvents = 0; + + /** Number of event generators to use. Each generates events in its own timeline. */ + @JsonProperty public int numEventGenerators = 1; + + /** Shape of event rate curve. */ + @JsonProperty public NexmarkUtils.RateShape rateShape = NexmarkUtils.RateShape.SQUARE; + + /** Initial overall event rate (in {@link #rateUnit}). */ + @JsonProperty public int firstEventRate = 10000; + + /** Next overall event rate (in {@link #rateUnit}). */ + @JsonProperty public int nextEventRate = 10000; + + /** Unit for rates. */ + @JsonProperty public NexmarkUtils.RateUnit rateUnit = NexmarkUtils.RateUnit.PER_SECOND; + + /** Overall period of rate shape, in seconds. */ + @JsonProperty public int ratePeriodSec = 600; + + /** + * Time in seconds to preload the subscription with data, at the initial input rate of the + * pipeline. + */ + @JsonProperty public int preloadSeconds = 0; + + /** Timeout for stream pipelines to stop in seconds. */ + @JsonProperty public int streamTimeout = 240; + + /** + * If true, and in streaming mode, generate events only when they are due according to their + * timestamp. + */ + @JsonProperty public boolean isRateLimited = false; + + /** + * If true, use wallclock time as event time. Otherwise, use a deterministic time in the past so + * that multiple runs will see exactly the same event streams and should thus have exactly the + * same results. + */ + @JsonProperty public boolean useWallclockEventTime = false; + + /** + * Person Proportion. + */ + @JsonProperty public int personProportion = 1; + + /** + * Auction Proportion. + */ + @JsonProperty public int auctionProportion = 3; + + /** + * Bid Proportion. + */ + @JsonProperty public int bidProportion = 46; + + /** Average idealized size of a 'new person' event, in bytes. */ + @JsonProperty public int avgPersonByteSize = 200; + + /** Average idealized size of a 'new auction' event, in bytes. */ + @JsonProperty public int avgAuctionByteSize = 500; + + /** Average idealized size of a 'bid' event, in bytes. */ + @JsonProperty public int avgBidByteSize = 100; + + /** Ratio of bids to 'hot' auctions compared to all other auctions. */ + @JsonProperty public int hotAuctionRatio = 2; + + /** Ratio of auctions for 'hot' sellers compared to all other people. */ + @JsonProperty public int hotSellersRatio = 4; + + /** Ratio of bids for 'hot' bidders compared to all other people. */ + @JsonProperty public int hotBiddersRatio = 4; + + /** Window size, in seconds, for queries 3, 5, 7 and 8. */ + @JsonProperty public long windowSizeSec = 10; + + /** Sliding window period, in seconds, for query 5. */ + @JsonProperty public long windowPeriodSec = 5; + + /** Number of seconds to hold back events according to their reported timestamp. */ + @JsonProperty public long watermarkHoldbackSec = 0; + + /** Average number of auction which should be inflight at any time, per generator. */ + @JsonProperty public int numInFlightAuctions = 100; + + /** Maximum number of people to consider as active for placing auctions or bids. */ + @JsonProperty public int numActivePeople = 1000; + + /** Length of occasional delay to impose on events (in seconds). */ + @JsonProperty public long occasionalDelaySec = 3; + + /** Probability that an event will be delayed by delayS. */ + @JsonProperty public double probDelayedEvent = 0.1; + + /** + * Number of events in out-of-order groups. 1 implies no out-of-order events. 1000 implies every + * 1000 events per generator are emitted in pseudo-random order. + */ + @JsonProperty public long outOfOrderGroupSize = 1; + + /** + * If true, the source will not finish and quit if all the events emit. + */ + @JsonProperty public boolean isSourceKeepAlive = false; + + /** + * The source will finish at number of event if it is not -1. + */ + @JsonProperty public long stopAtEvent = -1L; + + /** + * If true, the source will emit events as fast as possible. Otherwise, it will emit events + * according to the event time. + */ + @JsonProperty public boolean maxEmitSpeed = true; + + /** Return full description as a string. */ + @Override + public String toString() { + try { + return NexmarkUtils.MAPPER.writeValueAsString(this); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + NexmarkConfiguration that = (NexmarkConfiguration) o; + return numEvents == that.numEvents && + numEventGenerators == that.numEventGenerators && + firstEventRate == that.firstEventRate && + nextEventRate == that.nextEventRate && + ratePeriodSec == that.ratePeriodSec && + preloadSeconds == that.preloadSeconds && + streamTimeout == that.streamTimeout && + isRateLimited == that.isRateLimited && + useWallclockEventTime == that.useWallclockEventTime && + personProportion == that.personProportion && + auctionProportion == that.auctionProportion && + bidProportion == that.bidProportion && + avgPersonByteSize == that.avgPersonByteSize && + avgAuctionByteSize == that.avgAuctionByteSize && + avgBidByteSize == that.avgBidByteSize && + hotAuctionRatio == that.hotAuctionRatio && + hotSellersRatio == that.hotSellersRatio && + hotBiddersRatio == that.hotBiddersRatio && + windowSizeSec == that.windowSizeSec && + windowPeriodSec == that.windowPeriodSec && + watermarkHoldbackSec == that.watermarkHoldbackSec && + numInFlightAuctions == that.numInFlightAuctions && + numActivePeople == that.numActivePeople && + occasionalDelaySec == that.occasionalDelaySec && + Double.compare(that.probDelayedEvent, probDelayedEvent) == 0 && + outOfOrderGroupSize == that.outOfOrderGroupSize && + rateShape == that.rateShape && + rateUnit == that.rateUnit && + isSourceKeepAlive == that.isSourceKeepAlive && + stopAtEvent == that.stopAtEvent && + maxEmitSpeed == that.maxEmitSpeed; + } + + @Override + public int hashCode() { + return Objects.hash( + numEvents, + numEventGenerators, + rateShape, + firstEventRate, + nextEventRate, + rateUnit, + ratePeriodSec, + preloadSeconds, + streamTimeout, + isRateLimited, + useWallclockEventTime, + personProportion, + auctionProportion, + bidProportion, + avgPersonByteSize, + avgAuctionByteSize, + avgBidByteSize, + hotAuctionRatio, + hotSellersRatio, + hotBiddersRatio, + windowSizeSec, + windowPeriodSec, + watermarkHoldbackSec, + numInFlightAuctions, + numActivePeople, + occasionalDelaySec, + probDelayedEvent, + outOfOrderGroupSize, + isSourceKeepAlive, + stopAtEvent, + maxEmitSpeed); + } +} diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/QueryRunner.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/QueryRunner.java new file mode 100644 index 0000000..163dd43 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/QueryRunner.java @@ -0,0 +1,217 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink; + +import static com.github.nexmark.flink.Benchmark.CATEGORY_OA; + +import com.github.nexmark.core.SinkDdlProvider; +import com.github.nexmark.flink.metric.FlinkRestClient; +import com.github.nexmark.flink.metric.JobBenchmarkMetric; +import com.github.nexmark.flink.metric.MetricReporter; +import com.github.nexmark.flink.metric.Savepoint; +import com.github.nexmark.flink.utils.AutoClosableProcess; +import com.github.nexmark.flink.workload.Workload; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class QueryRunner { + + private static final Logger LOG = LoggerFactory.getLogger(QueryRunner.class); + + private final String queryName; + private final Workload workload; + private final Path location; + private final Path queryLocation; + private final Path flinkDist; + private final MetricReporter metricReporter; + private final FlinkRestClient flinkRestClient; + private final SinkDdlProvider sinkDdlProvider; + + public QueryRunner(String queryName, Workload workload, Path location, Path flinkDist, MetricReporter metricReporter, FlinkRestClient flinkRestClient, String category, SinkDdlProvider sinkDdlProvider) { + this.queryName = queryName; + this.workload = workload; + this.location = location; + this.queryLocation = + CATEGORY_OA.equals(category) ? location.resolve("queries") : location.resolve("queries-" + category); + this.flinkDist = flinkDist; + this.metricReporter = metricReporter; + this.flinkRestClient = flinkRestClient; + this.sinkDdlProvider = sinkDdlProvider; + } + + public JobBenchmarkMetric run() { + try { + Savepoint savepoint = null; + System.out.println("=================================================================="); + System.out.println("Start to run query " + queryName + " with workload " + workload.getSummaryString()); + LOG.info("=================================================================="); + LOG.info("Start to run query " + queryName + " with workload " + workload.getSummaryString()); + if (!"insert_kafka".equals(queryName) // no warmup for kafka source prepare + && (workload.getWarmupMills() > 0L || workload.getKafkaServers() == null) // when using kafka source we need a stop for warmup + && ((workload.getWarmupTps() > 0L && workload.getWarmupEvents() > 0L) || workload.getKafkaServers() != null) // otherwise we need a configuration for datagen source + ) { + System.out.println("Start the warmup for at most " + workload.getWarmupMills() + "ms and " + workload.getWarmupEvents() + " events."); + LOG.info("Start the warmup for at most " + workload.getWarmupMills() + "ms and " + workload.getWarmupEvents() + " events."); + String jobId = runWarmup(workload.getWarmupTps(), workload.getWarmupEvents()); + long waited = waitForOrJobFinish(jobId, workload.getWarmupMills()); + waited += cancelJob(jobId); + System.out.println("Stop the warmup, cost " + waited + "ms."); + LOG.info("Stop the warmup, cost " + waited + "."); + } + String jobId = runInternal(); + // blocking until collect enough metrics + JobBenchmarkMetric metrics = metricReporter.reportMetric(jobId, + workload.getEventsNum(), + workload.getKafkaServers() != null, + workload.getKafkaServers() != null); + // cancel job + System.out.println("Stop job query " + queryName); + LOG.info("Stop job query " + queryName); + cancelJob(jobId); + return metrics; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private long waitForOrJobFinish(String jobId, long timeout) { + long waited = 0L; + while ((timeout <= 0L || waited < timeout) && flinkRestClient.isJobRunning(jobId)) { + try { + Thread.sleep(100L); + waited += 100L; + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + return waited; + } + + private long cancelJob(String jobId) { + long cost = 0L; + while (!flinkRestClient.isJobCanceledOrFinished(jobId)) { + // make sure the job is canceled. + flinkRestClient.cancelJob(flinkRestClient.getCurrentJobId()); + try { + Thread.sleep(100L); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + return cost; + } + + private String runWarmup(long tps, long events) throws IOException { + Map varsMap = initializeVarsMap(); + if (workload.getKafkaServers() == null) { + varsMap.put("TPS", String.valueOf(tps)); + varsMap.put("EVENTS_NUM", String.valueOf(events)); + } + List sqlLines = initializeAllSqlLines(varsMap); + return submitSQLJob(sqlLines); + } + + private String runInternal() throws IOException { + Map varsMap = initializeVarsMap(); + List sqlLines = initializeAllSqlLines(varsMap); + return submitSQLJob(sqlLines); + } + + private Map initializeVarsMap() { + LocalDateTime currentTime = LocalDateTime.now(); + LocalDateTime submitTime = currentTime.minusNanos(currentTime.getNano()); + + Map varsMap = new HashMap<>(); + varsMap.put("NEXMARK_DIR", location.toFile().getAbsolutePath()); + varsMap.put("SUBMIT_TIME", submitTime.toString()); + varsMap.put("FLINK_HOME", flinkDist.toFile().getAbsolutePath()); + varsMap.put("TPS", String.valueOf(workload.getTps())); + varsMap.put("EVENTS_NUM", String.valueOf(workload.getEventsNum())); + varsMap.put("PERSON_PROPORTION", String.valueOf(workload.getPersonProportion())); + varsMap.put("AUCTION_PROPORTION", String.valueOf(workload.getAuctionProportion())); + varsMap.put("BID_PROPORTION", String.valueOf(workload.getBidProportion())); + varsMap.put("NEXMARK_TABLE", workload.getKafkaServers() == null ? "datagen" : "kafka"); + varsMap.put("BOOTSTRAP_SERVERS", workload.getKafkaServers() == null ? "" : workload.getKafkaServers()); + varsMap.put("SINK_DDL", sinkDdlProvider.getSinkDdl(queryName)); + return varsMap; + } + + private List initializeAllSqlLines(Map vars) throws IOException { + List allLines = new ArrayList<>(); + allLines.addAll(initializeSqlFileLines(vars, new File(queryLocation.toFile(), "ddl_gen.sql"))); + allLines.addAll(initializeSqlFileLines(vars, new File(queryLocation.toFile(), "ddl_kafka.sql"))); + allLines.addAll(initializeSqlFileLines(vars, new File(queryLocation.toFile(), "ddl_views.sql"))); + allLines.addAll(initializeSqlFileLines(vars, new File(queryLocation.toFile(), queryName + ".sql"))); + return allLines; + } + + private List initializeSqlFileLines(Map vars, File sqlFile) throws IOException { + List lines = Files.readAllLines(sqlFile.toPath()); + List result = new ArrayList<>(); + for (String line : lines) { + for (Map.Entry var : vars.entrySet()) { + line = line.replace("${" + var.getKey() + "}", var.getValue()); + } + result.add(line); + } + return result; + } + + public String submitSQLJob(List sqlLines) throws IOException { + Path flinkBin = flinkDist.resolve("bin"); + final List commands = new ArrayList<>(); + commands.add(flinkBin.resolve("sql-client.sh").toAbsolutePath().toString()); + commands.add("embedded"); + + LOG.info("\n================================================================================" + + "\nQuery {} is running." + + "\n--------------------------------------------------------------------------------" + + "\n" + , queryName); + + StringBuilder output = new StringBuilder(); + AutoClosableProcess + .create(commands.toArray(new String[0])) + .setStdInputs(sqlLines.toArray(new String[0])) + .setStdoutProcessor(output::append) // logging the SQL statements and error message + .runBlocking(); + + Pattern pattern = Pattern.compile("Job ID: ([A-Za-z0-9]{32})"); + Matcher matcher = pattern.matcher(output.toString()); + if (matcher.find()) { + return matcher.group(1); + } else { + throw new RuntimeException("Cannot find Job ID from the sql client output, maybe the job is not successfully submitted."); + } + } + +} diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/QueryRunnerV2.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/QueryRunnerV2.java new file mode 100644 index 0000000..c36fed5 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/QueryRunnerV2.java @@ -0,0 +1,255 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink; + +import static com.github.nexmark.flink.Benchmark.CATEGORY_OA; + +import com.github.nexmark.core.SinkDdlProvider; +import com.github.nexmark.flink.metric.FlinkRestClient; +import com.github.nexmark.flink.metric.JobBenchmarkMetric; +import com.github.nexmark.flink.metric.MetricReporter; +import com.github.nexmark.flink.metric.Savepoint; +import com.github.nexmark.flink.utils.AutoClosableProcess; +import com.github.nexmark.flink.workload.Workload; + +import org.apache.flink.api.java.tuple.Tuple2; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class QueryRunnerV2 { + + private static final Logger LOG = LoggerFactory.getLogger(QueryRunnerV2.class); + + private final String queryName; + private final Workload workload; + private final Path location; + private final Path queryLocation; + private final Path flinkDist; + private final MetricReporter metricReporter; + private final FlinkRestClient flinkRestClient; + private final SinkDdlProvider sinkDdlProvider; + + public QueryRunnerV2(String queryName, Workload workload, Path location, Path flinkDist, MetricReporter metricReporter, FlinkRestClient flinkRestClient, String category, SinkDdlProvider sinkDdlProvider) { + this.queryName = queryName; + this.workload = workload; + this.location = location; + this.queryLocation = + CATEGORY_OA.equals(category) ? location.resolve("queries") : location.resolve("queries-" + category); + this.flinkDist = flinkDist; + this.metricReporter = metricReporter; + this.flinkRestClient = flinkRestClient; + this.sinkDdlProvider = sinkDdlProvider; + } + + public JobBenchmarkMetric run() { + try { + Savepoint savepoint = null; + System.out.println("=================================================================="); + System.out.println("Start to run query " + queryName + " with workload " + workload.getSummaryString()); + LOG.info("=================================================================="); + LOG.info("Start to run query " + queryName + " with workload " + workload.getSummaryString()); + long totalEvents = workload.getWarmupEvents() + workload.getEventsNum(); + System.out.println("Start the warmup for " + workload.getWarmupEvents() + " events."); + LOG.info("Start the warmup for " + workload.getWarmupEvents() + " events."); + String warmupJob = runWarmup(workload.getWarmupEvents(), totalEvents); + long waited = waitForOrJobFinish(warmupJob, workload.getWarmupEvents()); + Tuple2 cancelResult = cancelJob(warmupJob, true); + savepoint = cancelResult.f0; + waited += cancelResult.f1; + System.out.println("Stop the warmup, cost " + waited + "ms."); + LOG.info("Stop the warmup, cost " + waited + "."); + + if (savepoint == null) { + System.out.println("The query set warmup with savepoint, but does not get any savepoint."); + LOG.error("The query set warmup with savepoint, but does not get any savepoint."); + throw new RuntimeException("The query set warmup with savepoint, but does not get any savepoint."); + } else { + System.out.println("Get warmup savepoint: " + savepoint); + LOG.info("Get warmup savepoint: " + savepoint); + } + + + String jobId = runInternal(totalEvents, savepoint); + // blocking until collect enough metrics + JobBenchmarkMetric metrics = metricReporter.reportMetric(jobId, + workload.getEventsNum(), + false, + workload.getKafkaServers() != null); + // cancel job + System.out.println("Stop job query " + queryName); + LOG.info("Stop job query " + queryName); + cancelJob(jobId, false); + return metrics; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private long waitForOrJobFinish(String jobId, long recordLimit) { + long start = System.currentTimeMillis(); + while (flinkRestClient.isJobRunning(jobId, recordLimit)) { + try { + Thread.sleep(100L); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + return System.currentTimeMillis() - start; + } + + private Tuple2 cancelJob(String jobId, boolean savepoint) { + System.out.println("Cancelling job " + jobId + " with checkpoint = " + savepoint); + long start = System.currentTimeMillis(); + boolean triggered = false; + String requestId = null; + + while (!flinkRestClient.isJobCanceledOrFinished(jobId)) { + // make sure the job is canceled. + if (savepoint) { + if (!triggered) { + requestId = flinkRestClient.triggerCheckpoint(jobId); + triggered = true; + } else { + Savepoint.Status status = flinkRestClient.checkCheckpointFinished(jobId, requestId); + if (status == Savepoint.Status.COMPLETED) { + // just wait for finished. + flinkRestClient.cancelJob(jobId); + } else if (status == Savepoint.Status.FAILED) { + triggered = false; + } + } + } else { + flinkRestClient.cancelJob(jobId); + } + try { + Thread.sleep(100L); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + return Tuple2.of( + savepoint ? flinkRestClient.getJobLastCheckpoint(jobId) : null, + System.currentTimeMillis() - start + ); + } + + private String runWarmup(long stopAtEvents, long totalEvents) throws IOException { + Map varsMap = initializeVarsMap(); + varsMap.put("EVENTS_NUM", String.valueOf(totalEvents)); + varsMap.put("STOP_AT", String.valueOf(stopAtEvents)); + varsMap.put("KEEP_ALIVE", "true"); + List sqlLines = initializeAllSqlLines(varsMap, queryName + "_warmup", null); + return submitSQLJob(sqlLines); + } + + private String runInternal(long totalEvents, Savepoint savepoint) throws IOException { + Map varsMap = initializeVarsMap(); + varsMap.put("EVENTS_NUM", String.valueOf(totalEvents)); + varsMap.put("STOP_AT", "-1"); + varsMap.put("KEEP_ALIVE", "false"); + List sqlLines = initializeAllSqlLines(varsMap, queryName, savepoint); + return submitSQLJob(sqlLines); + } + + private Map initializeVarsMap() { + LocalDateTime currentTime = LocalDateTime.now(); + LocalDateTime submitTime = currentTime.minusNanos(currentTime.getNano()); + + Map varsMap = new HashMap<>(); + varsMap.put("NEXMARK_DIR", location.toFile().getAbsolutePath()); + varsMap.put("SUBMIT_TIME", submitTime.toString()); + varsMap.put("FLINK_HOME", flinkDist.toFile().getAbsolutePath()); + varsMap.put("TPS", String.valueOf(workload.getTps())); + varsMap.put("PERSON_PROPORTION", String.valueOf(workload.getPersonProportion())); + varsMap.put("AUCTION_PROPORTION", String.valueOf(workload.getAuctionProportion())); + varsMap.put("BID_PROPORTION", String.valueOf(workload.getBidProportion())); + varsMap.put("NEXMARK_TABLE", "datagen"); + varsMap.put("SINK_DDL", sinkDdlProvider.getSinkDdl(queryName)); + return varsMap; + } + + private List initializeAllSqlLines(Map vars, String name, Savepoint savepoint) throws IOException { + List allLines = new ArrayList<>(); + if (savepoint != null) { + allLines.add("SET 'execution.savepoint.path' = '" + savepoint.getPath() + "';"); + allLines.add("SET 'execution.state-recovery.claim-mode' = 'CLAIM';"); + } + if (name != null && !name.isEmpty()) { + allLines.add("SET 'pipeline.name' = '" + name + "';"); + } + allLines.addAll(initializeSqlFileLines(vars, new File(queryLocation.toFile(), "ddl_gen_v2.sql"))); + allLines.addAll(initializeSqlFileLines(vars, new File(queryLocation.toFile(), "ddl_kafka.sql"))); + allLines.addAll(initializeSqlFileLines(vars, new File(queryLocation.toFile(), "ddl_views.sql"))); + allLines.addAll(initializeSqlFileLines(vars, new File(queryLocation.toFile(), queryName + ".sql"))); + return allLines; + } + + private List initializeSqlFileLines(Map vars, File sqlFile) throws IOException { + List lines = Files.readAllLines(sqlFile.toPath()); + List result = new ArrayList<>(); + for (String line : lines) { + for (Map.Entry var : vars.entrySet()) { + line = line.replace("${" + var.getKey() + "}", var.getValue()); + } + result.add(line); + } + return result; + } + + public String submitSQLJob(List sqlLines) throws IOException { + Path flinkBin = flinkDist.resolve("bin"); + final List commands = new ArrayList<>(); + commands.add(flinkBin.resolve("sql-client.sh").toAbsolutePath().toString()); + commands.add("embedded"); + + LOG.info("\n================================================================================" + + "\nQuery {} is running." + + "\n--------------------------------------------------------------------------------" + + "\n" + , queryName); + + StringBuilder output = new StringBuilder(); + AutoClosableProcess + .create(commands.toArray(new String[0])) + .setStdInputs(sqlLines.toArray(new String[0])) + .setStdoutProcessor(output::append) // logging the SQL statements and error message + .runBlocking(); + + Pattern pattern = Pattern.compile("Job ID: ([A-Za-z0-9]{32})"); + Matcher matcher = pattern.matcher(output.toString()); + if (matcher.find()) { + return matcher.group(1); + } else { + throw new RuntimeException("Cannot find Job ID from the sql client output, maybe the job is not successfully submitted."); + } + } + +} diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/generator/GeneratorConfig.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/generator/GeneratorConfig.java new file mode 100644 index 0000000..b01575b --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/generator/GeneratorConfig.java @@ -0,0 +1,354 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.nexmark.flink.generator; + +import com.github.nexmark.flink.model.Event; +import com.github.nexmark.flink.NexmarkConfiguration; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; + +/** Parameters controlling how {@link NexmarkGenerator} synthesizes {@link Event} elements. */ +public class GeneratorConfig implements Serializable { + + /** + * We start the ids at specific values to help ensure the queries find a match even on small + * synthesized dataset sizes. + */ + public static final long FIRST_AUCTION_ID = 1000L; + + public static final long FIRST_PERSON_ID = 1000L; + public static final long FIRST_CATEGORY_ID = 10L; + + /** Proportions of people/auctions/bids to synthesize. */ + public final int personProportion; + public final int auctionProportion; + public final int bidProportion; + public final int totalProportion; + + /** Environment options. */ + private final NexmarkConfiguration configuration; + + /** + * Delay between events, in microseconds. If the array has more than one entry then the rate is + * changed every {@link #stepLengthSec}, and wraps around. + */ + private final double[] interEventDelayUs; + + /** Delay before changing the current inter-event delay. */ + private final long stepLengthSec; + + /** Time for first event (ms since epoch). */ + public final long baseTime; + + /** + * Event id of first event to be generated. Event ids are unique over all generators, and are used + * as a seed to generate each event's data. + */ + public final long firstEventId; + + /** Maximum number of events to generate. */ + public final long maxEvents; + + /** The number of events that the generator should stop at. */ + public final long stopAtEvent; + + /** + * First event number. Generators running in parallel time may share the same event number, and + * the event number is used to determine the event timestamp. + */ + public final long firstEventNumber; + + /** + * True period of epoch in milliseconds. Derived from above. (Ie time to run through cycle for all + * interEventDelayUs entries). + */ + private final long epochPeriodMs; + + /** + * Number of events per epoch. Derived from above. (Ie number of events to run through cycle for + * all interEventDelayUs entries). + */ + private final long eventsPerEpoch; + + /** + * Whether to emit record at max speed, or to respect the inter-event delay. This is useful for + * testing the system under load. + */ + public final boolean maxEmitSpeed; + + public GeneratorConfig( + NexmarkConfiguration configuration, + long baseTime, + long firstEventId, + long maxEventsOrZero, + long stopAtEvent, + long firstEventNumber) { + + this.auctionProportion = configuration.auctionProportion; + this.personProportion = configuration.personProportion; + this.bidProportion = configuration.bidProportion; + this.totalProportion = this.auctionProportion + this.personProportion + this.bidProportion; + + this.configuration = configuration; + + this.interEventDelayUs = new double[1]; + this.interEventDelayUs[0] = 1000000.0 / configuration.firstEventRate * configuration.numEventGenerators; + this.stepLengthSec = configuration.rateShape.stepLengthSec(configuration.ratePeriodSec); + this.baseTime = baseTime; + this.firstEventId = firstEventId; + if (maxEventsOrZero == 0) { + // Scale maximum down to avoid overflow in getEstimatedSizeBytes. + this.maxEvents = + Long.MAX_VALUE + / (totalProportion + * Math.max( + Math.max(configuration.avgPersonByteSize, configuration.avgAuctionByteSize), + configuration.avgBidByteSize)); + } else { + this.maxEvents = maxEventsOrZero; + } + this.stopAtEvent = stopAtEvent; + this.firstEventNumber = firstEventNumber; + + long eventsPerEpoch = 0; + long epochPeriodMs = 0; + this.eventsPerEpoch = eventsPerEpoch; + this.epochPeriodMs = epochPeriodMs; + + this.maxEmitSpeed = configuration.maxEmitSpeed; + } + + public GeneratorConfig reconfigure(GeneratorConfig configuration, boolean clearStopAtEvent) { + return new GeneratorConfig( + configuration.configuration, baseTime, firstEventId, maxEvents, clearStopAtEvent ? -1L : stopAtEvent, firstEventNumber); + } + + /** Return a copy of this config. */ + public GeneratorConfig copy() { + GeneratorConfig result; + result = + new GeneratorConfig(configuration, baseTime, firstEventId, maxEvents, stopAtEvent, firstEventNumber); + return result; + } + + /** + * Split this config into {@code n} sub-configs with roughly equal number of possible events, but + * distinct value spaces. The generators will run on parallel timelines. This config should no + * longer be used. + */ + public List split(int n) { + List results = new ArrayList<>(); + if (n == 1) { + // No split required. + results.add(this); + } else { + long subMaxEvents = maxEvents / n; + long subFirstEventId = firstEventId; + long subValidEvents = stopAtEvent / n; + for (int i = 0; i < n; i++) { + if (i == n - 1) { + // Don't loose any events to round-down. + subMaxEvents = maxEvents - subMaxEvents * (n - 1); + subValidEvents = stopAtEvent - subValidEvents * (n - 1); + } + results.add(copyWith(subFirstEventId, subMaxEvents, subValidEvents, firstEventNumber)); + subFirstEventId += subMaxEvents; + } + } + return results; + } + + /** Return copy of this config except with given parameters. */ + public GeneratorConfig copyWith(long firstEventId, long maxEvents, long stopAtEvents, long firstEventNumber) { + return new GeneratorConfig( + configuration, baseTime, firstEventId, maxEvents, stopAtEvents, firstEventNumber); + } + + /** Return an estimate of the bytes needed by {@code numEvents}. */ + public long estimatedBytesForEvents(long numEvents) { + long numPersons = + (numEvents * personProportion) / totalProportion; + long numAuctions = (numEvents * auctionProportion) / totalProportion; + long numBids = (numEvents * bidProportion) / totalProportion; + return numPersons * configuration.avgPersonByteSize + + numAuctions * configuration.avgAuctionByteSize + + numBids * configuration.avgBidByteSize; + } + + public int getAvgPersonByteSize() { + return configuration.avgPersonByteSize; + } + + public int getNumActivePeople() { + return configuration.numActivePeople; + } + + public int getHotSellersRatio() { + return configuration.hotSellersRatio; + } + + public int getNumInFlightAuctions() { + return configuration.numInFlightAuctions; + } + + public int getHotAuctionRatio() { + return configuration.hotAuctionRatio; + } + + public int getHotBiddersRatio() { + return configuration.hotBiddersRatio; + } + + public int getAvgBidByteSize() { + return configuration.avgBidByteSize; + } + + public int getAvgAuctionByteSize() { + return configuration.avgAuctionByteSize; + } + + public double getProbDelayedEvent() { + return configuration.probDelayedEvent; + } + + public long getOccasionalDelaySec() { + return configuration.occasionalDelaySec; + } + + /** Return an estimate of the byte-size of all events a generator for this config would yield. */ + public long getEstimatedSizeBytes() { + return estimatedBytesForEvents(maxEvents); + } + + /** + * Return the first 'event id' which could be generated from this config. Though events don't have + * ids we can simulate them to help bookkeeping. + */ + public long getStartEventId() { + return firstEventId + firstEventNumber; + } + + /** Return one past the last 'event id' which could be generated from this config. */ + public long getStopEventId() { + return firstEventId + firstEventNumber + maxEvents; + } + + /** Return the next event number for a generator which has so far emitted {@code numEvents}. */ + public long nextEventNumber(long numEvents) { + return firstEventNumber + numEvents; + } + + /** + * Return the next event number for a generator which has so far emitted {@code numEvents}, but + * adjusted to account for {@code outOfOrderGroupSize}. + */ + public long nextAdjustedEventNumber(long numEvents) { + long n = configuration.outOfOrderGroupSize; + long eventNumber = nextEventNumber(numEvents); + long base = (eventNumber / n) * n; + long offset = (eventNumber * 953) % n; + return base + offset; + } + + /** + * Return the event number who's event time will be a suitable watermark for a generator which has + * so far emitted {@code numEvents}. + */ + public long nextEventNumberForWatermark(long numEvents) { + long n = configuration.outOfOrderGroupSize; + long eventNumber = nextEventNumber(numEvents); + return (eventNumber / n) * n; + } + + /** + * What timestamp should the event with {@code eventNumber} have for this generator? + */ + public long timestampForEvent(long eventNumber) { + return baseTime + (long)(eventNumber * interEventDelayUs[0]) / 1000L; + } + + public boolean isSourceKeepAlive() { + return configuration.isSourceKeepAlive; + } + + public boolean isSourceIgnoreStop() { + return stopAtEvent < 0; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + GeneratorConfig that = (GeneratorConfig) o; + return personProportion == that.personProportion && + auctionProportion == that.auctionProportion && + bidProportion == that.bidProportion && + totalProportion == that.totalProportion && + stepLengthSec == that.stepLengthSec && + firstEventId == that.firstEventId && + maxEvents == that.maxEvents && + firstEventNumber == that.firstEventNumber && + epochPeriodMs == that.epochPeriodMs && + eventsPerEpoch == that.eventsPerEpoch && + Objects.equals(configuration, that.configuration) && + Arrays.equals(interEventDelayUs, that.interEventDelayUs); + } + + @Override + public int hashCode() { + int result = Objects.hash(personProportion, auctionProportion, bidProportion, totalProportion, configuration, stepLengthSec, firstEventId, maxEvents, firstEventNumber, epochPeriodMs, eventsPerEpoch); + result = 31 * result + Arrays.hashCode(interEventDelayUs); + return result; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("GeneratorConfig"); + sb.append("{configuration:"); + sb.append(configuration.toString()); + sb.append(";interEventDelayUs=["); + for (int i = 0; i < interEventDelayUs.length; i++) { + if (i > 0) { + sb.append(","); + } + sb.append(interEventDelayUs[i]); + } + sb.append("]"); + sb.append(";stepLengthSec:"); + sb.append(stepLengthSec); + sb.append(";baseTime:"); + sb.append(baseTime); + sb.append(";firstEventId:"); + sb.append(firstEventId); + sb.append(";maxEvents:"); + sb.append(maxEvents); + sb.append(";firstEventNumber:"); + sb.append(firstEventNumber); + sb.append(";epochPeriodMs:"); + sb.append(epochPeriodMs); + sb.append(";eventsPerEpoch:"); + sb.append(eventsPerEpoch); + sb.append("}"); + return sb.toString(); + } +} diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/generator/NexmarkGenerator.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/generator/NexmarkGenerator.java new file mode 100644 index 0000000..1d3f3e3 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/generator/NexmarkGenerator.java @@ -0,0 +1,228 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.nexmark.flink.generator; + +import com.github.nexmark.flink.generator.model.AuctionGenerator; +import com.github.nexmark.flink.generator.model.PersonGenerator; +import com.github.nexmark.flink.model.Bid; +import com.github.nexmark.flink.model.Event; +import com.github.nexmark.flink.generator.model.BidGenerator; + +import java.io.Serializable; +import java.util.Iterator; +import java.util.Objects; +import java.util.SplittableRandom; + +import static org.apache.flink.util.Preconditions.checkNotNull; + + +/** + * A generator for synthetic events. We try to make the data vaguely reasonable. We also ensure most + * primary key/foreign key relations are correct. Eg: a {@link Bid} event will usually have valid + * auction and bidder ids which can be joined to already-generated Auction and Person events. + * + *

To help with testing, we generate timestamps relative to a given {@code baseTime}. Each new + * event is given a timestamp advanced from the previous timestamp by {@code interEventDelayUs} (in + * microseconds). The event stream is thus fully deterministic and does not depend on wallclock + * time. + */ +public class NexmarkGenerator implements Iterator, Serializable { + + /** + * The next event and its various timestamps. Ordered by increasing wallclock timestamp, then + * (arbitrary but stable) event hash order. + */ + public static class NextEvent implements Comparable { + /** When, in wallclock time, should this event be emitted? */ + public final long wallclockTimestamp; + + /** When, in event time, should this event be considered to have occured? */ + public final long eventTimestamp; + + /** The event itself. */ + public final Event event; + + /** The minimum of this and all future event timestamps. */ + public final long watermark; + + public NextEvent(long wallclockTimestamp, long eventTimestamp, Event event, long watermark) { + this.wallclockTimestamp = wallclockTimestamp; + this.eventTimestamp = eventTimestamp; + this.event = event; + this.watermark = watermark; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + NextEvent nextEvent = (NextEvent) o; + + return (wallclockTimestamp == nextEvent.wallclockTimestamp + && eventTimestamp == nextEvent.eventTimestamp + && watermark == nextEvent.watermark + && event.equals(nextEvent.event)); + } + + @Override + public int hashCode() { + return Objects.hash(wallclockTimestamp, eventTimestamp, watermark, event); + } + + @Override + public int compareTo(NextEvent other) { + int i = Long.compare(wallclockTimestamp, other.wallclockTimestamp); + if (i != 0) { + return i; + } + return Integer.compare(event.hashCode(), other.event.hashCode()); + } + } + + private final SplittableRandom random = new SplittableRandom(); + + /** + * Configuration to generate events against. + */ + private GeneratorConfig config; + + /** Number of events generated by this generator. */ + private long eventsCountSoFar; + + /** Wallclock time at which we emitted the first event (ms since epoch). Initially -1. */ + private long wallclockBaseTime; + + /** The max events that the generator will generate. */ + private long maxEvents; + + public NexmarkGenerator(GeneratorConfig config, long eventsCountSoFar, long wallclockBaseTime) { + checkNotNull(config); + this.config = config; + this.eventsCountSoFar = eventsCountSoFar; + this.wallclockBaseTime = wallclockBaseTime; + this.maxEvents = config.stopAtEvent < 0 ? config.maxEvents : Math.min(config.stopAtEvent, config.maxEvents); + } + + /** Create a fresh generator according to {@code config}. */ + public NexmarkGenerator(GeneratorConfig config) { + this(config, 0, -1); + } + + /** Return a deep copy of this generator. */ + public NexmarkGenerator copy() { + checkNotNull(config); + return new NexmarkGenerator(config, eventsCountSoFar, wallclockBaseTime); + } + + /** + * Return the current config for this generator. + */ + public GeneratorConfig getCurrentConfig() { + return config; + } + + /** + * Return the next 'event id'. Though events don't have ids we can simulate them to help with + * bookkeeping. + */ + public long getNextEventId() { + return config.firstEventId + config.nextAdjustedEventNumber(eventsCountSoFar); + } + + @Override + public boolean hasNext() { + return eventsCountSoFar < maxEvents; + } + + /** + * Return the next event. The outer timestamp is in wallclock time and corresponds to when the + * event should fire. The inner timestamp is in event-time and represents the time the event is + * purported to have taken place in the simulation. + */ + public NextEvent nextEvent() { + if (wallclockBaseTime < 0) { + wallclockBaseTime = System.currentTimeMillis(); + } + // When, in event time, we should generate the event. Monotonic. + long eventTimestamp = config.timestampForEvent(config.nextEventNumber(eventsCountSoFar)); + // When, in event time, the event should say it was generated. Depending on outOfOrderGroupSize + // may have local jitter. + long adjustedEventTimestamp = config.timestampForEvent(config.nextAdjustedEventNumber(eventsCountSoFar)); + // The minimum of this and all future adjusted event timestamps. Accounts for jitter in + // the event timestamp. + long watermark = config.timestampForEvent(config.nextEventNumberForWatermark(eventsCountSoFar)); + // When, in wallclock time, we should emit the event. + long wallclockTimestamp = wallclockBaseTime + (eventTimestamp - getCurrentConfig().baseTime); + + long newEventId = getNextEventId(); + long rem = newEventId % config.totalProportion; + + Event event; + if (rem < config.personProportion) { + event = + new Event(PersonGenerator.nextPerson(newEventId, random, adjustedEventTimestamp, config)); + } else if (rem < config.personProportion + config.auctionProportion) { + event = + new Event( + AuctionGenerator.nextAuction(eventsCountSoFar, newEventId, random, adjustedEventTimestamp, config)); + } else { + event = new Event(BidGenerator.nextBid(newEventId, random, adjustedEventTimestamp, config)); + } + + eventsCountSoFar++; + return new NextEvent(wallclockTimestamp, adjustedEventTimestamp, event, watermark); + } + + @Override + public NexmarkGenerator.NextEvent next() { + return nextEvent(); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + + /** Return an estimate of fraction of output consumed. */ + public double getFractionConsumed() { + return (double) eventsCountSoFar / config.maxEvents; + } + + /** + * Gets Number of events generated by this generator. + */ + public long getEventsCountSoFar() { + return eventsCountSoFar; + } + + public long getWallclockBaseTime() { + return wallclockBaseTime; + } + + @Override + public String toString() { + return String.format( + "Generator{config:%s; eventsCountSoFar:%d; wallclockBaseTime:%d}", + config, eventsCountSoFar, wallclockBaseTime); + } +} diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/generator/SideInputGenerator.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/generator/SideInputGenerator.java new file mode 100644 index 0000000..6577e6e --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/generator/SideInputGenerator.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink.generator; + + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.io.FileUtils; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Write data to be read as a side input. + * + *

Contains pairs of a number and its string representation to model lookups of some enrichment + * data by id. + * + *

Generated data covers the range {@code [0, sideInputRowCount)} so lookup joins on any + * desired id field can be modeled by looking up {@code id % sideInputRowCount}. + */ +public class SideInputGenerator { + + private static final Option PATH = new Option("p", "path", true, + "absolute path of the side input file."); + + private static final Option ROW_COUNT = new Option("n", "num", true, + "row count of side input."); + + public void prepareSideInput(int sideInputRowCount, String path) throws IOException { + List result = new ArrayList<>(); + for (int i = 0; i < sideInputRowCount; i++) { + result.add(i + "," + i); + } + FileUtils.writeLines(new File(path), "UTF-8", result); + } + + public static void main(String[] args) throws IOException, ParseException { + if (args == null || args.length == 0) { + throw new RuntimeException("Usage: -n 1000 -p /path/to/side_input.txt"); + } + Options options = getOptions(); + DefaultParser parser = new DefaultParser(); + CommandLine line = parser.parse(options, args, true); + String path = line.getOptionValue(PATH.getOpt()); + int sideInputRowCount = Integer.parseInt(line.getOptionValue(ROW_COUNT.getOpt())); + SideInputGenerator generator = new SideInputGenerator(); + generator.prepareSideInput(sideInputRowCount, path); + } + + private static Options getOptions() { + Options options = new Options(); + options.addOption(PATH); + options.addOption(ROW_COUNT); + return options; + } +} diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/generator/model/AuctionGenerator.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/generator/model/AuctionGenerator.java new file mode 100644 index 0000000..e24fc11 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/generator/model/AuctionGenerator.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.nexmark.flink.generator.model; + +import com.github.nexmark.flink.model.Auction; +import com.github.nexmark.flink.generator.GeneratorConfig; + +import java.time.Instant; +import java.util.SplittableRandom; + +/** AuctionGenerator. */ +public class AuctionGenerator { + /** + * Keep the number of categories small so the example queries will find results even with a small + * batch of events. + */ + private static final int NUM_CATEGORIES = 5; + + /** Number of yet-to-be-created people and auction ids allowed. */ + private static final int AUCTION_ID_LEAD = 10; + + /** + * Fraction of people/auctions which may be 'hot' sellers/bidders/auctions are 1 over these + * values. + */ + private static final int HOT_SELLER_RATIO = 100; + + /** Generate and return a random auction with next available id. */ + public static Auction nextAuction( + long eventsCountSoFar, long eventId, SplittableRandom random, long timestamp, GeneratorConfig config) { + + long id = lastBase0AuctionId(config, eventId) + GeneratorConfig.FIRST_AUCTION_ID; + + long seller; + // Here P(auction will be for a hot seller) = 1 - 1/hotSellersRatio. + if (random.nextInt(config.getHotSellersRatio()) > 0) { + // Choose the first person in the batch of last HOT_SELLER_RATIO people. + seller = (PersonGenerator.lastBase0PersonId(config, eventId) / HOT_SELLER_RATIO) * HOT_SELLER_RATIO; + } else { + seller = PersonGenerator.nextBase0PersonId(eventId, random, config); + } + seller += GeneratorConfig.FIRST_PERSON_ID; + + long category = GeneratorConfig.FIRST_CATEGORY_ID + random.nextInt(NUM_CATEGORIES); + long initialBid = PriceGenerator.nextPrice(random); + long expires = timestamp + nextAuctionLengthMs(eventsCountSoFar, random, timestamp, config); + String name = StringsGenerator.nextString(random, 20); + String desc = StringsGenerator.nextString(random, 100); + long reserve = initialBid + PriceGenerator.nextPrice(random); + int currentSize = 8 + name.length() + desc.length() + 8 + 8 + 8 + 8 + 8; + String extra = StringsGenerator.nextExtra(random, currentSize, config.getAvgAuctionByteSize()); + return new Auction( + id, + name, + desc, + initialBid, + reserve, + Instant.ofEpochMilli(timestamp), + Instant.ofEpochMilli(expires), + seller, + category, + extra); + } + + /** + * Return the last valid auction id (ignoring FIRST_AUCTION_ID). Will be the current auction id if + * due to generate an auction. + */ + public static long lastBase0AuctionId(GeneratorConfig config, long eventId) { + long epoch = eventId / config.totalProportion; + long offset = eventId % config.totalProportion; + if (offset < config.personProportion) { + // About to generate a person. + // Go back to the last auction in the last epoch. + epoch--; + offset = config.auctionProportion - 1; + } else if (offset >= config.personProportion + config.auctionProportion) { + // About to generate a bid. + // Go back to the last auction generated in this epoch. + offset = config.auctionProportion - 1; + } else { + // About to generate an auction. + offset -= config.personProportion; + } + return epoch * config.auctionProportion + offset; + } + + /** Return a random auction id (base 0). */ + public static long nextBase0AuctionId(long nextEventId, SplittableRandom random, GeneratorConfig config) { + + // Choose a random auction for any of those which are likely to still be in flight, + // plus a few 'leads'. + // Note that ideally we'd track non-expired auctions exactly, but that state + // is difficult to split. + long minAuction = + Math.max(lastBase0AuctionId(config, nextEventId) - config.getNumInFlightAuctions(), 0); + long maxAuction = lastBase0AuctionId(config, nextEventId); + return minAuction + LongGenerator.nextLong(random, maxAuction - minAuction + 1 + AUCTION_ID_LEAD); + } + + /** Return a random time delay, in milliseconds, for length of auctions. */ + private static long nextAuctionLengthMs( + long eventsCountSoFar, SplittableRandom random, long timestamp, GeneratorConfig config) { + + // What's our current event number? + long currentEventNumber = config.nextAdjustedEventNumber(eventsCountSoFar); + // How many events till we've generated numInFlightAuctions? + long numEventsForAuctions = + ((long) config.getNumInFlightAuctions() * config.totalProportion) + / config.auctionProportion; + // When will the auction numInFlightAuctions beyond now be generated? + long futureAuction = config.timestampForEvent(currentEventNumber + numEventsForAuctions); + // System.out.printf("*** auction will be for %dms (%d events ahead) ***\n", + // futureAuction - timestamp, numEventsForAuctions); + // Choose a length with average horizonMs. + long horizonMs = futureAuction - timestamp; + return 1L + LongGenerator.nextLong(random, Math.max(horizonMs * 2, 1L)); + } +} diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/generator/model/BidGenerator.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/generator/model/BidGenerator.java new file mode 100644 index 0000000..60678b1 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/generator/model/BidGenerator.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.nexmark.flink.generator.model; + +import com.github.nexmark.flink.generator.GeneratorConfig; +import com.github.nexmark.flink.model.Bid; + +import java.util.SplittableRandom; +import java.time.Instant; + +import static com.github.nexmark.flink.generator.model.StringsGenerator.nextString; + +/** Generates bids. */ +public class BidGenerator { + + /** + * Fraction of people/auctions which may be 'hot' sellers/bidders/auctions are 1 over these + * values. + */ + private static final int HOT_AUCTION_RATIO = 100; + + private static final int HOT_BIDDER_RATIO = 100; + + private static final int HOT_CHANNELS_RATIO = 2; + + private static final int CHANNELS_NUMBER = 10_000; + + private static final String[] HOT_CHANNELS = new String[] {"Google", "Facebook", "Baidu", "Apple"}; + private static final SplittableRandom random = new SplittableRandom(); + private static final String[] HOT_URLS = new String[] {getBaseUrl(random), getBaseUrl(random), getBaseUrl(random), getBaseUrl(random)}; + + private static final Tuple2[] CHANNEL_URL_CACHE = createChannelUrlCache(random); + + private static class Tuple2 { + public final T1 f0; + public final T2 f1; + public Tuple2(T1 f0, T2 f1){ + this.f0 = f0; + this.f1 = f1; + } + } + + /** Generate and return a random bid with next available id. */ + public static Bid nextBid(long eventId, SplittableRandom random, long timestamp, GeneratorConfig config) { + + long auction; + // Here P(bid will be for a hot auction) = 1 - 1/hotAuctionRatio. + if (random.nextInt(config.getHotAuctionRatio()) > 0) { + // Choose the first auction in the batch of last HOT_AUCTION_RATIO auctions. + auction = (AuctionGenerator.lastBase0AuctionId(config, eventId) / HOT_AUCTION_RATIO) * HOT_AUCTION_RATIO; + } else { + auction = AuctionGenerator.nextBase0AuctionId(eventId, random, config); + } + auction += GeneratorConfig.FIRST_AUCTION_ID; + + long bidder; + // Here P(bid will be by a hot bidder) = 1 - 1/hotBiddersRatio + if (random.nextInt(config.getHotBiddersRatio()) > 0) { + // Choose the second person (so hot bidders and hot sellers don't collide) in the batch of + // last HOT_BIDDER_RATIO people. + bidder = (PersonGenerator.lastBase0PersonId(config, eventId) / HOT_BIDDER_RATIO) * HOT_BIDDER_RATIO + 1; + } else { + bidder = PersonGenerator.nextBase0PersonId(eventId, random, config); + } + bidder += GeneratorConfig.FIRST_PERSON_ID; + + long price = PriceGenerator.nextPrice(random); + + String channel; + String url; + if (random.nextInt(HOT_CHANNELS_RATIO) > 0) { + int i = random.nextInt(HOT_CHANNELS.length); + channel = HOT_CHANNELS[i]; + url = HOT_URLS[i]; + } else { + Tuple2 channelAndUrl = getNextChannelAndurl(random); + channel = channelAndUrl.f0; + url = channelAndUrl.f1; + } + + bidder += GeneratorConfig.FIRST_PERSON_ID; + + int currentSize = 8 + 8 + 8 + 8; + String extra = StringsGenerator.nextExtra(random, currentSize, config.getAvgBidByteSize()); + return new Bid(auction, bidder, price, channel, url, Instant.ofEpochMilli(timestamp), extra); + } + + private static String getBaseUrl(SplittableRandom random) { + return "https://www.nexmark.com/" + + nextString(random, 5, '_') + '/' + + nextString(random, 5, '_') + '/' + + nextString(random, 5, '_') + '/' + + "item.htm?query=1"; + } + + private static Tuple2[] createChannelUrlCache(SplittableRandom random) { + Tuple2[] cache = new Tuple2[CHANNELS_NUMBER]; + for (int i = 0; i < CHANNELS_NUMBER; ++i) { + String url = getBaseUrl(random); + if (random.nextInt(10) > 0) { + url = url + "&channel_id=" + Math.abs(Integer.reverse(i)); + } + cache[i] = new Tuple2<>("channel-" + i, url); + } + return cache; + } + + private static Tuple2 getNextChannelAndurl(SplittableRandom random) { + int channelNumber = random.nextInt(CHANNELS_NUMBER); + return CHANNEL_URL_CACHE[channelNumber]; + } +} diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/generator/model/LongGenerator.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/generator/model/LongGenerator.java new file mode 100644 index 0000000..5148e32 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/generator/model/LongGenerator.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.nexmark.flink.generator.model; + +import java.util.SplittableRandom; + +/** LongGenerator. */ +public class LongGenerator { + + /** Return a random long from {@code [0, n)}. */ + public static long nextLong(SplittableRandom random, long n) { + if (n < Integer.MAX_VALUE) { + return random.nextInt((int) n); + } else { + // WARNING: Very skewed distribution! Bad! + return Math.abs(random.nextLong() % n); + } + } +} diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/generator/model/PersonGenerator.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/generator/model/PersonGenerator.java new file mode 100644 index 0000000..3ed7bc4 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/generator/model/PersonGenerator.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.nexmark.flink.generator.model; + +import com.github.nexmark.flink.model.Person; +import com.github.nexmark.flink.generator.GeneratorConfig; + +import java.time.Instant; +import java.util.Arrays; +import java.util.List; +import java.util.SplittableRandom; + +import static com.github.nexmark.flink.generator.model.StringsGenerator.nextExtra; +import static com.github.nexmark.flink.generator.model.StringsGenerator.nextString; + +/** Generates people. */ +public class PersonGenerator { + /** Number of yet-to-be-created people and auction ids allowed. */ + private static final int PERSON_ID_LEAD = 10; + + /** + * Keep the number of states small so that the example queries will find results even with a small + * batch of events. + */ + private static final List US_STATES = Arrays.asList("AZ,CA,ID,OR,WA,WY".split(",")); + + private static final List US_CITIES = + Arrays.asList( + "Phoenix,Los Angeles,San Francisco,Boise,Portland,Bend,Redmond,Seattle,Kent,Cheyenne" + .split(",")); + + private static final List FIRST_NAMES = + Arrays.asList("Peter,Paul,Luke,John,Saul,Vicky,Kate,Julie,Sarah,Deiter,Walter".split(",")); + + private static final List LAST_NAMES = + Arrays.asList("Shultz,Abrams,Spencer,White,Bartels,Walton,Smith,Jones,Noris".split(",")); + + private static final String[] CREDIT_CARD_STRINGS = createCreditCardStrings(); + + /** Generate and return a random person with next available id. */ + public static Person nextPerson( + long nextEventId, SplittableRandom random, long timestamp, GeneratorConfig config) { + + long id = lastBase0PersonId(config, nextEventId) + GeneratorConfig.FIRST_PERSON_ID; + String name = nextPersonName(random); + String email = nextEmail(random); + String creditCard = nextCreditCard(random); + String city = nextUSCity(random); + String state = nextUSState(random); + int currentSize = + 8 + name.length() + email.length() + creditCard.length() + city.length() + state.length(); + String extra = nextExtra(random, currentSize, config.getAvgPersonByteSize()); + return new Person(id, name, email, creditCard, city, state, Instant.ofEpochMilli(timestamp), extra); + } + + /** Return a random person id (base 0). */ + public static long nextBase0PersonId(long eventId, SplittableRandom random, GeneratorConfig config) { + // Choose a random person from any of the 'active' people, plus a few 'leads'. + // By limiting to 'active' we ensure the density of bids or auctions per person + // does not decrease over time for long running jobs. + // By choosing a person id ahead of the last valid person id we will make + // newPerson and newAuction events appear to have been swapped in time. + long numPeople = lastBase0PersonId(config, eventId) + 1; + long activePeople = Math.min(numPeople, config.getNumActivePeople()); + long n = LongGenerator.nextLong(random, activePeople + PERSON_ID_LEAD); + return numPeople - activePeople + n; + } + + /** + * Return the last valid person id (ignoring FIRST_PERSON_ID). Will be the current person id if + * due to generate a person. + */ + public static long lastBase0PersonId(GeneratorConfig config, long eventId) { + long epoch = eventId / config.totalProportion; + long offset = eventId % config.totalProportion; + if (offset >= config.personProportion) { + // About to generate an auction or bid. + // Go back to the last person generated in this epoch. + offset = config.personProportion - 1; + } + // About to generate a person. + return epoch * config.personProportion + offset; + } + + /** return a random US state. */ + private static String nextUSState(SplittableRandom random) { + return US_STATES.get(random.nextInt(US_STATES.size())); + } + + /** Return a random US city. */ + private static String nextUSCity(SplittableRandom random) { + return US_CITIES.get(random.nextInt(US_CITIES.size())); + } + + /** Return a random person name. */ + private static String nextPersonName(SplittableRandom random) { + return FIRST_NAMES.get(random.nextInt(FIRST_NAMES.size())) + + " " + + LAST_NAMES.get(random.nextInt(LAST_NAMES.size())); + } + + /** Return a random email address. */ + private static String nextEmail(SplittableRandom random) { + return nextString(random, 7) + "@" + nextString(random, 5) + ".com"; + } + + private static String[] createCreditCardStrings() { + String[] creditCardStrings = new String[10000]; + for (int i = 0; i < creditCardStrings.length; ++i) { + creditCardStrings[i] = String.format("%04d", i); + } + return creditCardStrings; + } + + /** Return a random credit card number. */ + private static String nextCreditCard(SplittableRandom random) { + StringBuilder sb = new StringBuilder(20); + for (int i = 0; i < 4; i++) { + if (i > 0) { + sb.append(' '); + } + sb.append(CREDIT_CARD_STRINGS[random.nextInt(CREDIT_CARD_STRINGS.length)]); + } + return sb.toString(); + } +} diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/generator/model/PriceGenerator.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/generator/model/PriceGenerator.java new file mode 100644 index 0000000..b12c08d --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/generator/model/PriceGenerator.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.nexmark.flink.generator.model; + +import java.util.SplittableRandom; + +/** Generates a random price. */ +public class PriceGenerator { + + /** Return a random price. */ + public static long nextPrice(SplittableRandom random) { + return Math.round(Math.pow(10.0, random.nextDouble() * 6.0) * 100.0); + } +} diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/generator/model/StringsGenerator.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/generator/model/StringsGenerator.java new file mode 100644 index 0000000..e3cad32 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/generator/model/StringsGenerator.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.nexmark.flink.generator.model; + +import java.util.SplittableRandom; + +/** Generates strings which are used for different field in other model objects. */ +public class StringsGenerator { + + /** Smallest random string size. */ + private static final int MIN_STRING_LENGTH = 3; + + private static final String REUSABLE_EXTRA_STRING = nextExactString(new SplittableRandom(), 1024 * 1024); + + /** Return a random string of up to {@code maxLength}. */ + public static String nextString(SplittableRandom random, int maxLength) { + return nextString(random, maxLength, ' '); + } + + public static String nextString(SplittableRandom random, int maxLength, char special) { + int len = MIN_STRING_LENGTH + random.nextInt(maxLength - MIN_STRING_LENGTH); + StringBuilder sb = new StringBuilder(len); + while (len-- > 0) { + if (random.nextInt(13) == 0) { + sb.append(special); + } else { + sb.append((char) ('a' + random.nextInt(26))); + } + } + return sb.toString().trim(); + } + + /** Return a random string of exactly {@code length}. */ + public static String nextExactString(SplittableRandom random, int length) { + if (REUSABLE_EXTRA_STRING != null && length < REUSABLE_EXTRA_STRING.length() / 2) { + int offset = random.nextInt(REUSABLE_EXTRA_STRING.length() - length); + return REUSABLE_EXTRA_STRING.substring(offset, offset + length); + } + + StringBuilder sb = new StringBuilder(length); + int rnd = 0; + int n = 0; // number of random characters left in rnd + while (length-- > 0) { + if (n == 0) { + rnd = random.nextInt(); + n = 6; // log_26(2^31) + } + sb.append((char) ('a' + rnd % 26)); + rnd /= 26; + n--; + } + return sb.toString(); + } + + /** + * Return a random {@code string} such that {@code currentSize + string.length()} is on average + * {@code averageSize}. + */ + public static String nextExtra(SplittableRandom random, int currentSize, int desiredAverageSize) { + if (currentSize > desiredAverageSize) { + return ""; + } + desiredAverageSize -= currentSize; + int delta = (int) Math.round(desiredAverageSize * 0.2); + int minSize = desiredAverageSize - delta; + int desiredSize = minSize + (delta == 0 ? 0 : random.nextInt(2 * delta)); + return nextExactString(random, desiredSize); + } +} diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/BenchmarkMetric.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/BenchmarkMetric.java new file mode 100644 index 0000000..ae79428 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/BenchmarkMetric.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink.metric; + +import java.text.DecimalFormat; +import java.text.NumberFormat; +import java.util.Map; +import java.util.NavigableMap; +import java.util.Objects; +import java.util.TreeMap; + +public class BenchmarkMetric { + private final long timestamp; + private final double tps; + private final double cpu; + private final long rss; + private final long vmem; + private final long netBytesRead; + private final long netBytesWritten; + private final long diskBytesRead; + private final long diskBytesWritten; + + /** Full constructor with all metrics. */ + public BenchmarkMetric( + long timestamp, + double tps, + double cpu, + long rss, + long vmem, + long netBytesRead, + long netBytesWritten, + long diskBytesRead, + long diskBytesWritten) { + this.timestamp = timestamp; + this.tps = tps; + this.cpu = cpu; + this.rss = rss; + this.vmem = vmem; + this.netBytesRead = netBytesRead; + this.netBytesWritten = netBytesWritten; + this.diskBytesRead = diskBytesRead; + this.diskBytesWritten = diskBytesWritten; + } + + public long getTimestamp() { + return timestamp; + } + + public double getTps() { + return tps; + } + + public String getPrettyTps() { + return formatLongValue((long) tps); + } + + public double getCpu() { + return cpu; + } + + public String getPrettyCpu() { + return NUMBER_FORMAT.format(cpu); + } + + public long getRss() { + return rss; + } + + public String getPrettyRss() { + return formatLongValue(rss); + } + + public long getVmem() { + return vmem; + } + + public String getPrettyVmem() { + return formatLongValue(vmem); + } + + public long getNetBytesRead() { + return netBytesRead; + } + + public long getNetBytesWritten() { + return netBytesWritten; + } + + public long getDiskBytesRead() { + return diskBytesRead; + } + + public long getDiskBytesWritten() { + return diskBytesWritten; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + BenchmarkMetric that = (BenchmarkMetric) o; + return timestamp == that.timestamp && + Double.compare(that.tps, tps) == 0 && + Double.compare(that.cpu, cpu) == 0 && + rss == that.rss && + vmem == that.vmem && + netBytesRead == that.netBytesRead && + netBytesWritten == that.netBytesWritten && + diskBytesRead == that.diskBytesRead && + diskBytesWritten == that.diskBytesWritten; + } + + @Override + public int hashCode() { + return Objects.hash(timestamp, tps, cpu, rss, vmem, netBytesRead, netBytesWritten, diskBytesRead, diskBytesWritten); + } + + @Override + public String toString() { + return "BenchmarkMetric{" + + "timestamp=" + timestamp + + ", tps=" + tps + + ", cpu=" + cpu + + ", rss=" + rss + + ", vmem=" + vmem + + ", netBytesRead=" + netBytesRead + + ", netBytesWritten=" + netBytesWritten + + ", diskBytesRead=" + diskBytesRead + + ", diskBytesWritten=" + diskBytesWritten + + '}'; + } + + + // ------------------------------------------------------------------------------------------- + // Pretty Utilities + // ------------------------------------------------------------------------------------------- + public static final NumberFormat NUMBER_FORMAT = NumberFormat.getInstance(); + private static final NavigableMap SUFFIXES = new TreeMap<>(); + static { + SUFFIXES.put(1_000L, "K"); + SUFFIXES.put(1_000_000L, "M"); + SUFFIXES.put(1_000_000_000L, "G"); + SUFFIXES.put(1_000_000_000_000L, "T"); + SUFFIXES.put(1_000_000_000_000_000L, "P"); + SUFFIXES.put(1_000_000_000_000_000_000L, "E"); + NUMBER_FORMAT.setMaximumFractionDigits(2); + } + + public static String formatLongValuePerSecond(long value) { + return formatLongValue(value) + "/s"; + } + + public static String formatLongValue(long value) { + //Long.MIN_VALUE == -Long.MIN_VALUE so we need an adjustment here + if (value == Long.MIN_VALUE) return formatLongValue(Long.MIN_VALUE + 1); + if (value < 0) return "-" + formatLongValue(-value); + if (value < 1000) return Long.toString(value); //deal with easy case + + Map.Entry e = SUFFIXES.floorEntry(value); + Long divideBy = e.getKey(); + String suffix = e.getValue(); + + DecimalFormat format = new DecimalFormat("0.##"); + return format.format(value / (double) divideBy) + " " + suffix; + } + + public static String formatDoubleValue(double value) { + return String.format("%.3f", value); + } +} diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/FlinkRestClient.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/FlinkRestClient.java new file mode 100644 index 0000000..bb30d47 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/FlinkRestClient.java @@ -0,0 +1,355 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink.metric; + +import com.github.nexmark.flink.metric.tps.TpsMetric; +import com.github.nexmark.flink.utils.NexmarkUtils; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonProcessingException; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ArrayNode; +import org.apache.http.Consts; +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.HttpStatus; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPatch; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpRequestBase; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.util.EntityUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.URI; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import static org.apache.flink.util.Preconditions.checkArgument; + +/** + * A HTTP client to request TPS metric to JobMaster REST API. + */ +public class FlinkRestClient { + + private static final Logger LOG = LoggerFactory.getLogger(FlinkRestClient.class); + + private static int CONNECT_TIMEOUT = 5000; + private static int SOCKET_TIMEOUT = 60000; + private static int CONNECTION_REQUEST_TIMEOUT = 10000; + private static int MAX_IDLE_TIME = 60000; + private static int MAX_CONN_TOTAL = 60; + private static int MAX_CONN_PER_ROUTE = 30; + + private final String jmEndpoint; + private final CloseableHttpClient httpClient; + private final Map jobIds; + private volatile String lastJobId; + + public FlinkRestClient(String jmAddress, int jmPort) { + this.jmEndpoint = jmAddress + ":" + jmPort; + + RequestConfig requestConfig = RequestConfig.custom() + .setSocketTimeout(SOCKET_TIMEOUT) + .setConnectTimeout(CONNECT_TIMEOUT) + .setConnectionRequestTimeout(CONNECTION_REQUEST_TIMEOUT) + .build(); + PoolingHttpClientConnectionManager httpClientConnectionManager = new PoolingHttpClientConnectionManager(); + httpClientConnectionManager.setValidateAfterInactivity(MAX_IDLE_TIME); + httpClientConnectionManager.setDefaultMaxPerRoute(MAX_CONN_PER_ROUTE); + httpClientConnectionManager.setMaxTotal(MAX_CONN_TOTAL); + + this.httpClient = HttpClientBuilder.create() + .setConnectionManager(httpClientConnectionManager) + .setDefaultRequestConfig(requestConfig) + .build(); + + this.jobIds = new ConcurrentHashMap<>(50); + this.lastJobId = ""; + } + + public synchronized void updateAllJobStatus() { + String url = String.format("http://%s/jobs", jmEndpoint); + String response = executeAsString(url); + try { + JsonNode jsonNode = NexmarkUtils.MAPPER.readTree(response); + JsonNode jobs = jsonNode.get("jobs"); + for (JsonNode job : jobs) { + String id = job.get("id").asText(); + if (jobIds.put(id, job.get("status").asText()) == null) { + lastJobId = id; + } + } + } catch (JsonProcessingException e) { + throw new RuntimeException("The response is not a valid JSON string:\n" + response, e); + } + } + + public void cancelJob(String jobId) { + LOG.info("Stopping Job: {}", jobId); + String url = String.format("http://%s/jobs/%s?mode=cancel", jmEndpoint, jobId); + patch(url); + } + + public String triggerCheckpoint(String jobId) { + String url = String.format("http://%s/jobs/%s/checkpoints", jmEndpoint, jobId); + String data = "{\"checkpointType\":\"CONFIGURED\"}"; + String response = post(url, data); + try { + JsonNode jsonNode = NexmarkUtils.MAPPER.readTree(response); + return jsonNode.get("request-id").asText(); + } catch (Exception e) { + throw new RuntimeException("The response is not a valid JSON string:\n" + response, e); + } + } + + public String stopWithSavepoint(String jobId) { + String url = String.format("http://%s/jobs/%s/stop", jmEndpoint, jobId); + String data = "{\"formatType\":\"NATIVE\", \"drain\":\"true\"}"; + String response = post(url, data); + try { + JsonNode jsonNode = NexmarkUtils.MAPPER.readTree(response); + return jsonNode.get("request-id").asText(); + } catch (Exception e) { + throw new RuntimeException("The response is not a valid JSON string:\n" + response, e); + } + } + + public String getCurrentJobId() { + updateAllJobStatus(); + return lastJobId; + } + + public synchronized boolean isJobRunning(String jobId, long readCount) { + String url = String.format("http://%s/jobs/%s", jmEndpoint, jobId); + String response = executeAsString(url); + try { + JsonNode jsonNode = NexmarkUtils.MAPPER.readTree(response); + String state = jsonNode.get("state").asText(); + if (!state.equalsIgnoreCase("RUNNING")) { + return false; + } + JsonNode vertices = jsonNode.get("vertices"); + if (vertices.isEmpty()) { + return false; + } + return vertices.get(0).get("metrics").get("read-records").asLong() < readCount; + } catch (JsonProcessingException e) { + throw new RuntimeException("The response is not a valid JSON string:\n" + response, e); + } + } + + public synchronized boolean isJobAndAllTasksRunning(String jobId) { + String url = String.format("http://%s/jobs/%s", jmEndpoint, jobId); + String response = executeAsString(url); + try { + JsonNode jsonNode = NexmarkUtils.MAPPER.readTree(response); + String state = jsonNode.get("state").asText(); + if (!state.equalsIgnoreCase("RUNNING")) { + return false; + } + JsonNode vertices = jsonNode.get("vertices"); + if (vertices.isEmpty()) { + return false; + } + for (JsonNode vertex : vertices) { + String status = vertex.get("status").asText().toUpperCase(); + if (status.equals("CANCELING") || status.equals("FAILED") || status.equals("CANCELED")) { + throw new RuntimeException("There is one task failed, canceling or canceled."); + } else if (!status.equals("RUNNING") && !status.equals("FINISHED")) { + return false; + } + } + return true; + } catch (JsonProcessingException e) { + throw new RuntimeException("The response is not a valid JSON string:\n" + response, e); + } + } + + public boolean isJobRunning(String jobId) { + updateAllJobStatus(); + return !isNullOrEmpty(jobId) && jobIds.get(jobId).equalsIgnoreCase("RUNNING"); + } + + public boolean isJobCanceledOrFinished(String jobId) { + updateAllJobStatus(); + if (!isNullOrEmpty(jobId)) { + String status = jobIds.get(jobId); + return status.equalsIgnoreCase("CANCELED") || status.equalsIgnoreCase("FINISHED"); + } + return true; + } + + private static boolean isNullOrEmpty(String string) { + return string == null || string.length() == 0; + } + + public String getSourceVertexId(String jobId) { + String url = String.format("http://%s/jobs/%s", jmEndpoint, jobId); + String response = executeAsString(url); + try { + JsonNode jsonNode = NexmarkUtils.MAPPER.readTree(response); + JsonNode vertices = jsonNode.get("vertices"); + JsonNode sourceVertex = vertices.get(0); + checkArgument( + sourceVertex.get("name").asText().startsWith("Source:"), + "The first vertex is not a source."); + return sourceVertex.get("id").asText(); + } catch (Exception e) { + throw new RuntimeException("The response is not a valid JSON string:\n" + response, e); + } + } + + public String getTpsMetricName(String jobId, String vertexId) { + String url = String.format("http://%s/jobs/%s/vertices/%s/subtasks/metrics", jmEndpoint, jobId, vertexId); + String response = executeAsString(url); + try { + ArrayNode arrayNode = (ArrayNode) NexmarkUtils.MAPPER.readTree(response); + for (JsonNode node : arrayNode) { + String metricName = node.get("id").asText(); + if (metricName.startsWith("Source_") && metricName.endsWith(".numRecordsOutPerSecond")) { + return metricName; + } + } + } catch (Exception e) { + throw new RuntimeException("The response is not a valid JSON string:\n" + response, e); + } + throw new RuntimeException("Can't find TPS metric name from the response:\n" + response); + } + + public synchronized TpsMetric getTpsMetric(String jobId, String vertexId, String tpsMetricName) { + String url = String.format( + "http://%s/jobs/%s/vertices/%s/subtasks/metrics?get=%s", + jmEndpoint, + jobId, + vertexId, + tpsMetricName); + String response = executeAsString(url); + return TpsMetric.fromJson(response); + } + + public Savepoint.Status checkCheckpointFinished(String jobId, String triggerId) { + String url = String.format("http://%s/jobs/%s/checkpoints/%s", jmEndpoint, jobId, triggerId); + String response = executeAsString(url); + try { + JsonNode jsonNode = NexmarkUtils.MAPPER.readTree(response); + String status = jsonNode.get("status").get("id").asText(); + return Savepoint.Status.valueOf(status); + } catch (Throwable e) { + throw new RuntimeException("The response is not a valid JSON string:\n" + response, e); + } + } + + public Savepoint getJobLastCheckpoint(String jobId) { + String url = String.format("http://%s/jobs/%s/checkpoints", jmEndpoint, jobId); + String response = executeAsString(url); + try { + JsonNode jsonNode = NexmarkUtils.MAPPER.readTree(response); + return new Savepoint( + Savepoint.Status.valueOf(jsonNode.get("latest").get("completed").get("status").asText()), + jsonNode.get("latest").get("completed").get("external_path").asText()); + } catch (Throwable e) { + throw new RuntimeException("The response is not a valid JSON string:\n" + response, e); + } + } + + private void patch(String url) { + HttpPatch httpPatch = new HttpPatch(); + httpPatch.setURI(URI.create(url)); + HttpResponse response; + try { + httpPatch.setHeader("Connection", "close"); + response = httpClient.execute(httpPatch); + int httpCode = response.getStatusLine().getStatusCode(); + if (httpCode != HttpStatus.SC_ACCEPTED) { + String msg = String.format("http execute failed,status code is %d", httpCode); + throw new RuntimeException(msg); + } + } catch (Exception e) { + httpPatch.abort(); + throw new RuntimeException(e); + } + } + + private String post(String url, String data) { + HttpPost httpPost = new HttpPost(); + httpPost.setURI(URI.create(url)); + HttpResponse response; + try { + httpPost.setHeader("Connection", "close"); + httpPost.setEntity(new StringEntity(data)); + response = httpClient.execute(httpPost); + int httpCode = response.getStatusLine().getStatusCode(); + if (httpCode != HttpStatus.SC_ACCEPTED) { + String msg = String.format("http execute failed, status code is %d, response: %s", httpCode, EntityUtils.toString(response.getEntity())); + throw new RuntimeException(msg); + } else { + return EntityUtils.toString(response.getEntity()); + } + } catch (Exception e) { + httpPost.abort(); + throw new RuntimeException(e); + } + } + + private String executeAsString(String url) { + HttpGet httpGet = new HttpGet(); + httpGet.setURI(URI.create(url)); + try { + HttpEntity entity = execute(httpGet).getEntity(); + if (entity != null) { + return EntityUtils.toString(entity, Consts.UTF_8); + } + } catch (Exception e) { + throw new RuntimeException("Failed to request URL " + url, e); + } + throw new RuntimeException(String.format("Response of URL %s is null.", url)); + } + + private HttpResponse execute(HttpRequestBase httpRequestBase) throws Exception { + HttpResponse response; + try { + httpRequestBase.setHeader("Connection", "close"); + response = httpClient.execute(httpRequestBase); + int httpCode = response.getStatusLine().getStatusCode(); + if (httpCode != HttpStatus.SC_OK) { + String msg = String.format("http execute failed,status code is %d", httpCode); + throw new RuntimeException(msg); + } + return response; + } catch (Exception e) { + httpRequestBase.abort(); + throw e; + } + } + + public synchronized void close() { + try { + if (httpClient != null) { + httpClient.close(); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/JobBenchmarkMetric.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/JobBenchmarkMetric.java new file mode 100644 index 0000000..7507dcf --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/JobBenchmarkMetric.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink.metric; + +import java.util.Objects; + +import static com.github.nexmark.flink.metric.BenchmarkMetric.NUMBER_FORMAT; +import static com.github.nexmark.flink.metric.BenchmarkMetric.formatLongValue; + +public class JobBenchmarkMetric { + private final double tps; + private final double cpu; + private final long eventsNum; + private final long timeMills; + private final long jobInitializedTimeMills; + + public JobBenchmarkMetric(double tps, double cpu) { + this(tps, cpu, 0, 0, 0); + } + + public JobBenchmarkMetric(double tps, double cpu, long eventsNum, long timeMills, long jobInitializedTimeMills) { + this.tps = tps; + this.eventsNum = eventsNum; + this.cpu = cpu; + this.timeMills = timeMills; + this.jobInitializedTimeMills = jobInitializedTimeMills; + } + + public String getPrettyTps() { + return formatLongValue((long) tps); + } + + public long getEventsNum() { + return eventsNum; + } + + public double getTimeSeconds() { + return timeMills / 1000D; + } + + public double getJobInitializedTimeSeconds() { + return jobInitializedTimeMills / 1000D; + } + + public String getPrettyCpu() { + return NUMBER_FORMAT.format(cpu); + } + + public double getCpu() { + return cpu; + } + + public String getPrettyTpsPerCore() { + return formatLongValue(getTpsPerCore()); + } + + public long getTpsPerCore() { + return (long) (tps / cpu); + } + + public double getCoresMultiplyTimeSeconds() { + return cpu * getTimeSeconds(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + JobBenchmarkMetric that = (JobBenchmarkMetric) o; + return Double.compare(that.tps, tps) == 0 && + eventsNum == that.eventsNum && + Double.compare(that.cpu, cpu) == 0 && + timeMills == that.timeMills; + } + + @Override + public int hashCode() { + return Objects.hash(tps, eventsNum, cpu, timeMills); + } + + @Override + public String toString() { + return "BenchmarkMetric{" + + "tps=" + tps + + ", eventsNum=" + eventsNum + + ", cpu=" + cpu + + ", timeMills=" + timeMills + + '}'; + } +} diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/MetricReporter.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/MetricReporter.java new file mode 100644 index 0000000..916d630 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/MetricReporter.java @@ -0,0 +1,314 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink.metric; + +import org.apache.flink.api.common.time.Deadline; +import org.apache.flink.api.java.tuple.Tuple2; + +import com.github.nexmark.flink.metric.process.ProcessMetricReceiver; +import com.github.nexmark.flink.metric.tps.TpsMetric; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import static com.github.nexmark.flink.metric.BenchmarkMetric.NUMBER_FORMAT; +import static com.github.nexmark.flink.metric.BenchmarkMetric.formatDoubleValue; + +/** + * A reporter to aggregate metrics and report summary results. + */ +public class MetricReporter { + + private static final Logger LOG = LoggerFactory.getLogger(MetricReporter.class); + + private final Duration monitorDelay; + private final Duration monitorInterval; + private final Duration monitorDuration; + private final FlinkRestClient flinkRestClient; + private final ProcessMetricReceiver processMetricReceiver; + private final List metrics; + private final ScheduledExecutorService service = Executors.newScheduledThreadPool(1); + private volatile Throwable error; + + public MetricReporter(FlinkRestClient flinkRestClient, ProcessMetricReceiver processMetricReceiver, Duration monitorDelay, Duration monitorInterval, Duration monitorDuration) { + this.monitorDelay = monitorDelay; + this.monitorInterval = monitorInterval; + this.monitorDuration = monitorDuration; + this.flinkRestClient = flinkRestClient; + this.processMetricReceiver = processMetricReceiver; + this.metrics = Collections.synchronizedList(new ArrayList<>()); + } + + private void submitMonitorThread(String jobId, long eventsNum) { + + String vertexId; + String metricName; + + while (true) { + Tuple2 jobInfo = getJobInformation(jobId); + if (jobInfo != null) { + vertexId = jobInfo.f0; + metricName = jobInfo.f1; + break; + } else { + // wait for the job startup + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + } + + this.service.scheduleWithFixedDelay( + new MetricCollector(jobId, vertexId, metricName, eventsNum), + 0L, + monitorInterval.toMillis(), + TimeUnit.MILLISECONDS + ); + } + + private Tuple2 getJobInformation(String jobId) { + try { + String vertexId = flinkRestClient.getSourceVertexId(jobId); + String metricName = flinkRestClient.getTpsMetricName(jobId, vertexId); + return Tuple2.of(vertexId, metricName); + } catch (Exception e) { + LOG.warn("Job metric is not ready yet.", e); + return null; + } + } + + private void waitFor(Duration duration) { + Deadline deadline = Deadline.fromNow(duration); + while (deadline.hasTimeLeft()) { + try { + Thread.sleep(100L); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + if (error != null) { + throw new RuntimeException(error); + } + } + } + + private boolean isJobRunning(String jobId) { + return flinkRestClient.isJobRunning(jobId); + } + + private void waitForOrJobFinish(String jobId, Duration duration, boolean isKafkaUsed) { + // The TPS drop to 0 which means job is finished or specific interval for tps mode + Deadline deadline = Deadline.fromNow(duration); + while (isJobRunning(jobId) && deadline.hasTimeLeft()) { + if (isKafkaUsed && jobIsFinished()) break; + try { + Thread.sleep(100L); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + if (error != null) { + throw new RuntimeException(error); + } + } + } + + private void waitForOrJobRunning(String jobId) { + while (!flinkRestClient.isJobAndAllTasksRunning(jobId)) { + try { + Thread.sleep(50L); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + } + + private boolean jobIsFinished() { + synchronized (metrics) { + if (metrics.size() <= 5) { + return false; + } + int lastPos = metrics.size() - 1; + BenchmarkMetric lastMetric = metrics.get(lastPos); + if (Double.compare(lastMetric.getTps(), 0.0) == 0) { + for (int i = 1; i < 5; i++) { + if (Double.compare(metrics.get(lastPos - i).getTps(), 0.0) != 0) { + return false; + } + } + return true; + } + return false; + } + } + + public JobBenchmarkMetric reportMetric(String jobId, long eventsNum, boolean trim, boolean isKafkaUsed) { + long startTime = System.currentTimeMillis(); + waitForOrJobRunning(jobId); + long jobStartTime = System.currentTimeMillis(); + System.out.println("Waited for job until running status for " + (jobStartTime - startTime) + "ms."); + System.out.printf("Monitor metrics after %s seconds.%n", monitorDelay.getSeconds()); + waitFor(monitorDelay); + if (eventsNum == 0) { + System.out.printf("Start to monitor metrics for %s seconds.%n", monitorDuration.getSeconds()); + } else { + System.out.println("Start to monitor metrics until job is finished."); + } + submitMonitorThread(jobId, eventsNum); + // monitorDuration is Long.MAX_VALUE in event number mode + waitForOrJobFinish(jobId, monitorDuration, isKafkaUsed); + + long endTime = System.currentTimeMillis(); + + // cleanup the resource + this.close(); + + if (metrics.isEmpty()) { + throw new RuntimeException("The metric reporter doesn't collect any metrics."); + } + double sumTps = 0.0; + double sumCpu = 0.0; + int realMetricSize = metrics.size(); + + // If the job finished, the tps will drop to 0, so we need to remove the effect of these metrics on the final result + if (trim) { + for (int i = metrics.size() - 1; i >= 0; i--) { + if (Double.compare(metrics.get(i).getTps(), 0.0) != 0) { + break; + } else { + realMetricSize--; + } + } + } + + List realMetrics = metrics.subList(0, realMetricSize); + for (BenchmarkMetric metric : realMetrics) { + sumTps += metric.getTps(); + sumCpu += metric.getCpu(); + } + + double avgTps = sumTps / realMetrics.size(); + double avgCpu = sumCpu / realMetrics.size(); + JobBenchmarkMetric metric = new JobBenchmarkMetric( + avgTps, avgCpu, eventsNum, endTime - jobStartTime, jobStartTime - startTime); + + String message; + if (eventsNum == 0) { + message = String.format("Summary Average: Throughput=%s, Cores=%s", + metric.getPrettyTps(), + metric.getPrettyCpu()); + } else { + message = String.format("Summary Average: EventsNum=%s, Cores=%s, Time=%s s, Initialize Time=%s s", + NUMBER_FORMAT.format(eventsNum), + metric.getPrettyCpu(), + formatDoubleValue(metric.getTimeSeconds()), + formatDoubleValue(metric.getJobInitializedTimeSeconds())); + } + System.out.println(message); + LOG.info(message); + return metric; + } + + public void close() { + service.shutdownNow(); + } + + /** + * Get the collected metric samples (for JSON output). + * Returns a trimmed copy excluding trailing 0-TPS samples. + */ + public List getMetricSamples() { + List snapshot; + synchronized (metrics) { + snapshot = new ArrayList<>(metrics); + } + // Trim trailing 0-TPS samples from the snapshot + int trimmedSize = snapshot.size(); + for (int i = snapshot.size() - 1; i >= 0; i--) { + if (Double.compare(snapshot.get(i).getTps(), 0.0) != 0) { + break; + } else { + trimmedSize--; + } + } + // Remove trailing samples from the copy so we don't expose a subList view + if (trimmedSize < snapshot.size()) { + snapshot.subList(trimmedSize, snapshot.size()).clear(); + } + return snapshot; + } + + private class MetricCollector implements Runnable { + private final String jobId; + private final String vertexId; + private final String metricName; + private final long eventsNum; + + private MetricCollector(String jobId, String vertexId, String metricName, long eventsNum) { + this.jobId = jobId; + this.vertexId = vertexId; + this.metricName = metricName; + this.eventsNum = eventsNum; + } + + @Override + public void run() { + try { + TpsMetric tps = flinkRestClient.getTpsMetric(jobId, vertexId, metricName); + double cpu = processMetricReceiver.getTotalCpu(); + int tms = processMetricReceiver.getNumberOfTM(); + long rss = processMetricReceiver.getTotalRss(); + long vmem = processMetricReceiver.getTotalVmem(); + long netBytesRead = processMetricReceiver.getTotalNetBytesRead(); + long netBytesWritten = processMetricReceiver.getTotalNetBytesWritten(); + long diskBytesRead = processMetricReceiver.getTotalDiskBytesRead(); + long diskBytesWritten = processMetricReceiver.getTotalDiskBytesWritten(); + + BenchmarkMetric metric = new BenchmarkMetric( + System.currentTimeMillis(), + tps.getSum(), + cpu, + rss, + vmem, + netBytesRead, + netBytesWritten, + diskBytesRead, + diskBytesWritten); + // it's thread-safe to update metrics + metrics.add(metric); + // logging + String message = eventsNum == 0 ? + String.format("Current Throughput=%s, Cores=%s, RSS=%s (%s TMs)", + metric.getPrettyTps(), metric.getPrettyCpu(), metric.getPrettyRss(), tms) : + String.format("Current Cores=%s, RSS=%s (%s TMs)", metric.getPrettyCpu(), metric.getPrettyRss(), tms); + System.out.println(message); + LOG.info(message); + } catch (Exception e) { + error = e; + } + } + } +} diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/Savepoint.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/Savepoint.java new file mode 100644 index 0000000..9be9c1a --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/Savepoint.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink.metric; + +public class Savepoint { + + public enum Status { + /** Checkpoint that is still in progress. */ + IN_PROGRESS, + /** Checkpoint that has successfully completed. */ + COMPLETED, + /** Checkpoint that failed. */ + FAILED; + } + + private final Status status; + + private final String path; + + public Savepoint(Status status, String path) { + this.status = status; + this.path = path; + } + + public String getPath() { + return path; + } + + public Status getStatus() { + return status; + } + + @Override + public String toString() { + return "Savepoint{path = " + path + ", status = " + status + "}"; + } +} diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/process/CpuTimeTracker.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/process/CpuTimeTracker.java new file mode 100644 index 0000000..bb1c30a --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/process/CpuTimeTracker.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink.metric.process; + +import java.math.BigInteger; + +/** + * Utility for sampling and computing CPU usage. + * Copied from CpuTimeTracker of YARN project. + */ +public class CpuTimeTracker { + public static final int UNAVAILABLE = -1; + private final long minimumTimeInterval; + + // CPU used time since system is on (ms) + private BigInteger cumulativeCpuTime = BigInteger.ZERO; + + // CPU used time read last time (ms) + private BigInteger lastCumulativeCpuTime = BigInteger.ZERO; + + // Unix timestamp while reading the CPU time (ms) + private long sampleTime; + private long lastSampleTime; + private float cpuUsage; + private BigInteger jiffyLengthInMillis; + + public CpuTimeTracker(long jiffyLengthInMillis) { + this.jiffyLengthInMillis = BigInteger.valueOf(jiffyLengthInMillis); + this.cpuUsage = UNAVAILABLE; + this.sampleTime = UNAVAILABLE; + this.lastSampleTime = UNAVAILABLE; + minimumTimeInterval = 10 * jiffyLengthInMillis; + } + + /** + * Return percentage of cpu time spent over the time since last update. + * CPU time spent is based on elapsed jiffies multiplied by amount of + * time for 1 core. Thus, if you use 2 cores completely you would have spent + * twice the actual time between updates and this will return 200%. + * + * @return Return percentage of cpu usage since last update, {@link + * CpuTimeTracker#UNAVAILABLE} if there haven't been 2 updates more than + * {@link CpuTimeTracker#minimumTimeInterval} apart + */ + public float getCpuTrackerUsagePercent() { + if (lastSampleTime == UNAVAILABLE || + lastSampleTime > sampleTime) { + // lastSampleTime > sampleTime may happen when the system time is changed + lastSampleTime = sampleTime; + lastCumulativeCpuTime = cumulativeCpuTime; + return cpuUsage; + } + // When lastSampleTime is sufficiently old, update cpuUsage. + // Also take a sample of the current time and cumulative CPU time for the + // use of the next calculation. + if (sampleTime > lastSampleTime + minimumTimeInterval) { + cpuUsage = + ((cumulativeCpuTime.subtract(lastCumulativeCpuTime)).floatValue()) + * 100F / ((float) (sampleTime - lastSampleTime)); + lastSampleTime = sampleTime; + lastCumulativeCpuTime = cumulativeCpuTime; + } + return cpuUsage; + } + + /** + * Obtain the cumulative CPU time since the system is on. + * + * @return cumulative CPU time in milliseconds + */ + public long getCumulativeCpuTime() { + return cumulativeCpuTime.longValue(); + } + + /** + * Apply delta to accumulators. + * + * @param elapsedJiffies updated jiffies + * @param newTime new sample time + */ + public void updateElapsedJiffies(BigInteger elapsedJiffies, long newTime) { + BigInteger newValue = elapsedJiffies.multiply(jiffyLengthInMillis); + cumulativeCpuTime = newValue.compareTo(cumulativeCpuTime) >= 0 ? + newValue : cumulativeCpuTime; + sampleTime = newTime; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("SampleTime ").append(this.sampleTime); + sb.append(" CummulativeCpuTime ").append(this.cumulativeCpuTime); + sb.append(" LastSampleTime ").append(this.lastSampleTime); + sb.append(" LastCummulativeCpuTime ").append(this.lastCumulativeCpuTime); + sb.append(" CpuUsage ").append(this.cpuUsage); + sb.append(" JiffyLengthMillisec ").append(this.jiffyLengthInMillis); + return sb.toString(); + } +} diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/process/OperatingSystem.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/process/OperatingSystem.java new file mode 100644 index 0000000..d4450b0 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/process/OperatingSystem.java @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink.metric.process; + +/** + * An enumeration indicating the operating system that the JVM runs on. + */ +public enum OperatingSystem { + + LINUX, + WINDOWS, + MAC_OS, + FREE_BSD, + SOLARIS, + UNKNOWN; + + // ------------------------------------------------------------------------ + + /** + * Gets the operating system that the JVM runs on from the java system properties. + * this method returns UNKNOWN, if the operating system was not successfully determined. + * + * @return The enum constant for the operating system, or UNKNOWN, if it was not possible to determine. + */ + public static OperatingSystem getCurrentOperatingSystem() { + return os; + } + + /** + * Checks whether the operating system this JVM runs on is Windows. + * + * @return true if the operating system this JVM runs on is + * Windows, false otherwise + */ + public static boolean isWindows() { + return getCurrentOperatingSystem() == WINDOWS; + } + + /** + * Checks whether the operating system this JVM runs on is Linux. + * + * @return true if the operating system this JVM runs on is + * Linux, false otherwise + */ + public static boolean isLinux() { + return getCurrentOperatingSystem() == LINUX; + } + + /** + * Checks whether the operating system this JVM runs on is Windows. + * + * @return true if the operating system this JVM runs on is + * Windows, false otherwise + */ + public static boolean isMac() { + return getCurrentOperatingSystem() == MAC_OS; + } + + /** + * Checks whether the operating system this JVM runs on is FreeBSD. + * + * @return true if the operating system this JVM runs on is + * FreeBSD, false otherwise + */ + public static boolean isFreeBSD() { + return getCurrentOperatingSystem() == FREE_BSD; + } + + /** + * Checks whether the operating system this JVM runs on is Solaris. + * + * @return true if the operating system this JVM runs on is + * Solaris, false otherwise + */ + public static boolean isSolaris() { + return getCurrentOperatingSystem() == SOLARIS; + } + + /** + * The enum constant for the operating system. + */ + private static final OperatingSystem os = readOSFromSystemProperties(); + + /** + * Parses the operating system that the JVM runs on from the java system properties. + * If the operating system was not successfully determined, this method returns {@code UNKNOWN}. + * + * @return The enum constant for the operating system, or {@code UNKNOWN}, if it was not possible to determine. + */ + private static OperatingSystem readOSFromSystemProperties() { + String osName = System.getProperty(OS_KEY); + + if (osName.startsWith(LINUX_OS_PREFIX)) { + return LINUX; + } + if (osName.startsWith(WINDOWS_OS_PREFIX)) { + return WINDOWS; + } + if (osName.startsWith(MAC_OS_PREFIX)) { + return MAC_OS; + } + if (osName.startsWith(FREEBSD_OS_PREFIX)) { + return FREE_BSD; + } + String osNameLowerCase = osName.toLowerCase(); + if (osNameLowerCase.contains(SOLARIS_OS_INFIX_1) || osNameLowerCase.contains(SOLARIS_OS_INFIX_2)) { + return SOLARIS; + } + + return UNKNOWN; + } + + // -------------------------------------------------------------------------------------------- + // Constants to extract the OS type from the java environment + // -------------------------------------------------------------------------------------------- + + /** + * The key to extract the operating system name from the system properties. + */ + private static final String OS_KEY = "os.name"; + + /** + * The expected prefix for Linux operating systems. + */ + private static final String LINUX_OS_PREFIX = "Linux"; + + /** + * The expected prefix for Windows operating systems. + */ + private static final String WINDOWS_OS_PREFIX = "Windows"; + + /** + * The expected prefix for Mac OS operating systems. + */ + private static final String MAC_OS_PREFIX = "Mac"; + + /** + * The expected prefix for FreeBSD. + */ + private static final String FREEBSD_OS_PREFIX = "FreeBSD"; + + /** + * One expected infix for Solaris. + */ + private static final String SOLARIS_OS_INFIX_1 = "sunos"; + + /** + * One expected infix for Solaris. + */ + private static final String SOLARIS_OS_INFIX_2 = "solaris"; +} diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/process/ProcessMetric.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/process/ProcessMetric.java new file mode 100644 index 0000000..824051b --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/process/ProcessMetric.java @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink.metric.process; + +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonIgnore; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonInclude; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonProcessingException; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ArrayNode; + +import com.github.nexmark.flink.utils.NexmarkUtils; + +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +public class ProcessMetric { + + private static final String FIELD_NAME_HOST = "host"; + private static final String FIELD_NAME_PID = "pid"; + private static final String FIELD_NAME_CPU = "cpu"; + private static final String FIELD_NAME_RSS = "rss"; + private static final String FIELD_NAME_VMEM = "vmem"; + private static final String FIELD_NAME_NET_BYTES_READ = "netBytesRead"; + private static final String FIELD_NAME_NET_BYTES_WRITTEN = "netBytesWritten"; + private static final String FIELD_NAME_DISK_BYTES_READ = "diskBytesRead"; + private static final String FIELD_NAME_DISK_BYTES_WRITTEN = "diskBytesWritten"; + + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonProperty(value = FIELD_NAME_HOST, required = true) + private final String host; + + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonProperty(value = FIELD_NAME_PID, required = true) + private final int pid; + + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonProperty(value = FIELD_NAME_CPU, required = true) + private final double cpu; + + @JsonProperty(value = FIELD_NAME_RSS) + private final long rss; + + @JsonProperty(value = FIELD_NAME_VMEM) + private final long vmem; + + @JsonProperty(value = FIELD_NAME_NET_BYTES_READ) + private final long netBytesRead; + + @JsonProperty(value = FIELD_NAME_NET_BYTES_WRITTEN) + private final long netBytesWritten; + + @JsonProperty(value = FIELD_NAME_DISK_BYTES_READ) + private final long diskBytesRead; + + @JsonProperty(value = FIELD_NAME_DISK_BYTES_WRITTEN) + private final long diskBytesWritten; + + @JsonCreator + public ProcessMetric( + @Nullable @JsonProperty(FIELD_NAME_HOST) String host, + @JsonProperty(FIELD_NAME_PID) int pid, + @JsonProperty(FIELD_NAME_CPU) double cpu, + @JsonProperty(FIELD_NAME_RSS) long rss, + @JsonProperty(FIELD_NAME_VMEM) long vmem, + @JsonProperty(FIELD_NAME_NET_BYTES_READ) long netBytesRead, + @JsonProperty(FIELD_NAME_NET_BYTES_WRITTEN) long netBytesWritten, + @JsonProperty(FIELD_NAME_DISK_BYTES_READ) long diskBytesRead, + @JsonProperty(FIELD_NAME_DISK_BYTES_WRITTEN) long diskBytesWritten) { + this.host = host; + this.pid = pid; + this.cpu = cpu; + this.rss = rss; + this.vmem = vmem; + this.netBytesRead = netBytesRead; + this.netBytesWritten = netBytesWritten; + this.diskBytesRead = diskBytesRead; + this.diskBytesWritten = diskBytesWritten; + } + + @JsonIgnore + public String getHost() { + return host; + } + + @JsonIgnore + public int getPid() { + return pid; + } + + @JsonIgnore + public double getCpu() { + return cpu; + } + + @JsonIgnore + public long getRss() { + return rss; + } + + @JsonIgnore + public long getVmem() { + return vmem; + } + + @JsonIgnore + public long getNetBytesRead() { + return netBytesRead; + } + + @JsonIgnore + public long getNetBytesWritten() { + return netBytesWritten; + } + + @JsonIgnore + public long getDiskBytesRead() { + return diskBytesRead; + } + + @JsonIgnore + public long getDiskBytesWritten() { + return diskBytesWritten; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ProcessMetric processMetric = (ProcessMetric) o; + return Double.compare(processMetric.cpu, cpu) == 0 && + pid == processMetric.pid && + rss == processMetric.rss && + vmem == processMetric.vmem && + netBytesRead == processMetric.netBytesRead && + netBytesWritten == processMetric.netBytesWritten && + diskBytesRead == processMetric.diskBytesRead && + diskBytesWritten == processMetric.diskBytesWritten && + Objects.equals(host, processMetric.host); + } + + @Override + public int hashCode() { + return Objects.hash(host, pid, cpu, rss, vmem, netBytesRead, netBytesWritten, diskBytesRead, diskBytesWritten); + } + + @Override + public String toString() { + try { + return NexmarkUtils.MAPPER.writeValueAsString(this); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + public static List fromJsonArray(String json) { + try { + ArrayNode arrayNode = (ArrayNode) NexmarkUtils.MAPPER.readTree(json); + List expected = new ArrayList<>(); + for (JsonNode jsonNode : arrayNode) { + expected.add(NexmarkUtils.MAPPER.convertValue(jsonNode, ProcessMetric.class)); + } + return expected; + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } +} diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/process/ProcessMetricReceiver.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/process/ProcessMetricReceiver.java new file mode 100644 index 0000000..0c7f37c --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/process/ProcessMetricReceiver.java @@ -0,0 +1,236 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink.metric.process; + +import org.apache.flink.configuration.Configuration; + +import com.github.nexmark.flink.FlinkNexmarkOptions; +import com.github.nexmark.flink.utils.NexmarkGlobalConfiguration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayOutputStream; +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStream; +import java.net.InetAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import static com.github.nexmark.flink.metric.process.ProcessMetricSender.DELIMITER; + +public class ProcessMetricReceiver implements Closeable { + + private static final Logger LOG = LoggerFactory.getLogger(ProcessMetricReceiver.class); + + /** + * Server socket to listen at. + */ + private final ServerSocket server; + + /** Stores full ProcessMetric per TaskManager (key: "host:pid"). */ + private final ConcurrentHashMap processMetrics = new ConcurrentHashMap<>(); + + private final ExecutorService service = Executors.newCachedThreadPool(); + + public ProcessMetricReceiver(String host, int port) { + try { + InetAddress address = InetAddress.getByName(host); + server = new ServerSocket(port, 10, address); + } catch (IOException e) { + throw new RuntimeException("Could not open socket to receive back process metrics.", e); + } + } + + public void runServer() { + service.submit(this::runServerBlocking); + } + + public void runServerBlocking() { + try { + //noinspection InfiniteLoopStatement + while (true) { + Socket socket = server.accept(); + try { + service.submit(new ServerThread(socket, processMetrics)); + } catch (Exception e) { + // Executor may be shut down, close socket to prevent leak + try { + socket.close(); + } catch (IOException ignored) { + } + throw e; + } + } + } catch (IOException e) { + LOG.error("Failed to start the socket server.", e); + try { + server.close(); + } catch (Throwable ignored) { + } + } + } + + public double getTotalCpu() { + double sumCpu = 0.0; + int size = 0; + for (ProcessMetric metric : processMetrics.values()) { + size++; + sumCpu += metric.getCpu(); + } + if (size == 0) { + LOG.warn("The process metric receiver doesn't receive any metrics."); + } + return sumCpu; + } + + public int getNumberOfTM() { + return processMetrics.size(); + } + + /** Get total RSS memory across all TaskManagers. */ + public long getTotalRss() { + long sum = 0; + for (ProcessMetric metric : processMetrics.values()) { + sum += metric.getRss(); + } + return sum; + } + + /** Get total virtual memory across all TaskManagers. */ + public long getTotalVmem() { + long sum = 0; + for (ProcessMetric metric : processMetrics.values()) { + sum += metric.getVmem(); + } + return sum; + } + + /** Get total network bytes read (deduplicated by host since it's system-wide). */ + public long getTotalNetBytesRead() { + java.util.Map perHost = new java.util.HashMap<>(); + for (ProcessMetric metric : processMetrics.values()) { + perHost.merge(metric.getHost(), metric.getNetBytesRead(), Math::max); + } + return perHost.values().stream().mapToLong(Long::longValue).sum(); + } + + /** Get total network bytes written (deduplicated by host since it's system-wide). */ + public long getTotalNetBytesWritten() { + java.util.Map perHost = new java.util.HashMap<>(); + for (ProcessMetric metric : processMetrics.values()) { + perHost.merge(metric.getHost(), metric.getNetBytesWritten(), Math::max); + } + return perHost.values().stream().mapToLong(Long::longValue).sum(); + } + + /** Get total disk bytes read (deduplicated by host since it's system-wide). */ + public long getTotalDiskBytesRead() { + java.util.Map perHost = new java.util.HashMap<>(); + for (ProcessMetric metric : processMetrics.values()) { + perHost.merge(metric.getHost(), metric.getDiskBytesRead(), Math::max); + } + return perHost.values().stream().mapToLong(Long::longValue).sum(); + } + + /** Get total disk bytes written (deduplicated by host since it's system-wide). */ + public long getTotalDiskBytesWritten() { + java.util.Map perHost = new java.util.HashMap<>(); + for (ProcessMetric metric : processMetrics.values()) { + perHost.merge(metric.getHost(), metric.getDiskBytesWritten(), Math::max); + } + return perHost.values().stream().mapToLong(Long::longValue).sum(); + } + + @Override + public void close() { + try { + server.close(); + } catch (Throwable ignored) { + } + + service.shutdownNow(); + } + + private static final class ServerThread implements Runnable { + + private final Socket socket; + private final ConcurrentHashMap processMetrics; + + private ServerThread(Socket socket, ConcurrentHashMap processMetrics) { + this.socket = socket; + this.processMetrics = processMetrics; + } + + @Override + public void run() { + try { + InputStream inStream = socket.getInputStream(); + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + int b; + while ((b = inStream.read()) >= 0) { + // buffer until delimiter + if (b != DELIMITER) { + buffer.write(b); + } + // decode and emit record + else { + byte[] bytes = buffer.toByteArray(); + String message = new String(bytes, StandardCharsets.UTF_8); + LOG.info("Received process metric report: {}", message); + List receivedMetrics = ProcessMetric.fromJsonArray(message); + for (ProcessMetric metric : receivedMetrics) { + processMetrics.put(metric.getHost() + ":" + metric.getPid(), metric); + } + buffer.reset(); + } + } + } catch (IOException e) { + LOG.error("Socket server error.", e); + } finally { + try { + socket.close(); + } catch (IOException ex) { + // ignore + } + } + } + } + + public static void main(String[] args) throws ExecutionException, InterruptedException { + // start metric servers + Configuration conf = NexmarkGlobalConfiguration.loadConfiguration(); + String reporterAddress = conf.getOptional(FlinkNexmarkOptions.METRIC_REPORTER_RECEIVING_HOST) + .filter(s -> !s.isEmpty()) + .orElseGet(() -> conf.get(FlinkNexmarkOptions.METRIC_REPORTER_HOST)); + int reporterPort = conf.getOptional(FlinkNexmarkOptions.METRIC_REPORTER_RECEIVING_PORT) + .orElseGet(() -> conf.get(FlinkNexmarkOptions.METRIC_REPORTER_PORT)); + ProcessMetricReceiver processMetricReceiver = new ProcessMetricReceiver(reporterAddress, reporterPort); + Runtime.getRuntime().addShutdownHook(new Thread(processMetricReceiver::close)); + processMetricReceiver.runServer(); + processMetricReceiver.service.awaitTermination(Long.MAX_VALUE, TimeUnit.MILLISECONDS); + } +} diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/process/ProcessMetricSender.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/process/ProcessMetricSender.java new file mode 100644 index 0000000..e386298 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/process/ProcessMetricSender.java @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink.metric.process; + +import org.apache.flink.configuration.Configuration; +import org.apache.flink.runtime.net.ConnectionUtils; + +import com.github.nexmark.flink.FlinkNexmarkOptions; +import com.github.nexmark.flink.utils.AutoClosableProcess; +import com.github.nexmark.flink.utils.NexmarkGlobalConfiguration; +import com.github.nexmark.flink.utils.NexmarkUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +public class ProcessMetricSender implements AutoCloseable { + + private static final Logger LOG = LoggerFactory.getLogger(ProcessMetricSender.class); + public static final char DELIMITER = '\n'; + + private final String serverHostIp; + private final int serverPort; + private final Duration interval; + private final ScheduledExecutorService service = Executors.newScheduledThreadPool(1); + private InetAddress serverAddress; + private String localHostIp; + private ConcurrentHashMap processTrees; + private SysInfoLinux sysInfo; + + public ProcessMetricSender(String serverHostIp, int serverPort, Duration interval) { + this.serverHostIp = serverHostIp; + this.serverPort = serverPort; + this.interval = interval; + } + + public void startClient() throws Exception { + List taskmanagers = getTaskManagerPidList(); + if (taskmanagers.isEmpty()) { + throw new RuntimeException("There is no Flink TaskManager is running."); + } + this.serverAddress = InetAddress.getByName(serverHostIp); + this.processTrees = new ConcurrentHashMap<>(); + for (Integer pid : taskmanagers) { + processTrees.put(pid, new ProcfsBasedProcessTree(String.valueOf(pid))); + } + LOG.info("Start to monitor process: {}", taskmanagers); + this.service.scheduleAtFixedRate( + this::reportProcessMetric, + 0L, + interval.toMillis(), + TimeUnit.MILLISECONDS); + } + + @Override + public void close() { + service.shutdownNow(); + } + + private void reportProcessMetric() { + try (Socket socket = new Socket(serverAddress, serverPort); + OutputStream outputStream = socket.getOutputStream()) { + InetAddress localAddress = ConnectionUtils.findConnectingAddress( + new InetSocketAddress(serverAddress, serverPort), + 10000L, + 400); + this.localHostIp = localAddress.getHostAddress(); + + List processMetrics = getProcessMetrics(); + String jsonMessage = NexmarkUtils.MAPPER.writeValueAsString(processMetrics); + outputStream.write(jsonMessage.getBytes(StandardCharsets.UTF_8)); + outputStream.write(DELIMITER); + LOG.info("Report process metric: {}", jsonMessage); + } catch (IOException e) { + LOG.warn("Can't connect to metric server. Skip to report metric for this round.", e); + } catch (Exception e) { + LOG.error("Report process metric error.", e); + } + } + + private List getProcessMetrics() { + // Collect system-wide I/O metrics once per host (lazily initialize cached instance) + if (sysInfo == null) { + sysInfo = new SysInfoLinux(); + } + long netBytesRead = sysInfo.getNetworkBytesRead(); + long netBytesWritten = sysInfo.getNetworkBytesWritten(); + long diskBytesRead = sysInfo.getStorageBytesRead(); + long diskBytesWritten = sysInfo.getStorageBytesWritten(); + + List processMetrics = new ArrayList<>(); + for (Map.Entry entry : processTrees.entrySet()) { + ProcfsBasedProcessTree processTree = entry.getValue(); + processTree.updateProcessTree(); + int pid = entry.getKey(); + double cpuCores = processTree.getCpuUsagePercent() / 100.0; + long rss = processTree.getRssMemorySize(); + long vmem = processTree.getVirtualMemorySize(); + processMetrics.add(new ProcessMetric( + localHostIp, pid, cpuCores, + rss, vmem, + netBytesRead, netBytesWritten, + diskBytesRead, diskBytesWritten)); + } + return processMetrics; + } + + public static List getTaskManagerPidList() throws IOException { + List javaProcessors = new ArrayList<>(); + AutoClosableProcess + .create("jps") + .setStdoutProcessor(javaProcessors::add) + .runBlocking(); + List taskManagers = new ArrayList<>(); + for (String processor : javaProcessors) { + if (processor.endsWith("TaskManagerRunner")) { + String pid = processor.split(" ")[0]; + taskManagers.add(Integer.parseInt(pid)); + } + } + return taskManagers; + } + + public static void main(String[] args) throws Exception { + // start metric servers + Configuration conf = NexmarkGlobalConfiguration.loadConfiguration(); + String reporterAddress = conf.get(FlinkNexmarkOptions.METRIC_REPORTER_HOST); + int reporterPort = conf.get(FlinkNexmarkOptions.METRIC_REPORTER_PORT); + Duration reportInterval = conf.get(FlinkNexmarkOptions.METRIC_MONITOR_INTERVAL); + + ProcessMetricSender sender = new ProcessMetricSender(reporterAddress, reporterPort, reportInterval); + Runtime.getRuntime().addShutdownHook(new Thread(sender::close)); + sender.startClient(); + sender.service.awaitTermination(Long.MAX_VALUE, TimeUnit.MILLISECONDS); + } +} diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/process/ProcessMonitor.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/process/ProcessMonitor.java new file mode 100644 index 0000000..200b276 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/process/ProcessMonitor.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink.metric.process; + +public class ProcessMonitor { + + public static void main(String[] args) throws InterruptedException { + ProcfsBasedProcessTree procfsBasedProcessTree = new ProcfsBasedProcessTree(args[0]); + procfsBasedProcessTree.updateProcessTree(); + while (true) { + Thread.sleep(100); + procfsBasedProcessTree.updateProcessTree(); + System.out.println(System.currentTimeMillis() / 1000 +" "+ procfsBasedProcessTree.getCpuUsagePercent()); + } + } +} diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/process/ProcfsBasedProcessTree.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/process/ProcfsBasedProcessTree.java new file mode 100644 index 0000000..a291589 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/process/ProcfsBasedProcessTree.java @@ -0,0 +1,997 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink.metric.process; + +import com.github.nexmark.flink.metric.process.clock.Clock; +import com.github.nexmark.flink.metric.process.clock.SystemClock; +import org.apache.commons.io.IOUtils; +import org.apache.commons.io.filefilter.AndFileFilter; +import org.apache.commons.io.filefilter.DirectoryFileFilter; +import org.apache.commons.io.filefilter.RegexFileFilter; +import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.StringUtils; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileFilter; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStreamReader; +import java.math.BigInteger; +import java.nio.charset.Charset; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * A Proc file-system based ProcessTree. Works only on Linux. + * Based on ProcfsBasedProcessTree from YARN project. + */ +public class ProcfsBasedProcessTree { + + private static final String PROCFS = "/proc/"; + + private static final String SELF = "self"; + + private static final Pattern PROCFS_STAT_FILE_FORMAT = Pattern.compile( + "^([\\d-]+)\\s\\((.*)\\)\\s[^\\s]\\s([\\d-]+)\\s([\\d-]+)\\s" + + "([\\d-]+)\\s([\\d-]+\\s){7}(\\d+)\\s(\\d+)\\s([\\d-]+\\s){7}(\\d+)\\s" + + "(\\d+)(\\s[\\d-]+){15}"); + + public static final int UNAVAILABLE = -1; + public static final String PROCFS_STAT_FILE = "stat"; + public static final String PROCFS_CMDLINE_FILE = "cmdline"; + public static final long PAGE_SIZE = SysInfoLinux.PAGE_SIZE; + public static final long JIFFY_LENGTH_IN_MILLIS = + SysInfoLinux.JIFFY_LENGTH_IN_MILLIS; // in millisecond + private final CpuTimeTracker cpuTimeTracker; + private Clock clock; + + enum MemInfo { + SIZE("Size"), RSS("Rss"), PSS("Pss"), SHARED_CLEAN("Shared_Clean"), + SHARED_DIRTY("Shared_Dirty"), PRIVATE_CLEAN("Private_Clean"), + PRIVATE_DIRTY("Private_Dirty"), REFERENCED("Referenced"), ANONYMOUS( + "Anonymous"), ANON_HUGE_PAGES("AnonHugePages"), SWAP("swap"), + KERNEL_PAGE_SIZE("kernelPageSize"), MMU_PAGE_SIZE("mmuPageSize"), INVALID( + "invalid"); + + private String name; + + private MemInfo(String name) { + this.name = name; + } + + public static MemInfo getMemInfoByName(String name) { + String searchName = StringUtils.trimToNull(name); + for (MemInfo info : MemInfo.values()) { + if (info.name.trim().equalsIgnoreCase(searchName)) { + return info; + } + } + return INVALID; + } + } + + public static final String SMAPS = "smaps"; + public static final int KB_TO_BYTES = 1024; + private static final String KB = "kB"; + private static final String READ_ONLY_WITH_SHARED_PERMISSION = "r--s"; + private static final String READ_EXECUTE_WITH_SHARED_PERMISSION = "r-xs"; + private static final Pattern ADDRESS_PATTERN = Pattern + .compile("([[a-f]|(0-9)]*)-([[a-f]|(0-9)]*)(\\s)*([rxwps\\-]*)"); + private static final Pattern MEM_INFO_PATTERN = Pattern + .compile("(^[A-Z].*):[\\s ]*(\\d+).*"); + + private boolean smapsEnabled; + + protected Map processSMAPTree = + new HashMap(); + + // to enable testing, using this variable which can be configured + // to a test directory. + private String procfsDir; + + static private String deadPid = "-1"; + private String pid = deadPid; + static private Pattern numberPattern = Pattern.compile("[1-9][0-9]*"); + private long cpuTime = UNAVAILABLE; + + protected Map processTree = + new HashMap(); + + public ProcfsBasedProcessTree() throws IOException { + this(new File(PROCFS, SELF).getCanonicalFile().getName()); + } + + public ProcfsBasedProcessTree(boolean smapsEnabled) throws IOException { + this(new File(PROCFS, SELF).getCanonicalFile().getName(), smapsEnabled); + } + + public ProcfsBasedProcessTree(String pid) { + this(pid, PROCFS); + } + + public ProcfsBasedProcessTree(String pid, boolean smapsEnabled) { + this(pid, PROCFS, SystemClock.getInstance(), smapsEnabled); + } + + public ProcfsBasedProcessTree(String pid, String procfsDir) { + this(pid, procfsDir, SystemClock.getInstance(), false); + } + + /** + * Build a new process tree rooted at the pid. + *

+ * This method is provided mainly for testing purposes, where + * the root of the proc file system can be adjusted. + * + * @param pid root of the process tree + * @param procfsDir the root of a proc file system - only used for testing. + * @param clock clock for controlling time for testing + */ + public ProcfsBasedProcessTree(String pid, String procfsDir, Clock clock, boolean smapsEnabled) { + this.clock = clock; + this.pid = getValidPID(pid); + this.procfsDir = procfsDir; + this.cpuTimeTracker = new CpuTimeTracker(JIFFY_LENGTH_IN_MILLIS); + this.smapsEnabled = smapsEnabled; + } + + public void setSmapsEnabled(boolean smapsEnabled) { + this.smapsEnabled = smapsEnabled; + } + + /** + * Update process-tree with latest state. If the root-process is not alive, + * tree will be empty. + */ + public void updateProcessTree() { + if (!pid.equals(deadPid)) { + // Get the list of processes + List processList = getProcessList(); + + Map allProcessInfo = new HashMap(); + + // cache the processTree to get the age for processes + Map oldProcs = + new HashMap(processTree); + processTree.clear(); + + ProcessInfo me = null; + for (String proc : processList) { + // Get information for each process + ProcessInfo pInfo = new ProcessInfo(proc); + if (constructProcessInfo(pInfo, procfsDir) != null) { + allProcessInfo.put(proc, pInfo); + if (proc.equals(this.pid)) { + me = pInfo; // cache 'me' + processTree.put(proc, pInfo); + } + } + } + + if (me == null) { + return; + } + + // Add each process to its parent. + for (Map.Entry entry : allProcessInfo.entrySet()) { + String pID = entry.getKey(); + if (!"1".equals(pID)) { + ProcessInfo pInfo = entry.getValue(); + String ppid = pInfo.getPpid(); + // If parent is init and process is not session leader, + // attach to sessionID + if ("1".equals(ppid)) { + String sid = pInfo.getSessionId().toString(); + if (!pID.equals(sid)) { + ppid = sid; + } + } + ProcessInfo parentPInfo = allProcessInfo.get(ppid); + if (parentPInfo != null) { + parentPInfo.addChild(pInfo); + } + } + } + + // now start constructing the process-tree + List children = me.getChildren(); + Queue pInfoQueue = new ArrayDeque(children); + while (!pInfoQueue.isEmpty()) { + ProcessInfo pInfo = pInfoQueue.remove(); + if (!processTree.containsKey(pInfo.getPid())) { + processTree.put(pInfo.getPid(), pInfo); + } + pInfoQueue.addAll(pInfo.getChildren()); + } + + // update age values and compute the number of jiffies since last update + for (Map.Entry procs : processTree.entrySet()) { + ProcessInfo oldInfo = oldProcs.get(procs.getKey()); + if (procs.getValue() != null) { + procs.getValue().updateJiffy(oldInfo); + if (oldInfo != null) { + procs.getValue().updateAge(oldInfo); + } + } + } + + if (smapsEnabled) { + // Update smaps info + processSMAPTree.clear(); + for (ProcessInfo p : processTree.values()) { + if (p != null) { + // Get information for each process + ProcessTreeSmapMemInfo memInfo = new ProcessTreeSmapMemInfo(p.getPid()); + constructProcessSMAPInfo(memInfo, procfsDir); + processSMAPTree.put(p.getPid(), memInfo); + } + } + } + } + } + + /** + * Verify that the given process id is same as its process group id. + * + * @return true if the process id matches else return false. + */ + public boolean checkPidPgrpidForMatch() { + return checkPidPgrpidForMatch(pid, PROCFS); + } + + public static boolean checkPidPgrpidForMatch(String _pid, String procfs) { + // Get information for this process + ProcessInfo pInfo = new ProcessInfo(_pid); + pInfo = constructProcessInfo(pInfo, procfs); + // null if process group leader finished execution; issue no warning + // make sure that pid and its pgrpId match + if (pInfo == null) { + return true; + } + String pgrpId = pInfo.getPgrpId().toString(); + return pgrpId.equals(_pid); + } + + private static final String PROCESSTREE_DUMP_FORMAT = + "\t|- %s %s %d %d %s %d %d %d %d %s%n"; + + public List getCurrentProcessIDs() { + return Collections.unmodifiableList(new ArrayList<>(processTree.keySet())); + } + + /** + * Get a dump of the process-tree. + * + * @return a string concatenating the dump of information of all the processes + * in the process-tree + */ + public String getProcessTreeDump() { + StringBuilder ret = new StringBuilder(); + // The header. + ret.append(String.format("\t|- PID PPID PGRPID SESSID CMD_NAME " + + "USER_MODE_TIME(MILLIS) SYSTEM_TIME(MILLIS) VMEM_USAGE(BYTES) " + + "RSSMEM_USAGE(PAGES) FULL_CMD_LINE%n")); + for (ProcessInfo p : processTree.values()) { + if (p != null) { + ret.append(String.format(PROCESSTREE_DUMP_FORMAT, p.getPid(), p + .getPpid(), p.getPgrpId(), p.getSessionId(), p.getName(), p + .getUtime(), p.getStime(), p.getVmem(), p.getRssmemPage(), p + .getCmdLine(procfsDir))); + } + } + return ret.toString(); + } + + public long getVirtualMemorySize() { + return getVirtualMemorySize(0); + } + + public long getVirtualMemorySize(int olderThanAge) { + long total = 0L; + boolean isAvailable = false; + for (ProcessInfo p : processTree.values()) { + if (p != null) { + isAvailable = true; + if (p.getAge() > olderThanAge) { + total += p.getVmem(); + } + } + } + return isAvailable ? total : UNAVAILABLE; + } + + public long getRssMemorySize() { + return getRssMemorySize(0); + } + + public long getRssMemorySize(int olderThanAge) { + if (PAGE_SIZE < 0) { + return UNAVAILABLE; + } + if (smapsEnabled) { + return getSmapBasedRssMemorySize(olderThanAge); + } + boolean isAvailable = false; + long totalPages = 0; + for (ProcessInfo p : processTree.values()) { + if (p != null) { + isAvailable = true; + if (p.getAge() > olderThanAge) { + totalPages += p.getRssmemPage(); + } + } + } + return isAvailable ? totalPages * PAGE_SIZE : UNAVAILABLE; // convert # pages to byte + } + + /** + * Get the resident set size (RSS) memory used by all the processes + * in the process-tree that are older than the passed in age. RSS is + * calculated based on SMAP information. Skip mappings with "r--s", "r-xs" + * permissions to get real RSS usage of the process. + * + * @param olderThanAge processes above this age are included in the memory addition + * @return rss memory used by the process-tree in bytes, for + * processes older than this age. return {@link #UNAVAILABLE} if it cannot + * be calculated. + */ + private long getSmapBasedRssMemorySize(int olderThanAge) { + long total = UNAVAILABLE; + for (ProcessInfo p : processTree.values()) { + if (p != null) { + // set resource to 0 instead of UNAVAILABLE + if (total == UNAVAILABLE) { + total = 0; + } + if (p.getAge() > olderThanAge) { + ProcessTreeSmapMemInfo procMemInfo = processSMAPTree.get(p.getPid()); + if (procMemInfo != null) { + for (ProcessSmapMemoryInfo info : procMemInfo.getMemoryInfoList()) { + // Do not account for r--s or r-xs mappings + if (info.getPermission().trim() + .equalsIgnoreCase(READ_ONLY_WITH_SHARED_PERMISSION) + || info.getPermission().trim() + .equalsIgnoreCase(READ_EXECUTE_WITH_SHARED_PERMISSION)) { + continue; + } + + // Account for anonymous to know the amount of + // memory reclaimable by killing the process + total += info.anonymous; + + } + } + } + } + } + if (total > 0) { + total *= KB_TO_BYTES; // convert to bytes + } + return total; // size + } + + public long getCumulativeCpuTime() { + if (JIFFY_LENGTH_IN_MILLIS < 0) { + return UNAVAILABLE; + } + long incJiffies = 0; + boolean isAvailable = false; + for (ProcessInfo p : processTree.values()) { + if (p != null) { + // data is available + isAvailable = true; + incJiffies += p.getDtime(); + } + } + if (isAvailable) { + // reset cpuTime to 0 instead of UNAVAILABLE + if (cpuTime == UNAVAILABLE) { + cpuTime = 0L; + } + cpuTime += incJiffies * JIFFY_LENGTH_IN_MILLIS; + } + return cpuTime; + } + + private BigInteger getTotalProcessJiffies() { + BigInteger totalStime = BigInteger.ZERO; + long totalUtime = 0; + for (ProcessInfo p : processTree.values()) { + if (p != null) { + totalUtime += p.getUtime(); + totalStime = totalStime.add(p.getStime()); + } + } + return totalStime.add(BigInteger.valueOf(totalUtime)); + } + + /** + * Get the CPU usage by all the processes in the process-tree in Unix. + * Note: UNAVAILABLE will be returned in case when CPU usage is not + * available. It is NOT advised to return any other error code. + * + * @return percentage CPU usage since the process-tree was created, + * {@link #UNAVAILABLE} if CPU usage cannot be calculated or not available. + */ + public float getCpuUsagePercent() { + BigInteger processTotalJiffies = getTotalProcessJiffies(); + cpuTimeTracker.updateElapsedJiffies(processTotalJiffies, + clock.absoluteTimeMillis()); + return cpuTimeTracker.getCpuTrackerUsagePercent(); + } + + private static String getValidPID(String pid) { + if (pid == null) { + return deadPid; + } + Matcher m = numberPattern.matcher(pid); + if (m.matches()) { + return pid; + } + return deadPid; + } + + /** + * Get the list of all processes in the system. + */ + private List getProcessList() { + List processList = Collections.emptyList(); + FileFilter procListFileFilter = new AndFileFilter( + DirectoryFileFilter.INSTANCE, new RegexFileFilter(numberPattern)); + + File dir = new File(procfsDir); + File[] processDirs = dir.listFiles(procListFileFilter); + + if (ArrayUtils.isNotEmpty(processDirs)) { + processList = new ArrayList(processDirs.length); + for (File processDir : processDirs) { + processList.add(processDir.getName()); + } + } + return processList; + } + + /** + * Construct the ProcessInfo using the process' PID and procfs rooted at the + * specified directory and return the same. It is provided mainly to assist + * testing purposes. + *

+ * Returns null on failing to read from procfs, + * + * @param pinfo ProcessInfo that needs to be updated + * @param procfsDir root of the proc file system + * @return updated ProcessInfo, null on errors. + */ + private static ProcessInfo constructProcessInfo(ProcessInfo pinfo, + String procfsDir) { + ProcessInfo ret = null; + // Read "procfsDir//stat" file - typically /proc//stat + BufferedReader in = null; + InputStreamReader fReader = null; + try { + File pidDir = new File(procfsDir, pinfo.getPid()); + fReader = new InputStreamReader( + new FileInputStream( + new File(pidDir, PROCFS_STAT_FILE)), Charset.forName("UTF-8")); + in = new BufferedReader(fReader); + } catch (FileNotFoundException f) { + // The process vanished in the interim! + return ret; + } + + ret = pinfo; + try { + String str = in.readLine(); // only one line + Matcher m = PROCFS_STAT_FILE_FORMAT.matcher(str); + boolean mat = m.find(); + if (mat) { + String processName = "(" + m.group(2) + ")"; + // Set (name) (ppid) (pgrpId) (session) (utime) (stime) (vsize) (rss) + pinfo.updateProcessInfo(processName, m.group(3), + Integer.parseInt(m.group(4)), Integer.parseInt(m.group(5)), + Long.parseLong(m.group(7)), new BigInteger(m.group(8)), + Long.parseLong(m.group(10)), Long.parseLong(m.group(11))); + } else { + ret = null; + } + } catch (IOException io) { + ret = null; + } finally { + // Close the streams + try { + fReader.close(); + try { + in.close(); + } catch (IOException i) { + } + } catch (IOException i) { + } + } + + return ret; + } + + /** + * Returns a string printing PIDs of process present in the + * ProcfsBasedProcessTree. Output format : [pid pid ..] + */ + @Override + public String toString() { + StringBuffer pTree = new StringBuffer("[ "); + for (String p : processTree.keySet()) { + pTree.append(p); + pTree.append(" "); + } + return pTree.substring(0, pTree.length()) + "]"; + } + + /** + * Returns boolean indicating whether pid + * is in process tree. + */ + public boolean contains(String pid) { + return processTree.containsKey(pid); + } + + /** + * Class containing information of a process. + */ + private static class ProcessInfo { + private String pid; // process-id + private String name; // command name + private Integer pgrpId; // process group-id + private String ppid; // parent process-id + private Integer sessionId; // session-id + private Long vmem; // virtual memory usage + private Long rssmemPage; // rss memory usage in # of pages + private Long utime = 0L; // # of jiffies in user mode + private final BigInteger MAX_LONG = BigInteger.valueOf(Long.MAX_VALUE); + private BigInteger stime = new BigInteger("0"); // # of jiffies in kernel mode + // how many times has this process been seen alive + private int age; + + // # of jiffies used since last update: + private Long dtime = 0L; + // dtime = (utime + stime) - (utimeOld + stimeOld) + // We need this to compute the cumulative CPU time + // because the subprocess may finish earlier than root process + + private List children = new ArrayList(); // list of children + + public ProcessInfo(String pid) { + this.pid = pid; + // seeing this the first time. + this.age = 1; + } + + public String getPid() { + return pid; + } + + public String getName() { + return name; + } + + public Integer getPgrpId() { + return pgrpId; + } + + public String getPpid() { + return ppid; + } + + public Integer getSessionId() { + return sessionId; + } + + public Long getVmem() { + return vmem; + } + + public Long getUtime() { + return utime; + } + + public BigInteger getStime() { + return stime; + } + + public Long getDtime() { + return dtime; + } + + public Long getRssmemPage() { // get rss # of pages + return rssmemPage; + } + + public int getAge() { + return age; + } + + public void updateProcessInfo(String name, String ppid, Integer pgrpId, + Integer sessionId, Long utime, BigInteger stime, Long vmem, Long rssmem) { + this.name = name; + this.ppid = ppid; + this.pgrpId = pgrpId; + this.sessionId = sessionId; + this.utime = utime; + this.stime = stime; + this.vmem = vmem; + this.rssmemPage = rssmem; + } + + public void updateJiffy(ProcessInfo oldInfo) { + if (oldInfo == null) { + BigInteger sum = this.stime.add(BigInteger.valueOf(this.utime)); + if (sum.compareTo(MAX_LONG) > 0) { + this.dtime = 0L; + } else { + this.dtime = sum.longValue(); + } + return; + } + this.dtime = (this.utime - oldInfo.utime + + this.stime.subtract(oldInfo.stime).longValue()); + } + + public void updateAge(ProcessInfo oldInfo) { + this.age = oldInfo.age + 1; + } + + public boolean addChild(ProcessInfo p) { + return children.add(p); + } + + public List getChildren() { + return children; + } + + public String getCmdLine(String procfsDir) { + String ret = "N/A"; + if (pid == null) { + return ret; + } + BufferedReader in = null; + InputStreamReader fReader = null; + try { + fReader = new InputStreamReader( + new FileInputStream( + new File(new File(procfsDir, pid.toString()), PROCFS_CMDLINE_FILE)), + Charset.forName("UTF-8")); + } catch (FileNotFoundException f) { + // The process vanished in the interim! + return ret; + } + + in = new BufferedReader(fReader); + + try { + ret = in.readLine(); // only one line + if (ret == null) { + ret = "N/A"; + } else { + ret = ret.replace('\0', ' '); // Replace each null char with a space + if (ret.isEmpty()) { + // The cmdline might be empty because the process is swapped out or + // is a zombie. + ret = "N/A"; + } + } + } catch (IOException io) { + ret = "N/A"; + } finally { + // Close the streams + try { + fReader.close(); + try { + in.close(); + } catch (IOException i) { + } + } catch (IOException i) { + } + } + + return ret; + } + } + + /** + * Update memory related information + * + * @param pInfo + * @param procfsDir + */ + private static void constructProcessSMAPInfo(ProcessTreeSmapMemInfo pInfo, + String procfsDir) { + BufferedReader in = null; + InputStreamReader fReader = null; + try { + File pidDir = new File(procfsDir, pInfo.getPid()); + File file = new File(pidDir, SMAPS); + if (!file.exists()) { + return; + } + fReader = new InputStreamReader( + new FileInputStream(file), Charset.forName("UTF-8")); + in = new BufferedReader(fReader); + ProcessSmapMemoryInfo memoryMappingInfo = null; + List lines = IOUtils.readLines(in); + for (String line : lines) { + line = line.trim(); + try { + Matcher address = ADDRESS_PATTERN.matcher(line); + if (address.find()) { + memoryMappingInfo = new ProcessSmapMemoryInfo(line); + memoryMappingInfo.setPermission(address.group(4)); + pInfo.getMemoryInfoList().add(memoryMappingInfo); + continue; + } + Matcher memInfo = MEM_INFO_PATTERN.matcher(line); + if (memInfo.find()) { + String key = memInfo.group(1).trim(); + String value = memInfo.group(2); + + if (memoryMappingInfo != null) { + memoryMappingInfo.setMemInfo(key, value); + } + } + } catch (Throwable t) { + } + } + } catch (Throwable t) { + } finally { + IOUtils.closeQuietly(in); + } + } + + /** + * Placeholder for process's SMAPS information + */ + static class ProcessTreeSmapMemInfo { + private String pid; + private List memoryInfoList; + + public ProcessTreeSmapMemInfo(String pid) { + this.pid = pid; + this.memoryInfoList = new LinkedList(); + } + + public List getMemoryInfoList() { + return memoryInfoList; + } + + public String getPid() { + return pid; + } + + public String toString() { + StringBuilder sb = new StringBuilder(); + for (ProcessSmapMemoryInfo info : memoryInfoList) { + sb.append("\n"); + sb.append(info); + } + return sb.toString(); + } + } + + /** + * 

+	 * Private Pages : Pages that were mapped only by the process
+	 * Shared Pages : Pages that were shared with other processes
+	 *
+	 * Clean Pages : Pages that have not been modified since they were mapped
+	 * Dirty Pages : Pages that have been modified since they were mapped
+	 *
+	 * Private RSS = Private Clean Pages + Private Dirty Pages
+	 * Shared RSS = Shared Clean Pages + Shared Dirty Pages
+	 * RSS = Private RSS + Shared RSS
+	 * PSS = The count of all pages mapped uniquely by the process,
+	 *  plus a fraction of each shared page, said fraction to be
+	 *  proportional to the number of processes which have mapped the page.
+	 *
+	 *
+ */ + static class ProcessSmapMemoryInfo { + private int size; + private int rss; + private int pss; + private int sharedClean; + private int sharedDirty; + private int privateClean; + private int privateDirty; + private int anonymous; + private int referenced; + private String regionName; + private String permission; + + public ProcessSmapMemoryInfo(String name) { + this.regionName = name; + } + + public String getName() { + return regionName; + } + + public void setPermission(String permission) { + this.permission = permission; + } + + public String getPermission() { + return permission; + } + + public int getSize() { + return size; + } + + public int getRss() { + return rss; + } + + public int getPss() { + return pss; + } + + public int getSharedClean() { + return sharedClean; + } + + public int getSharedDirty() { + return sharedDirty; + } + + public int getPrivateClean() { + return privateClean; + } + + public int getPrivateDirty() { + return privateDirty; + } + + public int getReferenced() { + return referenced; + } + + public int getAnonymous() { + return anonymous; + } + + public void setMemInfo(String key, String value) { + MemInfo info = MemInfo.getMemInfoByName(key); + int val = 0; + try { + val = Integer.parseInt(value.trim()); + } catch (NumberFormatException ne) { + return; + } + if (info == null) { + return; + } + switch (info) { + case SIZE: + size = val; + break; + case RSS: + rss = val; + break; + case PSS: + pss = val; + break; + case SHARED_CLEAN: + sharedClean = val; + break; + case SHARED_DIRTY: + sharedDirty = val; + break; + case PRIVATE_CLEAN: + privateClean = val; + break; + case PRIVATE_DIRTY: + privateDirty = val; + break; + case REFERENCED: + referenced = val; + break; + case ANONYMOUS: + anonymous = val; + break; + default: + break; + } + } + + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("\t").append(this.getName()).append("\n"); + sb.append("\t").append(MemInfo.SIZE.name + ":" + this.getSize()) + .append(" kB\n"); + sb.append("\t").append(MemInfo.PSS.name + ":" + this.getPss()) + .append(" kB\n"); + sb.append("\t").append(MemInfo.RSS.name + ":" + this.getRss()) + .append(" kB\n"); + sb.append("\t") + .append(MemInfo.SHARED_CLEAN.name + ":" + this.getSharedClean()) + .append(" kB\n"); + sb.append("\t") + .append(MemInfo.SHARED_DIRTY.name + ":" + this.getSharedDirty()) + .append(" kB\n"); + sb.append("\t") + .append(MemInfo.PRIVATE_CLEAN.name + ":" + this.getPrivateClean()) + .append(" kB\n"); + sb.append("\t") + .append(MemInfo.PRIVATE_DIRTY.name + ":" + this.getPrivateDirty()) + .append(" kB\n"); + sb.append("\t") + .append(MemInfo.REFERENCED.name + ":" + this.getReferenced()) + .append(" kB\n"); + sb.append("\t") + .append(MemInfo.ANONYMOUS.name + ":" + this.getAnonymous()) + .append(" kB\n"); + return sb.toString(); + } + } + + /** + * Test the {@link ProcfsBasedProcessTree} + * + * @param args + */ + public static void main(String[] args) { + if (args.length != 1) { + System.out.println("Provide "); + return; + } + + System.out.println("Creating ProcfsBasedProcessTree for process " + + args[0]); + ProcfsBasedProcessTree procfsBasedProcessTree = new ProcfsBasedProcessTree(args[0]); + procfsBasedProcessTree.updateProcessTree(); + + System.out.println(procfsBasedProcessTree.getProcessTreeDump()); + System.out.println("Get cpu usage " + procfsBasedProcessTree + .getCpuUsagePercent()); + + try { + // Sleep so we can compute the CPU usage + Thread.sleep(500L); + } catch (InterruptedException e) { + // do nothing + } + + procfsBasedProcessTree.updateProcessTree(); + + System.out.println(procfsBasedProcessTree.getProcessTreeDump()); + System.out.println("Cpu usage " + procfsBasedProcessTree + .getCpuUsagePercent()); + System.out.println("Vmem usage in bytes " + procfsBasedProcessTree + .getVirtualMemorySize()); + System.out.println("Rss mem usage in bytes " + procfsBasedProcessTree + .getRssMemorySize()); + } +} diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/process/ShellCommandExecutor.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/process/ShellCommandExecutor.java new file mode 100644 index 0000000..b8045ab --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/process/ShellCommandExecutor.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink.metric.process; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.Charset; + +/** + * The shell command executor. + */ +public class ShellCommandExecutor { + + private final String[] command; + + private final StringBuilder output = new StringBuilder(); + + private int exitCode; + + /** + * Instantiates a new Shell command executor. + * + * @param execString the exec string + */ + public ShellCommandExecutor(String[] execString) { + this.command = execString; + } + + /** + * Execute. + * + * @throws IOException the io exception + */ + public void execute() throws IOException, InterruptedException { + final ProcessBuilder builder = new ProcessBuilder(command); + final Process process = builder.start(); + exitCode = process.waitFor(); + try (BufferedReader inReader = new BufferedReader(new InputStreamReader(process.getInputStream(), Charset.defaultCharset()))) { + char[] buf = new char[512]; + int readCount; + while ((readCount = inReader.read(buf, 0, buf.length)) > 0) { + output.append(buf, 0, readCount); + } + } + } + + /** + * Gets exit code. + * + * @return the exit code + */ + public int getExitCode() { + return exitCode; + } + + /** + * Gets stdout. + * + * @return the output + */ + public String getOutput() { + return output.toString(); + } +} diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/process/SysInfoLinux.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/process/SysInfoLinux.java new file mode 100644 index 0000000..eb48d29 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/process/SysInfoLinux.java @@ -0,0 +1,676 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink.metric.process; + +import java.io.BufferedReader; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStreamReader; +import java.math.BigInteger; +import java.nio.charset.Charset; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Set; +import java.util.TreeSet; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Plugin to calculate resource information on Linux systems. + */ +public class SysInfoLinux { + + /** + * proc's meminfo virtual file has keys-values in the format + * "key:[ \t]*value[ \t]kB". + */ + private static final String PROCFS_MEMFILE = "/proc/meminfo"; + private static final Pattern PROCFS_MEMFILE_FORMAT = + Pattern.compile("^([a-zA-Z_()]*):[ \t]*([0-9]*)[ \t]*(kB)?"); + + // We need the values for the following keys in meminfo + private static final String MEMTOTAL_STRING = "MemTotal"; + private static final String SWAPTOTAL_STRING = "SwapTotal"; + private static final String MEMFREE_STRING = "MemFree"; + private static final String SWAPFREE_STRING = "SwapFree"; + private static final String INACTIVE_STRING = "Inactive"; + private static final String INACTIVEFILE_STRING = "Inactive(file)"; + private static final String HARDWARECORRUPTED_STRING = "HardwareCorrupted"; + private static final String HUGEPAGESTOTAL_STRING = "HugePages_Total"; + private static final String HUGEPAGESIZE_STRING = "Hugepagesize"; + + /** + * Patterns for parsing /proc/cpuinfo. + */ + private static final String PROCFS_CPUINFO = "/proc/cpuinfo"; + private static final Pattern PROCESSOR_FORMAT = + Pattern.compile("^processor[ \t]:[ \t]*([0-9]*)"); + private static final Pattern FREQUENCY_FORMAT = + Pattern.compile("^cpu MHz[ \t]*:[ \t]*([0-9.]*)"); + private static final Pattern PHYSICAL_ID_FORMAT = + Pattern.compile("^physical id[ \t]*:[ \t]*([0-9]*)"); + private static final Pattern CORE_ID_FORMAT = + Pattern.compile("^core id[ \t]*:[ \t]*([0-9]*)"); + + /** + * Pattern for parsing /proc/stat. + */ + private static final String PROCFS_STAT = "/proc/stat"; + private static final Pattern CPU_TIME_FORMAT = + Pattern.compile("^cpu[ \t]*([0-9]*)" + + "[ \t]*([0-9]*)[ \t]*([0-9]*)[ \t].*"); + private CpuTimeTracker cpuTimeTracker; + + /** + * Pattern for parsing /proc/net/dev. + */ + private static final String PROCFS_NETFILE = "/proc/net/dev"; + private static final Pattern PROCFS_NETFILE_FORMAT = + Pattern.compile("^[ \t]*([a-zA-Z]+[0-9]*):" + + "[ \t]*([0-9]+)[ \t]*([0-9]+)[ \t]*([0-9]+)[ \t]*([0-9]+)" + + "[ \t]*([0-9]+)[ \t]*([0-9]+)[ \t]*([0-9]+)[ \t]*([0-9]+)" + + "[ \t]*([0-9]+)[ \t]*([0-9]+)[ \t]*([0-9]+)[ \t]*([0-9]+)" + + "[ \t]*([0-9]+)[ \t]*([0-9]+)[ \t]*([0-9]+)[ \t]*([0-9]+).*"); + + /** + * Pattern for parsing /proc/diskstats. + */ + private static final String PROCFS_DISKSFILE = "/proc/diskstats"; + private static final Pattern PROCFS_DISKSFILE_FORMAT = + Pattern.compile("^[ \t]*([0-9]+)[ \t]*([0-9 ]+)" + + "(?!([a-zA-Z]+[0-9]+))([a-zA-Z]+)" + + "[ \t]*([0-9]+)[ \t]*([0-9]+)[ \t]*([0-9]+)[ \t]*([0-9]+)" + + "[ \t]*([0-9]+)[ \t]*([0-9]+)[ \t]*([0-9]+)[ \t]*([0-9]+)" + + "[ \t]*([0-9]+)[ \t]*([0-9]+)[ \t]*([0-9]+)"); + /** + * Pattern for parsing /sys/block/partition_name/queue/hw_sector_size. + */ + private static final Pattern PROCFS_DISKSECTORFILE_FORMAT = + Pattern.compile("^([0-9]+)"); + + private String procfsMemFile; + private String procfsCpuFile; + private String procfsStatFile; + private String procfsNetFile; + private String procfsDisksFile; + private long jiffyLengthInMillis; + + private long ramSize = 0; + private long swapSize = 0; + private long ramSizeFree = 0; // free ram space on the machine (kB) + private long swapSizeFree = 0; // free swap space on the machine (kB) + private long inactiveSize = 0; // inactive memory (kB) + private long inactiveFileSize = -1; // inactive cache memory, -1 if not there + private long hardwareCorruptSize = 0; // RAM corrupt and not available + private long hugePagesTotal = 0; // # of hugepages reserved + private long hugePageSize = 0; // # size of each hugepage + + /* number of logical processors on the system. */ + private int numProcessors = 0; + /* number of physical cores on the system. */ + private int numCores = 0; + private int numCpuSocket = 0; + private long cpuFrequency = 0L; // CPU frequency on the system (kHz) + private long numNetBytesRead = 0L; // aggregated bytes read from network + private long numNetBytesWritten = 0L; // aggregated bytes written to network + private long numDisksBytesRead = 0L; // aggregated bytes read from disks + private long numDisksBytesWritten = 0L; // aggregated bytes written to disks + + private boolean readMemInfoFile = false; + private boolean readCpuInfoFile = false; + + /* map for every disk its sector size */ + private HashMap perDiskSectorSize = null; + + public static final long PAGE_SIZE = getConf("PAGESIZE"); + public static final long JIFFY_LENGTH_IN_MILLIS = + Math.max(Math.round(1000D / getConf("CLK_TCK")), -1); + + private static long getConf(String attr) { + if (OperatingSystem.isLinux()) { + try { + final ShellCommandExecutor shellCommandExecutor = new ShellCommandExecutor(new String[] { "getconf", attr }); + shellCommandExecutor.execute(); + return Long.parseLong(shellCommandExecutor.getOutput().replace("\n", "")); + } catch (IOException | InterruptedException | NumberFormatException e) { + return -1; + } + } + return -1; + } + + /** + * Get current time. + * + * @return Unix time stamp in millisecond + */ + long getCurrentTime() { + return System.currentTimeMillis(); + } + + public SysInfoLinux() { + this(PROCFS_MEMFILE, PROCFS_CPUINFO, PROCFS_STAT, + PROCFS_NETFILE, PROCFS_DISKSFILE, JIFFY_LENGTH_IN_MILLIS); + } + + /** + * Constructor which allows assigning the /proc/ directories. This will be + * used only in unit tests. + * + * @param procfsMemFile fake file for /proc/meminfo + * @param procfsCpuFile fake file for /proc/cpuinfo + * @param procfsStatFile fake file for /proc/stat + * @param procfsNetFile fake file for /proc/net/dev + * @param procfsDisksFile fake file for /proc/diskstats + * @param jiffyLengthInMillis fake jiffy length value + */ + public SysInfoLinux(String procfsMemFile, + String procfsCpuFile, + String procfsStatFile, + String procfsNetFile, + String procfsDisksFile, + long jiffyLengthInMillis) { + this.procfsMemFile = procfsMemFile; + this.procfsCpuFile = procfsCpuFile; + this.procfsStatFile = procfsStatFile; + this.procfsNetFile = procfsNetFile; + this.procfsDisksFile = procfsDisksFile; + this.jiffyLengthInMillis = jiffyLengthInMillis; + this.cpuTimeTracker = new CpuTimeTracker(jiffyLengthInMillis); + this.perDiskSectorSize = new HashMap(); + } + + /** + * Read /proc/meminfo, parse and compute memory information only once. + */ + private void readProcMemInfoFile() { + readProcMemInfoFile(false); + } + + /** + * Read /proc/meminfo, parse and compute memory information. + * + * @param readAgain if false, read only on the first time + */ + private void readProcMemInfoFile(boolean readAgain) { + + if (readMemInfoFile && !readAgain) { + return; + } + + // Read "/proc/memInfo" file + BufferedReader in; + InputStreamReader fReader; + try { + fReader = new InputStreamReader( + new FileInputStream(procfsMemFile), Charset.forName("UTF-8")); + in = new BufferedReader(fReader); + } catch (FileNotFoundException f) { + // shouldn't happen.... + return; + } + + Matcher mat; + + try { + String str = in.readLine(); + while (str != null) { + mat = PROCFS_MEMFILE_FORMAT.matcher(str); + if (mat.find()) { + if (mat.group(1).equals(MEMTOTAL_STRING)) { + ramSize = Long.parseLong(mat.group(2)); + } else if (mat.group(1).equals(SWAPTOTAL_STRING)) { + swapSize = Long.parseLong(mat.group(2)); + } else if (mat.group(1).equals(MEMFREE_STRING)) { + ramSizeFree = Long.parseLong(mat.group(2)); + } else if (mat.group(1).equals(SWAPFREE_STRING)) { + swapSizeFree = Long.parseLong(mat.group(2)); + } else if (mat.group(1).equals(INACTIVE_STRING)) { + inactiveSize = Long.parseLong(mat.group(2)); + } else if (mat.group(1).equals(INACTIVEFILE_STRING)) { + inactiveFileSize = Long.parseLong(mat.group(2)); + } else if (mat.group(1).equals(HARDWARECORRUPTED_STRING)) { + hardwareCorruptSize = Long.parseLong(mat.group(2)); + } else if (mat.group(1).equals(HUGEPAGESTOTAL_STRING)) { + hugePagesTotal = Long.parseLong(mat.group(2)); + } else if (mat.group(1).equals(HUGEPAGESIZE_STRING)) { + hugePageSize = Long.parseLong(mat.group(2)); + } + } + str = in.readLine(); + } + } catch (IOException io) { + } finally { + // Close the streams + try { + fReader.close(); + try { + in.close(); + } catch (IOException i) { + } + } catch (IOException i) { + } + } + + readMemInfoFile = true; + } + + /** + * Read /proc/cpuinfo, parse and calculate CPU information. + */ + private void readProcCpuInfoFile() { + // This directory needs to be read only once + if (readCpuInfoFile) { + return; + } + HashSet coreIdSet = new HashSet<>(); + // Read "/proc/cpuinfo" file + BufferedReader in; + InputStreamReader fReader; + try { + fReader = new InputStreamReader( + new FileInputStream(procfsCpuFile), Charset.forName("UTF-8")); + in = new BufferedReader(fReader); + } catch (FileNotFoundException f) { + // shouldn't happen.... + return; + } + Matcher mat; + Set physicalIds = new TreeSet<>(); + try { + numProcessors = 0; + numCores = 1; + String currentPhysicalId = ""; + String str = in.readLine(); + while (str != null) { + mat = PROCESSOR_FORMAT.matcher(str); + if (mat.find()) { + numProcessors++; + } + mat = FREQUENCY_FORMAT.matcher(str); + if (mat.find()) { + cpuFrequency = (long) (Double.parseDouble(mat.group(1)) * 1000); // kHz + } + mat = PHYSICAL_ID_FORMAT.matcher(str); + if (mat.find()) { + currentPhysicalId = str; + physicalIds.add(currentPhysicalId); + } + mat = CORE_ID_FORMAT.matcher(str); + if (mat.find()) { + coreIdSet.add(currentPhysicalId + " " + str); + numCores = coreIdSet.size(); + } + str = in.readLine(); + } + numCpuSocket = physicalIds.size(); + } catch (IOException io) { + } finally { + // Close the streams + try { + fReader.close(); + try { + in.close(); + } catch (IOException i) { + } + } catch (IOException i) { + } + } + readCpuInfoFile = true; + } + + /** + * Read /proc/stat file, parse and calculate cumulative CPU. + */ + private void readProcStatFile() { + // Read "/proc/stat" file + BufferedReader in; + InputStreamReader fReader; + try { + fReader = new InputStreamReader( + new FileInputStream(procfsStatFile), Charset.forName("UTF-8")); + in = new BufferedReader(fReader); + } catch (FileNotFoundException f) { + // shouldn't happen.... + return; + } + + Matcher mat; + try { + String str = in.readLine(); + while (str != null) { + mat = CPU_TIME_FORMAT.matcher(str); + if (mat.find()) { + long uTime = Long.parseLong(mat.group(1)); + long nTime = Long.parseLong(mat.group(2)); + long sTime = Long.parseLong(mat.group(3)); + cpuTimeTracker.updateElapsedJiffies( + BigInteger.valueOf(uTime + nTime + sTime), + getCurrentTime()); + break; + } + str = in.readLine(); + } + } catch (IOException io) { + } finally { + // Close the streams + try { + fReader.close(); + try { + in.close(); + } catch (IOException i) { + } + } catch (IOException i) { + } + } + } + + /** + * Read /proc/net/dev file, parse and calculate amount + * of bytes read and written through the network. + */ + private void readProcNetInfoFile() { + + numNetBytesRead = 0L; + numNetBytesWritten = 0L; + + // Read "/proc/net/dev" file + BufferedReader in; + InputStreamReader fReader; + try { + fReader = new InputStreamReader( + new FileInputStream(procfsNetFile), Charset.forName("UTF-8")); + in = new BufferedReader(fReader); + } catch (FileNotFoundException f) { + return; + } + + Matcher mat; + try { + String str = in.readLine(); + while (str != null) { + mat = PROCFS_NETFILE_FORMAT.matcher(str); + if (mat.find()) { + assert mat.groupCount() >= 16; + + // ignore loopback interfaces + if (mat.group(1).equals("lo")) { + str = in.readLine(); + continue; + } + numNetBytesRead += Long.parseLong(mat.group(2)); + numNetBytesWritten += Long.parseLong(mat.group(10)); + } + str = in.readLine(); + } + } catch (IOException io) { + } finally { + // Close the streams + try { + fReader.close(); + try { + in.close(); + } catch (IOException i) { + } + } catch (IOException i) { + } + } + } + + /** + * Read /proc/diskstats file, parse and calculate amount + * of bytes read and written from/to disks. + */ + private void readProcDisksInfoFile() { + + numDisksBytesRead = 0L; + numDisksBytesWritten = 0L; + + // Read "/proc/diskstats" file + BufferedReader in; + try { + in = new BufferedReader(new InputStreamReader( + new FileInputStream(procfsDisksFile), Charset.forName("UTF-8"))); + } catch (FileNotFoundException f) { + return; + } + + Matcher mat; + try { + String str = in.readLine(); + while (str != null) { + mat = PROCFS_DISKSFILE_FORMAT.matcher(str); + if (mat.find()) { + String diskName = mat.group(4); + assert diskName != null; + // ignore loop or ram partitions + if (diskName.contains("loop") || diskName.contains("ram")) { + str = in.readLine(); + continue; + } + + Integer sectorSize; + synchronized (perDiskSectorSize) { + sectorSize = perDiskSectorSize.get(diskName); + if (null == sectorSize) { + // retrieve sectorSize + // if unavailable or error, assume 512 + sectorSize = readDiskBlockInformation(diskName, 512); + perDiskSectorSize.put(diskName, sectorSize); + } + } + + String sectorsRead = mat.group(7); + String sectorsWritten = mat.group(11); + if (null == sectorsRead || null == sectorsWritten) { + return; + } + numDisksBytesRead += Long.parseLong(sectorsRead) * sectorSize; + numDisksBytesWritten += Long.parseLong(sectorsWritten) * sectorSize; + } + str = in.readLine(); + } + } catch (IOException e) { + } finally { + // Close the streams + try { + in.close(); + } catch (IOException e) { + } + } + } + + /** + * Read /sys/block/diskName/queue/hw_sector_size file, parse and calculate + * sector size for a specific disk. + * + * @return sector size of specified disk, or defSector + */ + int readDiskBlockInformation(String diskName, int defSector) { + + assert perDiskSectorSize != null && diskName != null; + + String procfsDiskSectorFile = + "/sys/block/" + diskName + "/queue/hw_sector_size"; + + BufferedReader in; + try { + in = new BufferedReader(new InputStreamReader( + new FileInputStream(procfsDiskSectorFile), + Charset.forName("UTF-8"))); + } catch (FileNotFoundException f) { + return defSector; + } + + Matcher mat; + try { + String str = in.readLine(); + while (str != null) { + mat = PROCFS_DISKSECTORFILE_FORMAT.matcher(str); + if (mat.find()) { + String secSize = mat.group(1); + if (secSize != null) { + return Integer.parseInt(secSize); + } + } + str = in.readLine(); + } + return defSector; + } catch (IOException | NumberFormatException e) { + return defSector; + } finally { + // Close the streams + try { + in.close(); + } catch (IOException e) { + } + } + } + + public long getPhysicalMemorySize() { + readProcMemInfoFile(); + return (ramSize + - hardwareCorruptSize + - (hugePagesTotal * hugePageSize)) * 1024; + } + + public long getVirtualMemorySize() { + return getPhysicalMemorySize() + (swapSize * 1024); + } + + public long getAvailablePhysicalMemorySize() { + readProcMemInfoFile(true); + long inactive = inactiveFileSize != -1 + ? inactiveFileSize + : inactiveSize; + return (ramSizeFree + inactive) * 1024; + } + + public long getAvailableVirtualMemorySize() { + return getAvailablePhysicalMemorySize() + (swapSizeFree * 1024); + } + + public int getNumProcessors() { + readProcCpuInfoFile(); + return numProcessors; + } + + public int getNumCores() { + readProcCpuInfoFile(); + return numCores; + } + + public int getNumCpuSocket() { + readProcCpuInfoFile(); + return numCpuSocket; + } + + public long getCpuFrequency() { + readProcCpuInfoFile(); + return cpuFrequency; + } + + public long getCumulativeCpuTime() { + readProcStatFile(); + return cpuTimeTracker.getCumulativeCpuTime(); + } + + public float getCpuUsagePercentage() { + readProcStatFile(); + float overallCpuUsage = cpuTimeTracker.getCpuTrackerUsagePercent(); + if (overallCpuUsage != CpuTimeTracker.UNAVAILABLE) { + overallCpuUsage = overallCpuUsage / getNumProcessors(); + } + return overallCpuUsage; + } + + public float getNumVCoresUsed() { + readProcStatFile(); + float overallVCoresUsage = cpuTimeTracker.getCpuTrackerUsagePercent(); + if (overallVCoresUsage != CpuTimeTracker.UNAVAILABLE) { + overallVCoresUsage = overallVCoresUsage / 100F; + } + return overallVCoresUsage; + } + + public long getNetworkBytesRead() { + readProcNetInfoFile(); + return numNetBytesRead; + } + + public long getNetworkBytesWritten() { + readProcNetInfoFile(); + return numNetBytesWritten; + } + + public long getStorageBytesRead() { + readProcDisksInfoFile(); + return numDisksBytesRead; + } + + public long getStorageBytesWritten() { + readProcDisksInfoFile(); + return numDisksBytesWritten; + } + + /** + * Test the {@link SysInfoLinux}. + * + * @param args - arguments to this calculator test + */ + public static void main(String[] args) { + SysInfoLinux plugin = new SysInfoLinux(); + System.out.println("Physical memory Size (bytes) : " + + plugin.getPhysicalMemorySize()); + System.out.println("Total Virtual memory Size (bytes) : " + + plugin.getVirtualMemorySize()); + System.out.println("Available Physical memory Size (bytes) : " + + plugin.getAvailablePhysicalMemorySize()); + System.out.println("Total Available Virtual memory Size (bytes) : " + + plugin.getAvailableVirtualMemorySize()); + System.out.println("Number of Processors : " + plugin.getNumProcessors()); + System.out.println("CPU frequency (kHz) : " + plugin.getCpuFrequency()); + System.out.println("Cumulative CPU time (ms) : " + + plugin.getCumulativeCpuTime()); + System.out.println("Total network read (bytes) : " + + plugin.getNetworkBytesRead()); + System.out.println("Total network written (bytes) : " + + plugin.getNetworkBytesWritten()); + System.out.println("Total storage read (bytes) : " + + plugin.getStorageBytesRead()); + System.out.println("Total storage written (bytes) : " + + plugin.getStorageBytesWritten()); + try { + // Sleep so we can compute the CPU usage + Thread.sleep(500L); + } catch (InterruptedException e) { + // do nothing + } + System.out.println("CPU usage % : " + plugin.getCpuUsagePercentage()); + } + + void setReadCpuInfoFile(boolean readCpuInfoFileValue) { + this.readCpuInfoFile = readCpuInfoFileValue; + } + + public long getJiffyLengthInMillis() { + return this.jiffyLengthInMillis; + } +} diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/process/clock/Clock.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/process/clock/Clock.java new file mode 100644 index 0000000..35433a0 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/process/clock/Clock.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink.metric.process.clock; + +/** + * A clock that gives access to time. This clock returns two flavors of time: + * + *

Absolute Time: This refers to real world wall clock time, and it typically + * derived from a system clock. It is subject to clock drift and inaccuracy, and can jump + * if the system clock is adjusted. + * + *

Relative Time: This time advances at the same speed as the absolute time, + * but the timestamps can only be referred to relative to each other. The timestamps have + * no absolute meaning and cannot be compared across JVM processes. The source for the + * timestamps is not affected by adjustments to the system clock, so it never jumps. + */ +public abstract class Clock { + + public abstract long absoluteTimeMillis(); + + public abstract long relativeTimeMillis(); + + public abstract long relativeTimeNanos(); +} diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/process/clock/SystemClock.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/process/clock/SystemClock.java new file mode 100644 index 0000000..01c0cb6 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/process/clock/SystemClock.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink.metric.process.clock; + +/** + * A clock that returns the time of the system / process. + * + *

This clock uses {@link System#currentTimeMillis()} for absolute time + * and {@link System#nanoTime()} for relative time. + * + *

This SystemClock exists as a singleton instance. + */ +public class SystemClock extends Clock { + + private static final SystemClock INSTANCE = new SystemClock(); + + public static SystemClock getInstance() { + return INSTANCE; + } + + // ------------------------------------------------------------------------ + + @Override + public long absoluteTimeMillis() { + return System.currentTimeMillis(); + } + + @Override + public long relativeTimeMillis() { + return System.nanoTime() / 1_000_000; + } + + @Override + public long relativeTimeNanos() { + return System.nanoTime(); + } + + // ------------------------------------------------------------------------ + + private SystemClock() {} +} diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/tps/TpsMetric.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/tps/TpsMetric.java new file mode 100644 index 0000000..ac38031 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/metric/tps/TpsMetric.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink.metric.tps; + +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonIgnore; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonInclude; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonProcessingException; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; + +import com.github.nexmark.flink.utils.NexmarkUtils; + +import javax.annotation.Nullable; + +import java.util.Objects; + +import static java.util.Objects.requireNonNull; + +/** + * Response type for TPS aggregated metrics. Contains the metric name and optionally the sum, average, minimum and maximum. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class TpsMetric { + + private static final String FIELD_NAME_ID = "id"; + + private static final String FIELD_NAME_MIN = "min"; + + private static final String FIELD_NAME_MAX = "max"; + + private static final String FIELD_NAME_AVG = "avg"; + + private static final String FIELD_NAME_SUM = "sum"; + + @JsonProperty(value = FIELD_NAME_ID, required = true) + private final String id; + + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonProperty(FIELD_NAME_MIN) + private final Double min; + + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonProperty(FIELD_NAME_MAX) + private final Double max; + + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonProperty(FIELD_NAME_AVG) + private final Double avg; + + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonProperty(FIELD_NAME_SUM) + private final Double sum; + + @JsonCreator + public TpsMetric( + final @JsonProperty(value = FIELD_NAME_ID, required = true) String id, + final @Nullable @JsonProperty(FIELD_NAME_MIN) Double min, + final @Nullable @JsonProperty(FIELD_NAME_MAX) Double max, + final @Nullable @JsonProperty(FIELD_NAME_AVG) Double avg, + final @Nullable @JsonProperty(FIELD_NAME_SUM) Double sum) { + + this.id = requireNonNull(id, "id must not be null"); + this.min = min; + this.max = max; + this.avg = avg; + this.sum = sum; + } + + public TpsMetric(final @JsonProperty(value = FIELD_NAME_ID, required = true) String id) { + this(id, null, null, null, null); + } + + @JsonIgnore + public String getId() { + return id; + } + + @JsonIgnore + public Double getMin() { + return min; + } + + @JsonIgnore + public Double getMax() { + return max; + } + + @JsonIgnore + public Double getSum() { + return sum; + } + + @JsonIgnore + public Double getAvg() { + return avg; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + TpsMetric tpsMetric = (TpsMetric) o; + return Objects.equals(id, tpsMetric.id) && + Objects.equals(min, tpsMetric.min) && + Objects.equals(max, tpsMetric.max) && + Objects.equals(avg, tpsMetric.avg) && + Objects.equals(sum, tpsMetric.sum); + } + + @Override + public int hashCode() { + return Objects.hash(id, min, max, avg, sum); + } + + @Override + public String toString() { + return "AggregatedMetric{" + + "id='" + id + '\'' + + ", mim='" + min + '\'' + + ", max='" + max + '\'' + + ", avg='" + avg + '\'' + + ", sum='" + sum + '\'' + + '}'; + } + + public static TpsMetric fromJson(String json) { + try { + JsonNode jsonNode = NexmarkUtils.MAPPER.readTree(json); + return NexmarkUtils.MAPPER.convertValue(jsonNode.get(0), TpsMetric.class); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } +} diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/model/Auction.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/model/Auction.java new file mode 100644 index 0000000..72f989d --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/model/Auction.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink.model; + +import java.io.Serializable; +import java.time.Instant; +import java.util.Objects; + +/** An auction submitted by a person. */ +public class Auction implements Serializable { + + /** Id of auction. */ + public long id; // primary key + + /** Extra auction properties. */ + public String itemName; + + public String description; + + /** Initial bid price, in cents. */ + public long initialBid; + + /** Reserve price, in cents. */ + public long reserve; + + public Instant dateTime; + + /** When does auction expire? (ms since epoch). Bids at or after this time are ignored. */ + public Instant expires; + + /** Id of person who instigated auction. */ + public long seller; // foreign key: Person.id + + /** Id of category auction is listed under. */ + public long category; // foreign key: Category.id + + /** Additional arbitrary payload for performance testing. */ + public String extra; + + public Auction( + long id, + String itemName, + String description, + long initialBid, + long reserve, + Instant dateTime, + Instant expires, + long seller, + long category, + String extra) { + this.id = id; + this.itemName = itemName; + this.description = description; + this.initialBid = initialBid; + this.reserve = reserve; + this.dateTime = dateTime; + this.expires = expires; + this.seller = seller; + this.category = category; + this.extra = extra; + } + + @Override + public String toString() { + return "Auction{" + + "id=" + id + + ", itemName='" + itemName + '\'' + + ", description='" + description + '\'' + + ", initialBid=" + initialBid + + ", reserve=" + reserve + + ", dateTime=" + dateTime + + ", expires=" + expires + + ", seller=" + seller + + ", category=" + category + + ", extra='" + extra + '\'' + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Auction auction = (Auction) o; + return id == auction.id + && initialBid == auction.initialBid + && reserve == auction.reserve + && Objects.equals(dateTime, auction.dateTime) + && Objects.equals(expires, auction.expires) + && seller == auction.seller + && category == auction.category + && Objects.equals(itemName, auction.itemName) + && Objects.equals(description, auction.description) + && Objects.equals(extra, auction.extra); + } + + @Override + public int hashCode() { + return Objects.hash( + id, itemName, description, initialBid, reserve, dateTime, expires, seller, category, extra); + } +} diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/model/Bid.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/model/Bid.java new file mode 100644 index 0000000..a9b5458 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/model/Bid.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink.model; + +import java.io.Serializable; +import java.time.Instant; +import java.util.Objects; + +/** A bid for an item on auction. */ +public class Bid implements Serializable { + + /** Id of auction this bid is for. */ + public long auction; // foreign key: Auction.id + + /** Id of person bidding in auction. */ + public long bidder; // foreign key: Person.id + + /** Price of bid, in cents. */ + public long price; + + /** The channel introduced this bidding. */ + public String channel; + + /** The url of this bid. */ + public String url; + + /** + * Instant at which bid was made (ms since epoch). NOTE: This may be earlier than the system's + * event time. + */ + public Instant dateTime; + + /** Additional arbitrary payload for performance testing. */ + public String extra; + + public Bid(long auction, long bidder, long price, String channel, String url, Instant dateTime, String extra) { + this.auction = auction; + this.bidder = bidder; + this.price = price; + this.channel = channel; + this.url = url; + this.dateTime = dateTime; + this.extra = extra; + } + + @Override + public boolean equals(Object otherObject) { + if (this == otherObject) { + return true; + } + if (otherObject == null || getClass() != otherObject.getClass()) { + return false; + } + + Bid other = (Bid) otherObject; + return Objects.equals(auction, other.auction) + && Objects.equals(bidder, other.bidder) + && Objects.equals(price, other.price) + && Objects.equals(channel, other.channel) + && Objects.equals(url, other.url) + && Objects.equals(dateTime, other.dateTime) + && Objects.equals(extra, other.extra); + } + + @Override + public int hashCode() { + return Objects.hash(auction, bidder, price, channel, url, dateTime, extra); + } + + @Override + public String toString() { + return "Bid{" + + "auction=" + auction + + ", bidder=" + bidder + + ", price=" + price + + ", channel=" + channel + + ", url=" + url + + ", dateTime=" + dateTime + + ", extra='" + extra + '\'' + + '}'; + } +} diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/model/Event.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/model/Event.java new file mode 100644 index 0000000..d13d0b7 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/model/Event.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink.model; + +import javax.annotation.Nullable; + +import java.util.Objects; + +/** + * An event in the auction system, either a (new) {@link Person}, a (new) {@link Auction}, or a + * {@link Bid}. + */ +public class Event { + + public @Nullable Person newPerson; + public @Nullable Auction newAuction; + public @Nullable Bid bid; + public Type type; + + /** The type of object stored in this event. * */ + public enum Type { + PERSON(0), + AUCTION(1), + BID(2); + + public final int value; + + Type(int value) { + this.value = value; + } + } + + public Event(Person newPerson) { + this.newPerson = newPerson; + newAuction = null; + bid = null; + type = Type.PERSON; + } + + public Event(Auction newAuction) { + newPerson = null; + this.newAuction = newAuction; + bid = null; + type = Type.AUCTION; + } + + public Event(Bid bid) { + newPerson = null; + newAuction = null; + this.bid = bid; + type = Type.BID; + } + + @Override + public boolean equals(@Nullable Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Event event = (Event) o; + return Objects.equals(newPerson, event.newPerson) + && Objects.equals(newAuction, event.newAuction) + && Objects.equals(bid, event.bid); + } + + @Override + public int hashCode() { + return Objects.hash(newPerson, newAuction, bid); + } + + @Override + public String toString() { + if (newPerson != null) { + return newPerson.toString(); + } else if (newAuction != null) { + return newAuction.toString(); + } else if (bid != null) { + return bid.toString(); + } else { + throw new RuntimeException("invalid event"); + } + } +} diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/model/Person.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/model/Person.java new file mode 100644 index 0000000..87fa00a --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/model/Person.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink.model; + +import java.io.Serializable; +import java.time.Instant; +import java.util.Objects; + +/** A person either creating an auction or making a bid. */ +public class Person implements Serializable { + + /** Id of person. */ + public long id; // primary key + + /** Extra person properties. */ + public String name; + + public String emailAddress; + + public String creditCard; + + public String city; + + public String state; + + public Instant dateTime; + + /** Additional arbitrary payload for performance testing. */ + public String extra; + + public Person( + long id, + String name, + String emailAddress, + String creditCard, + String city, + String state, + Instant dateTime, + String extra) { + this.id = id; + this.name = name; + this.emailAddress = emailAddress; + this.creditCard = creditCard; + this.city = city; + this.state = state; + this.dateTime = dateTime; + this.extra = extra; + } + + @Override + public String toString() { + return "Person{" + + "id=" + id + + ", name='" + name + '\'' + + ", emailAddress='" + emailAddress + '\'' + + ", creditCard='" + creditCard + '\'' + + ", city='" + city + '\'' + + ", state='" + state + '\'' + + ", dateTime=" + dateTime + + ", extra='" + extra + '\'' + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Person person = (Person) o; + return id == person.id + && Objects.equals(dateTime, person.dateTime) + && Objects.equals(name, person.name) + && Objects.equals(emailAddress, person.emailAddress) + && Objects.equals(creditCard, person.creditCard) + && Objects.equals(city, person.city) + && Objects.equals(state, person.state) + && Objects.equals(extra, person.extra); + } + + @Override + public int hashCode() { + return Objects.hash(id, name, emailAddress, creditCard, city, state, dateTime, extra); + } +} diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/source/EventDeserializer.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/source/EventDeserializer.java new file mode 100644 index 0000000..d249479 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/source/EventDeserializer.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink.source; + +import com.github.nexmark.flink.model.Event; + +import java.io.Serializable; + +/** + * Parse {@link Event} into user specified type. + */ +public interface EventDeserializer extends Serializable { + + T deserialize(Event event); +} diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/source/NexmarkSource.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/source/NexmarkSource.java new file mode 100644 index 0000000..fc1b283 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/source/NexmarkSource.java @@ -0,0 +1,299 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink.source; + +import com.github.nexmark.flink.generator.GeneratorConfig; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.connector.source.Boundedness; +import org.apache.flink.api.connector.source.Source; +import org.apache.flink.api.connector.source.SourceReader; +import org.apache.flink.api.connector.source.SourceReaderContext; +import org.apache.flink.api.connector.source.SourceSplit; +import org.apache.flink.api.connector.source.SplitEnumerator; +import org.apache.flink.api.connector.source.SplitEnumeratorContext; +import org.apache.flink.api.java.typeutils.ResultTypeQueryable; +import org.apache.flink.core.io.SimpleVersionedSerializer; +import org.apache.flink.table.data.RowData; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nullable; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.Serializable; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedList; +import java.util.List; +import java.util.UUID; + +/** + * Nexmark source, using Source V2 of Flink. + */ +public class NexmarkSource implements Source>, + ResultTypeQueryable { + + private static final Logger LOG = LoggerFactory.getLogger(NexmarkSource.class); + + private final GeneratorConfig config; + private final TypeInformation outputType; + private final RowDataEventDeserializer deserializer; + + NexmarkSource(GeneratorConfig config, TypeInformation outputType) { + this.config = config; + this.outputType = outputType; + this.deserializer = new RowDataEventDeserializer(); + } + + @Override + public Boundedness getBoundedness() { + return Boundedness.CONTINUOUS_UNBOUNDED; + } + + @Override + public SplitEnumerator> createEnumerator( + SplitEnumeratorContext splitEnumeratorContext) throws Exception { + LOG.info("Creating Nexmark Enumerator"); + return new StaticSplitEnumerator(splitEnumeratorContext, getSplits(splitEnumeratorContext.currentParallelism())); + } + + @Override + public SplitEnumerator> restoreEnumerator( + SplitEnumeratorContext splitEnumeratorContext, + Collection nexmarkSourceSplits) throws Exception { + return new StaticSplitEnumerator(splitEnumeratorContext, nexmarkSourceSplits); + } + + @Override + public SimpleVersionedSerializer getSplitSerializer() { + return SimpleSplitSerializer.INSTANCE; + } + + @Override + public SimpleVersionedSerializer> getEnumeratorCheckpointSerializer() { + return new SimpleSplitCollectionSerializer(SimpleSplitSerializer.INSTANCE); + } + + @Override + public SourceReader createReader(SourceReaderContext sourceReaderContext) { + LOG.info("Creating Nexmark Reader"); + return new NexmarkSourceReader(sourceReaderContext, config, deserializer); + } + + @Override + public TypeInformation getProducedType() { + return outputType; + } + + List getSplits(int parallelism) { + List subConfigs = config.split(parallelism); + List splits = new ArrayList<>(parallelism); + for (GeneratorConfig subConfig : subConfigs) { + splits.add(new NexmarkSourceSplit(UUID.randomUUID().toString(), subConfig)); + } + return splits; + } + + public static class StaticSplitEnumerator + implements SplitEnumerator> { + + private static final Logger LOG = LoggerFactory.getLogger(StaticSplitEnumerator.class); + + private final SplitEnumeratorContext context; + private final LinkedList splits; + + StaticSplitEnumerator(SplitEnumeratorContext context, + Collection splits) { + this.context = context; + this.splits = new LinkedList<>(splits); + LOG.info("StaticSplitEnumerator init with {} splits", splits.size()); + } + + @Override + public void start() { + } + + @Override + public void handleSplitRequest(int subtask, @Nullable String hostname) { + if (!context.registeredReaders().containsKey(subtask)) { + // reader failed between sending the request and now. skip this request. + return; + } + + final NexmarkSourceSplit split = splits.removeFirst(); + if (split != null) { + context.assignSplit(split, subtask); + LOG.info("Assigned split to subtask {} : {}", subtask, split); + } else { + context.signalNoMoreSplits(subtask); + LOG.info("No more splits available for subtask {}", subtask); + } + } + + @Override + public void addSplitsBack(List list, int i) { + splits.addAll(list); + } + + @Override + public void addReader(int i) { + } + + @Override + public Collection snapshotState(long l) throws Exception { + return new ArrayList<>(splits); + } + + @Override + public void close() throws IOException { + } + } + + public static class NexmarkSourceSplit implements SourceSplit, Serializable { + + private final String id; + private final GeneratorConfig generatorConfig; + private volatile long numEmittedSoFar; + private long wallClockBaseTime = -1L; + + NexmarkSourceSplit(String id, GeneratorConfig generatorConfig) { + this.id = id; + this.generatorConfig = generatorConfig; + this.numEmittedSoFar = 0; + } + + @Override + public String splitId() { + return id; + } + + public GeneratorConfig getGeneratorConfig() { + return generatorConfig; + } + + public long getNumEmittedSoFar() { + return numEmittedSoFar; + } + + public long getWallClockBaseTime() { + return wallClockBaseTime; + } + + public void setNumEmittedSoFar(long numEmittedSoFar) { + this.numEmittedSoFar = numEmittedSoFar; + } + + public void setWallClockBaseTime(long wallClockBaseTime) { + this.wallClockBaseTime = wallClockBaseTime; + } + } + + public static class SimpleSplitSerializer implements SimpleVersionedSerializer { + + public static SimpleSplitSerializer INSTANCE = new SimpleSplitSerializer(); + + private SimpleSplitSerializer() { + } + + @Override + public int getVersion() { + return 1; + } + + @Override + public byte[] serialize(NexmarkSourceSplit nexmarkSourceSplit) throws IOException { + try (ByteArrayOutputStream output = new ByteArrayOutputStream(64); + ObjectOutputStream oos = new ObjectOutputStream(output)) { + oos.writeObject(nexmarkSourceSplit); + return output.toByteArray(); + } + } + + @Override + public NexmarkSourceSplit deserialize(int i, byte[] bytes) throws IOException { + try (ByteArrayInputStream bais = new ByteArrayInputStream(bytes); + ObjectInputStream ois = new ObjectInputStream(bais)) { + return (NexmarkSourceSplit) ois.readObject(); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + } + } + + public static class SimpleSplitCollectionSerializer + implements SimpleVersionedSerializer> { + + SimpleVersionedSerializer splitSerializer; + + public SimpleSplitCollectionSerializer(SimpleVersionedSerializer splitSerializer) { + this.splitSerializer = splitSerializer; + } + + @Override + public int getVersion() { + return splitSerializer.getVersion(); + } + + @Override + public byte[] serialize(Collection splits) throws IOException { + final ArrayList serializedSplits = new ArrayList<>(splits.size()); + int totalLen = 8; + for (NexmarkSourceSplit split : splits) { + final byte[] serSplit = splitSerializer.serialize(split); + serializedSplits.add(serSplit); + totalLen += serSplit.length + 4; // 4 bytes for the length field + } + + final byte[] result = new byte[totalLen]; + final ByteBuffer byteBuffer = ByteBuffer.wrap(result).order(ByteOrder.LITTLE_ENDIAN); + byteBuffer.putInt(getVersion()); + byteBuffer.putInt(splits.size()); + for (byte[] splitBytes : serializedSplits) { + byteBuffer.putInt(splitBytes.length); + byteBuffer.put(splitBytes); + } + return result; + } + + @Override + public Collection deserialize(int i, byte[] input) throws IOException { + final ByteBuffer bb = ByteBuffer.wrap(input).order(ByteOrder.LITTLE_ENDIAN); + + final int splitSerializerVersion = bb.getInt(); + final int numSplits = bb.getInt(); + + final ArrayList splits = new ArrayList<>(numSplits); + + for (int remaining = numSplits; remaining > 0; remaining--) { + final byte[] bytes = new byte[bb.getInt()]; + bb.get(bytes); + final NexmarkSourceSplit split = splitSerializer.deserialize(splitSerializerVersion, bytes); + splits.add(split); + } + return splits; + } + } +} diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/source/NexmarkSourceOptions.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/source/NexmarkSourceOptions.java new file mode 100644 index 0000000..52933c9 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/source/NexmarkSourceOptions.java @@ -0,0 +1,204 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink.source; + +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.configuration.ConfigOptions; +import org.apache.flink.configuration.MemorySize; +import org.apache.flink.configuration.ReadableConfig; + +import com.github.nexmark.flink.utils.NexmarkUtils; +import com.github.nexmark.flink.NexmarkConfiguration; + +import java.time.Duration; + +public class NexmarkSourceOptions { + + /** + * @see NexmarkConfiguration#rateShape + */ + public static final ConfigOption RATE_SHAPE = ConfigOptions + .key("rate.shape") + .enumType(NexmarkUtils.RateShape.class) + .defaultValue(NexmarkUtils.RateShape.SQUARE); + + /** + * @see NexmarkConfiguration#ratePeriodSec + */ + public static final ConfigOption RATE_PERIOD = ConfigOptions + .key("rate.period") + .durationType() + .defaultValue(Duration.ofSeconds(600)); + + /** + * @see NexmarkConfiguration#isRateLimited + */ + public static final ConfigOption RATE_LIMITED = ConfigOptions + .key("rate.limited") + .booleanType() + .defaultValue(false); + + /** + * @see NexmarkConfiguration#firstEventRate + */ + public static final ConfigOption FIRST_EVENT_RATE = ConfigOptions + .key("first-event.rate") + .intType() + .defaultValue(10000); + + /** + * @see NexmarkConfiguration#nextEventRate + */ + public static final ConfigOption NEXT_EVENT_RATE = ConfigOptions + .key("next-event.rate") + .intType() + .defaultValue(10000); + + /** + * @see NexmarkConfiguration#avgPersonByteSize + */ + public static final ConfigOption PERSON_AVG_SIZE = ConfigOptions + .key("person.avg-size") + .memoryType() + .defaultValue(MemorySize.parse("200b")); + + /** + * @see NexmarkConfiguration#avgAuctionByteSize + */ + public static final ConfigOption AUCTION_AVG_SIZE = ConfigOptions + .key("auction.avg-size") + .memoryType() + .defaultValue(MemorySize.parse("500b")); + + /** + * @see NexmarkConfiguration#avgBidByteSize + */ + public static final ConfigOption BID_AVG_SIZE = ConfigOptions + .key("bid.avg-size") + .memoryType() + .defaultValue(MemorySize.parse("100b")); + + /** + * @see NexmarkConfiguration#personProportion + */ + public static final ConfigOption PERSON_PROPORTION = ConfigOptions + .key("person.proportion") + .intType() + .defaultValue(1); + + /** + * @see NexmarkConfiguration#auctionProportion + */ + public static final ConfigOption AUCTION_PROPORTION = ConfigOptions + .key("auction.proportion") + .intType() + .defaultValue(3); + + /** + * @see NexmarkConfiguration#bidProportion + */ + public static final ConfigOption BID_PROPORTION = ConfigOptions + .key("bid.proportion") + .intType() + .defaultValue(46); + + /** + * @see NexmarkConfiguration#hotAuctionRatio + */ + public static final ConfigOption BID_HOT_RATIO_AUCTIONS = ConfigOptions + .key("bid.hot-ratio.auctions") + .intType() + .defaultValue(2); + + /** + * @see NexmarkConfiguration#hotBiddersRatio + */ + public static final ConfigOption BID_HOT_RATIO_BIDDERS = ConfigOptions + .key("bid.hot-ratio.bidders") + .intType() + .defaultValue(4); + + /** + * @see NexmarkConfiguration#hotSellersRatio + */ + public static final ConfigOption AUCTION_HOT_RATIO_SELLERS = ConfigOptions + .key("auction.hot-ratio.sellers") + .intType() + .defaultValue(4); + + /** + * @see NexmarkConfiguration#numEvents + */ + public static final ConfigOption EVENTS_NUM = ConfigOptions + .key("events.num") + .longType() + .defaultValue(0L); + + /** + * @see NexmarkConfiguration#isSourceKeepAlive + */ + public static final ConfigOption KEEP_ALIVE = ConfigOptions + .key("keep-alive") + .booleanType() + .defaultValue(false); + + /** + * @see NexmarkConfiguration#stopAtEvent + */ + public static final ConfigOption STOP_AT = ConfigOptions + .key("stop-at") + .longType() + .defaultValue(-1L); + + /** + * @see NexmarkConfiguration#maxEmitSpeed + */ + public static final ConfigOption MAX_EMIT_SPEED = ConfigOptions + .key("max-emit-speed") + .booleanType() + .defaultValue(true); + + public static NexmarkConfiguration convertToNexmarkConfiguration(ReadableConfig config) { + NexmarkConfiguration nexmarkConf = new NexmarkConfiguration(); + nexmarkConf.rateShape = config.get(RATE_SHAPE); + nexmarkConf.ratePeriodSec = (int) config.get(RATE_PERIOD).getSeconds(); + nexmarkConf.isRateLimited = config.get(RATE_LIMITED); + nexmarkConf.firstEventRate = config.get(FIRST_EVENT_RATE); + nexmarkConf.nextEventRate = config.get(NEXT_EVENT_RATE); + nexmarkConf.avgPersonByteSize = (int) config.get(PERSON_AVG_SIZE).getBytes(); + nexmarkConf.avgAuctionByteSize = (int) config.get(AUCTION_AVG_SIZE).getBytes(); + nexmarkConf.avgBidByteSize = (int) config.get(BID_AVG_SIZE).getBytes(); + nexmarkConf.personProportion = config.get(PERSON_PROPORTION); + nexmarkConf.auctionProportion = config.get(AUCTION_PROPORTION); + nexmarkConf.bidProportion = config.get(BID_PROPORTION); + nexmarkConf.hotAuctionRatio = config.get(BID_HOT_RATIO_AUCTIONS); + nexmarkConf.hotBiddersRatio = config.get(BID_HOT_RATIO_BIDDERS); + nexmarkConf.hotSellersRatio = config.get(AUCTION_HOT_RATIO_SELLERS); + nexmarkConf.numEvents = config.get(EVENTS_NUM); + nexmarkConf.isSourceKeepAlive = config.get(KEEP_ALIVE); + nexmarkConf.stopAtEvent = config.get(STOP_AT); + nexmarkConf.maxEmitSpeed = config.get(MAX_EMIT_SPEED); + + return nexmarkConf; + } + + + + +} diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/source/NexmarkSourceReader.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/source/NexmarkSourceReader.java new file mode 100644 index 0000000..0d6304e --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/source/NexmarkSourceReader.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink.source; + +import com.github.nexmark.flink.generator.GeneratorConfig; +import com.github.nexmark.flink.generator.NexmarkGenerator; +import org.apache.flink.api.connector.source.ReaderOutput; +import org.apache.flink.api.connector.source.SourceReader; +import org.apache.flink.api.connector.source.SourceReaderContext; +import org.apache.flink.core.io.InputStatus; +import org.apache.flink.metrics.Counter; +import org.apache.flink.table.data.RowData; +import org.apache.flink.util.Preconditions; + +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +public class NexmarkSourceReader implements SourceReader { + + private final SourceReaderContext context; + private final EventDeserializer deserializer; + private final Counter numRecordsInCounter; + private final boolean isKeepAlive; + private final GeneratorConfig config; + private NexmarkSource.NexmarkSourceSplit sourceSplit; + private NexmarkGenerator generator; + + NexmarkSourceReader(SourceReaderContext sourceReaderContext, + GeneratorConfig config, + EventDeserializer deserializer) { + this.context = sourceReaderContext; + this.isKeepAlive = config.isSourceKeepAlive(); + this.config = config; + this.deserializer = deserializer; + this.numRecordsInCounter = context.metricGroup().getIOMetricGroup().getNumRecordsInCounter(); + } + + @Override + public void start() { + if (sourceSplit == null) { + context.sendSplitRequest(); + } + } + + @Override + public InputStatus pollNext(ReaderOutput readerOutput) throws Exception { + if (sourceSplit == null || generator == null) { + return InputStatus.NOTHING_AVAILABLE; + } + if (!generator.hasNext()) { + return isKeepAlive ? InputStatus.NOTHING_AVAILABLE : InputStatus.END_OF_INPUT; + } + long now = System.currentTimeMillis(); + NexmarkGenerator.NextEvent nextEvent = generator.next(); + if (!config.maxEmitSpeed && nextEvent.wallclockTimestamp > now) { + Thread.sleep(nextEvent.wallclockTimestamp - now); + } + readerOutput.collect(deserializer.deserialize(nextEvent.event)); + numRecordsInCounter.inc(); + return InputStatus.MORE_AVAILABLE; + } + + @Override + public List snapshotState(long l) { + sourceSplit.setNumEmittedSoFar(generator.getEventsCountSoFar()); + sourceSplit.setWallClockBaseTime(generator.getWallclockBaseTime()); + return Collections.singletonList(sourceSplit); + } + + @Override + public CompletableFuture isAvailable() { + return CompletableFuture.completedFuture(null); + } + + @Override + public void addSplits(List list) { + Preconditions.checkState(list.size() == 1, "Only one split supported for one reader"); + Preconditions.checkState(sourceSplit == null, "We already have one split."); + sourceSplit = list.get(0); + generator = new NexmarkGenerator(sourceSplit.getGeneratorConfig().reconfigure(config, config.isSourceIgnoreStop()), sourceSplit.getNumEmittedSoFar(), sourceSplit.getWallClockBaseTime()); + } + + @Override + public void notifyNoMoreSplits() { + } + + @Override + public void close() throws Exception { + } +} diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/source/NexmarkTableSource.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/source/NexmarkTableSource.java new file mode 100644 index 0000000..0a43c70 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/source/NexmarkTableSource.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink.source; + +import com.github.nexmark.flink.generator.GeneratorConfig; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.table.api.Schema; +import org.apache.flink.table.catalog.ResolvedSchema; +import org.apache.flink.table.connector.ChangelogMode; +import org.apache.flink.table.connector.source.DynamicTableSource; +import org.apache.flink.table.connector.source.ScanTableSource; +import org.apache.flink.table.connector.source.SourceProvider; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.types.DataType; + +import java.util.Objects; +import java.util.stream.Collectors; + +import static org.apache.flink.table.api.DataTypes.BIGINT; +import static org.apache.flink.table.api.DataTypes.FIELD; +import static org.apache.flink.table.api.DataTypes.INT; +import static org.apache.flink.table.api.DataTypes.ROW; +import static org.apache.flink.table.api.DataTypes.STRING; +import static org.apache.flink.table.api.DataTypes.TIMESTAMP; + +/** + * Table source for Nexmark. + */ +public class NexmarkTableSource implements ScanTableSource { + + public static final Schema NEXMARK_SCHEMA = Schema.newBuilder() + .column("event_type", INT()) + .column("person", ROW( + FIELD("id", BIGINT()), + FIELD("name", STRING()), + FIELD("emailAddress", STRING()), + FIELD("creditCard", STRING()), + FIELD("city", STRING()), + FIELD("state", STRING()), + FIELD("dateTime", TIMESTAMP(3)), + FIELD("extra", STRING()))) + .column("auction", ROW( + FIELD("id", BIGINT()), + FIELD("itemName", STRING()), + FIELD("description", STRING()), + FIELD("initialBid", BIGINT()), + FIELD("reserve", BIGINT()), + FIELD("dateTime", TIMESTAMP(3)), + FIELD("expires", TIMESTAMP(3)), + FIELD("seller", BIGINT()), + FIELD("category", BIGINT()), + FIELD("extra", STRING()))) + .column("bid", ROW( + FIELD("auction", BIGINT()), + FIELD("bidder", BIGINT()), + FIELD("price", BIGINT()), + FIELD("channel", STRING()), + FIELD("url", STRING()), + FIELD("dateTime", TIMESTAMP(3)), + FIELD("extra", STRING()))) + .build(); + + public static final ResolvedSchema RESOLVED_SCHEMA = ResolvedSchema.physical( + NEXMARK_SCHEMA.getColumns().stream().map(Schema.UnresolvedColumn::getName).collect(Collectors.toList()), + NEXMARK_SCHEMA.getColumns().stream() + .map(unresolvedColumn -> + (DataType) ((Schema.UnresolvedPhysicalColumn) unresolvedColumn).getDataType()) + .collect(Collectors.toList())); + + + private final GeneratorConfig config; + + public NexmarkTableSource(GeneratorConfig config) { + this.config = config; + } + + @Override + public ChangelogMode getChangelogMode() { + return ChangelogMode.insertOnly(); + } + + @Override + public ScanRuntimeProvider getScanRuntimeProvider(ScanContext scanContext) { + TypeInformation outputType = scanContext + .createTypeInformation(RESOLVED_SCHEMA.toPhysicalRowDataType()); + return SourceProvider.of(new NexmarkSource(config, outputType)); + } + + @Override + public DynamicTableSource copy() { + return new NexmarkTableSource(config); + } + + @Override + public String asSummaryString() { + return "Nexmark Source"; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + NexmarkTableSource that = (NexmarkTableSource) o; + return Objects.equals(config, that.config); + } + + @Override + public int hashCode() { + return Objects.hash(config); + } + + @Override + public String toString() { + return "NexmarkTableSource{" + + "config=" + config + + '}'; + } +} diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/source/NexmarkTableSourceFactory.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/source/NexmarkTableSourceFactory.java new file mode 100644 index 0000000..28fb74e --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/source/NexmarkTableSourceFactory.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink.source; + +import com.github.nexmark.flink.NexmarkConfiguration; +import com.github.nexmark.flink.generator.GeneratorConfig; +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.configuration.CoreOptions; +import org.apache.flink.configuration.ReadableConfig; +import org.apache.flink.table.connector.source.DynamicTableSource; +import org.apache.flink.table.factories.DynamicTableSourceFactory; +import org.apache.flink.table.factories.FactoryUtil; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +public class NexmarkTableSourceFactory implements DynamicTableSourceFactory { + + @Override + public DynamicTableSource createDynamicTableSource(Context context) { + final FactoryUtil.TableFactoryHelper helper = FactoryUtil.createTableFactoryHelper(this, context); + final ReadableConfig config = helper.getOptions(); + helper.validate(); + + // for compatibility reason of "context.getCatalogTable()", do not validate schema. + // validateSchema(TableSchemaUtils.getPhysicalSchema(context.getCatalogTable().getSchema())); + + int parallelism = context.getConfiguration().get(CoreOptions.DEFAULT_PARALLELISM); + NexmarkConfiguration nexmarkConf = NexmarkSourceOptions.convertToNexmarkConfiguration(config); + nexmarkConf.numEventGenerators = parallelism; + GeneratorConfig generatorConfig = new GeneratorConfig( + nexmarkConf, + System.currentTimeMillis(), + 1, + nexmarkConf.numEvents, + nexmarkConf.stopAtEvent, + 1); + + return new NexmarkTableSource(generatorConfig); + } + + @Override + public String factoryIdentifier() { + return "nexmark"; + } + + @Override + public Set> requiredOptions() { + return Collections.emptySet(); + } + + @Override + public Set> optionalOptions() { + Set> sets = new HashSet<>(); + sets.add(NexmarkSourceOptions.RATE_SHAPE); + sets.add(NexmarkSourceOptions.RATE_PERIOD); + sets.add(NexmarkSourceOptions.RATE_LIMITED); + sets.add(NexmarkSourceOptions.FIRST_EVENT_RATE); + sets.add(NexmarkSourceOptions.NEXT_EVENT_RATE); + sets.add(NexmarkSourceOptions.PERSON_AVG_SIZE); + sets.add(NexmarkSourceOptions.AUCTION_AVG_SIZE); + sets.add(NexmarkSourceOptions.BID_AVG_SIZE); + sets.add(NexmarkSourceOptions.PERSON_PROPORTION); + sets.add(NexmarkSourceOptions.AUCTION_PROPORTION); + sets.add(NexmarkSourceOptions.BID_PROPORTION); + sets.add(NexmarkSourceOptions.BID_HOT_RATIO_AUCTIONS); + sets.add(NexmarkSourceOptions.BID_HOT_RATIO_BIDDERS); + sets.add(NexmarkSourceOptions.AUCTION_HOT_RATIO_SELLERS); + sets.add(NexmarkSourceOptions.EVENTS_NUM); + sets.add(NexmarkSourceOptions.KEEP_ALIVE); + sets.add(NexmarkSourceOptions.STOP_AT); + sets.add(NexmarkSourceOptions.MAX_EMIT_SPEED); + return sets; + } +} diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/source/RowDataEventDeserializer.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/source/RowDataEventDeserializer.java new file mode 100644 index 0000000..30066dd --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/source/RowDataEventDeserializer.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink.source; + +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.data.StringData; +import org.apache.flink.table.data.TimestampData; + +import com.github.nexmark.flink.model.Auction; +import com.github.nexmark.flink.model.Bid; +import com.github.nexmark.flink.model.Event; +import com.github.nexmark.flink.model.Person; + +public class RowDataEventDeserializer implements EventDeserializer { + + @Override + public RowData deserialize(Event event) { + return convertEvent(event); + } + + private RowData convertEvent(Event event) { + GenericRowData rowData = new GenericRowData(4); + rowData.setField(0, event.type.value); + if (event.type == Event.Type.PERSON) { + assert event.newPerson != null; + rowData.setField(1, convertPerson(event.newPerson)); + } else if (event.type == Event.Type.AUCTION) { + assert event.newAuction != null; + rowData.setField(2, convertAuction(event.newAuction)); + } else if (event.type == Event.Type.BID) { + assert event.bid != null; + rowData.setField(3, convertBid(event.bid)); + } else { + throw new UnsupportedOperationException("Unsupported event type: " + event.type.name()); + } + return rowData; + } + + private RowData convertPerson(Person person) { + GenericRowData rowData = new GenericRowData(8); + rowData.setField(0, person.id); + rowData.setField(1, StringData.fromString(person.name)); + rowData.setField(2, StringData.fromString(person.emailAddress)); + rowData.setField(3, StringData.fromString(person.creditCard)); + rowData.setField(4, StringData.fromString(person.city)); + rowData.setField(5, StringData.fromString(person.state)); + rowData.setField(6, TimestampData.fromInstant(person.dateTime)); + rowData.setField(7, StringData.fromString(person.extra)); + return rowData; + } + + private RowData convertAuction(Auction auction) { + GenericRowData rowData = new GenericRowData(10); + rowData.setField(0, auction.id); + rowData.setField(1, StringData.fromString(auction.itemName)); + rowData.setField(2, StringData.fromString(auction.description)); + rowData.setField(3, auction.initialBid); + rowData.setField(4, auction.reserve); + rowData.setField(5, TimestampData.fromInstant(auction.dateTime)); + rowData.setField(6, TimestampData.fromInstant(auction.expires)); + rowData.setField(7, auction.seller); + rowData.setField(8, auction.category); + rowData.setField(9, StringData.fromString(auction.extra)); + return rowData; + } + + private RowData convertBid(Bid bid) { + GenericRowData rowData = new GenericRowData(7); + rowData.setField(0, bid.auction); + rowData.setField(1, bid.bidder); + rowData.setField(2, bid.price); + rowData.setField(3, StringData.fromString(bid.channel)); + rowData.setField(4, StringData.fromString(bid.url)); + rowData.setField(5, TimestampData.fromInstant(bid.dateTime)); + rowData.setField(6, StringData.fromString(bid.extra)); + return rowData; + } + +} diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/udf/CountChar.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/udf/CountChar.java new file mode 100644 index 0000000..a70e935 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/udf/CountChar.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.nexmark.flink.udf; + +import org.apache.flink.table.annotation.DataTypeHint; +import org.apache.flink.table.annotation.FunctionHint; +import org.apache.flink.table.data.StringData; +import org.apache.flink.table.functions.ScalarFunction; + +/** + * User defined function tou count number of specified characters. + */ +public class CountChar extends ScalarFunction { + + public long eval(@DataTypeHint("STRING") StringData s, @DataTypeHint("STRING") StringData character) { + long count = 0; + if (null != s) { + byte[] bytes = s.toBytes(); + byte chr = character.toBytes()[0]; + for (byte aByte : bytes) { + if (aByte == chr) { + count++; + } + } + } + return count; + } +} diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/utils/AutoClosableProcess.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/utils/AutoClosableProcess.java new file mode 100644 index 0000000..667a0f1 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/utils/AutoClosableProcess.java @@ -0,0 +1,218 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink.utils; + +import org.apache.flink.api.common.time.Deadline; +import org.apache.flink.util.Preconditions; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nullable; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.PrintStream; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.Arrays; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.function.Consumer; + +import static org.apache.flink.util.Preconditions.checkArgument; +import static org.apache.flink.util.Preconditions.checkNotNull; + +/** + * Utility class to terminate a given {@link Process} when exiting a try-with-resources statement. + */ +public class AutoClosableProcess implements AutoCloseable { + + private static final Logger LOG = LoggerFactory.getLogger(AutoClosableProcess.class); + + private final Process process; + + private AutoClosableProcess(final Process process) { + Preconditions.checkNotNull(process); + this.process = process; + } + + public Process getProcess() { + return process; + } + + public static AutoClosableProcess runNonBlocking(String... commands) throws IOException { + return create(commands).runNonBlocking(); + } + + public static void runBlocking(String... commands) throws IOException { + create(commands).runBlocking(); + } + + public static AutoClosableProcessBuilder create(String... commands) { + return new AutoClosableProcessBuilder(commands); + } + + /** + * Builder for most sophisticated processes. + */ + public static final class AutoClosableProcessBuilder { + private final String[] commands; + private Consumer stdoutProcessor = LOG::debug; + private Consumer stderrProcessor = LOG::debug; + private @Nullable String[] stdInputs; + + AutoClosableProcessBuilder(final String... commands) { + this.commands = commands; + } + + public AutoClosableProcessBuilder setStdoutProcessor(final Consumer stdoutProcessor) { + this.stdoutProcessor = stdoutProcessor; + return this; + } + + public AutoClosableProcessBuilder setStderrProcessor(final Consumer stderrProcessor) { + this.stderrProcessor = stderrProcessor; + return this; + } + + public AutoClosableProcessBuilder setStdInputs(final String... inputLines) { + checkNotNull(inputLines); + checkArgument(inputLines.length >= 1); + this.stdInputs = inputLines; + return this; + } + + public void runBlocking() throws IOException { + runBlocking(Duration.ofSeconds(30)); + } + + public void runBlocking(final Duration timeout) throws IOException { + final StringWriter sw = new StringWriter(); + try (final PrintWriter printer = new PrintWriter(sw)) { + final Process process = createProcess(commands, stdoutProcessor, line -> { + stderrProcessor.accept(line); + printer.println(line); + }, + stdInputs); + + try (AutoClosableProcess autoProcess = new AutoClosableProcess(process)) { + final boolean success = process.waitFor(timeout.toMillis(), TimeUnit.MILLISECONDS); + if (!success) { + throw new TimeoutException("Process exceeded timeout of " + timeout.getSeconds() + "seconds."); + } + + if (process.exitValue() != 0) { + throw new IOException("Process execution failed due error. Error output:" + sw); + } + } catch (TimeoutException | InterruptedException e) { + throw new IOException("Process failed due to timeout."); + } + } + } + + public void runBlockingWithRetry(final int maxRetries, final Duration attemptTimeout, final Duration globalTimeout) throws IOException { + int retries = 0; + final Deadline globalDeadline = Deadline.fromNow(globalTimeout); + + while (true) { + try { + runBlocking(attemptTimeout); + break; + } catch (Exception e) { + if (++retries > maxRetries || !globalDeadline.hasTimeLeft()) { + String errMsg = String.format( + "Process (%s) exceeded timeout (%s) or number of retries (%s).", + Arrays.toString(commands), globalTimeout.toMillis(), maxRetries); + throw new IOException(errMsg, e); + } + } + } + } + + public AutoClosableProcess runNonBlocking() throws IOException { + return new AutoClosableProcess(createProcess(commands, stdoutProcessor, stderrProcessor, stdInputs)); + } + } + + private static Process createProcess( + final String[] commands, + Consumer stdoutProcessor, + Consumer stderrProcessor, + @Nullable String[] stdInputs) throws IOException { + final ProcessBuilder processBuilder = new ProcessBuilder(); + LOG.debug("Creating process: {}", Arrays.toString(commands)); + processBuilder.command(commands); + + final Process process = processBuilder.start(); + + consumeOutput(process.getInputStream(), stdoutProcessor); + consumeOutput(process.getErrorStream(), stderrProcessor); + if (stdInputs != null) { + produceInput(process.getOutputStream(), stdInputs); + } + + return process; + } + + private static void consumeOutput(final InputStream stream, final Consumer streamConsumer) { + new Thread(() -> { + try (BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = bufferedReader.readLine()) != null) { + streamConsumer.accept(line); + } + } catch (IOException e) { + LOG.error("Failure while processing process stdout/stderr.", e); + } + } + ).start(); + } + + private static void produceInput(final OutputStream stream, final String[] inputLines) { + new Thread(() -> { + // try with resource will close the OutputStream automatically, + // usually the process terminal will also be finished then. + try (PrintStream printStream = new PrintStream(stream, true, StandardCharsets.UTF_8.name())) { + for (String line : inputLines) { + printStream.println(line); + } + } catch (IOException e) { + LOG.error("Failure while processing process stdin.", e); + } + }).start(); + } + + @Override + public void close() throws IOException { + if (process.isAlive()) { + process.destroy(); + try { + process.waitFor(10, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + } +} diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/utils/NexmarkGlobalConfiguration.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/utils/NexmarkGlobalConfiguration.java new file mode 100644 index 0000000..7d37be2 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/utils/NexmarkGlobalConfiguration.java @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink.utils; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.IllegalConfigurationException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nullable; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; + +/** + * Global configuration object for Flink Nexmark. Similar to Java properties configuration + * objects it includes key-value pairs which represent the framework's configuration. + */ +@Internal +public final class NexmarkGlobalConfiguration { + + private static final Logger LOG = LoggerFactory.getLogger(org.apache.flink.configuration.GlobalConfiguration.class); + + public static final String NEXMARK_CONF_FILENAME = "nexmark.yaml"; + + // -------------------------------------------------------------------------------------------- + + private NexmarkGlobalConfiguration() {} + + // -------------------------------------------------------------------------------------------- + + /** + * Loads the global configuration from the environment. Fails if an error occurs during loading. Returns an + * empty configuration object if the environment variable is not set. In production this variable is set but + * tests and local execution/debugging don't have this environment variable set. That's why we should fail + * if it is not set. + * @return Returns the Configuration + */ + public static Configuration loadConfiguration() { + return loadConfiguration(new Configuration()); + } + + /** + * Loads the global configuration and adds the given dynamic properties + * configuration. + * + * @param dynamicProperties The given dynamic properties + * @return Returns the loaded global configuration with dynamic properties + */ + public static Configuration loadConfiguration(Configuration dynamicProperties) { + final String configDir = System.getenv("NEXMARK_CONF_DIR"); + if (configDir == null) { + return new Configuration(dynamicProperties); + } + + return loadConfiguration(configDir, dynamicProperties); + } + + /** + * Loads the configuration files from the specified directory. + * + *

YAML files are supported as configuration files. + * + * @param configDir + * the directory which contains the configuration files + */ + public static Configuration loadConfiguration(final String configDir) { + return loadConfiguration(configDir, null); + } + + /** + * Loads the configuration files from the specified directory. If the dynamic properties + * configuration is not null, then it is added to the loaded configuration. + * + * @param configDir directory to load the configuration from + * @param dynamicProperties configuration file containing the dynamic properties. Null if none. + * @return The configuration loaded from the given configuration directory + */ + public static Configuration loadConfiguration(final String configDir, @Nullable final Configuration dynamicProperties) { + + if (configDir == null) { + throw new IllegalArgumentException("Given configuration directory is null, cannot load configuration"); + } + + final File confDirFile = new File(configDir); + if (!(confDirFile.exists())) { + throw new IllegalConfigurationException( + "The given configuration directory name '" + configDir + + "' (" + confDirFile.getAbsolutePath() + ") does not describe an existing directory."); + } + + // get Nexmark yaml configuration file + final File yamlConfigFile = new File(confDirFile, NEXMARK_CONF_FILENAME); + + if (!yamlConfigFile.exists()) { + throw new IllegalConfigurationException( + "The Nexmark config file '" + yamlConfigFile + + "' (" + confDirFile.getAbsolutePath() + ") does not exist."); + } + + Configuration configuration = loadYAMLResource(yamlConfigFile); + + if (dynamicProperties != null) { + configuration.addAll(dynamicProperties); + } + + return configuration; + } + + /** + * Loads a YAML-file of key-value pairs. + * + *

Colon and whitespace ": " separate key and value (one per line). The hash tag "#" starts a single-line comment. + * + *

Example: + * + * 

+	 * jobmanager.rpc.address: localhost # network address for communication with the job manager
+	 * jobmanager.rpc.port   : 6123      # network port to connect to for communication with the job manager
+	 * taskmanager.rpc.port  : 6122      # network port the task manager expects incoming IPC connections
+	 *
+ * + *

This does not span the whole YAML specification, but only the *syntax* of simple YAML key-value pairs (see issue + * #113 on GitHub). If at any point in time, there is a need to go beyond simple key-value pairs syntax + * compatibility will allow to introduce a YAML parser library. + * + * @param file the YAML file to read from + * @see YAML 1.2 specification + */ + private static Configuration loadYAMLResource(File file) { + final Configuration config = new Configuration(); + + try (BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(file)))){ + + String line; + int lineNo = 0; + while ((line = reader.readLine()) != null) { + lineNo++; + // 1. check for comments + String[] comments = line.split("#", 2); + String conf = comments[0].trim(); + + // 2. get key and value + if (conf.length() > 0) { + String[] kv = conf.split(": ", 2); + + // skip line with no valid key-value pair + if (kv.length == 1) { + LOG.warn("Error while trying to split key and value in configuration file " + file + ":" + lineNo + ": \"" + line + "\""); + continue; + } + + String key = kv[0].trim(); + String value = kv[1].trim(); + + // sanity check + if (key.length() == 0 || value.length() == 0) { + LOG.warn("Error after splitting key and value in configuration file " + file + ":" + lineNo + ": \"" + line + "\""); + continue; + } + + LOG.info("Loading configuration property: {}, {}", key, value); + config.setString(key, value); + } + } + } catch (IOException e) { + throw new RuntimeException("Error parsing YAML configuration.", e); + } + + return config; + } +} diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/utils/NexmarkUtils.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/utils/NexmarkUtils.java new file mode 100644 index 0000000..59653bb --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/utils/NexmarkUtils.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink.utils; + +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Odd's 'n Ends used throughout queries and driver. */ +public class NexmarkUtils { + + private static final Logger LOG = LoggerFactory.getLogger(NexmarkUtils.class); + + /** Mapper for (de)serializing JSON. */ + public static final ObjectMapper MAPPER = new ObjectMapper(); + + /** Units for rates. */ + public enum RateUnit { + PER_SECOND(1_000_000L), + PER_MINUTE(60_000_000L); + + RateUnit(long usPerUnit) { + this.usPerUnit = usPerUnit; + } + + /** Number of microseconds per unit. */ + private final long usPerUnit; + + /** Number of microseconds between events at given rate. */ + public long rateToPeriodUs(long rate) { + return (usPerUnit + rate / 2) / rate; + } + } + + /** Shape of event rate. */ + public enum RateShape { + SQUARE, + SINE; + + /** Number of steps used to approximate sine wave. */ + private static final int N = 10; + + /** + * Return inter-event delay, in microseconds, for each generator to follow in order to achieve + * {@code rate} at {@code unit} using {@code numGenerators}. + */ + public long interEventDelayUs(int rate, RateUnit unit, int numGenerators) { + return unit.rateToPeriodUs(rate) * numGenerators; + } + + /** + * Return array of successive inter-event delays, in microseconds, for each generator to follow + * in order to achieve this shape with {@code firstRate/nextRate} at {@code unit} using {@code + * numGenerators}. + */ + public long[] interEventDelayUs(int firstRate, int nextRate, RateUnit unit, int numGenerators) { + if (firstRate == nextRate) { + long[] interEventDelayUs = new long[1]; + interEventDelayUs[0] = unit.rateToPeriodUs(firstRate) * numGenerators; + return interEventDelayUs; + } + + switch (this) { + case SQUARE: + { + long[] interEventDelayUs = new long[2]; + interEventDelayUs[0] = unit.rateToPeriodUs(firstRate) * numGenerators; + interEventDelayUs[1] = unit.rateToPeriodUs(nextRate) * numGenerators; + return interEventDelayUs; + } + case SINE: + { + double mid = (firstRate + nextRate) / 2.0; + double amp = (firstRate - nextRate) / 2.0; // may be -ve + long[] interEventDelayUs = new long[N]; + for (int i = 0; i < N; i++) { + double r = (2.0 * Math.PI * i) / N; + double rate = mid + amp * Math.cos(r); + interEventDelayUs[i] = unit.rateToPeriodUs(Math.round(rate)) * numGenerators; + } + return interEventDelayUs; + } + } + throw new RuntimeException(); // switch should be exhaustive + } + + /** + * Return delay between steps, in seconds, for result of {@link #interEventDelayUs}, so as to + * cycle through the entire sequence every {@code ratePeriodSec}. + */ + public int stepLengthSec(int ratePeriodSec) { + int n = 0; + switch (this) { + case SQUARE: + n = 2; + break; + case SINE: + n = N; + break; + } + return (ratePeriodSec + n - 1) / n; + } + } +} diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/workload/Workload.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/workload/Workload.java new file mode 100644 index 0000000..c123453 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/workload/Workload.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink.workload; + +import org.apache.flink.util.Preconditions; + +import com.github.nexmark.flink.FlinkNexmarkOptions; +import com.github.nexmark.flink.metric.BenchmarkMetric; + +import javax.annotation.Nullable; + +import java.time.Duration; +import java.util.Objects; + +public class Workload { + + private final long tps; + private final long eventsNum; + private final int personProportion; + private final int auctionProportion; + private final int bidProportion; + private final @Nullable String kafkaServers; + private final long warmupMills; + private final long warmupTps; + private final long warmupEvents; + + public Workload(long tps, long eventsNum, int personProportion, int auctionProportion, int bidProportion) { + this(tps, eventsNum, personProportion, auctionProportion, bidProportion, null, 0L, 0L, 0L); + } + + public Workload( + long tps, + long eventsNum, + int personProportion, + int auctionProportion, + int bidProportion, + @Nullable String kafkaServers, + long warmupMills, + long warmupTps, + long warmupEvents) { + this.tps = tps; + this.eventsNum = eventsNum; + this.personProportion = personProportion; + this.auctionProportion = auctionProportion; + this.bidProportion = bidProportion; + this.kafkaServers = kafkaServers; + this.warmupMills = warmupMills; + this.warmupTps = warmupTps; + this.warmupEvents = warmupEvents; + } + + public long getTps() { + return tps; + } + + public long getEventsNum() { + return eventsNum; + } + + public int getPersonProportion() { + return personProportion; + } + + public int getAuctionProportion() { + return auctionProportion; + } + + public int getBidProportion() { + return bidProportion; + } + + public String getKafkaServers() { + return kafkaServers; + } + + public long getWarmupMills() { + return warmupMills; + } + + public long getWarmupTps() { + return warmupTps; + } + + public long getWarmupEvents() { + return warmupEvents; + } + + public void validateWorkload(Duration monitorDuration) { + boolean unboundedMonitor = monitorDuration.toNanos() == Long.MAX_VALUE; + if (getEventsNum() == 0) { + // TPS mode + Preconditions.checkArgument( + !unboundedMonitor, + "You should configure '%s' in the TPS mode." + + " Otherwise, the job will never end.", + FlinkNexmarkOptions.METRIC_MONITOR_DURATION.key()); + } else { + // EventsNum mode + Preconditions.checkArgument( + unboundedMonitor, + "The configuration of '%s' is not supported" + + " in the events number mode.", + FlinkNexmarkOptions.METRIC_MONITOR_DURATION.key()); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Workload workload = (Workload) o; + return tps == workload.tps && + eventsNum == workload.eventsNum && + personProportion == workload.personProportion && + auctionProportion == workload.auctionProportion && + bidProportion == workload.bidProportion && + Objects.equals(kafkaServers, workload.kafkaServers); + } + + @Override + public int hashCode() { + return Objects.hash(tps, eventsNum, personProportion, auctionProportion, bidProportion, kafkaServers); + } + + public String getSummaryString() { + return String.format( + "[tps=%s, eventsNum=%s, percentage=bid:%s,auction:%s,person:%s,kafkaServers:%s]", + BenchmarkMetric.formatLongValue(tps), + BenchmarkMetric.formatLongValue(eventsNum), + bidProportion, + auctionProportion, + personProportion, + kafkaServers); + } + + @Override + public String toString() { + return "Workload{" + + "tps=" + tps + + ", eventsNum=" + eventsNum + + ", personProportion=" + personProportion + + ", auctionProportion=" + auctionProportion + + ", bidProportion=" + bidProportion + + ", kafkaServers=" + kafkaServers + + '}'; + } +} diff --git a/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/workload/WorkloadSuite.java b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/workload/WorkloadSuite.java new file mode 100644 index 0000000..2577291 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/java/com/github/nexmark/flink/workload/WorkloadSuite.java @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink.workload; + +import org.apache.flink.configuration.Configuration; + +import com.github.nexmark.flink.source.NexmarkSourceOptions; +import org.apache.flink.util.TimeUtils; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +import static com.github.nexmark.flink.Benchmark.CATEGORY_OA; + +public class WorkloadSuite { + + private static final String WORKLOAD_SUITE_CONF_PREFIX = "nexmark.workload.suite."; + private static final String QUERIES_CONF_SUFFIX = ".queries"; + private static final String TPS_CONF_SUFFIX = ".tps"; + private static final String EVENTS_NUM_CONF_SUFFIX = "." + NexmarkSourceOptions.EVENTS_NUM.key(); + private static final String WARMUP_SUFFIX = ".warmup"; + private static final String WARMUP_DURATION_SUFFIX = ".warmup.duration"; + private static final String KAFKA_BOOTSTRAP_SERVERS = "kafka.bootstrap.servers"; + + private final Map query2Workload; + + WorkloadSuite(Map query2Workload) { + this.query2Workload = query2Workload; + } + + public Workload getQueryWorkload(String queryName) { + return query2Workload.get(queryName); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + WorkloadSuite that = (WorkloadSuite) o; + return Objects.equals(query2Workload, that.query2Workload); + } + + @Override + public int hashCode() { + return Objects.hash(query2Workload); + } + + @Override + public String toString() { + return "WorkloadSuite{" + + "query2Workload=" + query2Workload + + '}'; + } + + public static WorkloadSuite fromConf(Configuration nexmarkConf, String category) { + Map confMap = nexmarkConf.toMap(); + Set suites = new HashSet<>(); + String kafkaServers = confMap.getOrDefault(KAFKA_BOOTSTRAP_SERVERS, null); + String categoryQueries = CATEGORY_OA.equals(category) ? QUERIES_CONF_SUFFIX : QUERIES_CONF_SUFFIX + "." + category; + confMap.keySet().forEach(k -> { + if (k.startsWith(WORKLOAD_SUITE_CONF_PREFIX) && k.endsWith(categoryQueries)) { + String suiteName = k.substring( + WORKLOAD_SUITE_CONF_PREFIX.length(), + k.indexOf(QUERIES_CONF_SUFFIX)); + suites.add(suiteName); + } + }); + + Map query2Workload = new HashMap<>(); + for (String suiteName : suites) { + long tps = Long.parseLong(confMap.getOrDefault( + WORKLOAD_SUITE_CONF_PREFIX + suiteName + TPS_CONF_SUFFIX, + NexmarkSourceOptions.NEXT_EVENT_RATE.defaultValue().toString())); + + long eventsNum = Long.parseLong(confMap.getOrDefault( + WORKLOAD_SUITE_CONF_PREFIX + suiteName + EVENTS_NUM_CONF_SUFFIX, + NexmarkSourceOptions.EVENTS_NUM.defaultValue().toString())); + + int personProportion = NexmarkSourceOptions.PERSON_PROPORTION.defaultValue(); + int auctionProportion = NexmarkSourceOptions.AUCTION_PROPORTION.defaultValue(); + int bidProportion = NexmarkSourceOptions.BID_PROPORTION.defaultValue(); + String percentageKey = WORKLOAD_SUITE_CONF_PREFIX + suiteName + ".percentage"; + if (confMap.containsKey(percentageKey)) { + String percentage = removeQuotes(confMap.get(percentageKey)); + String[] percentageArray = percentage.split(","); + for (String str : percentageArray) { + String part = str.trim(); + if (part.startsWith("bid:")) { + bidProportion = Integer.parseInt(part.substring("bid:".length())); + } else if (part.startsWith("auction:")) { + auctionProportion = Integer.parseInt(part.substring("auction:".length())); + } else if (part.startsWith("person:")) { + personProportion = Integer.parseInt(part.substring("person:".length())); + } else { + throw new IllegalArgumentException("Unable to parse suite percentage: " + percentage); + } + } + } + + Duration warmupDuration = TimeUtils.parseDuration(confMap.getOrDefault( + WORKLOAD_SUITE_CONF_PREFIX + suiteName + WARMUP_DURATION_SUFFIX, + "120s")); + + long warmupTps = Long.parseLong(confMap.getOrDefault( + WORKLOAD_SUITE_CONF_PREFIX + suiteName + WARMUP_SUFFIX + TPS_CONF_SUFFIX, + String.valueOf(tps))); + + long warmupEventsNum = Long.parseLong(confMap.getOrDefault( + WORKLOAD_SUITE_CONF_PREFIX + suiteName + WARMUP_SUFFIX + EVENTS_NUM_CONF_SUFFIX, + String.valueOf(eventsNum))); + + Workload load = new Workload( + tps, eventsNum, personProportion, auctionProportion, bidProportion, kafkaServers, warmupDuration.toMillis(), warmupTps, warmupEventsNum); + + String queriesKey = WORKLOAD_SUITE_CONF_PREFIX + suiteName + categoryQueries; + List queries = new ArrayList<>(); + if (confMap.containsKey(queriesKey)) { + String queriesString = removeQuotes(confMap.get(queriesKey)); + for (String q : queriesString.split(",")) { + queries.add(q.trim()); + } + } + + for (String q : queries) { + Workload old = query2Workload.put(q, load); + if (old != null) { + throw new IllegalArgumentException( + String.format("Query %s is defined in multiple suites.", q)); + } + } + } + + return new WorkloadSuite(query2Workload); + } + + private static String removeQuotes(String str) { + String result = str; + if (result.startsWith("\"")) { + result = result.substring(1); + } + if (result.endsWith("\"")) { + result = result.substring(0, result.length() - 1); + } + return result; + } + +} diff --git a/nexmark/nexmark-flink/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory b/nexmark/nexmark-flink/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory new file mode 100644 index 0000000..82b21bf --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +com.github.nexmark.flink.source.NexmarkTableSourceFactory diff --git a/nexmark/nexmark-flink/src/main/resources/bin/config.sh b/nexmark/nexmark-flink/src/main/resources/bin/config.sh new file mode 100644 index 0000000..005a8ff --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/bin/config.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +target="$0" +# For the case, the executable has been directly symlinked, figure out +# the correct bin path by following its symlink up to an upper bound. +# Note: we can't use the readlink utility here if we want to be POSIX +# compatible. +iteration=0 +while [ -L "$target" ]; do + if [ "$iteration" -gt 100 ]; then + echo "Cannot resolve path: You have a cyclic symlink in $target." + break + fi + ls=`ls -ld -- "$target"` + target=`expr "$ls" : '.* -> \(.*\)$'` + iteration=$((iteration + 1)) +done + +# Convert relative path to absolute path and resolve directory symlinks +bin=`dirname "$target"` +SYMLINK_RESOLVED_BIN=`cd "$bin"; pwd -P` +NEXMARK_HOME=`dirname "$SYMLINK_RESOLVED_BIN"` +NEXMARK_LIB_DIR=$NEXMARK_HOME/lib +NEXMARK_QUERY_DIR=$NEXMARK_HOME/queries +NEXMARK_LOG_DIR=$NEXMARK_HOME/log +NEXMARK_CONF_DIR=$NEXMARK_HOME/conf +NEXMARK_BIN_DIR=$NEXMARK_HOME/bin + +### Exported environment variables ### +export NEXMARK_HOME +export NEXMARK_LIB_DIR +export NEXMARK_QUERY_DIR +export NEXMARK_LOG_DIR +export NEXMARK_CONF_DIR +export NEXMARK_BIN_DIR + +# Auxilliary function which extracts the name of host from a line which +# also potentially includes topology information and the taskManager type +extractHostName() { + # handle comments: extract first part of string (before first # character) + WORKER=`echo $1 | cut -d'#' -f 1` + + # Extract the hostname from the network hierarchy + if [[ "$WORKER" =~ ^.*/([0-9a-zA-Z.-]+)$ ]]; then + WORKER=${BASH_REMATCH[1]} + fi + + echo $WORKER +} + +readWorkers() { + WORKERS_FILE="${FLINK_HOME}/conf/workers" + + if [[ ! -f "$WORKERS_FILE" ]]; then + echo "No workers file. Please specify workers in 'conf/workers'." + exit 1 + fi + + WORKERS=() + + WORKERS_ALL_LOCALHOST=true + GOON=true + while $GOON; do + read line || GOON=false + HOST=$( extractHostName $line) + if [ -n "$HOST" ] ; then + WORKERS+=(${HOST}) + if [ "${HOST}" != "localhost" ] && [ "${HOST}" != "127.0.0.1" ] ; then + WORKERS_ALL_LOCALHOST=false + fi + fi + done < <(sort -u "$WORKERS_FILE") +} + +# starts or stops TMs on all workers +# TMWorkers start|stop +TMWorkers() { + CMD=$1 + + readWorkers + + if [ ${WORKERS_ALL_LOCALHOST} = true ] ; then + # all-local setup + for worker in ${WORKERS[@]}; do + if [ "${CMD}" != "stop" ] ; then + "${NEXMARK_BIN_DIR}"/side_input_gen.sh + fi + "${NEXMARK_BIN_DIR}"/metric_client.sh "${CMD}" + done + else + # non-local setup + # start/stop TaskManager instance(s) + for worker in ${WORKERS[@]}; do + if [[ $CMD == "start" ]] ; then + ssh -n $worker -- "nohup /bin/bash -l $NEXMARK_BIN_DIR/side_input_gen.sh &" + echo "Generated side input data on $worker" + ssh -n $worker -- "nohup /bin/bash -l $NEXMARK_BIN_DIR/metric_client.sh start &>/dev/null &" + echo "Started metric monitor on $worker" + else + ssh -n $worker -- "nohup /bin/bash -l $NEXMARK_BIN_DIR/metric_client.sh stop &" + ssh -n $worker -- "nohup rm -rf $NEXMARK_HOME/data/output &" + fi + done + fi +} \ No newline at end of file diff --git a/nexmark/nexmark-flink/src/main/resources/bin/metric_client.sh b/nexmark/nexmark-flink/src/main/resources/bin/metric_client.sh new file mode 100644 index 0000000..73ecd47 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/bin/metric_client.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +USAGE="Usage: metric_client.sh (start|stop)" + +if [ $# -lt 1 ]; then + echo $USAGE +fi + +bin=`dirname "$0"` +bin=`cd "$bin"; pwd` + +. "$bin"/config.sh + +STARTSTOP=$1 + +case $STARTSTOP in + (start) + log=$NEXMARK_LOG_DIR/metric-client.log + log_setting=(-Dlog.file="$log" -Dlog4j.configuration=file:"$NEXMARK_CONF_DIR"/log4j.properties -Dlog4j.configurationFile=file:"$NEXMARK_CONF_DIR"/log4j.properties) + java "${log_setting[@]}" -cp "$NEXMARK_HOME/lib/*:$FLINK_HOME/lib/*" com.github.nexmark.flink.metric.process.ProcessMetricSender & + ;; + + (stop) + PID="$(jps | grep ProcessMetricSender | awk '{print $1}')" + kill -9 $PID + echo "$PID ProcessMetricSender has been killed." + ;; +esac \ No newline at end of file diff --git a/nexmark/nexmark-flink/src/main/resources/bin/metric_server.sh b/nexmark/nexmark-flink/src/main/resources/bin/metric_server.sh new file mode 100644 index 0000000..71f4fd4 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/bin/metric_server.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +USAGE="Usage: metric-server.sh (start|stop)" + +if [ $# -lt 1 ]; then + echo $USAGE +fi + +bin=`dirname "$0"` +bin=`cd "$bin"; pwd` + +. "$bin"/config.sh + +STARTSTOP=$1 + +case $STARTSTOP in + (start) + log=$NEXMARK_LOG_DIR/metric-server.log + log_setting=(-Dlog.file="$log" -Dlog4j.configuration=file:"$NEXMARK_CONF_DIR"/log4j.properties -Dlog4j.configurationFile=file:"$NEXMARK_CONF_DIR"/log4j.properties) + java "${log_setting[@]}" -cp "$NEXMARK_HOME/lib/*:$FLINK_HOME/lib/*" com.github.nexmark.flink.metric.process.ProcessMetricReceiver & + ;; + + (stop) + PID="$(jps | grep ProcessMetricReceiver | awk '{print $1}')" + kill -9 $PID + echo "$PID ProcessMetricReceiver has been killed." + ;; +esac + + + diff --git a/nexmark/nexmark-flink/src/main/resources/bin/run_query.sh b/nexmark/nexmark-flink/src/main/resources/bin/run_query.sh new file mode 100755 index 0000000..17324e8 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/bin/run_query.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +USAGE="Usage: run_query.sh (oa|cep) (all|q0|q1|...)" + +bin=`dirname "$0"` +bin=`cd "$bin"; pwd` + +. "$bin"/config.sh + +CATEGORY="oa" +QUERY="all" + +if [ $# -gt 1 ]; then + CATEGORY="$1" + QUERY="$2" +elif [ $# -gt 0 ]; then + QUERY="$1" +fi + +log=$NEXMARK_LOG_DIR/nexmark-flink.log +log_setting=(-Dlog.file="$log" -Dlog4j.configuration=file:"$NEXMARK_CONF_DIR"/log4j.properties -Dlog4j.configurationFile=file:"$NEXMARK_CONF_DIR"/log4j.properties) + +java "${log_setting[@]}" -cp "$NEXMARK_HOME/lib/*:$FLINK_HOME/lib/*" com.github.nexmark.flink.Benchmark --location "$NEXMARK_HOME" --queries "$QUERY" --category "$CATEGORY" \ No newline at end of file diff --git a/nexmark/nexmark-flink/src/main/resources/bin/setup_cluster.sh b/nexmark/nexmark-flink/src/main/resources/bin/setup_cluster.sh new file mode 100644 index 0000000..d6686f4 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/bin/setup_cluster.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +bin=`dirname "$0"` +bin=`cd "$bin"; pwd` + +. "$bin"/config.sh + +# Start metric client instance(s) +TMWorkers start \ No newline at end of file diff --git a/nexmark/nexmark-flink/src/main/resources/bin/shutdown_cluster.sh b/nexmark/nexmark-flink/src/main/resources/bin/shutdown_cluster.sh new file mode 100644 index 0000000..b11099b --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/bin/shutdown_cluster.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +bin=`dirname "$0"` +bin=`cd "$bin"; pwd` + +. "$bin"/config.sh + +# Start metric client instance(s) +TMWorkers stop \ No newline at end of file diff --git a/nexmark/nexmark-flink/src/main/resources/bin/side_input_gen.sh b/nexmark/nexmark-flink/src/main/resources/bin/side_input_gen.sh new file mode 100755 index 0000000..90d63a2 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/bin/side_input_gen.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# setup side input files +USAGE="Usage: side_input_gen.sh" + +bin=`dirname "$0"` +bin=`cd "$bin"; pwd` + +. "$bin"/config.sh + +java -cp "$NEXMARK_HOME/lib/*:$FLINK_HOME/lib/*" com.github.nexmark.flink.generator.SideInputGenerator --num 10000 --path "$FLINK_HOME/data/side_input.txt" \ No newline at end of file diff --git a/nexmark/nexmark-flink/src/main/resources/conf/config.yaml b/nexmark/nexmark-flink/src/main/resources/conf/config.yaml new file mode 100644 index 0000000..1998927 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/conf/config.yaml @@ -0,0 +1,60 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +taskmanager.memory.process.size: 8G +jobmanager.rpc.address: localhost +jobmanager.rpc.port: 6123 +jobmanager.memory.process.size: 8G +taskmanager.numberOfTaskSlots: 1 +parallelism.default: 8 +io.tmp.dirs: /tmp + +#============================================================================== +# JVM +#============================================================================== + +# JVM options for GC +env.java.opts: -verbose:gc -XX:NewRatio=3 -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:ParallelGCThreads=4 +env.java.opts.jobmanager: -Xloggc:$FLINK_LOG_DIR/jobmanager-gc.log -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=2 -XX:GCLogFileSize=512M +env.java.opts.taskmanager: -Xloggc:$FLINK_LOG_DIR/taskmanager-gc.log -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=2 -XX:GCLogFileSize=512M + +#============================================================================== +# State & Checkpoint +#============================================================================== + +state.backend: rocksdb +# for example, hdfs://benchmark01/checkpoint +state.checkpoints.dir: file:///path/to/checkpoint +state.backend.rocksdb.localdir: /tmp +state.backend.incremental: true +execution.checkpointing.interval: 180000 +execution.checkpointing.mode: EXACTLY_ONCE +state.backend.local-recovery: true + +#============================================================================== +# Runtime Others +#============================================================================== + +# configuration options for adjusting and tuning table programs. +table.exec.mini-batch.enabled: true +table.exec.mini-batch.allow-latency: 2s +table.exec.mini-batch.size: 50000 +table.optimizer.distinct-agg.split.enabled: true + +# disable final checkpoint to avoid test waiting for the last checkpoint complete +execution.checkpointing.checkpoints-after-tasks-finish.enabled: false \ No newline at end of file diff --git a/nexmark/nexmark-flink/src/main/resources/conf/config_v2.yaml b/nexmark/nexmark-flink/src/main/resources/conf/config_v2.yaml new file mode 100644 index 0000000..664fbbd --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/conf/config_v2.yaml @@ -0,0 +1,58 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +taskmanager.memory.process.size: 8G +jobmanager.rpc.address: localhost +jobmanager.rpc.port: 6123 +jobmanager.memory.process.size: 8G +taskmanager.numberOfTaskSlots: 1 +parallelism.default: 8 +io.tmp.dirs: /tmp + +#============================================================================== +# JVM +#============================================================================== + +# JVM options for GC +env.java.opts: -verbose:gc -XX:NewRatio=3 -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:ParallelGCThreads=4 +env.java.opts.jobmanager: -Xloggc:$FLINK_LOG_DIR/jobmanager-gc.log -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=2 -XX:GCLogFileSize=512M +env.java.opts.taskmanager: -Xloggc:$FLINK_LOG_DIR/taskmanager-gc.log -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=2 -XX:GCLogFileSize=512M + +#============================================================================== +# State & Checkpoint +#============================================================================== + +state.backend.type: forst +# for example, hdfs://benchmark01/checkpoint +execution.checkpointing.dir: hdfs:///path/to/checkpoint +execution.checkpointing.incremental: true +execution.checkpointing.interval: 30000 +execution.checkpointing.mode: EXACTLY_ONCE + +#============================================================================== +# Runtime Others +#============================================================================== + +# configuration options for adjusting and tuning table programs. +table.exec.mini-batch.enabled: true +table.exec.mini-batch.allow-latency: 2s +table.exec.mini-batch.size: 50000 +table.optimizer.distinct-agg.split.enabled: true + +# disable final checkpoint to avoid test waiting for the last checkpoint complete +execution.checkpointing.checkpoints-after-tasks-finish.enabled: false \ No newline at end of file diff --git a/nexmark/nexmark-flink/src/main/resources/conf/log4j.properties b/nexmark/nexmark-flink/src/main/resources/conf/log4j.properties new file mode 100644 index 0000000..59e9a7f --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/conf/log4j.properties @@ -0,0 +1,33 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# This affects logging for both user code and Flink +rootLogger.level = INFO +rootLogger.appenderRef.file.ref = MainAppender + +# Uncomment this if you want to _only_ change Flink's logging +#logger.flink.name = org.apache.flink +#logger.flink.level = INFO + +# Log all infos in the given file +appender.main.name = MainAppender +appender.main.type = File +appender.main.append = false +appender.main.fileName = ${sys:log.file} +appender.main.layout.type = PatternLayout +appender.main.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n diff --git a/nexmark/nexmark-flink/src/main/resources/conf/nexmark.yaml b/nexmark/nexmark-flink/src/main/resources/conf/nexmark.yaml new file mode 100644 index 0000000..2bbaff1 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/conf/nexmark.yaml @@ -0,0 +1,73 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +#============================================================================== +# Rest & web frontend +#============================================================================== + +# The metric reporter server host. +nexmark.metric.reporter.host: localhost +# The metric reporter server port. +nexmark.metric.reporter.port: 9098 + +#============================================================================== +# Benchmark workload configuration (events.num) +#============================================================================== + +nexmark.workload.suite.100m.events.num: 100000000 +nexmark.workload.suite.100m.tps: 10000000 +nexmark.workload.suite.100m.queries: "q0,q1,q2,q3,q4,q5,q7,q8,q9,q10,q11,q12,q13,q14,q15,q16,q17,q18,q19,q20,q21,q22,q23" +nexmark.workload.suite.100m.queries.cep: "q0,q1,q2,q3" +nexmark.workload.suite.100m.warmup.duration: 120s +nexmark.workload.suite.100m.warmup.events.num: 100000000 +nexmark.workload.suite.100m.warmup.tps: 10000000 + +#============================================================================== +# Benchmark workload configuration (tps, legacy mode) +# Without events.num and with monitor.duration +# NOTE: The numerical value of TPS is unstable +#============================================================================== + +# When to monitor the metrics, default 3min after job is started +# nexmark.metric.monitor.delay: 3min +# How long to monitor the metrics, default 3min, i.e. monitor from 3min to 6min after job is started +# nexmark.metric.monitor.duration: 3min + +# nexmark.workload.suite.10m.tps: 10000000 +# nexmark.workload.suite.10m.queries: "q0,q1,q2,q3,q4,q5,q7,q8,q9,q10,q11,q12,q13,q14,q15,q16,q17,q18,q19,q20,q21,q22" + +#============================================================================== +# Workload for data generation +#============================================================================== + +nexmark.workload.suite.datagen.tps: 10000000 +nexmark.workload.suite.datagen.queries: "insert_kafka" +nexmark.workload.suite.datagen.queries.cep: "insert_kafka" + +#============================================================================== +# Flink REST +#============================================================================== + +flink.rest.address: localhost +flink.rest.port: 8081 + +#============================================================================== +# Kafka config +#============================================================================== + +# kafka.bootstrap.servers: ***:9092 diff --git a/nexmark/nexmark-flink/src/main/resources/conf/nexmark_v2.yaml b/nexmark/nexmark-flink/src/main/resources/conf/nexmark_v2.yaml new file mode 100644 index 0000000..7f8f2d4 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/conf/nexmark_v2.yaml @@ -0,0 +1,67 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +#============================================================================== +# Rest & web frontend +#============================================================================== + +# The metric reporter server host. +nexmark.metric.reporter.host: localhost +# The metric reporter server port. +nexmark.metric.reporter.port: 9098 + +#============================================================================== +# Benchmark workload configuration (events.num) +#============================================================================== + +nexmark.workload.suite.tp-200m.events.num: 50000000 +nexmark.workload.suite.tp-200m.tps: 100000 +nexmark.workload.suite.tp-200m.queries: "q0,q1,q2,q3,q4,q5,q7,q8,q9,q10,q11,q12,q13,q14,q15,q16,q17,q18,q19,q20,q21,q22" +nexmark.workload.suite.tp-200m.queries.cep: "q0,q1,q2,q3" +nexmark.workload.suite.tp-200m.warmup.events.num: 150000000 + +#============================================================================== +# Benchmark workload configuration (tps, legacy mode) +# Without events.num and with monitor.duration +# NOTE: The numerical value of TPS is unstable +#============================================================================== + +# Runner version 2 + +nexmark.runner.version: v2 + +# When to monitor the metrics, default 3min after job is started +# nexmark.metric.monitor.delay: 3min +# How long to monitor the metrics, default 3min, i.e. monitor from 3min to 6min after job is started +# nexmark.metric.monitor.duration: 3min + +# nexmark.workload.suite.10m.tps: 10000000 +# nexmark.workload.suite.10m.queries: "q0,q1,q2,q3,q4,q5,q7,q8,q9,q10,q11,q12,q13,q14,q15,q16,q17,q18,q19,q20,q21,q22" + +#============================================================================== +# Flink REST +#============================================================================== + +flink.rest.address: localhost +flink.rest.port: 8081 + +#============================================================================== +# Kafka config +#============================================================================== + +# kafka.bootstrap.servers: ***:9092 diff --git a/nexmark/nexmark-flink/src/main/resources/queries-cep/ddl_gen.sql b/nexmark/nexmark-flink/src/main/resources/queries-cep/ddl_gen.sql new file mode 100644 index 0000000..24f47aa --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/queries-cep/ddl_gen.sql @@ -0,0 +1,46 @@ +CREATE TABLE datagen ( + event_type int, + person ROW< + id BIGINT, + name VARCHAR, + emailAddress VARCHAR, + creditCard VARCHAR, + city VARCHAR, + state VARCHAR, + `dateTime` TIMESTAMP(3), + extra VARCHAR>, + auction ROW< + id BIGINT, + itemName VARCHAR, + description VARCHAR, + initialBid BIGINT, + reserve BIGINT, + `dateTime` TIMESTAMP(3), + expires TIMESTAMP(3), + seller BIGINT, + category BIGINT, + extra VARCHAR>, + bid ROW< + auction BIGINT, + bidder BIGINT, + price BIGINT, + channel VARCHAR, + url VARCHAR, + `dateTime` TIMESTAMP(3), + extra VARCHAR>, + `dateTime` AS + CASE + WHEN event_type = 0 THEN person.`dateTime` + WHEN event_type = 1 THEN auction.`dateTime` + ELSE bid.`dateTime` + END, + WATERMARK FOR `dateTime` AS `dateTime` - INTERVAL '4' SECOND +) WITH ( + 'connector' = 'nexmark', + 'first-event.rate' = '${TPS}', + 'next-event.rate' = '${TPS}', + 'events.num' = '${EVENTS_NUM}', + 'person.proportion' = '${PERSON_PROPORTION}', + 'auction.proportion' = '${AUCTION_PROPORTION}', + 'bid.proportion' = '${BID_PROPORTION}' +); diff --git a/nexmark/nexmark-flink/src/main/resources/queries-cep/ddl_kafka.sql b/nexmark/nexmark-flink/src/main/resources/queries-cep/ddl_kafka.sql new file mode 100644 index 0000000..1e9e464 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/queries-cep/ddl_kafka.sql @@ -0,0 +1,46 @@ +CREATE TABLE kafka ( + event_type int, + person ROW< + id BIGINT, + name VARCHAR, + emailAddress VARCHAR, + creditCard VARCHAR, + city VARCHAR, + state VARCHAR, + `dateTime` TIMESTAMP(3), + extra VARCHAR>, + auction ROW< + id BIGINT, + itemName VARCHAR, + description VARCHAR, + initialBid BIGINT, + reserve BIGINT, + `dateTime` TIMESTAMP(3), + expires TIMESTAMP(3), + seller BIGINT, + category BIGINT, + extra VARCHAR>, + bid ROW< + auction BIGINT, + bidder BIGINT, + price BIGINT, + channel VARCHAR, + url VARCHAR, + `dateTime` TIMESTAMP(3), + extra VARCHAR>, + `dateTime` AS + CASE + WHEN event_type = 0 THEN person.`dateTime` + WHEN event_type = 1 THEN auction.`dateTime` + ELSE bid.`dateTime` + END, + WATERMARK FOR `dateTime` AS `dateTime` - INTERVAL '4' SECOND +) WITH ( + 'connector' = 'kafka', + 'topic' = 'nexmark', + 'properties.bootstrap.servers' = '${BOOTSTRAP_SERVERS}', + 'properties.group.id' = 'nexmark', + 'scan.startup.mode' = 'earliest-offset', + 'sink.partitioner' = 'fixed', + 'format' = 'json' +); diff --git a/nexmark/nexmark-flink/src/main/resources/queries-cep/ddl_views.sql b/nexmark/nexmark-flink/src/main/resources/queries-cep/ddl_views.sql new file mode 100644 index 0000000..36f368d --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/queries-cep/ddl_views.sql @@ -0,0 +1,36 @@ +CREATE VIEW person AS +SELECT + person.id, + person.name, + person.emailAddress, + person.creditCard, + person.city, + person.state, + `dateTime`, + person.extra +FROM ${NEXMARK_TABLE} WHERE event_type = 0; + +CREATE VIEW auction AS +SELECT + auction.id, + auction.itemName, + auction.description, + auction.initialBid, + auction.reserve, + `dateTime`, + auction.expires, + auction.seller, + auction.category, + auction.extra +FROM ${NEXMARK_TABLE} WHERE event_type = 1; + +CREATE VIEW bid AS +SELECT + bid.auction, + bid.bidder, + bid.price, + bid.channel, + bid.url, + `dateTime`, + bid.extra +FROM ${NEXMARK_TABLE} WHERE event_type = 2; diff --git a/nexmark/nexmark-flink/src/main/resources/queries-cep/insert_kafka.sql b/nexmark/nexmark-flink/src/main/resources/queries-cep/insert_kafka.sql new file mode 100644 index 0000000..5f7a173 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/queries-cep/insert_kafka.sql @@ -0,0 +1,6 @@ +-- ------------------------------------------------------------------------------------------------- +-- Insert into kafka from nexmark data generator. +-- ------------------------------------------------------------------------------------------------- + +INSERT INTO kafka +SELECT event_type, person, auction, bid FROM datagen; diff --git a/nexmark/nexmark-flink/src/main/resources/queries-cep/q0.sql b/nexmark/nexmark-flink/src/main/resources/queries-cep/q0.sql new file mode 100644 index 0000000..3b472e9 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/queries-cep/q0.sql @@ -0,0 +1,38 @@ +-- ------------------------------------------------------------------------------------------------- +-- Query 0: Periods of a constantly decreasing price (Not in original suite) +-- ------------------------------------------------------------------------------------------------- +-- Finds periods of a constantly decreasing price of a single bidder. +-- Illustrates a typical Pattern in MATCH_RECOGNIZE. +-- ------------------------------------------------------------------------------------------------- + +CREATE TABLE discard_sink ( + auction BIGINT, + bidder BIGINT, + start_tstamp TIMESTAMP(3), + bottom_tstamp TIMESTAMP(3), + end_tstamp TIMESTAMP(3) +) WITH ( + 'connector' = 'blackhole' +); + +INSERT INTO discard_sink +SELECT + auction, bidder, start_tstamp, bottom_tstamp, end_tstamp +FROM bid +MATCH_RECOGNIZE ( + PARTITION BY auction, bidder + ORDER BY `dateTime` + MEASURES + START_ROW.`dateTime` AS start_tstamp, + LAST(PRICE_DOWN.`dateTime`) AS bottom_tstamp, + LAST(PRICE_UP.`dateTime`) AS end_tstamp + ONE ROW PER MATCH + AFTER MATCH SKIP TO LAST PRICE_UP + PATTERN (START_ROW PRICE_DOWN+ PRICE_UP) + DEFINE + PRICE_DOWN AS + (LAST(PRICE_DOWN.price, 1) IS NULL AND PRICE_DOWN.price < START_ROW.price) OR + PRICE_DOWN.price < LAST(PRICE_DOWN.price, 1), + PRICE_UP AS + PRICE_UP.price > LAST(PRICE_DOWN.price, 1) +); diff --git a/nexmark/nexmark-flink/src/main/resources/queries-cep/q1.sql b/nexmark/nexmark-flink/src/main/resources/queries-cep/q1.sql new file mode 100644 index 0000000..9b1e174 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/queries-cep/q1.sql @@ -0,0 +1,34 @@ +-- ------------------------------------------------------------------------------------------------- +-- Query 1: Longest period of time for average price (Not in original suite) +-- ------------------------------------------------------------------------------------------------- +-- Finds the longest period of time for which the average price of a bidder did not go below certain threshold. +-- Illustrates Aggregations and a AFTER MATCH strategy SKIP PAST LAST ROW in MATCH_RECOGNIZE. +-- ------------------------------------------------------------------------------------------------- + +CREATE TABLE discard_sink ( + auction BIGINT, + bidder BIGINT, + start_tstamp TIMESTAMP(3), + end_tstamp TIMESTAMP(3), + avg_price BIGINT +) WITH ( + 'connector' = 'blackhole' +); + +INSERT INTO discard_sink +SELECT + auction, bidder, start_tstamp, end_tstamp, avg_price +FROM bid +MATCH_RECOGNIZE( + PARTITION BY auction, bidder + ORDER BY `dateTime` + MEASURES + FIRST(A.`dateTime`) AS start_tstamp, + LAST(A.`dateTime`) AS end_tstamp, + AVG(A.price) AS avg_price + ONE ROW PER MATCH + AFTER MATCH SKIP PAST LAST ROW + PATTERN (A+ B) + DEFINE + A AS AVG(A.price) < 10000 +); diff --git a/nexmark/nexmark-flink/src/main/resources/queries-cep/q2.sql b/nexmark/nexmark-flink/src/main/resources/queries-cep/q2.sql new file mode 100644 index 0000000..71ec286 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/queries-cep/q2.sql @@ -0,0 +1,34 @@ +-- ------------------------------------------------------------------------------------------------- +-- Query 2: Longest period of time for average price (Not in original suite) +-- ------------------------------------------------------------------------------------------------- +-- Finds the longest period of time for which the average price of a bidder did not go below certain threshold. +-- Illustrates Aggregations and a AFTER MATCH strategy SKIP TO NEXT ROW in MATCH_RECOGNIZE. +-- ------------------------------------------------------------------------------------------------- + +CREATE TABLE discard_sink ( + auction BIGINT, + bidder BIGINT, + start_tstamp TIMESTAMP(3), + end_tstamp TIMESTAMP(3), + avg_price BIGINT +) WITH ( + 'connector' = 'blackhole' +); + +INSERT INTO discard_sink +SELECT + auction, bidder, start_tstamp, end_tstamp, avg_price +FROM bid +MATCH_RECOGNIZE( + PARTITION BY auction, bidder + ORDER BY `dateTime` + MEASURES + FIRST(A.`dateTime`) AS start_tstamp, + LAST(A.`dateTime`) AS end_tstamp, + AVG(A.price) AS avg_price + ONE ROW PER MATCH + AFTER MATCH SKIP TO NEXT ROW + PATTERN (A+ B) + DEFINE + A AS AVG(A.price) < 10000 +); diff --git a/nexmark/nexmark-flink/src/main/resources/queries-cep/q3.sql b/nexmark/nexmark-flink/src/main/resources/queries-cep/q3.sql new file mode 100644 index 0000000..ce3a95c --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/queries-cep/q3.sql @@ -0,0 +1,33 @@ +-- ------------------------------------------------------------------------------------------------- +-- Query 4: Price drop within an interval (Not in original suite) +-- ------------------------------------------------------------------------------------------------- +-- Detects a price drop of 50 that happens within an interval of 5 second time window. +-- Illustrates WITHIN clause in MATCH_RECOGNIZE. +-- ------------------------------------------------------------------------------------------------- + +CREATE TABLE discard_sink ( + auction BIGINT, + bidder BIGINT, + drop_time TIMESTAMP(3), + drop_diff BIGINT +) WITH ( + 'connector' = 'blackhole' +); + +INSERT INTO discard_sink +SELECT + auction, bidder, drop_time, drop_diff +FROM bid +MATCH_RECOGNIZE( + PARTITION BY auction, bidder + ORDER BY `dateTime` + MEASURES + C.`dateTime` AS drop_time, + A.price - C.price AS drop_diff + ONE ROW PER MATCH + AFTER MATCH SKIP PAST LAST ROW + PATTERN (A B* C) WITHIN INTERVAL '5' SECOND + DEFINE + B AS B.price > A.price - 50, + C AS C.price < A.price - 50 +); diff --git a/nexmark/nexmark-flink/src/main/resources/queries/ddl_gen.sql b/nexmark/nexmark-flink/src/main/resources/queries/ddl_gen.sql new file mode 100644 index 0000000..70ecc93 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/queries/ddl_gen.sql @@ -0,0 +1,46 @@ +CREATE TABLE datagen ( + event_type int, + person ROW< + id BIGINT, + name VARCHAR, + emailAddress VARCHAR, + creditCard VARCHAR, + city VARCHAR, + state VARCHAR, + `dateTime` TIMESTAMP(3), + extra VARCHAR>, + auction ROW< + id BIGINT, + itemName VARCHAR, + description VARCHAR, + initialBid BIGINT, + reserve BIGINT, + `dateTime` TIMESTAMP(3), + expires TIMESTAMP(3), + seller BIGINT, + category BIGINT, + extra VARCHAR>, + bid ROW< + auction BIGINT, + bidder BIGINT, + price BIGINT, + channel VARCHAR, + url VARCHAR, + `dateTime` TIMESTAMP(3), + extra VARCHAR>, + `dateTime` AS + CASE + WHEN event_type = 0 THEN person.`dateTime` + WHEN event_type = 1 THEN auction.`dateTime` + ELSE bid.`dateTime` + END, + WATERMARK FOR `dateTime` AS `dateTime` - INTERVAL '4' SECOND +) WITH ( + 'connector' = 'nexmark', + 'first-event.rate' = '${TPS}', + 'next-event.rate' = '${TPS}', + 'events.num' = '${EVENTS_NUM}', + 'person.proportion' = '${PERSON_PROPORTION}', + 'auction.proportion' = '${AUCTION_PROPORTION}', + 'bid.proportion' = '${BID_PROPORTION}' +); \ No newline at end of file diff --git a/nexmark/nexmark-flink/src/main/resources/queries/ddl_gen_v2.sql b/nexmark/nexmark-flink/src/main/resources/queries/ddl_gen_v2.sql new file mode 100644 index 0000000..68b583b --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/queries/ddl_gen_v2.sql @@ -0,0 +1,49 @@ +CREATE TABLE datagen ( + event_type int, + person ROW< + id BIGINT, + name VARCHAR, + emailAddress VARCHAR, + creditCard VARCHAR, + city VARCHAR, + state VARCHAR, + `dateTime` TIMESTAMP(3), + extra VARCHAR>, + auction ROW< + id BIGINT, + itemName VARCHAR, + description VARCHAR, + initialBid BIGINT, + reserve BIGINT, + `dateTime` TIMESTAMP(3), + expires TIMESTAMP(3), + seller BIGINT, + category BIGINT, + extra VARCHAR>, + bid ROW< + auction BIGINT, + bidder BIGINT, + price BIGINT, + channel VARCHAR, + url VARCHAR, + `dateTime` TIMESTAMP(3), + extra VARCHAR>, + `dateTime` AS + CASE + WHEN event_type = 0 THEN person.`dateTime` + WHEN event_type = 1 THEN auction.`dateTime` + ELSE bid.`dateTime` + END, + WATERMARK FOR `dateTime` AS `dateTime` - INTERVAL '4' SECOND +) WITH ( + 'connector' = 'nexmark', + 'first-event.rate' = '${TPS}', + 'next-event.rate' = '${TPS}', + 'events.num' = '${EVENTS_NUM}', + 'person.proportion' = '${PERSON_PROPORTION}', + 'auction.proportion' = '${AUCTION_PROPORTION}', + 'bid.proportion' = '${BID_PROPORTION}', + 'keep-alive' = '${KEEP_ALIVE}', + 'stop-at' = '${STOP_AT}', + 'max-emit-speed' = 'true' +); \ No newline at end of file diff --git a/nexmark/nexmark-flink/src/main/resources/queries/ddl_kafka.sql b/nexmark/nexmark-flink/src/main/resources/queries/ddl_kafka.sql new file mode 100644 index 0000000..28affcd --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/queries/ddl_kafka.sql @@ -0,0 +1,46 @@ +CREATE TABLE kafka ( + event_type int, + person ROW< + id BIGINT, + name VARCHAR, + emailAddress VARCHAR, + creditCard VARCHAR, + city VARCHAR, + state VARCHAR, + `dateTime` TIMESTAMP(3), + extra VARCHAR>, + auction ROW< + id BIGINT, + itemName VARCHAR, + description VARCHAR, + initialBid BIGINT, + reserve BIGINT, + `dateTime` TIMESTAMP(3), + expires TIMESTAMP(3), + seller BIGINT, + category BIGINT, + extra VARCHAR>, + bid ROW< + auction BIGINT, + bidder BIGINT, + price BIGINT, + channel VARCHAR, + url VARCHAR, + `dateTime` TIMESTAMP(3), + extra VARCHAR>, + `dateTime` AS + CASE + WHEN event_type = 0 THEN person.`dateTime` + WHEN event_type = 1 THEN auction.`dateTime` + ELSE bid.`dateTime` + END, + WATERMARK FOR `dateTime` AS `dateTime` - INTERVAL '4' SECOND +) WITH ( + 'connector' = 'kafka', + 'topic' = 'nexmark', + 'properties.bootstrap.servers' = '${BOOTSTRAP_SERVERS}', + 'properties.group.id' = 'nexmark', + 'scan.startup.mode' = 'earliest-offset', + 'sink.partitioner' = 'round-robin', + 'format' = 'json' +); \ No newline at end of file diff --git a/nexmark/nexmark-flink/src/main/resources/queries/ddl_views.sql b/nexmark/nexmark-flink/src/main/resources/queries/ddl_views.sql new file mode 100644 index 0000000..c74732d --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/queries/ddl_views.sql @@ -0,0 +1,36 @@ +CREATE VIEW person AS +SELECT + person.id, + person.name, + person.emailAddress, + person.creditCard, + person.city, + person.state, + `dateTime`, + person.extra +FROM ${NEXMARK_TABLE} WHERE event_type = 0; + +CREATE VIEW auction AS +SELECT + auction.id, + auction.itemName, + auction.description, + auction.initialBid, + auction.reserve, + `dateTime`, + auction.expires, + auction.seller, + auction.category, + auction.extra +FROM ${NEXMARK_TABLE} WHERE event_type = 1; + +CREATE VIEW bid AS +SELECT + bid.auction, + bid.bidder, + bid.price, + bid.channel, + bid.url, + `dateTime`, + bid.extra +FROM ${NEXMARK_TABLE} WHERE event_type = 2; \ No newline at end of file diff --git a/nexmark/nexmark-flink/src/main/resources/queries/insert_kafka.sql b/nexmark/nexmark-flink/src/main/resources/queries/insert_kafka.sql new file mode 100644 index 0000000..4b3341c --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/queries/insert_kafka.sql @@ -0,0 +1,6 @@ +-- ------------------------------------------------------------------------------------------------- +-- Insert into kafka from nexmark data generator. +-- ------------------------------------------------------------------------------------------------- + +INSERT INTO kafka +SELECT event_type, person, auction, bid FROM datagen; \ No newline at end of file diff --git a/nexmark/nexmark-flink/src/main/resources/queries/q0.sql b/nexmark/nexmark-flink/src/main/resources/queries/q0.sql new file mode 100644 index 0000000..08fc765 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/queries/q0.sql @@ -0,0 +1,19 @@ +-- ------------------------------------------------------------------------------------------------- +-- Query 0: Pass through (Not in original suite) +-- ------------------------------------------------------------------------------------------------- +-- This measures the monitoring overhead of the Flink SQL implementation including the source generator. +-- Using `bid` events here, as they are most numerous with default configuration. +-- ------------------------------------------------------------------------------------------------- + +CREATE TABLE nexmark_q0 ( + auction BIGINT, + bidder BIGINT, + price BIGINT, + `dateTime` TIMESTAMP(3), + extra VARCHAR +) WITH ( + ${SINK_DDL} +); + +INSERT INTO nexmark_q0 +SELECT auction, bidder, price, `dateTime`, extra FROM bid; \ No newline at end of file diff --git a/nexmark/nexmark-flink/src/main/resources/queries/q1.sql b/nexmark/nexmark-flink/src/main/resources/queries/q1.sql new file mode 100644 index 0000000..aa5c044 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/queries/q1.sql @@ -0,0 +1,24 @@ +-- ------------------------------------------------------------------------------------------------- +-- Query1: Currency conversion +-- ------------------------------------------------------------------------------------------------- +-- Convert each bid value from dollars to euros. Illustrates a simple transformation. +-- ------------------------------------------------------------------------------------------------- + +CREATE TABLE nexmark_q1 ( + auction BIGINT, + bidder BIGINT, + price DECIMAL(23, 3), + `dateTime` TIMESTAMP(3), + extra VARCHAR +) WITH ( + ${SINK_DDL} +); + +INSERT INTO nexmark_q1 +SELECT + auction, + bidder, + 0.908 * price as price, -- convert dollar to euro + `dateTime`, + extra +FROM bid; \ No newline at end of file diff --git a/nexmark/nexmark-flink/src/main/resources/queries/q10.sql b/nexmark/nexmark-flink/src/main/resources/queries/q10.sql new file mode 100644 index 0000000..ba8893f --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/queries/q10.sql @@ -0,0 +1,23 @@ +-- ------------------------------------------------------------------------------------------------- +-- Query 10: Log to File System (Not in original suite) +-- ------------------------------------------------------------------------------------------------- +-- Log all events to file system. Illustrates windows streaming data into partitioned file system. +-- +-- Every minute, save all events from the last period into partitioned log files. +-- ------------------------------------------------------------------------------------------------- + +CREATE TABLE nexmark_q10 ( + auction BIGINT, + bidder BIGINT, + price BIGINT, + `dateTime` TIMESTAMP(3), + extra VARCHAR, + dt STRING, + hm STRING +) PARTITIONED BY (dt, hm) WITH ( + ${SINK_DDL} +); + +INSERT INTO nexmark_q10 +SELECT auction, bidder, price, `dateTime`, extra, DATE_FORMAT(`dateTime`, 'yyyy-MM-dd'), DATE_FORMAT(`dateTime`, 'HH:mm') +FROM bid; \ No newline at end of file diff --git a/nexmark/nexmark-flink/src/main/resources/queries/q11.sql b/nexmark/nexmark-flink/src/main/resources/queries/q11.sql new file mode 100644 index 0000000..3af581b --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/queries/q11.sql @@ -0,0 +1,26 @@ +-- ------------------------------------------------------------------------------------------------- +-- Query 11: User Sessions (Not in original suite) +-- ------------------------------------------------------------------------------------------------- +-- How many bids did a user make in each session they were active? Illustrates session windows. +-- +-- Group bids by the same user into sessions with max session gap. +-- Emit the number of bids per session. +-- ------------------------------------------------------------------------------------------------- + +CREATE TABLE nexmark_q11 ( + bidder BIGINT, + bid_count BIGINT, + starttime TIMESTAMP(3), + endtime TIMESTAMP(3) +) WITH ( + ${SINK_DDL} +); + +INSERT INTO nexmark_q11 +SELECT + B.bidder, + count(*) as bid_count, + SESSION_START(B.`dateTime`, INTERVAL '10' SECOND) as starttime, + SESSION_END(B.`dateTime`, INTERVAL '10' SECOND) as endtime +FROM bid B +GROUP BY B.bidder, SESSION(B.`dateTime`, INTERVAL '10' SECOND); \ No newline at end of file diff --git a/nexmark/nexmark-flink/src/main/resources/queries/q12.sql b/nexmark/nexmark-flink/src/main/resources/queries/q12.sql new file mode 100644 index 0000000..9ba3f06 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/queries/q12.sql @@ -0,0 +1,30 @@ +-- ------------------------------------------------------------------------------------------------- +-- Query 12: Processing Time Windows (Not in original suite) +-- ------------------------------------------------------------------------------------------------- +-- How many bids does a user make within a fixed processing time limit? +-- Illustrates working in processing time window. +-- +-- Group bids by the same user into processing time windows of 10 seconds. +-- Emit the count of bids per window. +-- ------------------------------------------------------------------------------------------------- + +CREATE TABLE nexmark_q12 ( + bidder BIGINT, + bid_count BIGINT, + starttime TIMESTAMP(3), + endtime TIMESTAMP(3) +) WITH ( + ${SINK_DDL} +); + +CREATE VIEW B AS SELECT *, PROCTIME() as p_time FROM bid; + +INSERT INTO nexmark_q12 +SELECT + bidder, + count(*) as bid_count, + window_start AS starttime, + window_end AS endtime +FROM TABLE( + TUMBLE(TABLE B, DESCRIPTOR(p_time), INTERVAL '10' SECOND)) +GROUP BY bidder, window_start, window_end; \ No newline at end of file diff --git a/nexmark/nexmark-flink/src/main/resources/queries/q13.sql b/nexmark/nexmark-flink/src/main/resources/queries/q13.sql new file mode 100644 index 0000000..5285475 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/queries/q13.sql @@ -0,0 +1,36 @@ +-- ------------------------------------------------------------------------------------------------- +-- Query 13: Bounded Side Input Join (Not in original suite) +-- ------------------------------------------------------------------------------------------------- +-- Joins a stream to a bounded side input, modeling basic stream enrichment. +-- ------------------------------------------------------------------------------------------------- + +-- TODO: use the new "filesystem" connector once FLINK-17397 is done +CREATE TABLE side_input ( + key BIGINT, + `value` VARCHAR +) WITH ( + 'connector.type' = 'filesystem', + 'connector.path' = 'file://${FLINK_HOME}/data/side_input.txt', + 'format.type' = 'csv' +); + +CREATE TABLE nexmark_q13 ( + auction BIGINT, + bidder BIGINT, + price BIGINT, + `dateTime` TIMESTAMP(3), + `value` VARCHAR +) WITH ( + ${SINK_DDL} +); + +INSERT INTO nexmark_q13 +SELECT + B.auction, + B.bidder, + B.price, + B.`dateTime`, + S.`value` +FROM (SELECT *, PROCTIME() as p_time FROM bid) B +JOIN side_input FOR SYSTEM_TIME AS OF B.p_time AS S +ON mod(B.auction, 10000) = S.key; \ No newline at end of file diff --git a/nexmark/nexmark-flink/src/main/resources/queries/q14.sql b/nexmark/nexmark-flink/src/main/resources/queries/q14.sql new file mode 100644 index 0000000..102fcf0 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/queries/q14.sql @@ -0,0 +1,36 @@ +-- ------------------------------------------------------------------------------------------------- +-- Query 14: Calculation (Not in original suite) +-- ------------------------------------------------------------------------------------------------- +-- Convert bid timestamp into types and find bids with specific price. +-- Illustrates duplicate expressions and usage of user-defined-functions. +-- ------------------------------------------------------------------------------------------------- + +CREATE FUNCTION count_char AS 'com.github.nexmark.flink.udf.CountChar'; + +CREATE TABLE nexmark_q14 ( + auction BIGINT, + bidder BIGINT, + price DECIMAL(23, 3), + bidTimeType VARCHAR, + `dateTime` TIMESTAMP(3), + extra VARCHAR, + c_counts BIGINT +) WITH ( + ${SINK_DDL} +); + +INSERT INTO nexmark_q14 +SELECT + auction, + bidder, + 0.908 * price as price, + CASE + WHEN HOUR(`dateTime`) >= 8 AND HOUR(`dateTime`) <= 18 THEN 'dayTime' + WHEN HOUR(`dateTime`) <= 6 OR HOUR(`dateTime`) >= 20 THEN 'nightTime' + ELSE 'otherTime' + END AS bidTimeType, + `dateTime`, + extra, + count_char(extra, 'c') AS c_counts +FROM bid +WHERE 0.908 * price > 1000000 AND 0.908 * price < 50000000; \ No newline at end of file diff --git a/nexmark/nexmark-flink/src/main/resources/queries/q15.sql b/nexmark/nexmark-flink/src/main/resources/queries/q15.sql new file mode 100644 index 0000000..ed280f8 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/queries/q15.sql @@ -0,0 +1,42 @@ +-- ------------------------------------------------------------------------------------------------- +-- Query 15: Bidding Statistics Report (Not in original suite) +-- ------------------------------------------------------------------------------------------------- +-- How many distinct users join the bidding for different level of price? +-- Illustrates multiple distinct aggregations with filters. +-- ------------------------------------------------------------------------------------------------- + +CREATE TABLE nexmark_q15 ( + `day` VARCHAR, + total_bids BIGINT, + rank1_bids BIGINT, + rank2_bids BIGINT, + rank3_bids BIGINT, + total_bidders BIGINT, + rank1_bidders BIGINT, + rank2_bidders BIGINT, + rank3_bidders BIGINT, + total_auctions BIGINT, + rank1_auctions BIGINT, + rank2_auctions BIGINT, + rank3_auctions BIGINT +) WITH ( + ${SINK_DDL} +); + +INSERT INTO nexmark_q15 +SELECT + DATE_FORMAT(`dateTime`, 'yyyy-MM-dd') as `day`, + count(*) AS total_bids, + count(*) filter (where price < 10000) AS rank1_bids, + count(*) filter (where price >= 10000 and price < 1000000) AS rank2_bids, + count(*) filter (where price >= 1000000) AS rank3_bids, + count(distinct bidder) AS total_bidders, + count(distinct bidder) filter (where price < 10000) AS rank1_bidders, + count(distinct bidder) filter (where price >= 10000 and price < 1000000) AS rank2_bidders, + count(distinct bidder) filter (where price >= 1000000) AS rank3_bidders, + count(distinct auction) AS total_auctions, + count(distinct auction) filter (where price < 10000) AS rank1_auctions, + count(distinct auction) filter (where price >= 10000 and price < 1000000) AS rank2_auctions, + count(distinct auction) filter (where price >= 1000000) AS rank3_auctions +FROM bid +GROUP BY DATE_FORMAT(`dateTime`, 'yyyy-MM-dd'); \ No newline at end of file diff --git a/nexmark/nexmark-flink/src/main/resources/queries/q16.sql b/nexmark/nexmark-flink/src/main/resources/queries/q16.sql new file mode 100644 index 0000000..bde1696 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/queries/q16.sql @@ -0,0 +1,46 @@ +-- ------------------------------------------------------------------------------------------------- +-- Query 16: Channel Statistics Report (Not in original suite) +-- ------------------------------------------------------------------------------------------------- +-- How many distinct users join the bidding for different level of price for a channel? +-- Illustrates multiple distinct aggregations with filters for multiple keys. +-- ------------------------------------------------------------------------------------------------- + +CREATE TABLE nexmark_q16 ( + channel VARCHAR, + `day` VARCHAR, + `minute` VARCHAR, + total_bids BIGINT, + rank1_bids BIGINT, + rank2_bids BIGINT, + rank3_bids BIGINT, + total_bidders BIGINT, + rank1_bidders BIGINT, + rank2_bidders BIGINT, + rank3_bidders BIGINT, + total_auctions BIGINT, + rank1_auctions BIGINT, + rank2_auctions BIGINT, + rank3_auctions BIGINT +) WITH ( + ${SINK_DDL} +); + +INSERT INTO nexmark_q16 +SELECT + channel, + DATE_FORMAT(`dateTime`, 'yyyy-MM-dd') as `day`, + max(DATE_FORMAT(`dateTime`, 'HH:mm')) as `minute`, + count(*) AS total_bids, + count(*) filter (where price < 10000) AS rank1_bids, + count(*) filter (where price >= 10000 and price < 1000000) AS rank2_bids, + count(*) filter (where price >= 1000000) AS rank3_bids, + count(distinct bidder) AS total_bidders, + count(distinct bidder) filter (where price < 10000) AS rank1_bidders, + count(distinct bidder) filter (where price >= 10000 and price < 1000000) AS rank2_bidders, + count(distinct bidder) filter (where price >= 1000000) AS rank3_bidders, + count(distinct auction) AS total_auctions, + count(distinct auction) filter (where price < 10000) AS rank1_auctions, + count(distinct auction) filter (where price >= 10000 and price < 1000000) AS rank2_auctions, + count(distinct auction) filter (where price >= 1000000) AS rank3_auctions +FROM bid +GROUP BY channel, DATE_FORMAT(`dateTime`, 'yyyy-MM-dd'); \ No newline at end of file diff --git a/nexmark/nexmark-flink/src/main/resources/queries/q17.sql b/nexmark/nexmark-flink/src/main/resources/queries/q17.sql new file mode 100644 index 0000000..e91f192 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/queries/q17.sql @@ -0,0 +1,36 @@ +-- ------------------------------------------------------------------------------------------------- +-- Query 17: Auction Statistics Report (Not in original suite) +-- ------------------------------------------------------------------------------------------------- +-- How many bids on an auction made a day and what is the price? +-- Illustrates an unbounded group aggregation. +-- ------------------------------------------------------------------------------------------------- + +CREATE TABLE nexmark_q17 ( + auction BIGINT, + `day` VARCHAR, + total_bids BIGINT, + rank1_bids BIGINT, + rank2_bids BIGINT, + rank3_bids BIGINT, + min_price BIGINT, + max_price BIGINT, + avg_price BIGINT, + sum_price BIGINT +) WITH ( + ${SINK_DDL} +); + +INSERT INTO nexmark_q17 +SELECT + auction, + DATE_FORMAT(`dateTime`, 'yyyy-MM-dd') as `day`, + count(*) AS total_bids, + count(*) filter (where price < 10000) AS rank1_bids, + count(*) filter (where price >= 10000 and price < 1000000) AS rank2_bids, + count(*) filter (where price >= 1000000) AS rank3_bids, + min(price) AS min_price, + max(price) AS max_price, + avg(price) AS avg_price, + sum(price) AS sum_price +FROM bid +GROUP BY auction, DATE_FORMAT(`dateTime`, 'yyyy-MM-dd'); \ No newline at end of file diff --git a/nexmark/nexmark-flink/src/main/resources/queries/q18.sql b/nexmark/nexmark-flink/src/main/resources/queries/q18.sql new file mode 100644 index 0000000..63c833c --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/queries/q18.sql @@ -0,0 +1,24 @@ +-- ------------------------------------------------------------------------------------------------- +-- Query 18: Find last bid (Not in original suite) +-- ------------------------------------------------------------------------------------------------- +-- What's a's last bid for bidder to auction? +-- Illustrates a Deduplicate query. +-- ------------------------------------------------------------------------------------------------- + +CREATE TABLE nexmark_q18 ( + auction BIGINT, + bidder BIGINT, + price BIGINT, + channel VARCHAR, + url VARCHAR, + `dateTime` TIMESTAMP(3), + extra VARCHAR +) WITH ( + ${SINK_DDL} +); + +INSERT INTO nexmark_q18 +SELECT auction, bidder, price, channel, url, `dateTime`, extra + FROM (SELECT *, ROW_NUMBER() OVER (PARTITION BY bidder, auction ORDER BY `dateTime` DESC) AS rank_number + FROM bid) + WHERE rank_number <= 1; \ No newline at end of file diff --git a/nexmark/nexmark-flink/src/main/resources/queries/q19.sql b/nexmark/nexmark-flink/src/main/resources/queries/q19.sql new file mode 100644 index 0000000..54d31b2 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/queries/q19.sql @@ -0,0 +1,24 @@ +-- ------------------------------------------------------------------------------------------------- +-- Query 19: Auction TOP-10 Price (Not in original suite) +-- ------------------------------------------------------------------------------------------------- +-- What's the top price 10 bids of an auction? +-- Illustrates a TOP-N query. +-- ------------------------------------------------------------------------------------------------- + +CREATE TABLE nexmark_q19 ( + auction BIGINT, + bidder BIGINT, + price BIGINT, + channel VARCHAR, + url VARCHAR, + `dateTime` TIMESTAMP(3), + extra VARCHAR, + rank_number BIGINT +) WITH ( + ${SINK_DDL} +); + +INSERT INTO nexmark_q19 +SELECT * FROM +(SELECT *, ROW_NUMBER() OVER (PARTITION BY auction ORDER BY price DESC) AS rank_number FROM bid) +WHERE rank_number <= 10; \ No newline at end of file diff --git a/nexmark/nexmark-flink/src/main/resources/queries/q2.sql b/nexmark/nexmark-flink/src/main/resources/queries/q2.sql new file mode 100644 index 0000000..0b2b438 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/queries/q2.sql @@ -0,0 +1,24 @@ +-- ------------------------------------------------------------------------------------------------- +-- Query2: Selection +-- ------------------------------------------------------------------------------------------------- +-- Find bids with specific auction ids and show their bid price. +-- +-- In original Nexmark queries, Query2 is as following (in CQL syntax): +-- +-- SELECT Rstream(auction, price) +-- FROM Bid [NOW] +-- WHERE auction = 1007 OR auction = 1020 OR auction = 2001 OR auction = 2019 OR auction = 2087; +-- +-- However, that query will only yield a few hundred results over event streams of arbitrary size. +-- To make it more interesting we instead choose bids for every 123'th auction. +-- ------------------------------------------------------------------------------------------------- + +CREATE TABLE nexmark_q2 ( + auction BIGINT, + price BIGINT +) WITH ( + ${SINK_DDL} +); + +INSERT INTO nexmark_q2 +SELECT auction, price FROM bid WHERE MOD(auction, 123) = 0; \ No newline at end of file diff --git a/nexmark/nexmark-flink/src/main/resources/queries/q20.sql b/nexmark/nexmark-flink/src/main/resources/queries/q20.sql new file mode 100644 index 0000000..ea628a6 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/queries/q20.sql @@ -0,0 +1,36 @@ +-- ------------------------------------------------------------------------------------------------- +-- Query 20: Expand bid with auction (Not in original suite) +-- ------------------------------------------------------------------------------------------------- +-- Get bids with the corresponding auction information where category is 10. +-- Illustrates a filter join. +-- ------------------------------------------------------------------------------------------------- + +CREATE TABLE nexmark_q20 ( + auction BIGINT, + bidder BIGINT, + price BIGINT, + channel VARCHAR, + url VARCHAR, + bid_dateTime TIMESTAMP(3), + bid_extra VARCHAR, + + itemName VARCHAR, + description VARCHAR, + initialBid BIGINT, + reserve BIGINT, + auction_dateTime TIMESTAMP(3), + expires TIMESTAMP(3), + seller BIGINT, + category BIGINT, + auction_extra VARCHAR +) WITH ( + ${SINK_DDL} +); + +INSERT INTO nexmark_q20 +SELECT + auction, bidder, price, channel, url, B.`dateTime`, B.extra, + itemName, description, initialBid, reserve, A.`dateTime`, expires, seller, category, A.extra +FROM + bid AS B INNER JOIN auction AS A on B.auction = A.id +WHERE A.category = 10; \ No newline at end of file diff --git a/nexmark/nexmark-flink/src/main/resources/queries/q21.sql b/nexmark/nexmark-flink/src/main/resources/queries/q21.sql new file mode 100644 index 0000000..bfeb10b --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/queries/q21.sql @@ -0,0 +1,30 @@ +-- ------------------------------------------------------------------------------------------------- +-- Query 21: Add channel id (Not in original suite) +-- ------------------------------------------------------------------------------------------------- +-- Add a channel_id column to the bid table. +-- Illustrates a 'CASE WHEN' + 'REGEXP_EXTRACT' SQL. +-- ------------------------------------------------------------------------------------------------- + +CREATE TABLE nexmark_q21 ( + auction BIGINT, + bidder BIGINT, + price BIGINT, + channel VARCHAR, + channel_id VARCHAR +) WITH ( + ${SINK_DDL} +); + +INSERT INTO nexmark_q21 +SELECT + auction, bidder, price, channel, + CASE + WHEN lower(channel) = 'apple' THEN '0' + WHEN lower(channel) = 'google' THEN '1' + WHEN lower(channel) = 'facebook' THEN '2' + WHEN lower(channel) = 'baidu' THEN '3' + ELSE REGEXP_EXTRACT(url, '(&|^)channel_id=([^&]*)', 2) + END + AS channel_id FROM bid + where REGEXP_EXTRACT(url, '(&|^)channel_id=([^&]*)', 2) is not null or + lower(channel) in ('apple', 'google', 'facebook', 'baidu'); \ No newline at end of file diff --git a/nexmark/nexmark-flink/src/main/resources/queries/q22.sql b/nexmark/nexmark-flink/src/main/resources/queries/q22.sql new file mode 100644 index 0000000..47619dc --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/queries/q22.sql @@ -0,0 +1,25 @@ +-- ------------------------------------------------------------------------------------------------- +-- Query 22: Get URL Directories (Not in original suite) +-- ------------------------------------------------------------------------------------------------- +-- What is the directory structure of the URL? +-- Illustrates a SPLIT_INDEX SQL. +-- ------------------------------------------------------------------------------------------------- + +CREATE TABLE nexmark_q22 ( + auction BIGINT, + bidder BIGINT, + price BIGINT, + channel VARCHAR, + dir1 VARCHAR, + dir2 VARCHAR, + dir3 VARCHAR +) WITH ( + ${SINK_DDL} +); + +INSERT INTO nexmark_q22 +SELECT + auction, bidder, price, channel, + SPLIT_INDEX(url, '/', 3) as dir1, + SPLIT_INDEX(url, '/', 4) as dir2, + SPLIT_INDEX(url, '/', 5) as dir3 FROM bid; diff --git a/nexmark/nexmark-flink/src/main/resources/queries/q23.sql b/nexmark/nexmark-flink/src/main/resources/queries/q23.sql new file mode 100644 index 0000000..239c1c9 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/queries/q23.sql @@ -0,0 +1,66 @@ +-- ------------------------------------------------------------------------------------------------- +-- Query 23: Expand bid with person and auction (Not in original suite) +-- ------------------------------------------------------------------------------------------------- +-- Find all bids made by a person who has also listed an item for auction +-- Illustrates a multi-way join. +-- ------------------------------------------------------------------------------------------------- + +CREATE TABLE nexmark_q23 +( + bidder BIGINT, + price BIGINT, + channel VARCHAR, + url VARCHAR, + bid_extra VARCHAR, + + person_id BIGINT, + name VARCHAR, + emailAddress VARCHAR, + creditCard VARCHAR, + city VARCHAR, + state VARCHAR, + person_extra VARCHAR, + + itemName VARCHAR, + description VARCHAR, + initialBid BIGINT, + reserve BIGINT, + auction_dateTime TIMESTAMP(3), + expires TIMESTAMP(3), + seller BIGINT, + category BIGINT, + auction_extra VARCHAR +) WITH ( + ${SINK_DDL} +); + +INSERT INTO nexmark_q23 +SELECT bidder, + price, + channel, + url, + B.extra AS bid_extra, + + P.id AS person_id, + name, + emailAddress, + creditCard, + city, + state, + P.extra AS person_extra, + itemName, + + description, + initialBid, + reserve, + A.dateTime AS auction_dateTime, + expires, + seller, + category, + A.extra AS auction_extra +FROM bid B + JOIN + person P ON P.id = B.bidder + JOIN + auction A ON A.seller = B.bidder; + diff --git a/nexmark/nexmark-flink/src/main/resources/queries/q3.sql b/nexmark/nexmark-flink/src/main/resources/queries/q3.sql new file mode 100644 index 0000000..16d5445 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/queries/q3.sql @@ -0,0 +1,23 @@ +-- ------------------------------------------------------------------------------------------------- +-- Query 3: Local Item Suggestion +-- ------------------------------------------------------------------------------------------------- +-- Who is selling in OR, ID or CA in category 10, and for what auction ids? +-- Illustrates an incremental join (using per-key state and timer) and filter. +-- ------------------------------------------------------------------------------------------------- + +CREATE TABLE nexmark_q3 ( + name VARCHAR, + city VARCHAR, + state VARCHAR, + id BIGINT +) WITH ( + ${SINK_DDL} +); + +INSERT INTO nexmark_q3 +SELECT + P.name, P.city, P.state, A.id +FROM + auction AS A INNER JOIN person AS P on A.seller = P.id +WHERE + A.category = 10 and (P.state = 'OR' OR P.state = 'ID' OR P.state = 'CA'); \ No newline at end of file diff --git a/nexmark/nexmark-flink/src/main/resources/queries/q4.sql b/nexmark/nexmark-flink/src/main/resources/queries/q4.sql new file mode 100644 index 0000000..f076edd --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/queries/q4.sql @@ -0,0 +1,25 @@ +-- ------------------------------------------------------------------------------------------------- +-- Query 4: Average Price for a Category +-- ------------------------------------------------------------------------------------------------- +-- Select the average of the wining bid prices for all auctions in each category. +-- Illustrates complex join and aggregation. +-- ------------------------------------------------------------------------------------------------- + +CREATE TABLE nexmark_q4 ( + id BIGINT, + final BIGINT +) WITH ( + ${SINK_DDL} +); + +INSERT INTO nexmark_q4 +SELECT + Q.category, + AVG(Q.final) +FROM ( + SELECT MAX(B.price) AS final, A.category + FROM auction A, bid B + WHERE A.id = B.auction AND B.`dateTime` BETWEEN A.`dateTime` AND A.expires + GROUP BY A.id, A.category +) Q +GROUP BY Q.category; \ No newline at end of file diff --git a/nexmark/nexmark-flink/src/main/resources/queries/q5.sql b/nexmark/nexmark-flink/src/main/resources/queries/q5.sql new file mode 100644 index 0000000..ce89d10 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/queries/q5.sql @@ -0,0 +1,49 @@ +-- ------------------------------------------------------------------------------------------------- +-- Query 5: Hot Items +-- ------------------------------------------------------------------------------------------------- +-- Which auctions have seen the most bids in the last period? +-- Illustrates sliding windows and combiners. +-- +-- The original Nexmark Query5 calculate the hot items in the last hour (updated every minute). +-- To make things a bit more dynamic and easier to test we use much shorter windows, +-- i.e. in the last 10 seconds and update every 2 seconds. +-- ------------------------------------------------------------------------------------------------- + +CREATE TABLE nexmark_q5 ( + auction BIGINT, + num BIGINT +) WITH ( + ${SINK_DDL} +); + +INSERT INTO nexmark_q5 +SELECT AuctionBids.auction, AuctionBids.num + FROM ( + SELECT + auction, + count(*) AS num, + window_start AS starttime, + window_end AS endtime + FROM TABLE( + HOP(TABLE bid, DESCRIPTOR(`dateTime`), INTERVAL '2' SECOND, INTERVAL '10' SECOND)) + GROUP BY auction, window_start, window_end + ) AS AuctionBids + JOIN ( + SELECT + max(CountBids.num) AS maxn, + CountBids.starttime, + CountBids.endtime + FROM ( + SELECT + count(*) AS num, + window_start AS starttime, + window_end AS endtime + FROM TABLE( + HOP(TABLE bid, DESCRIPTOR(`dateTime`), INTERVAL '2' SECOND, INTERVAL '10' SECOND)) + GROUP BY auction, window_start, window_end + ) AS CountBids + GROUP BY CountBids.starttime, CountBids.endtime + ) AS MaxBids + ON AuctionBids.starttime = MaxBids.starttime AND + AuctionBids.endtime = MaxBids.endtime AND + AuctionBids.num >= MaxBids.maxn; \ No newline at end of file diff --git a/nexmark/nexmark-flink/src/main/resources/queries/q6.sql b/nexmark/nexmark-flink/src/main/resources/queries/q6.sql new file mode 100644 index 0000000..9356d65 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/queries/q6.sql @@ -0,0 +1,30 @@ +-- ------------------------------------------------------------------------------------------------- +-- Query 6: Average Selling Price by Seller +-- ------------------------------------------------------------------------------------------------- +-- What is the average selling price per seller for their last 10 closed auctions. +-- Shares the same ‘winning bids’ core as for Query4, and illustrates a specialized combiner. +-- ------------------------------------------------------------------------------------------------- + +CREATE TABLE nexmark_q6 ( + seller VARCHAR, + avg_price BIGINT +) WITH ( + ${SINK_DDL} +); + +-- TODO: this query is not supported yet in Flink SQL, because the OVER WINDOW operator doesn't +-- support to consume retractions. +INSERT INTO nexmark_q6 +SELECT + Q.seller, + AVG(Q.price) OVER + (PARTITION BY Q.seller ORDER BY Q.`dateTime` ROWS BETWEEN 10 PRECEDING AND CURRENT ROW) +FROM ( + SELECT *, ROW_NUMBER() OVER (PARTITION BY A.id, A.seller ORDER BY B.price DESC) AS rownum + FROM (SELECT A.id, A.seller, B.price, B.`dateTime` + FROM auction AS A, + bid AS B + WHERE A.id = B.auction + and B.`dateTime` between A.`dateTime` and A.expires) + WHERE rownum <= 1 +) AS Q; \ No newline at end of file diff --git a/nexmark/nexmark-flink/src/main/resources/queries/q7.sql b/nexmark/nexmark-flink/src/main/resources/queries/q7.sql new file mode 100644 index 0000000..ab14af5 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/queries/q7.sql @@ -0,0 +1,31 @@ +-- ------------------------------------------------------------------------------------------------- +-- Query 7: Highest Bid +-- ------------------------------------------------------------------------------------------------- +-- What are the highest bids per period? +-- Deliberately implemented using a side input to illustrate fanout. +-- +-- The original Nexmark Query7 calculate the highest bids in the last minute. +-- We will use a shorter window (10 seconds) to help make testing easier. +-- ------------------------------------------------------------------------------------------------- + +CREATE TABLE nexmark_q7 ( + auction BIGINT, + bidder BIGINT, + price BIGINT, + `dateTime` TIMESTAMP(3), + extra VARCHAR +) WITH ( + ${SINK_DDL} +); + +INSERT INTO nexmark_q7 +SELECT B.auction, B.price, B.bidder, B.`dateTime`, B.extra +from bid B +JOIN ( + SELECT MAX(price) AS maxprice, window_end as `dateTime` + FROM TABLE( + TUMBLE(TABLE bid, DESCRIPTOR(`dateTime`), INTERVAL '10' SECOND)) + GROUP BY window_start, window_end +) B1 +ON B.price = B1.maxprice +WHERE B.`dateTime` BETWEEN B1.`dateTime` - INTERVAL '10' SECOND AND B1.`dateTime`; \ No newline at end of file diff --git a/nexmark/nexmark-flink/src/main/resources/queries/q8.sql b/nexmark/nexmark-flink/src/main/resources/queries/q8.sql new file mode 100644 index 0000000..7ad2464 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/queries/q8.sql @@ -0,0 +1,37 @@ +-- ------------------------------------------------------------------------------------------------- +-- Query 8: Monitor New Users +-- ------------------------------------------------------------------------------------------------- +-- Select people who have entered the system and created auctions in the last period. +-- Illustrates a simple join. +-- +-- The original Nexmark Query8 monitors the new users the last 12 hours, updated every 12 hours. +-- To make things a bit more dynamic and easier to test we use much shorter windows (10 seconds). +-- ------------------------------------------------------------------------------------------------- + +CREATE TABLE nexmark_q8 ( + id BIGINT, + name VARCHAR, + stime TIMESTAMP(3) +) WITH ( + ${SINK_DDL} +); + +INSERT INTO nexmark_q8 +SELECT P.id, P.name, P.starttime +FROM ( + SELECT id, name, + window_start AS starttime, + window_end AS endtime + FROM TABLE( + TUMBLE(TABLE person, DESCRIPTOR(`dateTime`), INTERVAL '10' SECOND)) + GROUP BY id, name, window_start, window_end +) P +JOIN ( + SELECT seller, + window_start AS starttime, + window_end AS endtime + FROM TABLE( + TUMBLE(TABLE auction, DESCRIPTOR(`dateTime`), INTERVAL '10' SECOND)) + GROUP BY seller, window_start, window_end +) A +ON P.id = A.seller AND P.starttime = A.starttime AND P.endtime = A.endtime; \ No newline at end of file diff --git a/nexmark/nexmark-flink/src/main/resources/queries/q9.sql b/nexmark/nexmark-flink/src/main/resources/queries/q9.sql new file mode 100644 index 0000000..32d5e25 --- /dev/null +++ b/nexmark/nexmark-flink/src/main/resources/queries/q9.sql @@ -0,0 +1,37 @@ +-- ------------------------------------------------------------------------------------------------- +-- Query 9: Winning Bids (Not in original suite) +-- ------------------------------------------------------------------------------------------------- +-- Find the winning bid for each auction. +-- ------------------------------------------------------------------------------------------------- + +CREATE TABLE nexmark_q9 ( + id BIGINT, + itemName VARCHAR, + description VARCHAR, + initialBid BIGINT, + reserve BIGINT, + `dateTime` TIMESTAMP(3), + expires TIMESTAMP(3), + seller BIGINT, + category BIGINT, + extra VARCHAR, + auction BIGINT, + bidder BIGINT, + price BIGINT, + bid_dateTime TIMESTAMP(3), + bid_extra VARCHAR +) WITH ( + ${SINK_DDL} +); + +INSERT INTO nexmark_q9 +SELECT + id, itemName, description, initialBid, reserve, `dateTime`, expires, seller, category, extra, + auction, bidder, price, bid_dateTime, bid_extra +FROM ( + SELECT A.*, B.auction, B.bidder, B.price, B.`dateTime` AS bid_dateTime, B.extra AS bid_extra, + ROW_NUMBER() OVER (PARTITION BY A.id ORDER BY B.price DESC, B.`dateTime` ASC) AS rownum + FROM auction A, bid B + WHERE A.id = B.auction AND B.`dateTime` BETWEEN A.`dateTime` AND A.expires +) +WHERE rownum <= 1; \ No newline at end of file diff --git a/nexmark/nexmark-flink/src/test/java/com/github/nexmark/flink/BenchmarkTest.java b/nexmark/nexmark-flink/src/test/java/com/github/nexmark/flink/BenchmarkTest.java new file mode 100644 index 0000000..484bd30 --- /dev/null +++ b/nexmark/nexmark-flink/src/test/java/com/github/nexmark/flink/BenchmarkTest.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink; + +import com.github.nexmark.flink.metric.JobBenchmarkMetric; +import org.junit.Ignore; +import org.junit.Test; + +import java.util.LinkedHashMap; + +@Ignore +public class BenchmarkTest { + + @Test + public void testPrintSummary() { + LinkedHashMap totalMetrics = new LinkedHashMap<>(); + totalMetrics.put("q0", new JobBenchmarkMetric(1, 8.4)); + totalMetrics.put("q1", new JobBenchmarkMetric(10, 18.4)); + totalMetrics.put("q2", new JobBenchmarkMetric(100.0, 8.4)); + totalMetrics.put("q3", new JobBenchmarkMetric(1000.0, 8.4)); + totalMetrics.put("q4", new JobBenchmarkMetric(10_000.0, 8.4)); + totalMetrics.put("q5", new JobBenchmarkMetric(100_000.0, 8.4)); + totalMetrics.put("q6", new JobBenchmarkMetric(1_000_000.0, 8.4)); + totalMetrics.put("q7", new JobBenchmarkMetric(10_000_000.0, 8.23)); + totalMetrics.put("q8", new JobBenchmarkMetric(100_000_000.0, 8.4)); + totalMetrics.put("q9", new JobBenchmarkMetric(1_000_000_000.0, 8.4)); + totalMetrics.put("q10", new JobBenchmarkMetric(10_000_000_000.0, 8.4)); + totalMetrics.put("q11", new JobBenchmarkMetric(100_000_000_000.0, 8.419)); + totalMetrics.put("q12", new JobBenchmarkMetric(100_000_000.0, 8.4)); + totalMetrics.put("q13", new JobBenchmarkMetric(10_000_000.0, 8.4)); + totalMetrics.put("q14", new JobBenchmarkMetric(1_000_000.0, 8.4)); + Benchmark.printSummary(totalMetrics); + } + +} \ No newline at end of file diff --git a/nexmark/nexmark-flink/src/test/java/com/github/nexmark/flink/generator/NexmarkGeneratorTest.java b/nexmark/nexmark-flink/src/test/java/com/github/nexmark/flink/generator/NexmarkGeneratorTest.java new file mode 100644 index 0000000..2aafb6f --- /dev/null +++ b/nexmark/nexmark-flink/src/test/java/com/github/nexmark/flink/generator/NexmarkGeneratorTest.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink.generator; + +import com.github.nexmark.flink.NexmarkConfiguration; +import com.github.nexmark.flink.model.Event; +import org.junit.Test; + +public class NexmarkGeneratorTest { + + @Test + public void testGenerate() { + NexmarkConfiguration nexmarkConfiguration = new NexmarkConfiguration(); + nexmarkConfiguration.bidProportion = 46; + GeneratorConfig generatorConfig = new GeneratorConfig( + nexmarkConfiguration, System.currentTimeMillis(), 1, 100, 0L, 1); + NexmarkGenerator generator = new NexmarkGenerator(generatorConfig); + int count = 0; + while (generator.hasNext()) { + Event event = generator.next().event; + count ++; + System.out.println(event); + } + System.out.println("Total event:" + count); + } +} diff --git a/nexmark/nexmark-flink/src/test/java/com/github/nexmark/flink/metric/BenchmarkMetricTest.java b/nexmark/nexmark-flink/src/test/java/com/github/nexmark/flink/metric/BenchmarkMetricTest.java new file mode 100644 index 0000000..59db148 --- /dev/null +++ b/nexmark/nexmark-flink/src/test/java/com/github/nexmark/flink/metric/BenchmarkMetricTest.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink.metric; + +import org.junit.Assert; +import org.junit.Test; + +import static com.github.nexmark.flink.metric.BenchmarkMetric.formatLongValue; +import static org.junit.Assert.assertEquals; + +public class BenchmarkMetricTest { + + @Test + public void testFormatLongValue() { + Assert.assertEquals("1.64 M", formatLongValue(1_636_000)); + Assert.assertEquals("1.6 M", formatLongValue(1_600_000)); + Assert.assertEquals("232", formatLongValue(232)); + Assert.assertEquals("23.21 K", formatLongValue(23213)); + } + + @Test + public void testBenchmarkMetricWithAllFields() { + long timestamp = 1706284800000L; + double tps = 150000.0; + double cpu = 8.5; + long rss = 1073741824L; // 1GB + long vmem = 2147483648L; // 2GB + long netRead = 52428800L; // 50MB + long netWrite = 10485760L; // 10MB + long diskRead = 104857600L; // 100MB + long diskWrite = 52428800L; // 50MB + + BenchmarkMetric metric = new BenchmarkMetric( + timestamp, tps, cpu, rss, vmem, netRead, netWrite, diskRead, diskWrite); + + assertEquals(timestamp, metric.getTimestamp()); + assertEquals(tps, metric.getTps(), 0.001); + assertEquals(cpu, metric.getCpu(), 0.001); + assertEquals(rss, metric.getRss()); + assertEquals(vmem, metric.getVmem()); + assertEquals(netRead, metric.getNetBytesRead()); + assertEquals(netWrite, metric.getNetBytesWritten()); + assertEquals(diskRead, metric.getDiskBytesRead()); + assertEquals(diskWrite, metric.getDiskBytesWritten()); + + // Test pretty formatters + assertEquals("150 K", metric.getPrettyTps()); + assertEquals("1.07 G", metric.getPrettyRss()); + assertEquals("2.15 G", metric.getPrettyVmem()); + } + + @Test + public void testBenchmarkMetricWithDefaultMemoryAndIo() { + // Test creating metric with zeros for memory and I/O fields + long timestamp = System.currentTimeMillis(); + BenchmarkMetric metric = new BenchmarkMetric(timestamp, 150000.0, 8.5, 0, 0, 0, 0, 0, 0); + + assertEquals(150000.0, metric.getTps(), 0.001); + assertEquals(8.5, metric.getCpu(), 0.001); + assertEquals(0L, metric.getRss()); + assertEquals(0L, metric.getVmem()); + assertEquals(0L, metric.getNetBytesRead()); + assertEquals(0L, metric.getNetBytesWritten()); + assertEquals(0L, metric.getDiskBytesRead()); + assertEquals(0L, metric.getDiskBytesWritten()); + assertEquals(timestamp, metric.getTimestamp()); + } + + @Test + public void testBenchmarkMetricEquality() { + long timestamp = 1706284800000L; + BenchmarkMetric metric1 = new BenchmarkMetric( + timestamp, 150000.0, 8.5, 1073741824L, 2147483648L, + 52428800L, 10485760L, 104857600L, 52428800L); + BenchmarkMetric metric2 = new BenchmarkMetric( + timestamp, 150000.0, 8.5, 1073741824L, 2147483648L, + 52428800L, 10485760L, 104857600L, 52428800L); + + assertEquals(metric1, metric2); + assertEquals(metric1.hashCode(), metric2.hashCode()); + } +} diff --git a/nexmark/nexmark-flink/src/test/java/com/github/nexmark/flink/metric/FlinkRestClientTest.java b/nexmark/nexmark-flink/src/test/java/com/github/nexmark/flink/metric/FlinkRestClientTest.java new file mode 100644 index 0000000..f729274 --- /dev/null +++ b/nexmark/nexmark-flink/src/test/java/com/github/nexmark/flink/metric/FlinkRestClientTest.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink.metric; + +import com.github.nexmark.flink.metric.tps.TpsMetric; +import org.junit.Ignore; +import org.junit.Test; + +@Ignore +public class FlinkRestClientTest { + + @Test + public void testMetricsClient() { + FlinkRestClient client = new FlinkRestClient("localhost", 8081); + String jobId = client.getCurrentJobId(); + System.out.println("jobId: " + jobId); + + String vertexId = client.getSourceVertexId(jobId); + System.out.println("vertexId: " + vertexId); + + String metricName = client.getTpsMetricName(jobId, vertexId); + System.out.println("metricName: " + metricName); + + TpsMetric tps = client.getTpsMetric(jobId, vertexId, metricName); + System.out.println("tps: " + tps); + } + + @Test + public void testCancelJob() { + FlinkRestClient client = new FlinkRestClient("localhost", 8081); + String jobId = client.getCurrentJobId(); + System.out.println("jobId: " + jobId); + client.cancelJob(jobId); + } + +} diff --git a/nexmark/nexmark-flink/src/test/java/com/github/nexmark/flink/metric/ProcessMetricReceiverTest.java b/nexmark/nexmark-flink/src/test/java/com/github/nexmark/flink/metric/ProcessMetricReceiverTest.java new file mode 100644 index 0000000..231e50c --- /dev/null +++ b/nexmark/nexmark-flink/src/test/java/com/github/nexmark/flink/metric/ProcessMetricReceiverTest.java @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink.metric; + +import com.github.nexmark.flink.metric.process.ProcessMetric; +import com.github.nexmark.flink.metric.process.ProcessMetricReceiver; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.util.concurrent.ConcurrentHashMap; + +import static org.junit.Assert.assertEquals; + +/** + * Tests for {@link ProcessMetricReceiver} aggregation methods. + */ +public class ProcessMetricReceiverTest { + + private ProcessMetricReceiver receiver; + private ConcurrentHashMap metricsMap; + + @Before + public void setUp() throws Exception { + // Use an ephemeral port to avoid conflicts in parallel/CI test runs + receiver = new ProcessMetricReceiver("localhost", 0); + + // Access the private processMetrics map via reflection for testing + Field metricsField = ProcessMetricReceiver.class.getDeclaredField("processMetrics"); + metricsField.setAccessible(true); + @SuppressWarnings("unchecked") + ConcurrentHashMap map = + (ConcurrentHashMap) metricsField.get(receiver); + metricsMap = map; + } + + @After + public void tearDown() { + if (receiver != null) { + receiver.close(); + } + } + + /** Test convenience method to create a ProcessMetric with only CPU data. */ + private static ProcessMetric createProcessMetric(String host, int pid, double cpu) { + return new ProcessMetric(host, pid, cpu, 0, 0, 0, 0, 0, 0); + } + + @Test + public void testGetTotalCpu() { + // Add metrics from two TaskManagers on the same host + metricsMap.put("host1:1001", createProcessMetric("host1", 1001, 2.5)); + metricsMap.put("host1:1002", createProcessMetric("host1", 1002, 3.5)); + + assertEquals(6.0, receiver.getTotalCpu(), 0.001); + assertEquals(2, receiver.getNumberOfTM()); + } + + @Test + public void testGetTotalRssAndVmem() { + // Add metrics with memory info + metricsMap.put("host1:1001", new ProcessMetric( + "host1", 1001, 2.5, + 1073741824L, 2147483648L, // 1GB RSS, 2GB VMEM + 0, 0, 0, 0)); + metricsMap.put("host1:1002", new ProcessMetric( + "host1", 1002, 3.5, + 2147483648L, 4294967296L, // 2GB RSS, 4GB VMEM + 0, 0, 0, 0)); + + // RSS and VMEM should sum across all TMs + assertEquals(3221225472L, receiver.getTotalRss()); // 1GB + 2GB = 3GB + assertEquals(6442450944L, receiver.getTotalVmem()); // 2GB + 4GB = 6GB + } + + @Test + public void testGetTotalNetworkIoDeduplication() { + // Network I/O is system-wide, so should be deduplicated by host + // Two TMs on the same host should only count the host's I/O once + metricsMap.put("host1:1001", new ProcessMetric( + "host1", 1001, 2.5, + 1073741824L, 2147483648L, + 52428800L, 10485760L, // 50MB read, 10MB write + 0, 0)); + metricsMap.put("host1:1002", new ProcessMetric( + "host1", 1002, 3.5, + 2147483648L, 4294967296L, + 52428800L, 10485760L, // Same values (system-wide) + 0, 0)); + + // Should only count once since both TMs are on same host + assertEquals(52428800L, receiver.getTotalNetBytesRead()); + assertEquals(10485760L, receiver.getTotalNetBytesWritten()); + } + + @Test + public void testGetTotalNetworkIoMultipleHosts() { + // Network I/O from different hosts should sum + metricsMap.put("host1:1001", new ProcessMetric( + "host1", 1001, 2.5, + 1073741824L, 2147483648L, + 52428800L, 10485760L, // 50MB read, 10MB write + 0, 0)); + metricsMap.put("host2:2001", new ProcessMetric( + "host2", 2001, 3.5, + 2147483648L, 4294967296L, + 104857600L, 20971520L, // 100MB read, 20MB write + 0, 0)); + + // Should sum across different hosts + assertEquals(157286400L, receiver.getTotalNetBytesRead()); // 50MB + 100MB + assertEquals(31457280L, receiver.getTotalNetBytesWritten()); // 10MB + 20MB + } + + @Test + public void testGetTotalDiskIoDeduplication() { + // Disk I/O is system-wide, so should be deduplicated by host + metricsMap.put("host1:1001", new ProcessMetric( + "host1", 1001, 2.5, + 1073741824L, 2147483648L, + 0, 0, + 104857600L, 52428800L)); // 100MB read, 50MB write + metricsMap.put("host1:1002", new ProcessMetric( + "host1", 1002, 3.5, + 2147483648L, 4294967296L, + 0, 0, + 104857600L, 52428800L)); // Same values (system-wide) + + // Should only count once since both TMs are on same host + assertEquals(104857600L, receiver.getTotalDiskBytesRead()); + assertEquals(52428800L, receiver.getTotalDiskBytesWritten()); + } + + @Test + public void testGetTotalDiskIoMultipleHosts() { + // Disk I/O from different hosts should sum + metricsMap.put("host1:1001", new ProcessMetric( + "host1", 1001, 2.5, + 1073741824L, 2147483648L, + 0, 0, + 104857600L, 52428800L)); // 100MB read, 50MB write + metricsMap.put("host2:2001", new ProcessMetric( + "host2", 2001, 3.5, + 2147483648L, 4294967296L, + 0, 0, + 209715200L, 104857600L)); // 200MB read, 100MB write + + // Should sum across different hosts + assertEquals(314572800L, receiver.getTotalDiskBytesRead()); // 100MB + 200MB + assertEquals(157286400L, receiver.getTotalDiskBytesWritten()); // 50MB + 100MB + } + + @Test + public void testEmptyMetrics() { + // All aggregation methods should handle empty map gracefully + assertEquals(0.0, receiver.getTotalCpu(), 0.001); + assertEquals(0, receiver.getNumberOfTM()); + assertEquals(0L, receiver.getTotalRss()); + assertEquals(0L, receiver.getTotalVmem()); + assertEquals(0L, receiver.getTotalNetBytesRead()); + assertEquals(0L, receiver.getTotalNetBytesWritten()); + assertEquals(0L, receiver.getTotalDiskBytesRead()); + assertEquals(0L, receiver.getTotalDiskBytesWritten()); + } +} diff --git a/nexmark/nexmark-flink/src/test/java/com/github/nexmark/flink/metric/ProcessMetricSenderTest.java b/nexmark/nexmark-flink/src/test/java/com/github/nexmark/flink/metric/ProcessMetricSenderTest.java new file mode 100644 index 0000000..66f6fa6 --- /dev/null +++ b/nexmark/nexmark-flink/src/test/java/com/github/nexmark/flink/metric/ProcessMetricSenderTest.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink.metric; + +import com.github.nexmark.flink.metric.process.ProcessMetricSender; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.IOException; + +@Ignore +public class ProcessMetricSenderTest { + + @Test + public void testGetTaskManagerPid() throws IOException { + System.out.println(ProcessMetricSender.getTaskManagerPidList()); + } +} diff --git a/nexmark/nexmark-flink/src/test/java/com/github/nexmark/flink/metric/ProcessMetricTest.java b/nexmark/nexmark-flink/src/test/java/com/github/nexmark/flink/metric/ProcessMetricTest.java new file mode 100644 index 0000000..e1ffdd6 --- /dev/null +++ b/nexmark/nexmark-flink/src/test/java/com/github/nexmark/flink/metric/ProcessMetricTest.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink.metric; + +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonProcessingException; + +import com.github.nexmark.flink.metric.process.ProcessMetric; +import com.github.nexmark.flink.utils.NexmarkUtils; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +public class ProcessMetricTest { + + @Test + public void testProcessMetric() throws JsonProcessingException { + List processMetrics = new ArrayList<>(); + processMetrics.add(new ProcessMetric("10.0.0.12", 37927, 1.01, 0, 0, 0, 0, 0, 0)); + processMetrics.add(new ProcessMetric("10.1.0.33", 54389, 2.3, 0, 0, 0, 0, 0, 0)); + processMetrics.add(new ProcessMetric("10.2.0.44", 4401, 0.4, 0, 0, 0, 0, 0, 0)); + String result = NexmarkUtils.MAPPER.writeValueAsString(processMetrics); + + List expected = ProcessMetric.fromJsonArray(result); + assertEquals(expected, processMetrics); + } +} diff --git a/nexmark/nexmark-flink/src/test/java/com/github/nexmark/flink/metric/TpsMetricTest.java b/nexmark/nexmark-flink/src/test/java/com/github/nexmark/flink/metric/TpsMetricTest.java new file mode 100644 index 0000000..7e79548 --- /dev/null +++ b/nexmark/nexmark-flink/src/test/java/com/github/nexmark/flink/metric/TpsMetricTest.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink.metric; + +import com.github.nexmark.flink.metric.tps.TpsMetric; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + + +public class TpsMetricTest { + + @Test + public void testParseJson() { + String json = "[\n" + + "{\n" + + "\"id\": \"Source__TableSourceScan(table=[[default_catalog__default_database__nexmark]]__fi.numRecordsOutPerSecond\",\n" + + "\"min\": 5003.2,\n" + + "\"max\": 5003.2,\n" + + "\"avg\": 5003.2,\n" + + "\"sum\": 10006.3\n" + + "}\n" + + "]"; + + TpsMetric tps = TpsMetric.fromJson(json); + TpsMetric expected = new TpsMetric( + "Source__TableSourceScan(table=[[default_catalog__default_database__nexmark]]__fi.numRecordsOutPerSecond", + 5003.2, + 5003.2, + 5003.2, + 10006.3); + assertEquals(expected, tps); + } +} diff --git a/nexmark/nexmark-flink/src/test/java/com/github/nexmark/flink/source/NexmarkSourceFunctionITCase.java b/nexmark/nexmark-flink/src/test/java/com/github/nexmark/flink/source/NexmarkSourceFunctionITCase.java new file mode 100644 index 0000000..21705f5 --- /dev/null +++ b/nexmark/nexmark-flink/src/test/java/com/github/nexmark/flink/source/NexmarkSourceFunctionITCase.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink.source; + +import com.github.nexmark.flink.NexmarkConfiguration; +import com.github.nexmark.flink.generator.GeneratorConfig; +import org.apache.flink.api.common.eventtime.WatermarkStrategy; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.runtime.connector.source.ScanRuntimeProviderContext; +import org.junit.Test; + +import static com.github.nexmark.flink.source.NexmarkTableSource.RESOLVED_SCHEMA; + +public class NexmarkSourceFunctionITCase { + + @Test + @SuppressWarnings("unchecked") + public void testDataStreamSource() throws Exception { + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setParallelism(4); + NexmarkConfiguration nexmarkConfiguration = new NexmarkConfiguration(); + nexmarkConfiguration.bidProportion = 46; + GeneratorConfig generatorConfig = new GeneratorConfig( + nexmarkConfiguration, System.currentTimeMillis(), 1, 100, 0L, 1); + TypeInformation typeInformation = + (TypeInformation) ScanRuntimeProviderContext.INSTANCE + .createTypeInformation(RESOLVED_SCHEMA.toPhysicalRowDataType()); + env.fromSource(new NexmarkSource(generatorConfig, typeInformation), + WatermarkStrategy.noWatermarks(), + "Source") + .print(); + + env.execute(); + } +} diff --git a/nexmark/nexmark-flink/src/test/java/com/github/nexmark/flink/source/NexmarkTableSourceFactoryTest.java b/nexmark/nexmark-flink/src/test/java/com/github/nexmark/flink/source/NexmarkTableSourceFactoryTest.java new file mode 100644 index 0000000..5826b2e --- /dev/null +++ b/nexmark/nexmark-flink/src/test/java/com/github/nexmark/flink/source/NexmarkTableSourceFactoryTest.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink.source; + +import org.apache.flink.configuration.Configuration; +import org.apache.flink.table.catalog.CatalogTable; +import org.apache.flink.table.catalog.ObjectIdentifier; +import org.apache.flink.table.catalog.ResolvedCatalogTable; +import org.apache.flink.table.catalog.ResolvedSchema; +import org.apache.flink.table.connector.source.DynamicTableSource; +import org.apache.flink.table.factories.FactoryUtil; + +import com.github.nexmark.flink.NexmarkConfiguration; +import com.github.nexmark.flink.generator.GeneratorConfig; +import com.github.nexmark.flink.utils.NexmarkUtils; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static com.github.nexmark.flink.source.NexmarkTableSource.NEXMARK_SCHEMA; +import static com.github.nexmark.flink.source.NexmarkTableSource.RESOLVED_SCHEMA; +import static org.junit.Assert.assertEquals; + +/** + * Test for {@link NexmarkTableSource} created by {@link NexmarkTableSourceFactory}. + */ +public class NexmarkTableSourceFactoryTest { + + @Test + public void testCommonProperties() { + Map properties = getAllOptions(); + + // validation for source + DynamicTableSource actualSource = createTableSource(properties); + GeneratorConfig config = new GeneratorConfig( + new NexmarkConfiguration(), + System.currentTimeMillis(), + 1, + 0, + 0L, + 1 + ); + NexmarkTableSource expectedSource = new NexmarkTableSource(config); + assertEquals(expectedSource, actualSource); + } + + @Test + public void testCustomProperties() { + Map properties = getAllOptions(); + properties.put("rate.shape", "SQUARE"); + properties.put("rate.period", "11 min"); + properties.put("rate.limited", "true"); + properties.put("first-event.rate", "99"); + properties.put("next-event.rate", "199"); + properties.put("person.avg-size", "1kb"); + properties.put("auction.avg-size", "5kb"); + properties.put("bid.avg-size", "8kb"); + properties.put("person.proportion", "30"); + properties.put("auction.proportion", "15"); + properties.put("bid.proportion", "5"); + properties.put("bid.hot-ratio.auctions", "3"); + properties.put("bid.hot-ratio.bidders", "5"); + properties.put("auction.hot-ratio.sellers", "8"); + properties.put("events.num", "100"); + + DynamicTableSource actualSource = createTableSource(properties); + NexmarkConfiguration nexmarkConf = new NexmarkConfiguration(); + nexmarkConf.rateShape = NexmarkUtils.RateShape.SQUARE; + nexmarkConf.ratePeriodSec = 11 * 60; + nexmarkConf.isRateLimited = true; + nexmarkConf.firstEventRate = 99; + nexmarkConf.nextEventRate = 199; + nexmarkConf.avgPersonByteSize = 1024; + nexmarkConf.avgAuctionByteSize = 5 * 1024; + nexmarkConf.avgBidByteSize = 8 * 1024; + nexmarkConf.personProportion = 30; + nexmarkConf.auctionProportion = 15; + nexmarkConf.bidProportion = 5; + nexmarkConf.hotAuctionRatio = 3; + nexmarkConf.hotBiddersRatio = 5; + nexmarkConf.hotSellersRatio = 8; + nexmarkConf.numEvents = 100; + + GeneratorConfig config = new GeneratorConfig( + nexmarkConf, + System.currentTimeMillis(), + 1, + nexmarkConf.numEvents, + 0L, + 1 + ); + NexmarkTableSource expectedSource = new NexmarkTableSource(config); + assertEquals(expectedSource, actualSource); + } + + private Map getAllOptions() { + Map options = new HashMap<>(); + options.put("connector", "nexmark"); + return options; + } + + private static DynamicTableSource createTableSource(Map options) { + return FactoryUtil.createDynamicTableSource( + null, + ObjectIdentifier.of("default", "default", "t1"), + new ResolvedCatalogTable( + CatalogTable.newBuilder() + .schema(NEXMARK_SCHEMA) + .comment("mock source") + .partitionKeys(new ArrayList<>()) + .options(options) + .build(), + RESOLVED_SCHEMA), + Collections.emptyMap(), + new Configuration(), + NexmarkTableSourceFactoryTest.class.getClassLoader(), + false); + } +} diff --git a/nexmark/nexmark-flink/src/test/java/com/github/nexmark/flink/source/NexmarkTableSourceITCase.java b/nexmark/nexmark-flink/src/test/java/com/github/nexmark/flink/source/NexmarkTableSourceITCase.java new file mode 100644 index 0000000..4a9cea6 --- /dev/null +++ b/nexmark/nexmark-flink/src/test/java/com/github/nexmark/flink/source/NexmarkTableSourceITCase.java @@ -0,0 +1,335 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink.source; + +import org.apache.flink.streaming.api.CheckpointingMode; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.table.api.TableResult; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; +import org.apache.flink.types.Row; +import org.apache.flink.util.CloseableIterator; + +import org.junit.Before; +import org.junit.Test; + +public class NexmarkTableSourceITCase { + + private StreamTableEnvironment tEnv; + + @Before + public void before() { + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + tEnv = StreamTableEnvironment.create(env); + env.setParallelism(4); + env.getCheckpointConfig().setCheckpointingMode(CheckpointingMode.EXACTLY_ONCE); + env.getCheckpointConfig().setCheckpointInterval(1000L); + + tEnv.executeSql("CREATE TABLE nexmark (\n" + + " event_type int,\n" + + " person ROW<\n" + + " id BIGINT,\n" + + " name VARCHAR,\n" + + " emailAddress VARCHAR,\n" + + " creditCard VARCHAR,\n" + + " city VARCHAR,\n" + + " state VARCHAR,\n" + + " `dateTime` TIMESTAMP(3),\n" + + " extra VARCHAR>,\n" + + " auction ROW<\n" + + " id BIGINT,\n" + + " itemName VARCHAR,\n" + + " description VARCHAR,\n" + + " initialBid BIGINT,\n" + + " reserve BIGINT,\n" + + " `dateTime` TIMESTAMP(3),\n" + + " expires TIMESTAMP(3),\n" + + " seller BIGINT,\n" + + " category BIGINT,\n" + + " extra VARCHAR>,\n" + + " bid ROW<\n" + + " auction BIGINT,\n" + + " bidder BIGINT,\n" + + " price BIGINT,\n" + + " channel VARCHAR,\n" + + " url VARCHAR,\n" + + " `dateTime` TIMESTAMP(3),\n" + + " extra VARCHAR>,\n" + + " `dateTime` AS\n" + + " CASE\n" + + " WHEN event_type = 0 THEN person.`dateTime`\n" + + " WHEN event_type = 1 THEN auction.`dateTime`\n" + + " ELSE bid.`dateTime`\n" + + " END,\n" + + " WATERMARK FOR `dateTime` AS `dateTime` - INTERVAL '4' SECOND" + + ") WITH (\n" + + " 'connector' = 'nexmark',\n" + + " 'events.num' = '500'\n" + + ")"); + tEnv.executeSql("CREATE VIEW person AS\n" + + "SELECT person.id,\n" + + " person.name,\n" + + " person.emailAddress,\n" + + " person.creditCard,\n" + + " person.city,\n" + + " person.state,\n" + + " `dateTime`,\n" + + " person.extra FROM nexmark AS t WHERE event_type = 0"); + tEnv.executeSql("CREATE VIEW auction AS\n" + + "SELECT auction.id,\n" + + " auction.itemName,\n" + + " auction.description,\n" + + " auction.initialBid,\n" + + " auction.reserve,\n" + + " `dateTime`,\n" + + " auction.expires,\n" + + " auction.seller,\n" + + " auction.category,\n" + + " auction.extra FROM nexmark AS t WHERE event_type = 1"); + tEnv.executeSql("CREATE VIEW bid AS\n" + + "SELECT bid.auction,\n" + + " bid.bidder,\n" + + " bid.price,\n" + + " bid.channel,\n" + + " bid.url,\n" + + " `dateTime`,\n" + + " bid.extra FROM nexmark AS t WHERE event_type = 2"); + } + + @Test + public void testAllEvents() { + print(tEnv.executeSql("SELECT * FROM nexmark")); + } + + @Test + public void testPersonSource() { + print(tEnv.executeSql("SELECT * FROM person")); + } + + @Test + public void testAuctionSource() { + print(tEnv.executeSql("SELECT * FROM auction")); + } + + @Test + public void testBidSource() { + print(tEnv.executeSql("SELECT * FROM bid")); + } + + @Test + public void q16() { + print(tEnv.executeSql("SELECT\n" + + " channel,\n" + + " DATE_FORMAT(`dateTime`, 'yyyy-MM-dd') as `day`,\n" + + " max(DATE_FORMAT(`dateTime`, 'HH:mm')) as `minute`,\n" + + " count(*) AS total_bids,\n" + + " count(*) filter (where price < 10000) AS rank1_bids,\n" + + " count(*) filter (where price >= 10000 and price < 1000000) AS rank2_bids,\n" + + " count(*) filter (where price >= 1000000) AS rank3_bids,\n" + + " count(distinct bidder) AS total_bidders,\n" + + " count(distinct bidder) filter (where price < 10000) AS rank1_bidders,\n" + + " count(distinct bidder) filter (where price >= 10000 and price < 1000000) AS rank2_bidders,\n" + + " count(distinct bidder) filter (where price >= 1000000) AS rank3_bidders,\n" + + " count(distinct auction) AS total_auctions,\n" + + " count(distinct auction) filter (where price < 10000) AS rank1_auctions,\n" + + " count(distinct auction) filter (where price >= 10000 and price < 1000000) AS rank2_auctions,\n" + + " count(distinct auction) filter (where price >= 1000000) AS rank3_auctions\n" + + "FROM bid\n" + + "GROUP BY channel, DATE_FORMAT(`dateTime`, 'yyyy-MM-dd')")); + } + + @Test + public void q17() { + print(tEnv.executeSql("SELECT\n" + + " auction,\n" + + " DATE_FORMAT(`dateTime`, 'yyyy-MM-dd') as `day`,\n" + + " count(*) AS total_bids,\n" + + " count(*) filter (where price < 10000) AS rank1_bids,\n" + + " count(*) filter (where price >= 10000 and price < 1000000) AS rank2_bids,\n" + + " count(*) filter (where price >= 1000000) AS rank3_bids,\n" + + " min(price) AS min_price,\n" + + " max(price) AS max_price,\n" + + " avg(price) AS avg_price,\n" + + " sum(price) AS sum_price\n" + + "FROM bid\n" + + "GROUP BY auction, DATE_FORMAT(`dateTime`, 'yyyy-MM-dd')")); + } + + @Test + public void q18() { + print(tEnv.executeSql("SELECT auction, bidder, price, channel, url, `dateTime`, extra\n" + + " FROM (SELECT *, ROW_NUMBER() OVER (PARTITION BY bidder, auction ORDER BY `dateTime` DESC) AS rank_number\n" + + " FROM bid)\n" + + " WHERE rank_number <= 1")); + } + + @Test + public void q19() { + print(tEnv.executeSql("SELECT * FROM\n" + + "(SELECT *, ROW_NUMBER() OVER (PARTITION BY auction ORDER BY price DESC) AS rank_number FROM bid)\n" + + "WHERE rank_number <= 10")); + } + + @Test + public void q20() { + removeRowTime(); + print(tEnv.executeSql("SELECT\n" + + " auction, bidder, price, channel, url, B.`dateTime`, B.extra,\n" + + " itemName, description, initialBid, reserve, A.`dateTime`, expires, seller, category, A.extra\n" + + "FROM\n" + + " bid AS B INNER JOIN auction AS A on B.auction = A.id\n" + + "WHERE A.category = 10")); + } + + @Test + public void q21() { + print(tEnv.executeSql("SELECT\n" + + " auction, bidder, price, channel,\n" + + " CASE\n" + + " WHEN lower(channel) = 'apple' THEN '0'\n" + + " WHEN lower(channel) = 'google' THEN '1'\n" + + " WHEN lower(channel) = 'facebook' THEN '2'\n" + + " WHEN lower(channel) = 'baidu' THEN '3'\n" + + " ELSE REGEXP_EXTRACT(url, '(&|^)channel_id=([^&]*)', 2)\n" + + " END\n" + + " AS channel_id FROM bid\n" + + " where REGEXP_EXTRACT(url, '(&|^)channel_id=([^&]*)', 2) is not null or\n" + + " lower(channel) in ('apple', 'google', 'facebook', 'baidu')")); + } + + @Test + public void q22() { + print(tEnv.executeSql("SELECT\n" + + " auction, bidder, price, channel,\n" + + " SPLIT_INDEX(url, '/', 3) as dir1,\n" + + " SPLIT_INDEX(url, '/', 4) as dir2,\n" + + " SPLIT_INDEX(url, '/', 5) as dir3 FROM bid")); + } + + @Test + public void cepQ0() { + print( + tEnv.executeSql( + "SELECT\n" + + " auction, bidder, start_tstamp, bottom_tstamp, end_tstamp\n" + + "FROM bid\n" + + "MATCH_RECOGNIZE (\n" + + " PARTITION BY auction, bidder\n" + + " ORDER BY `dateTime`\n" + + " MEASURES\n" + + " START_ROW.`dateTime` AS start_tstamp,\n" + + " LAST(PRICE_DOWN.`dateTime`) AS bottom_tstamp,\n" + + " LAST(PRICE_UP.`dateTime`) AS end_tstamp\n" + + " ONE ROW PER MATCH\n" + + " AFTER MATCH SKIP TO LAST PRICE_UP\n" + + " PATTERN (START_ROW PRICE_DOWN+ PRICE_UP)\n" + + " DEFINE\n" + + " PRICE_DOWN AS\n" + + " (LAST(PRICE_DOWN.price, 1) IS NULL AND PRICE_DOWN.price < START_ROW.price) OR\n" + + " PRICE_DOWN.price < LAST(PRICE_DOWN.price, 1),\n" + + " PRICE_UP AS\n" + + " PRICE_UP.price > LAST(PRICE_DOWN.price, 1)\n" + + ")")); + } + + @Test + public void cepQ1() { + print( + tEnv.executeSql( + "SELECT\n" + + " auction, bidder, start_tstamp, end_tstamp, avg_price\n" + + "FROM bid\n" + + "MATCH_RECOGNIZE (\n" + + " PARTITION BY auction, bidder\n" + + " ORDER BY `dateTime`\n" + + " MEASURES\n" + + " FIRST(A.`dateTime`) AS start_tstamp,\n" + + " LAST(A.`dateTime`) AS end_tstamp,\n" + + " AVG(A.price) AS avg_price\n" + + " ONE ROW PER MATCH\n" + + " AFTER MATCH SKIP PAST LAST ROW\n" + + " PATTERN (A+ B)\n" + + " DEFINE\n" + + " A AS AVG(A.price) < 10000\n" + + ")")); + } + + @Test + public void cepQ2() { + print( + tEnv.executeSql( + "SELECT\n" + + " auction, bidder, start_tstamp, end_tstamp, avg_price\n" + + "FROM bid\n" + + "MATCH_RECOGNIZE (\n" + + " PARTITION BY auction, bidder\n" + + " ORDER BY `dateTime`\n" + + " MEASURES\n" + + " FIRST(A.`dateTime`) AS start_tstamp,\n" + + " LAST(A.`dateTime`) AS end_tstamp,\n" + + " AVG(A.price) AS avg_price\n" + + " ONE ROW PER MATCH\n" + + " AFTER MATCH SKIP TO NEXT ROW\n" + + " PATTERN (A+ B)\n" + + " DEFINE\n" + + " A AS AVG(A.price) < 10000\n" + + ")")); + } + + @Test + public void cepQ3() { + print( + tEnv.executeSql( + "SELECT\n" + + " auction, bidder, drop_time, drop_diff\n" + + "FROM bid\n" + + "MATCH_RECOGNIZE(\n" + + " PARTITION BY auction, bidder\n" + + " ORDER BY `dateTime`\n" + + " MEASURES\n" + + " C.`dateTime` AS drop_time,\n" + + " A.price - C.price AS drop_diff\n" + + " ONE ROW PER MATCH\n" + + " AFTER MATCH SKIP PAST LAST ROW\n" + + " PATTERN (A B* C) WITHIN INTERVAL '5' SECOND\n" + + " DEFINE\n" + + " B AS B.price > A.price - 50,\n" + + " C AS C.price < A.price - 50\n" + + ")")); + } + + private void print(TableResult result) { + // TableResult.print will truncate string value + try (CloseableIterator iter = result.collect()) { + while (iter.hasNext()) { + System.out.println(iter.next()); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private void removeRowTime() { + tEnv.executeSql("DROP VIEW IF EXISTS person"); + tEnv.executeSql("DROP VIEW IF EXISTS auction"); + tEnv.executeSql("DROP VIEW IF EXISTS bid"); + tEnv.executeSql("CREATE VIEW person AS SELECT person.* FROM nexmark WHERE event_type = 0"); + tEnv.executeSql("CREATE VIEW auction AS SELECT auction.* FROM nexmark WHERE event_type = 1"); + tEnv.executeSql("CREATE VIEW bid AS SELECT bid.* FROM nexmark WHERE event_type = 2"); + } +} diff --git a/nexmark/nexmark-flink/src/test/java/com/github/nexmark/flink/utils/NexmarkGlobalConfigurationTest.java b/nexmark/nexmark-flink/src/test/java/com/github/nexmark/flink/utils/NexmarkGlobalConfigurationTest.java new file mode 100644 index 0000000..2eb2eb6 --- /dev/null +++ b/nexmark/nexmark-flink/src/test/java/com/github/nexmark/flink/utils/NexmarkGlobalConfigurationTest.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink.utils; + +import org.apache.flink.configuration.Configuration; + +import org.junit.Test; + +import java.net.URL; + +import static com.github.nexmark.flink.FlinkNexmarkOptions.FLINK_REST_ADDRESS; +import static com.github.nexmark.flink.FlinkNexmarkOptions.FLINK_REST_PORT; +import static org.junit.Assert.assertEquals; + +public class NexmarkGlobalConfigurationTest { + + @Test + public void testLoadConfiguration() { + URL confDir = NexmarkGlobalConfigurationTest.class.getClassLoader().getResource("conf"); + assert confDir != null; + Configuration conf = NexmarkGlobalConfiguration.loadConfiguration(confDir.getPath()); + assertEquals((Integer) 8081, conf.get(FLINK_REST_PORT, -1)); + assertEquals("localhost", conf.get(FLINK_REST_ADDRESS, "")); + } +} \ No newline at end of file diff --git a/nexmark/nexmark-flink/src/test/java/com/github/nexmark/flink/workload/WorkloadSuiteTest.java b/nexmark/nexmark-flink/src/test/java/com/github/nexmark/flink/workload/WorkloadSuiteTest.java new file mode 100644 index 0000000..f2b1a5d --- /dev/null +++ b/nexmark/nexmark-flink/src/test/java/com/github/nexmark/flink/workload/WorkloadSuiteTest.java @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.nexmark.flink.workload; + +import org.apache.flink.configuration.Configuration; + +import com.github.nexmark.flink.FlinkNexmarkOptions; +import com.github.nexmark.flink.utils.NexmarkGlobalConfiguration; +import com.github.nexmark.flink.utils.NexmarkGlobalConfigurationTest; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.net.URL; +import java.time.Duration; +import java.util.HashMap; +import java.util.Map; + +import static com.github.nexmark.flink.Benchmark.CATEGORY_OA; +import static org.junit.Assert.assertEquals; + +public class WorkloadSuiteTest { + + @Rule + public ExpectedException exception = ExpectedException.none(); + + private static final String CATEGORY_CEP = "cep"; + + @Test + public void testCustomizedConf() { + Configuration conf = new Configuration(); + conf.setString("nexmark.workload.suite.8m.tps", "8000000"); + conf.setString("nexmark.workload.suite.8m.events.num", "80000000"); + conf.setString("nexmark.workload.suite.8m.queries", "q0,q1,q2,q10,q12,q13,q14"); + conf.setString("nexmark.workload.suite.8m.queries.cep", "q0"); + conf.setString("nexmark.workload.suite.2m-no-bid.tps", "2000000"); + conf.setString("nexmark.workload.suite.2m-no-bid.percentage", "bid:0, auction:9, person:1"); + conf.setString("nexmark.workload.suite.2m-no-bid.queries", "q3,q8"); + conf.setString("nexmark.workload.suite.2m-no-bid.queries.cep", "q1"); + conf.setString("nexmark.workload.suite.2m.tps", "2000000"); + conf.setString("nexmark.workload.suite.2m.queries", "q5,q15"); + conf.setString("nexmark.workload.suite.2m.queries.cep", "q2"); + conf.setString("nexmark.workload.suite.1m.tps", "1000000"); + conf.setString("nexmark.workload.suite.1m.queries", "q4,q7,q9,q11"); + conf.setString("nexmark.workload.suite.1m.queries.cep", "q3"); + + Workload load8m = new Workload(8000000, 80000000, 1, 3, 46); + Workload load2mNoBid = new Workload(2000000, 0, 1, 9, 0); + Workload load2m = new Workload(2000000, 0, 1, 3, 46); + Workload load1m = new Workload(1000000, 0, 1, 3, 46); + + Map query2Workload = new HashMap<>(); + query2Workload.put("q0", load8m); + query2Workload.put("q1", load8m); + query2Workload.put("q2", load8m); + query2Workload.put("q10", load8m); + query2Workload.put("q12", load8m); + query2Workload.put("q13", load8m); + query2Workload.put("q14", load8m); + + query2Workload.put("q3", load2mNoBid); + query2Workload.put("q8", load2mNoBid); + + query2Workload.put("q5", load2m); + + query2Workload.put("q4", load1m); + query2Workload.put("q7", load1m); + query2Workload.put("q9", load1m); + query2Workload.put("q11", load1m); + query2Workload.put("q15", load2m); + + WorkloadSuite expected = new WorkloadSuite(query2Workload); + + assertEquals(expected.toString(), WorkloadSuite.fromConf(conf, CATEGORY_OA).toString()); + + query2Workload = new HashMap<>(); + + query2Workload.put("q0", load8m); + + query2Workload.put("q1", load2mNoBid); + + query2Workload.put("q2", load2m); + + query2Workload.put("q3", load1m); + + expected = new WorkloadSuite(query2Workload); + + assertEquals(expected.toString(), WorkloadSuite.fromConf(conf, CATEGORY_CEP).toString()); + } + + @Test + public void testDefaultConf() { + URL confDir = NexmarkGlobalConfigurationTest.class.getClassLoader().getResource("conf"); + assert confDir != null; + Configuration conf = NexmarkGlobalConfiguration.loadConfiguration(confDir.getPath()); + + Workload load = new Workload(10000000, 100000000, 1, 3, 46); + + Map query2Workload = new HashMap<>(); + query2Workload.put("q0", load); + query2Workload.put("q1", load); + query2Workload.put("q2", load); + query2Workload.put("q3", load); + query2Workload.put("q4", load); + query2Workload.put("q5", load); + query2Workload.put("q7", load); + query2Workload.put("q8", load); + query2Workload.put("q9", load); + query2Workload.put("q10", load); + query2Workload.put("q11", load); + query2Workload.put("q12", load); + query2Workload.put("q13", load); + query2Workload.put("q14", load); + query2Workload.put("q15", load); + query2Workload.put("q16", load); + query2Workload.put("q17", load); + query2Workload.put("q18", load); + query2Workload.put("q19", load); + query2Workload.put("q20", load); + query2Workload.put("q21", load); + query2Workload.put("q22", load); + query2Workload.put("q23", load); + query2Workload.put("insert_kafka", new Workload(10000000, 0, 1, 3, 46)); + + WorkloadSuite expected = new WorkloadSuite(query2Workload); + + assertEquals(expected.toString(), WorkloadSuite.fromConf(conf, CATEGORY_OA).toString()); + + query2Workload = new HashMap<>(); + query2Workload.put("q0", load); + query2Workload.put("q1", load); + query2Workload.put("q2", load); + query2Workload.put("q3", load); + query2Workload.put("insert_kafka", new Workload(10000000, 0, 1, 3, 46)); + + expected = new WorkloadSuite(query2Workload); + + assertEquals(expected.toString(), WorkloadSuite.fromConf(conf, CATEGORY_CEP).toString()); + } + + @Test + public void testTPSValidation() { + exception.expectMessage("You should configure 'nexmark.metric.monitor.duration'" + + " in the TPS mode. Otherwise, the job will never end."); + // TPS mode + long eventsNum = 0L; + new Workload(0L, eventsNum,0, 0, 0) + .validateWorkload(FlinkNexmarkOptions.METRIC_MONITOR_DURATION.defaultValue()); + } + + @Test + public void testEventsNumValidation() { + exception.expectMessage("The configuration of 'nexmark.metric.monitor.duration'" + + " is not supported in the events number mode."); + // EventsNum mode + long eventsNum = 100L; + new Workload(0L, eventsNum,0, 0, 0) + .validateWorkload(Duration.ofMillis(1000)); + } + + @Test + public void testTPSMode() { + // TPS mode + long eventsNum = 0L; + new Workload(0L, eventsNum,0, 0, 0) + .validateWorkload(Duration.ofMillis(1000)); + } + + @Test + public void testEventsNumMode() { + // EventsNum mode + long eventsNum = 100L; + new Workload(0L, eventsNum,0, 0, 0) + .validateWorkload(FlinkNexmarkOptions.METRIC_MONITOR_DURATION.defaultValue()); + } +} \ No newline at end of file diff --git a/nexmark/nexmark-spark/pom.xml b/nexmark/nexmark-spark/pom.xml new file mode 100644 index 0000000..bbb2c50 --- /dev/null +++ b/nexmark/nexmark-spark/pom.xml @@ -0,0 +1,33 @@ + + + + + nexmark + com.github.nexmark + 0.3-SNAPSHOT + + 4.0.0 + + nexmark-spark + + + \ No newline at end of file diff --git a/nexmark/pom.xml b/nexmark/pom.xml new file mode 100644 index 0000000..7757c2b --- /dev/null +++ b/nexmark/pom.xml @@ -0,0 +1,47 @@ + + + + 4.0.0 + + com.github.nexmark + nexmark + pom + 0.3-SNAPSHOT + + + 2.0-preview1 + 1.7.15 + 1.2.17 + 1.8 + 2.12 + ${java.version} + ${java.version} + + + + nexmark-core + nexmark-flink + nexmark-spark + + + + \ No newline at end of file