From c6fd96e24a726553bc585dc01aa39ce20bffdaf8 Mon Sep 17 00:00:00 2001 From: Tianchen Wu Date: Sat, 26 Jul 2025 14:59:44 +0000 Subject: [PATCH 1/3] edit creditcard-s3 data --- .../creditcard-s3/cardassignment.table.sql | 12 ++++++++ .../creditcard-s3/merchant.table.sql | 12 ++++++++ .../creditcard-s3/merchantreward.table.sql | 16 ++++++++++ .../creditcard-s3/rewardsink.table.sql | 13 +++++++++ .../creditcard-s3/transaction.table.sql | 13 +++++++++ .../creditcard_analytics_package_s3.json | 29 +++++++++++++++++++ 6 files changed, 95 insertions(+) create mode 100755 finance-credit-card-chatbot/credit-card-analytics/creditcard-s3/cardassignment.table.sql create mode 100755 finance-credit-card-chatbot/credit-card-analytics/creditcard-s3/merchant.table.sql create mode 100755 finance-credit-card-chatbot/credit-card-analytics/creditcard-s3/merchantreward.table.sql create mode 100644 finance-credit-card-chatbot/credit-card-analytics/creditcard-s3/rewardsink.table.sql create mode 100755 finance-credit-card-chatbot/credit-card-analytics/creditcard-s3/transaction.table.sql create mode 100644 finance-credit-card-chatbot/credit-card-analytics/creditcard_analytics_package_s3.json diff --git a/finance-credit-card-chatbot/credit-card-analytics/creditcard-s3/cardassignment.table.sql b/finance-credit-card-chatbot/credit-card-analytics/creditcard-s3/cardassignment.table.sql new file mode 100755 index 00000000..cbe1e28f --- /dev/null +++ b/finance-credit-card-chatbot/credit-card-analytics/creditcard-s3/cardassignment.table.sql @@ -0,0 +1,12 @@ +CREATE TABLE CardAssignment ( + customerId BIGINT NOT NULL, + cardNo STRING NOT NULL, + cardType STRING NOT NULL, + `timestamp` TIMESTAMP_LTZ(3) NOT NULL METADATA FROM 'timestamp', + WATERMARK FOR `timestamp` AS `timestamp` - INTERVAL '1' SECOND +) WITH ( + 'connector' = 'filesystem', + 'format' = 'flexible-json', + 'path' = 's3://${S3_DATA_BUCKET}/cardAssignment.jsonl', + 'source.monitor-interval' = '1 min' + ); diff --git a/finance-credit-card-chatbot/credit-card-analytics/creditcard-s3/merchant.table.sql b/finance-credit-card-chatbot/credit-card-analytics/creditcard-s3/merchant.table.sql new file mode 100755 index 00000000..2d8acc12 --- /dev/null +++ b/finance-credit-card-chatbot/credit-card-analytics/creditcard-s3/merchant.table.sql @@ -0,0 +1,12 @@ +CREATE TABLE Merchant ( + merchantId BIGINT NOT NULL, + name STRING NOT NULL, + category STRING NOT NULL, + updatedTime TIMESTAMP_LTZ(3) NOT NULL METADATA FROM 'timestamp', + WATERMARK FOR `updatedTime` AS `updatedTime` - INTERVAL '1' SECOND +) WITH ( + 'connector' = 'filesystem', + 'format' = 'flexible-json', + 'path' = 's3://${S3_DATA_BUCKET}/merchant.jsonl', + 'source.monitor-interval' = '1 min' + ); \ No newline at end of file diff --git a/finance-credit-card-chatbot/credit-card-analytics/creditcard-s3/merchantreward.table.sql b/finance-credit-card-chatbot/credit-card-analytics/creditcard-s3/merchantreward.table.sql new file mode 100755 index 00000000..3ec657fd --- /dev/null +++ b/finance-credit-card-chatbot/credit-card-analytics/creditcard-s3/merchantreward.table.sql @@ -0,0 +1,16 @@ +CREATE TABLE MerchantReward ( + merchantId BIGINT NOT NULL, + rewardsByCard ARRAY> NOT NULL, + updatedTime TIMESTAMP_LTZ(3) NOT NULL METADATA FROM 'timestamp', + WATERMARK FOR `updatedTime` AS `updatedTime` - INTERVAL '1' SECOND +) WITH ( + 'connector' = 'filesystem', + 'format' = 'flexible-json', + 'path' = 's3://${S3_DATA_BUCKET}/merchantReward.jsonl', + 'source.monitor-interval' = '1 min' + ); \ No newline at end of file diff --git a/finance-credit-card-chatbot/credit-card-analytics/creditcard-s3/rewardsink.table.sql b/finance-credit-card-chatbot/credit-card-analytics/creditcard-s3/rewardsink.table.sql new file mode 100644 index 00000000..5bf9dba7 --- /dev/null +++ b/finance-credit-card-chatbot/credit-card-analytics/creditcard-s3/rewardsink.table.sql @@ -0,0 +1,13 @@ +CREATE TABLE CustomerReward ( + PRIMARY KEY (`customerId`) NOT ENFORCED +) WITH ( + 'connector' = 'kafka', + 'properties.bootstrap.servers' = '${PROPERTIES_BOOTSTRAP_SERVERS}', + 'properties.group.id' = 'mygroupid', + 'scan.startup.mode' = 'group-offsets', + 'properties.auto.offset.reset' = 'earliest', + 'key.format' = 'raw', + 'key.fields' = 'customerId', + 'value.format' = 'flexible-json', + 'topic' = 'customerreward' + ); \ No newline at end of file diff --git a/finance-credit-card-chatbot/credit-card-analytics/creditcard-s3/transaction.table.sql b/finance-credit-card-chatbot/credit-card-analytics/creditcard-s3/transaction.table.sql new file mode 100755 index 00000000..287920ac --- /dev/null +++ b/finance-credit-card-chatbot/credit-card-analytics/creditcard-s3/transaction.table.sql @@ -0,0 +1,13 @@ +CREATE TABLE Transaction ( + transactionId BIGINT NOT NULL, + cardNo STRING NOT NULL, + `time` TIMESTAMP_LTZ(3) NOT NULL METADATA FROM 'timestamp', + amount DOUBLE NOT NULL, + merchantId BIGINT NOT NULL, + WATERMARK FOR `time` AS `time` - INTERVAL '1' SECOND +) WITH ( + 'connector' = 'filesystem', + 'format' = 'flexible-json', + 'path' = 's3://${S3_DATA_BUCKET}/transaction.jsonl', + 'source.monitor-interval' = '1 min' + ); \ No newline at end of file diff --git a/finance-credit-card-chatbot/credit-card-analytics/creditcard_analytics_package_s3.json b/finance-credit-card-chatbot/credit-card-analytics/creditcard_analytics_package_s3.json new file mode 100644 index 00000000..7bd928a6 --- /dev/null +++ b/finance-credit-card-chatbot/credit-card-analytics/creditcard_analytics_package_s3.json @@ -0,0 +1,29 @@ +{ + "version": "1", + "enabled-engines": [ + "vertx", + "postgres", + "kafka", + "flink" + ], + "script": { + "main": "creditcard_analytics.sqrl" + }, + "engines": { + "flink": { + "config": { + "table.exec.source.idle-timeout": "10 s" + } + }, + "snowflake": { + "catalog-name": "MyCatalog", + "external-volume": "MyNewVolume", + "url": "jdbc:snowflake://${SNOWFLAKE_ID}.snowflakecomputing.com/?user=${SNOWFLAKE_USER}&password=${SNOWFLAKE_PASSWORD}&warehouse=COMPUTE_WH&db=MYSNOWFLAKEDB&schema=public&disableSslHostnameVerification=true" + } + }, + "dependencies": { + "creditcard-data": { + "folder": "creditcard-testdata" + } + } +} \ No newline at end of file From 41680840a7b2b86c93a526463f1b1dc29130b98b Mon Sep 17 00:00:00 2001 From: Tianchen Wu Date: Sun, 27 Jul 2025 13:58:47 +0000 Subject: [PATCH 2/3] adding creditcard_analytics_package_s3.json --- .../creditcard-s3/rewardsink.table.sql | 13 ------------- .../creditcard_analytics_package_s3.json | 12 +++++++----- 2 files changed, 7 insertions(+), 18 deletions(-) delete mode 100644 finance-credit-card-chatbot/credit-card-analytics/creditcard-s3/rewardsink.table.sql diff --git a/finance-credit-card-chatbot/credit-card-analytics/creditcard-s3/rewardsink.table.sql b/finance-credit-card-chatbot/credit-card-analytics/creditcard-s3/rewardsink.table.sql deleted file mode 100644 index 5bf9dba7..00000000 --- a/finance-credit-card-chatbot/credit-card-analytics/creditcard-s3/rewardsink.table.sql +++ /dev/null @@ -1,13 +0,0 @@ -CREATE TABLE CustomerReward ( - PRIMARY KEY (`customerId`) NOT ENFORCED -) WITH ( - 'connector' = 'kafka', - 'properties.bootstrap.servers' = '${PROPERTIES_BOOTSTRAP_SERVERS}', - 'properties.group.id' = 'mygroupid', - 'scan.startup.mode' = 'group-offsets', - 'properties.auto.offset.reset' = 'earliest', - 'key.format' = 'raw', - 'key.fields' = 'customerId', - 'value.format' = 'flexible-json', - 'topic' = 'customerreward' - ); \ No newline at end of file diff --git a/finance-credit-card-chatbot/credit-card-analytics/creditcard_analytics_package_s3.json b/finance-credit-card-chatbot/credit-card-analytics/creditcard_analytics_package_s3.json index 7bd928a6..c2b5500d 100644 --- a/finance-credit-card-chatbot/credit-card-analytics/creditcard_analytics_package_s3.json +++ b/finance-credit-card-chatbot/credit-card-analytics/creditcard_analytics_package_s3.json @@ -14,11 +14,13 @@ "config": { "table.exec.source.idle-timeout": "10 s" } - }, - "snowflake": { - "catalog-name": "MyCatalog", - "external-volume": "MyNewVolume", - "url": "jdbc:snowflake://${SNOWFLAKE_ID}.snowflakecomputing.com/?user=${SNOWFLAKE_USER}&password=${SNOWFLAKE_PASSWORD}&warehouse=COMPUTE_WH&db=MYSNOWFLAKEDB&schema=public&disableSslHostnameVerification=true" + } + }, + "connectors": { + "filesystem": { + "warehouse": "warehouse", + "catalog-type": "hadoop", + "catalog-name": "mydatabase" } }, "dependencies": { From a68660eda6cdb11d4df4be6f5801a44eeceb3d48 Mon Sep 17 00:00:00 2001 From: Tianchen Wu Date: Mon, 28 Jul 2025 14:28:15 +0000 Subject: [PATCH 3/3] try cmd run --- .../creditcard-s3/cardassignment.table.sql | 2 +- .../creditcard-s3/merchant.table.sql | 2 +- .../creditcard-s3/merchantreward.table.sql | 2 +- .../creditcard-s3/transaction.table.sql | 2 +- .../creditcard_analytics_package_s3.json | 10 +--------- 5 files changed, 5 insertions(+), 13 deletions(-) diff --git a/finance-credit-card-chatbot/credit-card-analytics/creditcard-s3/cardassignment.table.sql b/finance-credit-card-chatbot/credit-card-analytics/creditcard-s3/cardassignment.table.sql index cbe1e28f..b1cc612f 100755 --- a/finance-credit-card-chatbot/credit-card-analytics/creditcard-s3/cardassignment.table.sql +++ b/finance-credit-card-chatbot/credit-card-analytics/creditcard-s3/cardassignment.table.sql @@ -7,6 +7,6 @@ CREATE TABLE CardAssignment ( ) WITH ( 'connector' = 'filesystem', 'format' = 'flexible-json', - 'path' = 's3://${S3_DATA_BUCKET}/cardAssignment.jsonl', + 'path' = 's3://example-data.dev.datasqrl.com/mvp/cardAssignment.jsonl', 'source.monitor-interval' = '1 min' ); diff --git a/finance-credit-card-chatbot/credit-card-analytics/creditcard-s3/merchant.table.sql b/finance-credit-card-chatbot/credit-card-analytics/creditcard-s3/merchant.table.sql index 2d8acc12..ec987c61 100755 --- a/finance-credit-card-chatbot/credit-card-analytics/creditcard-s3/merchant.table.sql +++ b/finance-credit-card-chatbot/credit-card-analytics/creditcard-s3/merchant.table.sql @@ -7,6 +7,6 @@ CREATE TABLE Merchant ( ) WITH ( 'connector' = 'filesystem', 'format' = 'flexible-json', - 'path' = 's3://${S3_DATA_BUCKET}/merchant.jsonl', + 'path' = 's3://example-data.dev.datasqrl.com/mvp/merchant.jsonl', 'source.monitor-interval' = '1 min' ); \ No newline at end of file diff --git a/finance-credit-card-chatbot/credit-card-analytics/creditcard-s3/merchantreward.table.sql b/finance-credit-card-chatbot/credit-card-analytics/creditcard-s3/merchantreward.table.sql index 3ec657fd..8336374f 100755 --- a/finance-credit-card-chatbot/credit-card-analytics/creditcard-s3/merchantreward.table.sql +++ b/finance-credit-card-chatbot/credit-card-analytics/creditcard-s3/merchantreward.table.sql @@ -11,6 +11,6 @@ CREATE TABLE MerchantReward ( ) WITH ( 'connector' = 'filesystem', 'format' = 'flexible-json', - 'path' = 's3://${S3_DATA_BUCKET}/merchantReward.jsonl', + 'path' = 's3://example-data.dev.datasqrl.com/mvp/merchantReward.jsonl', 'source.monitor-interval' = '1 min' ); \ No newline at end of file diff --git a/finance-credit-card-chatbot/credit-card-analytics/creditcard-s3/transaction.table.sql b/finance-credit-card-chatbot/credit-card-analytics/creditcard-s3/transaction.table.sql index 287920ac..12354ce3 100755 --- a/finance-credit-card-chatbot/credit-card-analytics/creditcard-s3/transaction.table.sql +++ b/finance-credit-card-chatbot/credit-card-analytics/creditcard-s3/transaction.table.sql @@ -8,6 +8,6 @@ CREATE TABLE Transaction ( ) WITH ( 'connector' = 'filesystem', 'format' = 'flexible-json', - 'path' = 's3://${S3_DATA_BUCKET}/transaction.jsonl', + 'path' = 's3://example-data.dev.datasqrl.com/mvp/transaction.jsonl', 'source.monitor-interval' = '1 min' ); \ No newline at end of file diff --git a/finance-credit-card-chatbot/credit-card-analytics/creditcard_analytics_package_s3.json b/finance-credit-card-chatbot/credit-card-analytics/creditcard_analytics_package_s3.json index c2b5500d..32217411 100644 --- a/finance-credit-card-chatbot/credit-card-analytics/creditcard_analytics_package_s3.json +++ b/finance-credit-card-chatbot/credit-card-analytics/creditcard_analytics_package_s3.json @@ -3,7 +3,6 @@ "enabled-engines": [ "vertx", "postgres", - "kafka", "flink" ], "script": { @@ -16,16 +15,9 @@ } } }, - "connectors": { - "filesystem": { - "warehouse": "warehouse", - "catalog-type": "hadoop", - "catalog-name": "mydatabase" - } - }, "dependencies": { "creditcard-data": { - "folder": "creditcard-testdata" + "folder": "creditcard-s3" } } } \ No newline at end of file