From 4ce6893f557bc209aa41ad6d99384e13bf0814e6 Mon Sep 17 00:00:00 2001 From: Sam Sternberg Date: Sun, 21 Dec 2025 10:54:28 -0500 Subject: [PATCH] update data for more interesting dashboard --- samples/databricks.ipynb | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/samples/databricks.ipynb b/samples/databricks.ipynb index 93ee107..cac56d0 100644 --- a/samples/databricks.ipynb +++ b/samples/databricks.ipynb @@ -45,27 +45,7 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": [ - "from pyspark.sql.functions import expr, current_timestamp\n", - "\n", - "# Configure number of test rows\n", - "NUM_ROWS = 100\n", - "\n", - "# Create test data\n", - "test_df = spark.range(NUM_ROWS).select(\n", - " (expr(\"id + 1\").alias(\"user_id\")),\n", - " expr(\"concat('user_', id)\").alias(\"username\"),\n", - " expr(\"concat('user', id, '@example.com')\").alias(\"email\"),\n", - " expr(\"concat('+1-555-', LPAD(id % 1000, 3, '0'), '-', LPAD((id * 7) % 10000, 4, '0'))\").alias(\"phone\"),\n", - " current_timestamp().alias(\"created_at\")\n", - ")\n", - "\n", - "# Save as table\n", - "test_df.write.mode(\"overwrite\").saveAsTable(f\"{CATALOG}.{SCHEMA}.raw_users\")\n", - "\n", - "print(f\"āœ“ Created {CATALOG}.{SCHEMA}.raw_users table with {NUM_ROWS} rows\")\n", - "display(spark.table(f\"{CATALOG}.{SCHEMA}.raw_users\").limit(10))" - ] + "source": "from pyspark.sql.functions import expr, current_timestamp, date_sub, rand, when\n\n# Configure number of test rows\nNUM_ROWS = 100\n\n# Create test data with diverse email domains and varied registration dates\ntest_df = spark.range(NUM_ROWS).select(\n (expr(\"id + 1\").alias(\"user_id\")),\n expr(\"concat('user_', id)\").alias(\"username\"),\n \n # Diverse email domains (40% gmail, 25% yahoo, 15% hotmail, 10% company, 10% outlook)\n when(expr(\"id % 10 < 4\"), expr(\"concat('user', id, '@gmail.com')\"))\n .when(expr(\"id % 10 < 6\"), expr(\"concat('user', id, '@yahoo.com')\"))\n .when(expr(\"id % 10 < 7\"), expr(\"concat('user', id, '@hotmail.com')\"))\n .when(expr(\"id % 10 < 9\"), expr(\"concat('user', id, '@company.com')\"))\n .otherwise(expr(\"concat('user', id, '@outlook.com')\"))\n .alias(\"email\"),\n \n # Varied phone numbers with different area codes\n expr(\"concat('+1-', LPAD((id % 9 + 1) * 100 + (id % 50), 3, '0'), '-', LPAD((id * 7) % 1000, 3, '0'), '-', LPAD((id * 13) % 10000, 4, '0'))\").alias(\"phone\"),\n \n # Registration dates spread across last 90 days (creates timeline variation)\n date_sub(current_timestamp(), expr(\"cast(rand() * 90 as int)\")).alias(\"created_at\")\n)\n\n# Save as table\ntest_df.write.mode(\"overwrite\").saveAsTable(f\"{CATALOG}.{SCHEMA}.raw_users\")\n\nprint(f\"āœ“ Created {CATALOG}.{SCHEMA}.raw_users table with {NUM_ROWS} rows\")\nprint(f\"\\nšŸ“Š Data Characteristics:\")\nprint(f\" - Email Domains: gmail.com (40%), yahoo.com (25%), hotmail.com (15%),\")\nprint(f\" company.com (10%), outlook.com (10%)\")\nprint(f\" - Registration Dates: Randomly spread across last 90 days\")\nprint(f\" - Phone Area Codes: Varied across users\")\nprint(f\"\\nThis diverse data will create vibrant dashboard visualizations!\")\ndisplay(spark.table(f\"{CATALOG}.{SCHEMA}.raw_users\").limit(10))" }, { "cell_type": "markdown",