pyspark practice

Pyspark and SQL code/577. Employee Bonus.ipynb

@@ -0,0 +1,118 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 0,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "implicitDf": true,
+      "rowLimit": 10000
+     },
+     "inputWidgets": {},
+     "nuid": "9459dd11-478d-40f5-9ad8-bced4453bd42",
+     "showTitle": false,
+     "tableResultSettingsMap": {},
+     "title": ""
+    }
+   },
+   "outputs": [],
+   "source": [
+    "%sql\n",
+    "\n",
+    "CREATE OR REPLACE TABLE Employee (\n",
+    "  empID INT,\n",
+    "  name VARCHAR(30),\n",
+    "  supervisor INT,\n",
+    "  salary INT\n",
+    ");\n",
+    "\n",
+    "INSERT INTO Employee VALUES \n",
+    "(3, \"Brad\", null, 4000 ),\n",
+    "(1, \"John\", 3, 1000   ),\n",
+    "(2, \"Dan\", 3, 2000),\n",
+    "(4, \"Thomas\", 3, 4000);\n",
+    "\n",
+    "CREATE OR REPLACE TABLE Bonus (\n",
+    "  empId INT,\n",
+    "  bonus INT\n",
+    ");\n",
+    "\n",
+    "INSERT INTO Bonus (empId, bonus) VALUES\n",
+    "(2, 500),\n",
+    "(4, 2000);\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 0,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "inputWidgets": {},
+     "nuid": "be98d11e-5ca2-437c-9e99-0b0481aae0bc",
+     "showTitle": false,
+     "tableResultSettingsMap": {},
+     "title": ""
+    }
+   },
+   "outputs": [],
+   "source": [
+    "from pyspark.sql import functions as F\n",
+    "\n",
+    "e = spark.table(\"Employee\").alias(\"e\")\n",
+    "b = spark.table(\"Bonus\").alias(\"b\")\n",
+    "\n",
+    "result = (\n",
+    "    e.join(\n",
+    "        b, \n",
+    "        F.col(\"e.empID\") == F.col(\"b.empID\"),\n",
+    "        \"left\"\n",
+    "    )\n",
+    "    .filter(\n",
+    "        (F.col(\"b.bonus\") < 1000) |\n",
+    "        (F.col(\"b.bonus\").isNull())\n",
+    "    )\n",
+    "    .select(\n",
+    "        F.col(\"e.name\"),\n",
+    "        F.col(\"b.bonus\")\n",
+    "    )\n",
+    ")\n",
+    "\n",
+    "display(result)"
+   ]
+  }
+ ],
+ "metadata": {
+  "application/vnd.databricks.v1+notebook": {
+   "computePreferences": null,
+   "dashboards": [],
+   "environmentMetadata": {
+    "base_environment": "",
+    "environment_version": "5"
+   },
+   "inputWidgetPreferences": null,
+   "language": "python",
+   "notebookMetadata": {
+    "mostRecentlyExecutedCommandWithImplicitDF": {
+     "commandId": 8384752393431496,
+     "dataframes": [
+      "_sqldf"
+     ]
+    },
+    "pythonIndentUnit": 4
+   },
+   "notebookName": "577. Employee Bonus",
+   "widgets": {}
+  },
+  "language_info": {
+   "name": "python"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}