Footer improvements, duration fix, and version bump to 0.9.0

spark-plugin/build.sbt

@@ -1,7 +1,7 @@
 import xerial.sbt.Sonatype._
 import sbtassembly.AssemblyPlugin.autoImport._
 
-lazy val versionNum: String = "0.8.10"
+lazy val versionNum: String = "0.9.0"
 lazy val scala212 = "2.12.20"
 lazy val scala213 = "2.13.16"
 lazy val supportedScalaVersions = List(scala212, scala213)

spark-plugin/clean-and-setup.sh

@@ -38,6 +38,6 @@ echo "1. Refresh your IntelliJ IDEA project (File -> Reload Gradle Project or si
 echo "2. If you still get conflicts, try: File -> Invalidate Caches and Restart"
 echo ""
 echo "📦 Fat JARs created:"
-echo "- Spark 3.x: pluginspark3/target/scala-2.12/dataflint-spark3_2.12-0.8.6-SNAPSHOT.jar"
-echo "- Spark 4.x: pluginspark4/target/scala-2.13/dataflint-spark4_2.13-0.8.6-SNAPSHOT.jar"
+echo "- Spark 3.x: pluginspark3/target/scala-2.12/dataflint-spark3_2.12-0.9.0-SNAPSHOT.jar"
+echo "- Spark 4.x: pluginspark4/target/scala-2.13/dataflint-spark4_2.13-0.9.0-SNAPSHOT.jar"
 

spark-plugin/pyspark-testing/dataflint_pyspark_example.py

@@ -45,10 +45,10 @@ def sleep(seconds):
 
 # Select the appropriate plugin JAR based on Spark version
 if spark_major_version == 4:
-    _plugin_jar = _project_root / "pluginspark4" / "target" / "scala-2.13" / "dataflint-spark4_2.13-0.8.10.jar"
+    _plugin_jar = _project_root / "pluginspark4" / "target" / "scala-2.13" / "dataflint-spark4_2.13-0.9.0.jar"
     _plugin_module = "pluginspark4"
 else:
-    _plugin_jar = _project_root / "pluginspark3" / "target" / "scala-2.12" / "spark_2.12-0.8.10.jar"
+    _plugin_jar = _project_root / "pluginspark3" / "target" / "scala-2.12" / "spark_2.12-0.9.0.jar"
     _plugin_module = "pluginspark3"
 
 if not _plugin_jar.exists():

spark-plugin/pyspark-testing/dataflint_pyspark_example_test.py

@@ -45,10 +45,10 @@
 
 # Select the appropriate plugin JAR based on Spark version
 if spark_major_version == 4:
-    _plugin_jar = _project_root / "pluginspark4" / "target" / "scala-2.13" / "dataflint-spark4_2.13-0.8.10.jar"
+    _plugin_jar = _project_root / "pluginspark4" / "target" / "scala-2.13" / "dataflint-spark4_2.13-0.9.0.jar"
     _plugin_module = "pluginspark4"
 else:
-    _plugin_jar = _project_root / "pluginspark3" / "target" / "scala-2.12" / "spark_2.12-0.8.10.jar"
+    _plugin_jar = _project_root / "pluginspark3" / "target" / "scala-2.12" / "spark_2.12-0.9.0.jar"
     _plugin_module = "pluginspark3"
 
 if not _plugin_jar.exists():

spark-ui/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dataflint-ui",
-  "version": "0.8.10",
+  "version": "0.9.0",
   "homepage": "./",
   "private": true,
   "dependencies": {

spark-ui/src/components/Footer.tsx

@@ -1,10 +1,45 @@
 import GitHubIcon from "@mui/icons-material/GitHub";
 import MenuBookIcon from "@mui/icons-material/MenuBook";
+import OndemandVideoIcon from "@mui/icons-material/OndemandVideo";
 import StarIcon from "@mui/icons-material/Star";
-import { Box, Button, Typography } from "@mui/material";
+import NewReleasesIcon from "@mui/icons-material/NewReleases";
+import { Box, Button, Chip, Typography } from "@mui/material";
 import * as React from "react";
 
+function isNewerVersion(latest: string, current: string): boolean {
+    const latestParts = latest.split(".").map(Number);
+    const currentParts = current.split(".").map(Number);
+    for (let i = 0; i < Math.max(latestParts.length, currentParts.length); i++) {
+        const l = latestParts[i] || 0;
+        const c = currentParts[i] || 0;
+        if (l > c) return true;
+        if (l < c) return false;
+    }
+    return false;
+}
+
+function useLatestVersion(): string | null {
+    const [latestVersion, setLatestVersion] = React.useState<string | null>(null);
+
+    React.useEffect(() => {
+        const url = "https://central.sonatype.com/api/internal/browse/component/versions?sortField=normalizedVersion&sortDirection=desc&page=0&size=1&filter=namespace:io.dataflint,name:spark_2.12";
+        fetch(url)
+            .then(res => res.json())
+            .then(data => {
+                const version = data?.components?.[0]?.version;
+                if (version) setLatestVersion(version);
+            })
+            .catch(() => {});
+    }, []);
+
+    return latestVersion;
+}
+
 export default function Footer() {
+    const currentVersion = process.env.REACT_APP_VERSION;
+    const latestVersion = useLatestVersion();
+    const hasUpdate = latestVersion && currentVersion && isNewerVersion(latestVersion, currentVersion);
+
     const onGitHubClick = (): void => {
         window.open("https://github.com/dataflint/spark", "_blank");
     };
@@ -13,10 +48,18 @@ export default function Footer() {
         window.open("https://dataflint.gitbook.io/dataflint-for-spark/", "_blank");
     };
 
+    const onYouTubeClick = (): void => {
+        window.open("https://www.youtube.com/watch?v=4d_jBCmodKQ", "_blank");
+    };
+
     const onDataFlintClick = (): void => {
         window.open("https://www.dataflint.io/", "_blank");
     };
 
+    const onUpdateClick = (): void => {
+        window.open("https://github.com/dataflint/spark/releases", "_blank");
+    };
+
     return (
         <Box
             sx={{
@@ -32,6 +75,18 @@ export default function Footer() {
                 minHeight: "40px"
             }}
         >
+            {hasUpdate && (
+                <Chip
+                    icon={<NewReleasesIcon />}
+                    label={`New version ${latestVersion} available!`}
+                    size="small"
+                    color="primary"
+                    variant="outlined"
+                    onClick={onUpdateClick}
+                    sx={{ fontSize: "11px", cursor: "pointer" }}
+                />
+            )}
+
             <Button
                 onClick={onGitHubClick}
                 color="inherit"
@@ -53,6 +108,16 @@ export default function Footer() {
                 Docs
             </Button>
 
+            <Button
+                onClick={onYouTubeClick}
+                color="inherit"
+                size="small"
+                startIcon={<OndemandVideoIcon />}
+                sx={{ fontSize: "11px", textTransform: "none" }}
+            >
+                YouTube Tutorial
+            </Button>
+
             <Button
                 onClick={onDataFlintClick}
                 color="inherit"
@@ -76,4 +141,4 @@ export default function Footer() {
             </Button>
         </Box>
     );
-} 
+}

spark-ui/src/reducers/SQLNodeStageReducer.ts

@@ -121,7 +121,7 @@ export function calculateSqlStage(
   // Assign durations + percentages + exchange write/read durations
   const nodesWithDuration: EnrichedSqlNode[] = nodesWithStages.map((node) => {
     const nd = durationMap.get(node.nodeId);
-    const duration = nd !== undefined && nd.durationMs > 0 ? nd.durationMs : undefined;
+    const duration = nd !== undefined && nd.durationMs >= 0 ? nd.durationMs : undefined;
 
     const durationPercentage =
       duration !== undefined && sql.stageMetrics !== undefined