Fix SQL Server profiler row counts, system objects and procedure names

docs/lakebridge/docs/assessment/profiler/mssql.mdx

@@ -45,22 +45,29 @@ Permissions differ depending on whether you are running against an **on-premises
 
 #### On-Premises SQL Server
 
-On a self-hosted SQL Server instance, a server-level grant is sufficient:
+On a self-hosted SQL Server instance, grant one server-level permission to the login, then map it to a
+user with `VIEW DEFINITION` in each database you profile:
 
 ```sql
--- Grant access to Dynamic Management Views (DMVs)
-GRANT VIEW SERVER STATE TO [<user_name>];
+-- Server level: access to Dynamic Management Views (DMVs)
+USE master;
+GRANT VIEW SERVER STATE TO [<login_name>];
 
--- Grant access to object definitions (routines, views)
+-- In each database you profile
+USE [<target_database>];
+CREATE USER [<user_name>] FROM LOGIN [<login_name>];
 GRANT VIEW DEFINITION TO [<user_name>];
-
--- Grant read access to INFORMATION_SCHEMA views
-GRANT SELECT ON SCHEMA::INFORMATION_SCHEMA TO [<user_name>];
 ```
 
 :::tip
-`VIEW SERVER STATE` is a server-level permission required to query `sys.dm_*` Dynamic Management Views.
-`VIEW DEFINITION` allows the user to see the definitions of stored procedures and views.
+`VIEW SERVER STATE` is a server-level permission required to query `sys.dm_*` Dynamic Management Views. It
+also gives `VIEW DATABASE STATE` in every database, which `sys.dm_db_partition_stats` (table sizes) needs.
+`VIEW DEFINITION` makes a database's tables, views, columns and stored procedures visible to the profiler.
+Without it, that database's inventory and table sizes are empty or incomplete, and its stored procedure
+names in `proc_stats` are missing. Granting it per database keeps databases you don't profile out of reach.
+On SQL Server 2022 and later, these two grants also cover the finer-grained performance and security
+definition permissions. `INFORMATION_SCHEMA` views need no grant: every user can query them, and what they
+return depends on `VIEW DEFINITION`.
 These grants should be executed by a `sysadmin` or a login with `CONTROL SERVER` permission.
 :::
 
@@ -90,7 +97,6 @@ USE [<target_database>];
 CREATE USER [<user_name>] FROM LOGIN [<user_name>];
 GRANT VIEW DATABASE STATE TO [<user_name>];
 GRANT VIEW DEFINITION TO [<user_name>];
-GRANT SELECT ON SCHEMA::INFORMATION_SCHEMA TO [<user_name>];
 ```
 
 :::tip

src/databricks/labs/lakebridge/resources/assessments/mssql/columns.sql

@@ -2,6 +2,7 @@
  * Retrieves column-level metadata for all tables and views in the specified
  * database by querying INFORMATION_SCHEMA.COLUMNS. Returns column attributes
  * along with a timestamp indicating when the data was extracted.
+ * System objects (is_ms_shipped) and the sys and cdc (change data capture) schemas are excluded.
  */
 SELECT DB_NAME() AS DATABASE_NAME,
        TABLE_CATALOG,
@@ -27,4 +28,12 @@ SELECT DB_NAME() AS DATABASE_NAME,
        DOMAIN_CATALOG,
        DOMAIN_SCHEMA,
        DOMAIN_NAME
-FROM   INFORMATION_SCHEMA.COLUMNS;
+FROM   INFORMATION_SCHEMA.COLUMNS
+WHERE  TABLE_SCHEMA NOT IN (N'sys', N'cdc')
+       AND NOT EXISTS (SELECT 1
+                       FROM   sys.all_objects AS o
+                              JOIN sys.schemas AS s
+                                ON s.schema_id = o.schema_id
+                       WHERE  o.is_ms_shipped = 1
+                              AND s.[name] = TABLE_SCHEMA
+                              AND o.[name] = TABLE_NAME);

src/databricks/labs/lakebridge/resources/assessments/mssql/multi_db/columns.sql

@@ -3,6 +3,7 @@
  * database by dynamically UNION-ing INFORMATION_SCHEMA.COLUMNS from each database, tagging each row
  * with its source `database_name`. The WHERE 1 = 0 anchor guarantees a single, correctly-typed result
  * set. Requires three-part naming (on-prem SQL Server / Azure SQL Managed Instance only).
+ * System objects (is_ms_shipped) and the sys and cdc (change data capture) schemas are excluded.
  */
 SET NOCOUNT ON;
 
@@ -30,6 +31,9 @@ DECLARE @sql NVARCHAR(MAX) =
 SELECT @sql = @sql + ISNULL((
         SELECT ' UNION ALL SELECT N' + QUOTENAME([name], '''') + ' AS DATABASE_NAME, ' + @cols
                + ' FROM ' + QUOTENAME([name]) + '.INFORMATION_SCHEMA.COLUMNS'
+               + ' WHERE TABLE_SCHEMA NOT IN (N''sys'', N''cdc'') AND NOT EXISTS (SELECT 1 FROM ' + QUOTENAME([name]) + '.sys.all_objects AS o'
+               + ' JOIN ' + QUOTENAME([name]) + '.sys.schemas AS s ON s.schema_id = o.schema_id'
+               + ' WHERE o.is_ms_shipped = 1 AND s.[name] = TABLE_SCHEMA AND o.[name] = TABLE_NAME)'
         FROM   sys.databases
         WHERE  state_desc = 'ONLINE'
                AND HAS_DBACCESS([name]) = 1

src/databricks/labs/lakebridge/resources/assessments/mssql/multi_db/routines.sql

@@ -4,6 +4,7 @@
  * database, tagging each row with its source `database_name`. ROUTINE_DEFINITION is redacted, matching
  * the single-database extract. The WHERE 1 = 0 anchor guarantees a single, correctly-typed result set.
  * Requires three-part naming (on-prem SQL Server / Azure SQL Managed Instance only).
+ * System objects (is_ms_shipped) and the sys and cdc (change data capture) schemas are excluded.
  */
 SET NOCOUNT ON;
 
@@ -30,6 +31,9 @@ DECLARE @sql NVARCHAR(MAX) =
 SELECT @sql = @sql + ISNULL((
         SELECT ' UNION ALL SELECT N' + QUOTENAME([name], '''') + ' AS DATABASE_NAME, ' + @cols
                + ' FROM ' + QUOTENAME([name]) + '.INFORMATION_SCHEMA.ROUTINES'
+               + ' WHERE ROUTINE_SCHEMA NOT IN (N''sys'', N''cdc'') AND NOT EXISTS (SELECT 1 FROM ' + QUOTENAME([name]) + '.sys.all_objects AS o'
+               + ' JOIN ' + QUOTENAME([name]) + '.sys.schemas AS s ON s.schema_id = o.schema_id'
+               + ' WHERE o.is_ms_shipped = 1 AND s.[name] = ROUTINE_SCHEMA AND o.[name] = ROUTINE_NAME)'
         FROM   sys.databases
         WHERE  state_desc = 'ONLINE'
                AND HAS_DBACCESS([name]) = 1

src/databricks/labs/lakebridge/resources/assessments/mssql/multi_db/table_sizes.sql

@@ -3,7 +3,8 @@
  * accessible, online user database. sys.dm_db_partition_stats is a database-scoped DMV that resolves
  * against the current database context, so this cannot use three-part naming; instead it loops the
  * databases and runs the extract under USE <db> for each, accumulating rows tagged with their source
- * `database_name`. On-prem SQL Server / Azure SQL Managed Instance only.
+ * `database_name`. On-prem SQL Server / Azure SQL Managed Instance only. As in the single-database
+ * query, rows come from the heap or clustered index only and system tables are excluded.
  */
 SET NOCOUNT ON;
 
@@ -31,7 +32,7 @@ WHILE @@FETCH_STATUS = 0
 BEGIN
     SET @sql = N'USE ' + QUOTENAME(@name) + N';'
         + N' SELECT DB_NAME() AS DATABASE_NAME, o.[name] AS TableName,'
-        + N' SUM(ps.row_count) AS [RowCount],'
+        + N' SUM(CASE WHEN ps.index_id < 2 THEN ps.row_count ELSE 0 END) AS [RowCount],'
         + N' SUM(ps.reserved_page_count) * 8 / 1024 AS ReservedMB,'
         + N' SUM(ps.used_page_count) * 8 / 1024 AS UsedMB,'
         + N' (SUM(ps.reserved_page_count) - SUM(ps.used_page_count)) * 8 / 1024 AS UnusedMB,'
@@ -41,7 +42,7 @@ BEGIN
         + N' SYSDATETIME() AS extract_ts'
         + N' FROM sys.dm_db_partition_stats AS ps'
         + N' JOIN sys.objects AS o ON ps.object_id = o.object_id'
-        + N' WHERE o.type = ''U'''
+        + N' WHERE o.type = ''U'' AND o.is_ms_shipped = 0'
         + N' GROUP BY SCHEMA_NAME(o.schema_id), o.[name]';
     INSERT INTO @results
     EXEC sys.sp_executesql @sql;

src/databricks/labs/lakebridge/resources/assessments/mssql/multi_db/tables.sql

@@ -4,6 +4,7 @@
  * tagged with its source database via the projected `database_name` literal. The leading WHERE 1 = 0
  * anchor guarantees a single, correctly-typed result set even when no user database is accessible.
  * Requires three-part naming (on-prem SQL Server / Azure SQL Managed Instance only).
+ * System objects (is_ms_shipped) and the sys and cdc (change data capture) schemas are excluded.
  */
 SET NOCOUNT ON;
 
@@ -18,6 +19,9 @@ DECLARE @sql NVARCHAR(MAX) =
 SELECT @sql = @sql + ISNULL((
         SELECT ' UNION ALL SELECT N' + QUOTENAME([name], '''') + ' AS DATABASE_NAME, ' + @cols
                + ' FROM ' + QUOTENAME([name]) + '.INFORMATION_SCHEMA.TABLES'
+               + ' WHERE TABLE_SCHEMA NOT IN (N''sys'', N''cdc'') AND NOT EXISTS (SELECT 1 FROM ' + QUOTENAME([name]) + '.sys.all_objects AS o'
+               + ' JOIN ' + QUOTENAME([name]) + '.sys.schemas AS s ON s.schema_id = o.schema_id'
+               + ' WHERE o.is_ms_shipped = 1 AND s.[name] = TABLE_SCHEMA AND o.[name] = TABLE_NAME)'
         FROM   sys.databases
         WHERE  state_desc = 'ONLINE'
                AND HAS_DBACCESS([name]) = 1

src/databricks/labs/lakebridge/resources/assessments/mssql/multi_db/views.sql

@@ -3,6 +3,7 @@
  * dynamically UNION-ing INFORMATION_SCHEMA.VIEWS from each database, tagging each row with its source
  * `database_name`. VIEW_DEFINITION is redacted, matching the single-database extract. The WHERE 1 = 0
  * anchor guarantees a single, correctly-typed result set. Requires three-part naming (on-prem / MI).
+ * System objects (is_ms_shipped) and the sys and cdc (change data capture) schemas are excluded.
  */
 SET NOCOUNT ON;
 
@@ -18,6 +19,9 @@ DECLARE @sql NVARCHAR(MAX) =
 SELECT @sql = @sql + ISNULL((
         SELECT ' UNION ALL SELECT N' + QUOTENAME([name], '''') + ' AS DATABASE_NAME, ' + @cols
                + ' FROM ' + QUOTENAME([name]) + '.INFORMATION_SCHEMA.VIEWS'
+               + ' WHERE TABLE_SCHEMA NOT IN (N''sys'', N''cdc'') AND NOT EXISTS (SELECT 1 FROM ' + QUOTENAME([name]) + '.sys.all_objects AS o'
+               + ' JOIN ' + QUOTENAME([name]) + '.sys.schemas AS s ON s.schema_id = o.schema_id'
+               + ' WHERE o.is_ms_shipped = 1 AND s.[name] = TABLE_SCHEMA AND o.[name] = TABLE_NAME)'
         FROM   sys.databases
         WHERE  state_desc = 'ONLINE'
                AND HAS_DBACCESS([name]) = 1

src/databricks/labs/lakebridge/resources/assessments/mssql/proc_stats.sql

@@ -2,6 +2,9 @@
  * Retrieves execution statistics for stored procedures from `sys.dm_exec_procedure_stats`,
  * including execution counts, total CPU and elapsed time, last execution timestamp,
  * and maps object and database IDs to their names. Results are ordered by most recent execution.
+ * The hidden Resource database (database_id 32767) is excluded: it holds SQL Server's own system
+ * procedures, whose names never resolve. Names of user procedures resolve only where the profiling
+ * login has VIEW DEFINITION in that database.
  */
 SELECT database_id,
        Db_name(database_id)                AS db_name,
@@ -14,4 +17,5 @@ SELECT database_id,
        total_elapsed_time,
        Sysdatetime()                       AS extract_ts
 FROM   sys.dm_exec_procedure_stats
+WHERE  database_id <> 32767
 ORDER  BY last_execution_time

src/databricks/labs/lakebridge/resources/assessments/mssql/routines.sql

@@ -2,6 +2,7 @@
  * Retrieves metadata for all routines (stored procedures and functions) in the
  * specified database by querying INFORMATION_SCHEMA.ROUTINES. Returns routine
  * details along with a timestamp indicating when the data was extracted.
+ * System objects (is_ms_shipped) and the sys and cdc (change data capture) schemas are excluded.
  */
 SELECT DB_NAME() AS DATABASE_NAME,
        CREATED,
@@ -26,4 +27,12 @@ SELECT DB_NAME() AS DATABASE_NAME,
        SPECIFIC_NAME,
        SPECIFIC_SCHEMA,
        SQL_DATA_ACCESS
-FROM   INFORMATION_SCHEMA.ROUTINES;
+FROM   INFORMATION_SCHEMA.ROUTINES
+WHERE  ROUTINE_SCHEMA NOT IN (N'sys', N'cdc')
+       AND NOT EXISTS (SELECT 1
+                       FROM   sys.all_objects AS o
+                              JOIN sys.schemas AS s
+                                ON s.schema_id = o.schema_id
+                       WHERE  o.is_ms_shipped = 1
+                              AND s.[name] = ROUTINE_SCHEMA
+                              AND o.[name] = ROUTINE_NAME);

src/databricks/labs/lakebridge/resources/assessments/mssql/table_sizes.sql

@@ -3,12 +3,17 @@
  * database by querying sys.dm_db_partition_stats and sys.objects. Returns table
  * name, total rows, reserved, used, and unused space (MB), breakdown of data vs.
  * index space, and a timestamp indicating when the data was extracted.
+ * Rows are counted from the heap or clustered index only (index_id 0 or 1): every nonclustered index
+ * repeats the table's rows. System tables (is_ms_shipped, e.g. change data capture) are excluded.
  */
 SELECT DB_NAME()                                                            AS
        DATABASE_NAME,
        o.NAME                                                               AS
        TableName,
-       Sum(ps.row_count)                                                    AS
+       Sum(CASE
+             WHEN ps.index_id < 2 THEN ps.row_count
+             ELSE 0
+           END)                                                             AS
        [RowCount],
        Sum(ps.reserved_page_count) * 8 / 1024                               AS
        ReservedMB,
@@ -34,5 +39,6 @@ FROM   sys.dm_db_partition_stats AS ps
        JOIN sys.objects AS o
          ON ps.object_id = o.object_id
 WHERE  o.type = 'U'
+       AND o.is_ms_shipped = 0
 GROUP  BY Schema_name(o.schema_id),
           o.NAME;

src/databricks/labs/lakebridge/resources/assessments/mssql/tables.sql

@@ -2,10 +2,19 @@
  * Retrieves metadata for all tables in the specified database by querying
  * INFORMATION_SCHEMA.TABLES. Returns table definitions along with a timestamp
  * indicating when the data was extracted.
+ * System objects (is_ms_shipped) and the sys and cdc (change data capture) schemas are excluded.
  */
 SELECT DB_NAME() AS DATABASE_NAME,
        TABLE_CATALOG,
        TABLE_SCHEMA,
        TABLE_NAME,
        TABLE_TYPE
-FROM   INFORMATION_SCHEMA.TABLES;
+FROM   INFORMATION_SCHEMA.TABLES
+WHERE  TABLE_SCHEMA NOT IN (N'sys', N'cdc')
+       AND NOT EXISTS (SELECT 1
+                       FROM   sys.all_objects AS o
+                              JOIN sys.schemas AS s
+                                ON s.schema_id = o.schema_id
+                       WHERE  o.is_ms_shipped = 1
+                              AND s.[name] = TABLE_SCHEMA
+                              AND o.[name] = TABLE_NAME);

src/databricks/labs/lakebridge/resources/assessments/mssql/views.sql

@@ -2,6 +2,7 @@
  * Retrieves metadata for all views in the specified database by querying
  * `INFORMATION_SCHEMA.VIEWS`. Returns view definitions along with a timestamp
  * indicating when the data was extracted.
+ * System objects (is_ms_shipped) and the sys and cdc (change data capture) schemas are excluded.
  */
 SELECT DB_NAME() AS DATABASE_NAME,
        TABLE_CATALOG,
@@ -11,3 +12,11 @@ SELECT DB_NAME() AS DATABASE_NAME,
        IS_UPDATABLE,
        N'[REDACTED]' AS VIEW_DEFINITION
 FROM   INFORMATION_SCHEMA.VIEWS
+WHERE  TABLE_SCHEMA NOT IN (N'sys', N'cdc')
+       AND NOT EXISTS (SELECT 1
+                       FROM   sys.all_objects AS o
+                              JOIN sys.schemas AS s
+                                ON s.schema_id = o.schema_id
+                       WHERE  o.is_ms_shipped = 1
+                              AND s.[name] = TABLE_SCHEMA
+                              AND o.[name] = TABLE_NAME)

tests/unit/assessment/test_mssql_sql_rules.py

@@ -100,3 +100,29 @@ def test_sys_info_defaults_version_dependent_columns(column: str) -> None:
     # Without a NULL default, the whole sys_info query fails to compile on versions that lack the column.
     sql = (_MSSQL / "sys_info.sql").read_text(encoding="utf-8")
     assert re.search(rf"CAST\(NULL AS [\w() ]+\)\s+AS\s+{column}\b", sql)
+
+
+@pytest.mark.parametrize("query", [_MSSQL / "table_sizes.sql", _MSSQL / "multi_db/table_sizes.sql"], ids=_name)
+def test_table_sizes_count_rows_from_heap_or_clustered_index(query: Path) -> None:
+    # Every nonclustered index repeats the table's rows, so summing all indexes multiplies the row count.
+    sql = _without_comments(query.read_text(encoding="utf-8"))
+    assert re.search(r"WHEN ps\.index_id < 2 THEN ps\.row_count", sql, re.IGNORECASE)
+    assert not re.search(r"SUM\(\s*ps\.row_count\s*\)", sql, re.IGNORECASE)
+
+
+_INVENTORY = [
+    _MSSQL / folder / f"{name}.sql"
+    for folder in ("", "multi_db")
+    for name in ("tables", "views", "columns", "routines", "table_sizes")
+]
+
+
+@pytest.mark.parametrize("query", _INVENTORY, ids=_name)
+def test_inventory_excludes_system_objects(query: Path) -> None:
+    # Change data capture and change tracking add system tables and routines to user databases.
+    assert "is_ms_shipped" in _without_comments(query.read_text(encoding="utf-8"))
+
+
+def test_proc_stats_excludes_resource_database() -> None:
+    # Procedures in the hidden Resource database are SQL Server's own, and their names never resolve.
+    assert "database_id <> 32767" in _without_comments((_MSSQL / "proc_stats.sql").read_text(encoding="utf-8"))