diff --git a/docs/lakebridge/docs/assessment/profiler/mssql.mdx b/docs/lakebridge/docs/assessment/profiler/mssql.mdx index 25c42646f0..b77c950aeb 100644 --- a/docs/lakebridge/docs/assessment/profiler/mssql.mdx +++ b/docs/lakebridge/docs/assessment/profiler/mssql.mdx @@ -45,22 +45,29 @@ Permissions differ depending on whether you are running against an **on-premises #### On-Premises SQL Server -On a self-hosted SQL Server instance, a server-level grant is sufficient: +On a self-hosted SQL Server instance, grant one server-level permission to the login, then map it to a +user with `VIEW DEFINITION` in each database you profile: ```sql --- Grant access to Dynamic Management Views (DMVs) -GRANT VIEW SERVER STATE TO []; +-- Server level: access to Dynamic Management Views (DMVs) +USE master; +GRANT VIEW SERVER STATE TO []; --- Grant access to object definitions (routines, views) +-- In each database you profile +USE []; +CREATE USER [] FROM LOGIN []; GRANT VIEW DEFINITION TO []; - --- Grant read access to INFORMATION_SCHEMA views -GRANT SELECT ON SCHEMA::INFORMATION_SCHEMA TO []; ``` :::tip -`VIEW SERVER STATE` is a server-level permission required to query `sys.dm_*` Dynamic Management Views. -`VIEW DEFINITION` allows the user to see the definitions of stored procedures and views. +`VIEW SERVER STATE` is a server-level permission required to query `sys.dm_*` Dynamic Management Views. It +also gives `VIEW DATABASE STATE` in every database, which `sys.dm_db_partition_stats` (table sizes) needs. +`VIEW DEFINITION` makes a database's tables, views, columns and stored procedures visible to the profiler. +Without it, that database's inventory and table sizes are empty or incomplete, and its stored procedure +names in `proc_stats` are missing. Granting it per database keeps databases you don't profile out of reach. +On SQL Server 2022 and later, these two grants also cover the finer-grained performance and security +definition permissions. `INFORMATION_SCHEMA` views need no grant: every user can query them, and what they +return depends on `VIEW DEFINITION`. These grants should be executed by a `sysadmin` or a login with `CONTROL SERVER` permission. ::: @@ -90,7 +97,6 @@ USE []; CREATE USER [] FROM LOGIN []; GRANT VIEW DATABASE STATE TO []; GRANT VIEW DEFINITION TO []; -GRANT SELECT ON SCHEMA::INFORMATION_SCHEMA TO []; ``` :::tip diff --git a/src/databricks/labs/lakebridge/resources/assessments/mssql/columns.sql b/src/databricks/labs/lakebridge/resources/assessments/mssql/columns.sql index de9c008148..d7ebb41304 100644 --- a/src/databricks/labs/lakebridge/resources/assessments/mssql/columns.sql +++ b/src/databricks/labs/lakebridge/resources/assessments/mssql/columns.sql @@ -2,6 +2,7 @@ * Retrieves column-level metadata for all tables and views in the specified * database by querying INFORMATION_SCHEMA.COLUMNS. Returns column attributes * along with a timestamp indicating when the data was extracted. + * System objects (is_ms_shipped) and the sys and cdc (change data capture) schemas are excluded. */ SELECT DB_NAME() AS DATABASE_NAME, TABLE_CATALOG, @@ -27,4 +28,12 @@ SELECT DB_NAME() AS DATABASE_NAME, DOMAIN_CATALOG, DOMAIN_SCHEMA, DOMAIN_NAME -FROM INFORMATION_SCHEMA.COLUMNS; +FROM INFORMATION_SCHEMA.COLUMNS +WHERE TABLE_SCHEMA NOT IN (N'sys', N'cdc') + AND NOT EXISTS (SELECT 1 + FROM sys.all_objects AS o + JOIN sys.schemas AS s + ON s.schema_id = o.schema_id + WHERE o.is_ms_shipped = 1 + AND s.[name] = TABLE_SCHEMA + AND o.[name] = TABLE_NAME); diff --git a/src/databricks/labs/lakebridge/resources/assessments/mssql/multi_db/columns.sql b/src/databricks/labs/lakebridge/resources/assessments/mssql/multi_db/columns.sql index bf49b425dc..b05ec928e6 100644 --- a/src/databricks/labs/lakebridge/resources/assessments/mssql/multi_db/columns.sql +++ b/src/databricks/labs/lakebridge/resources/assessments/mssql/multi_db/columns.sql @@ -3,6 +3,7 @@ * database by dynamically UNION-ing INFORMATION_SCHEMA.COLUMNS from each database, tagging each row * with its source `database_name`. The WHERE 1 = 0 anchor guarantees a single, correctly-typed result * set. Requires three-part naming (on-prem SQL Server / Azure SQL Managed Instance only). + * System objects (is_ms_shipped) and the sys and cdc (change data capture) schemas are excluded. */ SET NOCOUNT ON; @@ -30,6 +31,9 @@ DECLARE @sql NVARCHAR(MAX) = SELECT @sql = @sql + ISNULL(( SELECT ' UNION ALL SELECT N' + QUOTENAME([name], '''') + ' AS DATABASE_NAME, ' + @cols + ' FROM ' + QUOTENAME([name]) + '.INFORMATION_SCHEMA.COLUMNS' + + ' WHERE TABLE_SCHEMA NOT IN (N''sys'', N''cdc'') AND NOT EXISTS (SELECT 1 FROM ' + QUOTENAME([name]) + '.sys.all_objects AS o' + + ' JOIN ' + QUOTENAME([name]) + '.sys.schemas AS s ON s.schema_id = o.schema_id' + + ' WHERE o.is_ms_shipped = 1 AND s.[name] = TABLE_SCHEMA AND o.[name] = TABLE_NAME)' FROM sys.databases WHERE state_desc = 'ONLINE' AND HAS_DBACCESS([name]) = 1 diff --git a/src/databricks/labs/lakebridge/resources/assessments/mssql/multi_db/routines.sql b/src/databricks/labs/lakebridge/resources/assessments/mssql/multi_db/routines.sql index c809b139c8..269f1bbf03 100644 --- a/src/databricks/labs/lakebridge/resources/assessments/mssql/multi_db/routines.sql +++ b/src/databricks/labs/lakebridge/resources/assessments/mssql/multi_db/routines.sql @@ -4,6 +4,7 @@ * database, tagging each row with its source `database_name`. ROUTINE_DEFINITION is redacted, matching * the single-database extract. The WHERE 1 = 0 anchor guarantees a single, correctly-typed result set. * Requires three-part naming (on-prem SQL Server / Azure SQL Managed Instance only). + * System objects (is_ms_shipped) and the sys and cdc (change data capture) schemas are excluded. */ SET NOCOUNT ON; @@ -30,6 +31,9 @@ DECLARE @sql NVARCHAR(MAX) = SELECT @sql = @sql + ISNULL(( SELECT ' UNION ALL SELECT N' + QUOTENAME([name], '''') + ' AS DATABASE_NAME, ' + @cols + ' FROM ' + QUOTENAME([name]) + '.INFORMATION_SCHEMA.ROUTINES' + + ' WHERE ROUTINE_SCHEMA NOT IN (N''sys'', N''cdc'') AND NOT EXISTS (SELECT 1 FROM ' + QUOTENAME([name]) + '.sys.all_objects AS o' + + ' JOIN ' + QUOTENAME([name]) + '.sys.schemas AS s ON s.schema_id = o.schema_id' + + ' WHERE o.is_ms_shipped = 1 AND s.[name] = ROUTINE_SCHEMA AND o.[name] = ROUTINE_NAME)' FROM sys.databases WHERE state_desc = 'ONLINE' AND HAS_DBACCESS([name]) = 1 diff --git a/src/databricks/labs/lakebridge/resources/assessments/mssql/multi_db/table_sizes.sql b/src/databricks/labs/lakebridge/resources/assessments/mssql/multi_db/table_sizes.sql index 2524c29106..bfca9941a9 100644 --- a/src/databricks/labs/lakebridge/resources/assessments/mssql/multi_db/table_sizes.sql +++ b/src/databricks/labs/lakebridge/resources/assessments/mssql/multi_db/table_sizes.sql @@ -3,7 +3,8 @@ * accessible, online user database. sys.dm_db_partition_stats is a database-scoped DMV that resolves * against the current database context, so this cannot use three-part naming; instead it loops the * databases and runs the extract under USE for each, accumulating rows tagged with their source - * `database_name`. On-prem SQL Server / Azure SQL Managed Instance only. + * `database_name`. On-prem SQL Server / Azure SQL Managed Instance only. As in the single-database + * query, rows come from the heap or clustered index only and system tables are excluded. */ SET NOCOUNT ON; @@ -31,7 +32,7 @@ WHILE @@FETCH_STATUS = 0 BEGIN SET @sql = N'USE ' + QUOTENAME(@name) + N';' + N' SELECT DB_NAME() AS DATABASE_NAME, o.[name] AS TableName,' - + N' SUM(ps.row_count) AS [RowCount],' + + N' SUM(CASE WHEN ps.index_id < 2 THEN ps.row_count ELSE 0 END) AS [RowCount],' + N' SUM(ps.reserved_page_count) * 8 / 1024 AS ReservedMB,' + N' SUM(ps.used_page_count) * 8 / 1024 AS UsedMB,' + N' (SUM(ps.reserved_page_count) - SUM(ps.used_page_count)) * 8 / 1024 AS UnusedMB,' @@ -41,7 +42,7 @@ BEGIN + N' SYSDATETIME() AS extract_ts' + N' FROM sys.dm_db_partition_stats AS ps' + N' JOIN sys.objects AS o ON ps.object_id = o.object_id' - + N' WHERE o.type = ''U''' + + N' WHERE o.type = ''U'' AND o.is_ms_shipped = 0' + N' GROUP BY SCHEMA_NAME(o.schema_id), o.[name]'; INSERT INTO @results EXEC sys.sp_executesql @sql; diff --git a/src/databricks/labs/lakebridge/resources/assessments/mssql/multi_db/tables.sql b/src/databricks/labs/lakebridge/resources/assessments/mssql/multi_db/tables.sql index fe7065a55f..543a4f3869 100644 --- a/src/databricks/labs/lakebridge/resources/assessments/mssql/multi_db/tables.sql +++ b/src/databricks/labs/lakebridge/resources/assessments/mssql/multi_db/tables.sql @@ -4,6 +4,7 @@ * tagged with its source database via the projected `database_name` literal. The leading WHERE 1 = 0 * anchor guarantees a single, correctly-typed result set even when no user database is accessible. * Requires three-part naming (on-prem SQL Server / Azure SQL Managed Instance only). + * System objects (is_ms_shipped) and the sys and cdc (change data capture) schemas are excluded. */ SET NOCOUNT ON; @@ -18,6 +19,9 @@ DECLARE @sql NVARCHAR(MAX) = SELECT @sql = @sql + ISNULL(( SELECT ' UNION ALL SELECT N' + QUOTENAME([name], '''') + ' AS DATABASE_NAME, ' + @cols + ' FROM ' + QUOTENAME([name]) + '.INFORMATION_SCHEMA.TABLES' + + ' WHERE TABLE_SCHEMA NOT IN (N''sys'', N''cdc'') AND NOT EXISTS (SELECT 1 FROM ' + QUOTENAME([name]) + '.sys.all_objects AS o' + + ' JOIN ' + QUOTENAME([name]) + '.sys.schemas AS s ON s.schema_id = o.schema_id' + + ' WHERE o.is_ms_shipped = 1 AND s.[name] = TABLE_SCHEMA AND o.[name] = TABLE_NAME)' FROM sys.databases WHERE state_desc = 'ONLINE' AND HAS_DBACCESS([name]) = 1 diff --git a/src/databricks/labs/lakebridge/resources/assessments/mssql/multi_db/views.sql b/src/databricks/labs/lakebridge/resources/assessments/mssql/multi_db/views.sql index 2ddd7adb60..9a1a560500 100644 --- a/src/databricks/labs/lakebridge/resources/assessments/mssql/multi_db/views.sql +++ b/src/databricks/labs/lakebridge/resources/assessments/mssql/multi_db/views.sql @@ -3,6 +3,7 @@ * dynamically UNION-ing INFORMATION_SCHEMA.VIEWS from each database, tagging each row with its source * `database_name`. VIEW_DEFINITION is redacted, matching the single-database extract. The WHERE 1 = 0 * anchor guarantees a single, correctly-typed result set. Requires three-part naming (on-prem / MI). + * System objects (is_ms_shipped) and the sys and cdc (change data capture) schemas are excluded. */ SET NOCOUNT ON; @@ -18,6 +19,9 @@ DECLARE @sql NVARCHAR(MAX) = SELECT @sql = @sql + ISNULL(( SELECT ' UNION ALL SELECT N' + QUOTENAME([name], '''') + ' AS DATABASE_NAME, ' + @cols + ' FROM ' + QUOTENAME([name]) + '.INFORMATION_SCHEMA.VIEWS' + + ' WHERE TABLE_SCHEMA NOT IN (N''sys'', N''cdc'') AND NOT EXISTS (SELECT 1 FROM ' + QUOTENAME([name]) + '.sys.all_objects AS o' + + ' JOIN ' + QUOTENAME([name]) + '.sys.schemas AS s ON s.schema_id = o.schema_id' + + ' WHERE o.is_ms_shipped = 1 AND s.[name] = TABLE_SCHEMA AND o.[name] = TABLE_NAME)' FROM sys.databases WHERE state_desc = 'ONLINE' AND HAS_DBACCESS([name]) = 1 diff --git a/src/databricks/labs/lakebridge/resources/assessments/mssql/proc_stats.sql b/src/databricks/labs/lakebridge/resources/assessments/mssql/proc_stats.sql index d056291b6d..6f858f1eb9 100644 --- a/src/databricks/labs/lakebridge/resources/assessments/mssql/proc_stats.sql +++ b/src/databricks/labs/lakebridge/resources/assessments/mssql/proc_stats.sql @@ -2,6 +2,9 @@ * Retrieves execution statistics for stored procedures from `sys.dm_exec_procedure_stats`, * including execution counts, total CPU and elapsed time, last execution timestamp, * and maps object and database IDs to their names. Results are ordered by most recent execution. + * The hidden Resource database (database_id 32767) is excluded: it holds SQL Server's own system + * procedures, whose names never resolve. Names of user procedures resolve only where the profiling + * login has VIEW DEFINITION in that database. */ SELECT database_id, Db_name(database_id) AS db_name, @@ -14,4 +17,5 @@ SELECT database_id, total_elapsed_time, Sysdatetime() AS extract_ts FROM sys.dm_exec_procedure_stats +WHERE database_id <> 32767 ORDER BY last_execution_time diff --git a/src/databricks/labs/lakebridge/resources/assessments/mssql/routines.sql b/src/databricks/labs/lakebridge/resources/assessments/mssql/routines.sql index d97e45c386..1b7bf1961e 100644 --- a/src/databricks/labs/lakebridge/resources/assessments/mssql/routines.sql +++ b/src/databricks/labs/lakebridge/resources/assessments/mssql/routines.sql @@ -2,6 +2,7 @@ * Retrieves metadata for all routines (stored procedures and functions) in the * specified database by querying INFORMATION_SCHEMA.ROUTINES. Returns routine * details along with a timestamp indicating when the data was extracted. + * System objects (is_ms_shipped) and the sys and cdc (change data capture) schemas are excluded. */ SELECT DB_NAME() AS DATABASE_NAME, CREATED, @@ -26,4 +27,12 @@ SELECT DB_NAME() AS DATABASE_NAME, SPECIFIC_NAME, SPECIFIC_SCHEMA, SQL_DATA_ACCESS -FROM INFORMATION_SCHEMA.ROUTINES; +FROM INFORMATION_SCHEMA.ROUTINES +WHERE ROUTINE_SCHEMA NOT IN (N'sys', N'cdc') + AND NOT EXISTS (SELECT 1 + FROM sys.all_objects AS o + JOIN sys.schemas AS s + ON s.schema_id = o.schema_id + WHERE o.is_ms_shipped = 1 + AND s.[name] = ROUTINE_SCHEMA + AND o.[name] = ROUTINE_NAME); diff --git a/src/databricks/labs/lakebridge/resources/assessments/mssql/table_sizes.sql b/src/databricks/labs/lakebridge/resources/assessments/mssql/table_sizes.sql index 2b27672e4c..7627890aae 100644 --- a/src/databricks/labs/lakebridge/resources/assessments/mssql/table_sizes.sql +++ b/src/databricks/labs/lakebridge/resources/assessments/mssql/table_sizes.sql @@ -3,12 +3,17 @@ * database by querying sys.dm_db_partition_stats and sys.objects. Returns table * name, total rows, reserved, used, and unused space (MB), breakdown of data vs. * index space, and a timestamp indicating when the data was extracted. + * Rows are counted from the heap or clustered index only (index_id 0 or 1): every nonclustered index + * repeats the table's rows. System tables (is_ms_shipped, e.g. change data capture) are excluded. */ SELECT DB_NAME() AS DATABASE_NAME, o.NAME AS TableName, - Sum(ps.row_count) AS + Sum(CASE + WHEN ps.index_id < 2 THEN ps.row_count + ELSE 0 + END) AS [RowCount], Sum(ps.reserved_page_count) * 8 / 1024 AS ReservedMB, @@ -34,5 +39,6 @@ FROM sys.dm_db_partition_stats AS ps JOIN sys.objects AS o ON ps.object_id = o.object_id WHERE o.type = 'U' + AND o.is_ms_shipped = 0 GROUP BY Schema_name(o.schema_id), o.NAME; diff --git a/src/databricks/labs/lakebridge/resources/assessments/mssql/tables.sql b/src/databricks/labs/lakebridge/resources/assessments/mssql/tables.sql index f471726fb1..b41bdc0a91 100644 --- a/src/databricks/labs/lakebridge/resources/assessments/mssql/tables.sql +++ b/src/databricks/labs/lakebridge/resources/assessments/mssql/tables.sql @@ -2,10 +2,19 @@ * Retrieves metadata for all tables in the specified database by querying * INFORMATION_SCHEMA.TABLES. Returns table definitions along with a timestamp * indicating when the data was extracted. + * System objects (is_ms_shipped) and the sys and cdc (change data capture) schemas are excluded. */ SELECT DB_NAME() AS DATABASE_NAME, TABLE_CATALOG, TABLE_SCHEMA, TABLE_NAME, TABLE_TYPE -FROM INFORMATION_SCHEMA.TABLES; +FROM INFORMATION_SCHEMA.TABLES +WHERE TABLE_SCHEMA NOT IN (N'sys', N'cdc') + AND NOT EXISTS (SELECT 1 + FROM sys.all_objects AS o + JOIN sys.schemas AS s + ON s.schema_id = o.schema_id + WHERE o.is_ms_shipped = 1 + AND s.[name] = TABLE_SCHEMA + AND o.[name] = TABLE_NAME); diff --git a/src/databricks/labs/lakebridge/resources/assessments/mssql/views.sql b/src/databricks/labs/lakebridge/resources/assessments/mssql/views.sql index 56c42297d9..09a90800b0 100644 --- a/src/databricks/labs/lakebridge/resources/assessments/mssql/views.sql +++ b/src/databricks/labs/lakebridge/resources/assessments/mssql/views.sql @@ -2,6 +2,7 @@ * Retrieves metadata for all views in the specified database by querying * `INFORMATION_SCHEMA.VIEWS`. Returns view definitions along with a timestamp * indicating when the data was extracted. + * System objects (is_ms_shipped) and the sys and cdc (change data capture) schemas are excluded. */ SELECT DB_NAME() AS DATABASE_NAME, TABLE_CATALOG, @@ -11,3 +12,11 @@ SELECT DB_NAME() AS DATABASE_NAME, IS_UPDATABLE, N'[REDACTED]' AS VIEW_DEFINITION FROM INFORMATION_SCHEMA.VIEWS +WHERE TABLE_SCHEMA NOT IN (N'sys', N'cdc') + AND NOT EXISTS (SELECT 1 + FROM sys.all_objects AS o + JOIN sys.schemas AS s + ON s.schema_id = o.schema_id + WHERE o.is_ms_shipped = 1 + AND s.[name] = TABLE_SCHEMA + AND o.[name] = TABLE_NAME) diff --git a/tests/unit/assessment/test_mssql_sql_rules.py b/tests/unit/assessment/test_mssql_sql_rules.py index 7257c510b8..78649a01ae 100644 --- a/tests/unit/assessment/test_mssql_sql_rules.py +++ b/tests/unit/assessment/test_mssql_sql_rules.py @@ -100,3 +100,29 @@ def test_sys_info_defaults_version_dependent_columns(column: str) -> None: # Without a NULL default, the whole sys_info query fails to compile on versions that lack the column. sql = (_MSSQL / "sys_info.sql").read_text(encoding="utf-8") assert re.search(rf"CAST\(NULL AS [\w() ]+\)\s+AS\s+{column}\b", sql) + + +@pytest.mark.parametrize("query", [_MSSQL / "table_sizes.sql", _MSSQL / "multi_db/table_sizes.sql"], ids=_name) +def test_table_sizes_count_rows_from_heap_or_clustered_index(query: Path) -> None: + # Every nonclustered index repeats the table's rows, so summing all indexes multiplies the row count. + sql = _without_comments(query.read_text(encoding="utf-8")) + assert re.search(r"WHEN ps\.index_id < 2 THEN ps\.row_count", sql, re.IGNORECASE) + assert not re.search(r"SUM\(\s*ps\.row_count\s*\)", sql, re.IGNORECASE) + + +_INVENTORY = [ + _MSSQL / folder / f"{name}.sql" + for folder in ("", "multi_db") + for name in ("tables", "views", "columns", "routines", "table_sizes") +] + + +@pytest.mark.parametrize("query", _INVENTORY, ids=_name) +def test_inventory_excludes_system_objects(query: Path) -> None: + # Change data capture and change tracking add system tables and routines to user databases. + assert "is_ms_shipped" in _without_comments(query.read_text(encoding="utf-8")) + + +def test_proc_stats_excludes_resource_database() -> None: + # Procedures in the hidden Resource database are SQL Server's own, and their names never resolve. + assert "database_id <> 32767" in _without_comments((_MSSQL / "proc_stats.sql").read_text(encoding="utf-8"))