Describe CSV

WHAT-WE-LEARNED.md

@@ -119,3 +119,7 @@ Selectize menus should overflow the containing card.
 */
 .card, .card-body { overflow: visible !important; }
 ```
+
+## Hard to make elements with controls display conditionally.
+
+I tried moving a `ui.input_selectize` out of the top-level UI function because I needed it to be conditional, but the event that should have updated the list no longer worked: I guess it's not visible if it's not part of the static render?

dp_wizard/shiny/__init__.py

@@ -18,7 +18,7 @@
 from dp_wizard.types import AppState, Product
 from dp_wizard.utils import config
 from dp_wizard.utils.argparse_helpers import CLIInfo
-from dp_wizard.utils.csv_helper import read_csv_names
+from dp_wizard.utils.csv_helper import read_csv_names, read_csv_numeric_names
 
 _shiny_root = package_root / "shiny"
 _assets_root = _shiny_root / "assets"
@@ -211,10 +211,14 @@ def server(input: Inputs, output: Outputs, session: Session):  # pragma: no cove
             initial_private_csv_path = package_root / "tmp/sample.csv"
             _make_sample_csv(initial_private_csv_path, initial_contributions)
             initial_column_names = read_csv_names(Path(initial_private_csv_path))
+            initial_numeric_column_names = read_csv_numeric_names(
+                Path(initial_private_csv_path)
+            )
         else:
             initial_contributions = 1
             initial_private_csv_path = ""
             initial_column_names = []
+            initial_numeric_column_names = []
 
         initial_product = Product.STATISTICS
 
@@ -236,7 +240,8 @@ def server(input: Inputs, output: Outputs, session: Session):  # pragma: no cove
             initial_product=initial_product,
             product=reactive.value(initial_product),
             # Analysis choices:
-            column_names=reactive.value(initial_column_names),
+            all_column_names=reactive.value(initial_column_names),
+            numeric_column_names=reactive.value(initial_numeric_column_names),
             groups=reactive.value([]),
             epsilon=reactive.value(1.0),
             # Per-column choices:

dp_wizard/shiny/components/summaries.py

@@ -8,7 +8,7 @@
     product_icon,
     unit_of_privacy_icon,
 )
-from dp_wizard.types import AppState
+from dp_wizard.types import AppState, Product
 
 _css = "display: block; padding: 0 1em 1em 1em;"
 
@@ -47,10 +47,16 @@ def analysis_summary(state: AppState):  # pragma: no cover
     budget = state.epsilon()
 
     return tags.small(
-        columns_icon,
-        f"Columns: {columns}; ",
-        groups_icon,
-        f"Groups: {groups}; ",
+        (
+            []
+            if state.product() == Product.CSV_DESCRIPTION
+            else [
+                columns_icon,
+                f"Columns: {columns}; ",
+                groups_icon,
+                f"Groups: {groups}; ",
+            ]
+        ),
         budget_icon,
         f"Privacy Budget: {budget} epsilon.",
         style=_css,

dp_wizard/shiny/panels/analysis_panel/__init__.py

@@ -21,7 +21,7 @@
 )
 from dp_wizard.shiny.components.summaries import dataset_summary
 from dp_wizard.shiny.panels.analysis_panel.column_module import column_server, column_ui
-from dp_wizard.types import AppState
+from dp_wizard.types import AppState, Product
 from dp_wizard.utils.code_generators import make_privacy_loss_block
 from dp_wizard.utils.csv_helper import (
     get_csv_row_count,
@@ -36,17 +36,19 @@ def analysis_ui():
         ui.output_ui("analysis_requirements_warning_ui"),
         ui.output_ui("analysis_release_warning_ui"),
         ui.output_ui("previous_summary_ui"),
+        ui.output_ui("conditional_css_ui"),
         ui.layout_columns(
             ui.card(
                 ui.card_header(columns_icon, "Columns"),
-                ui.markdown("Select columns to calculate statistics on."),
+                ui.markdown("Select numeric columns to calculate statistics on."),
                 ui.input_selectize(
                     "columns_selectize",
                     "Columns",
                     [],
                     multiple=True,
                 ),
                 ui.output_ui("columns_selectize_tutorial_ui"),
+                class_="columns-card",
             ),
             ui.card(
                 ui.card_header(groups_icon, "Grouping"),
@@ -66,6 +68,7 @@ def analysis_ui():
                     multiple=True,
                 ),
                 ui.output_ui("groups_selectize_tutorial_ui"),
+                class_="grouping-card",
             ),
             ui.card(
                 ui.card_header(budget_icon, "Privacy Budget"),
@@ -83,10 +86,12 @@ def analysis_ui():
                 log_slider("log_epsilon_slider", 0.1, 10.0),
                 ui.output_ui("epsilon_ui"),
                 ui.output_ui("privacy_loss_python_ui"),
+                class_="budget-card",
             ),
             ui.card(
                 ui.card_header(simulation_icon, "Simulation"),
                 ui.output_ui("simulation_card_ui"),
+                class_="simulation-card",
             ),
             col_widths={
                 "sm": [12, 12, 12, 12],  # 4 rows
@@ -152,10 +157,11 @@ def analysis_server(
     # contributions_entity = state.contributions_entity
     max_rows = state.max_rows
     # initial_product = state.initial_product
-    # product = state.product
+    product = state.product
 
     # Analysis choices:
-    column_names = state.column_names
+    all_column_names = state.all_column_names
+    numeric_column_names = state.numeric_column_names
     groups = state.groups
     epsilon = state.epsilon
 
@@ -173,26 +179,37 @@ def analysis_server(
 
     @reactive.calc
     def button_enabled():
+        # TODO: Get this in sync with results panel warning:
+        # https://github.com/opendp/dp-wizard/issues/562
         at_least_one_column = bool(weights())
         no_errors = not any(analysis_errors().values())
-        return at_least_one_column and no_errors
+        return (
+            at_least_one_column and no_errors
+        ) or product() == Product.CSV_DESCRIPTION
 
     @reactive.effect
     def _update_columns():
-        csv_ids_labels = {
+        all_ids_labels = {
             # Cast to string for type checking.
-            str(k): v
-            for k, v in csv_ids_labels_calc().items()
+            str(col_id): label
+            for col_id, label in csv_ids_labels_calc().items()
         }
         ui.update_selectize(
             "groups_selectize",
             label=None,
-            choices=csv_ids_labels,
+            choices=all_ids_labels,
         )
+
+        numeric_column_ids = id_names_dict_from_names(numeric_column_names()).keys()
+        numeric_ids_labels = {
+            col_id: label
+            for col_id, label in all_ids_labels.items()
+            if col_id in numeric_column_ids
+        }
         ui.update_selectize(
             "columns_selectize",
             label=None,
-            choices=csv_ids_labels,
+            choices=numeric_ids_labels,
         )
 
     @reactive.effect
@@ -205,7 +222,7 @@ def _on_groups_change():
     @render.ui
     def analysis_requirements_warning_ui():
         return hide_if(
-            bool(column_names()),
+            bool(all_column_names()),
             info_md_box(
                 """
                 Please select your dataset on the previous tab
@@ -227,6 +244,26 @@ def analysis_release_warning_ui():
             ),
         )
 
+    @render.ui
+    def conditional_css_ui():
+        # This is hacky, but other approaches for conditional card display
+        # didn't work for me.
+        # - Adding a wrapping element caused the card not to fill the whole height.
+        # - The selectize lists for columns and groups weren't updating.
+        # If we can find something better, great!
+        if product() == Product.CSV_DESCRIPTION:
+            return ui.tags.style(
+                """
+                .bslib-grid-item:has(
+                    .columns-card,
+                    .grouping-card,
+                    .simulation-card
+                ) {
+                    display: none;
+                }
+                """
+            )
+
     @render.ui
     def previous_summary_ui():
         return dataset_summary(state)
@@ -359,11 +396,11 @@ def columns_ui():
 
     @reactive.calc
     def csv_ids_names_calc():
-        return id_names_dict_from_names(column_names())
+        return id_names_dict_from_names(all_column_names())
 
     @reactive.calc
     def csv_ids_labels_calc():
-        return id_labels_dict_from_names(column_names())
+        return id_labels_dict_from_names(all_column_names())
 
     @reactive.effect
     @reactive.event(input.log_epsilon_slider)

dp_wizard/shiny/panels/dataset_panel/__init__.py

@@ -26,7 +26,11 @@
     PUBLIC_TEXT,
 )
 from dp_wizard.utils.code_generators import make_privacy_unit_block
-from dp_wizard.utils.csv_helper import get_csv_names_mismatch, read_csv_names
+from dp_wizard.utils.csv_helper import (
+    get_csv_names_mismatch,
+    read_csv_names,
+    read_csv_numeric_names,
+)
 
 dataset_panel_id = "dataset_panel"
 
@@ -127,7 +131,8 @@ def dataset_server(
     product = state.product
 
     # Analysis choices:
-    column_names = state.column_names
+    all_column_names = state.all_column_names
+    numeric_column_names = state.numeric_column_names
     # groups = state.groups
     # epsilon = state.epsilon
 
@@ -148,25 +153,27 @@ def dataset_server(
     def _on_public_csv_path_change():
         path = input.public_csv_path()[0]["datapath"]
         public_csv_path.set(path)
-        column_names.set(read_csv_names(Path(path)))
+        all_column_names.set(read_csv_names(Path(path)))
+        numeric_column_names.set(read_csv_numeric_names(Path(path)))
 
     @reactive.effect
     @reactive.event(input.private_csv_path)
     def _on_private_csv_path_change():
         path = input.private_csv_path()[0]["datapath"]
         private_csv_path.set(path)
-        column_names.set(read_csv_names(Path(path)))
+        all_column_names.set(read_csv_names(Path(path)))
+        numeric_column_names.set(read_csv_numeric_names(Path(path)))
 
     @reactive.effect
-    @reactive.event(input.column_names)
+    @reactive.event(input.all_column_names)
     def _on_column_names_change():
-        column_names.set(
-            [
-                clean
-                for line in input.column_names().splitlines()
-                if (clean := line.strip())
-            ]
-        )
+        column_names = [
+            clean
+            for line in input.all_column_names().splitlines()
+            if (clean := line.strip())
+        ]
+        all_column_names.set(column_names)
+        numeric_column_names.set(column_names)
 
     @reactive.calc
     def csv_column_mismatch_calc() -> Optional[tuple[set, set]]:
@@ -239,7 +246,7 @@ def csv_or_columns_ui():
                     """,
                     responsive=False,
                 ),
-                ui.input_text_area("column_names", "CSV Column Names", rows=5),
+                ui.input_text_area("all_column_names", "CSV Column Names", rows=5),
             ]
         else:
             content = [
@@ -452,7 +459,7 @@ def button_enabled():
         return (
             contributions_valid()
             and not get_row_count_errors(max_rows())
-            and len(column_names()) > 0
+            and len(all_column_names()) > 0
             and (in_cloud or not csv_column_mismatch_calc())
         )
 
@@ -578,17 +585,20 @@ def product_ui():
             ),
             tutorial_box(
                 is_tutorial_mode(),
-                """
+                f"""
                 Although the underlying OpenDP library is very flexible,
                 DP Wizard offers only a few analysis options:
 
-                - The **DP Statistics** option supports
+                - The **{Product.STATISTICS}** option supports
                   grouping, histograms, mean, median, and count.
-                - With **DP Synthetic Data**, your privacy budget is used
+                - With **{Product.SYNTHETIC_DATA}**, your privacy budget is used
                   to infer the distributions of values within the
                   selected columns, and the correlations between columns.
                   This is less accurate than calculating the desired
                   statistics directly, but can be easier to work with downstream.
+                - The **{Product.CSV_DESCRIPTION}** summarizes the contents of CSVs
+                  with a large number of columns, without revealing details
+                  from individual rows.
                 """,
                 responsive=False,
             ),

dp_wizard/shiny/panels/results_panel/__init__.py

@@ -16,7 +16,7 @@
     tutorial_box,
 )
 from dp_wizard.shiny.components.summaries import analysis_summary, dataset_summary
-from dp_wizard.types import AppState
+from dp_wizard.types import AppState, Product
 from dp_wizard.utils.code_generators import AnalysisPlan, AnalysisPlanColumn
 from dp_wizard.utils.code_generators.notebook_generator import (
     PLACEHOLDER_CSV_NAME,
@@ -115,7 +115,8 @@ def results_server(
     product = state.product
 
     # Analysis choices:
-    # column_names = state.column_names
+    # all_column_names = state.all_column_names
+    # numeric_column_names = state.numeric_column_names
     groups = state.groups
     epsilon = state.epsilon
 
@@ -134,7 +135,9 @@ def results_server(
     @render.ui
     def results_requirements_warning_ui():
         return hide_if(
-            bool(weights()),
+            # TODO: Get this in sync with analysis_panel validation
+            # https://github.com/opendp/dp-wizard/issues/562
+            bool(weights()) or product() == Product.CSV_DESCRIPTION,
             info_md_box(
                 """
                 Please define your analysis on the previous tab
@@ -195,7 +198,7 @@ def clean_download_stem() -> str:
     def download_results_ui():
         if in_cloud:
             return None
-        disabled = not weights()
+        disabled = not (weights() or product() == Product.CSV_DESCRIPTION)
         return [
             ui.h3("Download Results"),
             tutorial_box(