diff --git a/deployments/apple_iconic_scenes_photo_selections_2023.yaml b/deployments/apple_iconic_scenes_photo_selections_2023.yaml index b46362b0..a6aa569a 100644 --- a/deployments/apple_iconic_scenes_photo_selections_2023.yaml +++ b/deployments/apple_iconic_scenes_photo_selections_2023.yaml @@ -1,6 +1,4 @@ url_slug: iconic-scenes-and-photo-sections-for-ios-apps-apple-2023 -status: Draft -tier: 3 deployment: basic: # Tier 1: @@ -9,12 +7,12 @@ deployment: - data_curator_name: 'Apple' data_curator_type: 'Industry' + description: This deployment identifies frequently photographed iconic locations across users, enabling Apple’s Photos app to automatically select representative images for features such as Memories. The system is designed to protect individual user data while providing personalized content. intended_use: Identify frequently photographed iconic locations across users, enabling Apple’s Photos app to automatically select representative images for features such as Memories, all while protecting individual user data. data_product_type: Machine learning model data_product_region: Global - description: This deployment identifies frequently photographed iconic locations across users, enabling Apple’s Photos app to automatically select representative images for features such as Memories. The system is designed to protect individual user data while providing personalized content. - publication_date: '2023-07-21' data_product_sector: Technology + publication_date: '2023-07-21' # Tiers 2 and 3: dp_variant: @@ -32,19 +30,12 @@ deployment: deployment_model: model_name: Local - model_name_details: Local Differential Privacy - # actors: '' release_type: Many releases release_type_details: Image data is collected from users who opted in to the Analytics & Improvements feature on their Apple devices and be used for improving features in Apple’s Photos app. data_source_type: Dynamic - # data_source_type_details: '' access_type: Non-interactive # access_type_details: '' - # accounting: - # post_processing: '' # Tier 3 - # composition: '' # Tier 3 - implementation: # preprocessing_and_hyperparameter_tuning: '' # Tier 3 mechanisms: | @@ -58,5 +49,7 @@ deployment: sources: | - Blog post: https://machinelearning.apple.com/research/scenes-differential-privacy # notes: '' # other extra information + status: Draft registry_authors: - Michael Khavkin + tier: 3 diff --git a/deployments/apple_popular_emojis.yaml b/deployments/apple_popular_emojis.yaml index 65473aac..047dd48a 100644 --- a/deployments/apple_popular_emojis.yaml +++ b/deployments/apple_popular_emojis.yaml @@ -1,6 +1,4 @@ url_slug: popular-emojis-apple-2017 -status: Pending -tier: 3 deployment: basic: name: Popular Emojis @@ -8,12 +6,12 @@ deployment: - data_curator_name: 'Apple' data_curator_type: 'Industry' + description: Summary statistics of frequency histograms of emoji usage per locale. intended_use: For measurement (within Apple) of which emojis are most used, and their relative frequencies, across keyboard locales, to improve QuickType’s predictive emoji suggestions. data_product_type: Summary statistics data_product_region: Global - description: Summary statistics of frequency histograms of emoji usage per locale. - publication_date: '2017-01-01' data_product_sector: Technology + publication_date: '2017-01-01' dp_variant: variant_name: Pure DP @@ -47,15 +45,6 @@ deployment: access_type: Non-interactive # access_type_details: '' - accounting: - post_processing: | - - Server discards IP addresses associated with each record. - - Aggregation and Sketch Matrix Construction: The server aggregates the anonymized, private records into a sketch matrix, which is a summary grid with 65,536 rows and 1,024 columns. Each row corresponds to a possible hash function that the user’s device could have used. For each private report received, the server uses the included index to identify which row to update , and then adds the user's noisy data to that specific row. The random noise tends to average out, allowing the underlying patterns of popular emojis to be estimated in the next step. - - Frequency Estimation and Debiasing: The server estimates the frequency of each emoji in the dictionary based on the sketch matrix, averages these values, and debiases the result, producing a final estimated count for that emoji. - - Final Histogram Generation and Thresholding: The estimated counts for all emojis are assembled into a histogram, and thresholding is performed so that only emojis whose estimated counts are above a predetermined threshold are included in the final data product. - - composition: No information was provided about composition. - implementation: preprocessing_and_hyperparameter_tuning: 'Three parameters are explicitly set: \\(\epsilon = 4\\), number of hash functions k = 65,546 and the size of the privatized vector m = 1024. These were predetermined by the authors based on an analysis of the tradeoff between utility, server computation cost (related to k), and device bandwidth (related to m).' mechanisms: | @@ -69,18 +58,25 @@ deployment: - The privatized records are temporarily stored on-device using data protection, rather than being immediately transmitted to the server. After a delay based on device conditions, the system randomly samples from the differentially private records subject to a predetermined limit on the number of privatized records that can be transmitted daily for each use case, and sends the sampled records to the server. These records do not include device identifiers or timestamps of when events were generated. - After the data is privatized, the resulting records are not sent instantaneously, but are stored on-device using data protection and sent as part of a random batch once a day. - The privatized records are sent to the server with TLS encryption. - + composition: No information was provided about composition. + post_processing: | + - Server discards IP addresses associated with each record. + - Aggregation and Sketch Matrix Construction: The server aggregates the anonymized, private records into a sketch matrix, which is a summary grid with 65,536 rows and 1,024 columns. Each row corresponds to a possible hash function that the user’s device could have used. For each private report received, the server uses the included index to identify which row to update , and then adds the user's noisy data to that specific row. The random noise tends to average out, allowing the underlying patterns of popular emojis to be estimated in the next step. + - Frequency Estimation and Debiasing: The server estimates the frequency of each emoji in the dictionary based on the sketch matrix, averages these values, and debiases the result, producing a final estimated count for that emoji. + - Final Histogram Generation and Thresholding: The estimated counts for all emojis are assembled into a histogram, and thresholding is performed so that only emojis whose estimated counts are above a predetermined threshold are included in the final data product. justification: | - The choice of \\(\epsilon\\) was “based on the privacy characteristics of the underlying dataset” and is “consistent with the parameters proposed in the differential privacy research community”. - The CMS algorithm yields a high number of hash collisions (mapping 2,600 emojis to 1024 bits) providing further plausible deniability. + additional_information: | + - Under "Security Measures": "The privatized records are temporarily stored on-device using data protection"; the paper points to this link https://www.apple.com/business/docs/iOS_Security_Guide.pdf + - The paper covers collecting aggregate statistics for different use cases: trending new words, popular emojis, popular HealthKit data types, Safari media-playback preferences, and high energy/memory domains. We only treated the popular emojis use case, treating it as one deployment. administrative: sources: | - Paper: https://docs-assets.developer.apple.com/ml-research/papers/learning-with-privacy-at-scale.pdf - notes: | - - Under "Security Measures": "The privatized records are temporarily stored on-device using data protection"; the paper points to this link https://www.apple.com/business/docs/iOS_Security_Guide.pdf - - The paper covers collecting aggregate statistics for different use cases: trending new words, popular emojis, popular HealthKit data types, Safari media-playback preferences, and high energy/memory domains. We only treated the popular emojis use case, treating it as one deployment. + status: Pending registry_authors: - Elena Ghazi - Nicolas Berrios - Jack Fitzsimons + tier: 3 diff --git a/deployments/assistive_ai.yaml b/deployments/assistive_ai.yaml index 4e96a007..db507119 100644 --- a/deployments/assistive_ai.yaml +++ b/deployments/assistive_ai.yaml @@ -1,6 +1,4 @@ url_slug: assistive-ai-microsoft-2020 -status: Pending -tier: 2 deployment: basic: name: Assistive AI @@ -8,12 +6,12 @@ deployment: - data_curator_name: 'Microsoft' data_curator_type: 'Industry' + description: Privately refining the set of responses suggested for email and chat. intended_use: The AI system was trained internally at Microsoft to be deployed in their products. The model provides suggestions for replies to emails and texts, and is intended to make responding to emails and texts easier for users. The underlying data are “hundreds of millions of Messages (Emails or chats) and their Replies called Message-Reply (MR) pairs.” data_product_type: Machine learning model data_product_region: Global - description: Privately refining the set of responses suggested for email and chat. - publication_date: '2020-01-01' data_product_sector: Technology + publication_date: '2020-01-01' dp_variant: variant_name: Approximate DP @@ -32,14 +30,17 @@ deployment: preprocessing_and_hyperparameter_tuning: '' # TODO: Fill in correct value mechanisms: Differentially Private Set Union justification: '' # TODO: Fill in correct value + additional_information: | + - Restricted to some languages (EN, ES, FR, PR, IT) + administrative: sources: | - Blog post: https://www.microsoft.com/en-us/research/articles/assistive-ai-makes-replying-easier-2/ - Paper: https://arxiv.org/pdf/2002.09745 - notes: | - - Restricted to some languages (EN, ES, FR, PR, IT) + status: Pending registry_authors: - Priyanka Nanayakkara - Nicolas Berrios - Jack Fitzsimons - James Honaker + tier: 2 diff --git a/deployments/audience_engagement.yaml b/deployments/audience_engagement.yaml index e4c82230..0f2d8335 100644 --- a/deployments/audience_engagement.yaml +++ b/deployments/audience_engagement.yaml @@ -1,6 +1,4 @@ url_slug: audience-engagement-api-linkedin-2020 -status: Pending -tier: 1 deployment: basic: name: Audience Engagement API @@ -8,12 +6,12 @@ deployment: - data_curator_name: 'LinkedIn' data_curator_type: 'Industry' + description: Aggregated summary statistics about content engagement with LinkedIn posts. intended_use: For data analysts to learn about the LinkedIn user base. For example, marketers may want to learn about the kind of content LinkedIn users engage with before they create sponsored content. data_product_type: Interactive API data_product_region: Global - description: Aggregated summary statistics about content engagement with LinkedIn posts. - publication_date: '2020-01-01' data_product_sector: Technology + publication_date: '2020-01-01' dp_variant: variant_name: Approximate DP data_domain: '' # TODO @@ -36,13 +34,16 @@ deployment: preprocessing_and_hyperparameter_tuning: '' # TODO: Fill in correct value mechanisms: Laplace, Gumbel justification: '' # TODO: Fill in correct value + additional_information: | + - The paper states that “data is purged within 30 days of a user leaving the ecosystem because it has a 30 day retention.” We take this to mean that the data are refreshed on a monthly basis, i.e., that leaving the ecosystem means leaving a single user session on LinkedIn as opposed to account deletion. + administrative: sources: | - Paper: https://arxiv.org/pdf/2002.05839 - notes: | - - The paper states that “data is purged within 30 days of a user leaving the ecosystem because it has a 30 day retention.” We take this to mean that the data are refreshed on a monthly basis, i.e., that leaving the ecosystem means leaving a single user session on LinkedIn as opposed to account deletion. + status: Pending registry_authors: - Priyanka Nanayakkara - Nicolas Berrios - Jack Fitzsimons - James Honaker + tier: 1 diff --git a/deployments/autoplay_intent.yaml b/deployments/autoplay_intent.yaml index de99dd65..ec99b582 100644 --- a/deployments/autoplay_intent.yaml +++ b/deployments/autoplay_intent.yaml @@ -1,6 +1,4 @@ url_slug: autoplay-intent-apple-2017 -status: Pending -tier: 1 deployment: basic: name: AutoPlay Intent @@ -8,12 +6,12 @@ deployment: - data_curator_name: 'Apple' data_curator_type: 'Industry' + description: Identify which websites users typically prefer to have auto-play with sound for video elements. intended_use: Video Browser Play data_product_type: Summary statistics # TODO: Is this correct? data_product_region: Not Specified - description: Identify which websites users typically prefer to have auto-play with sound for video elements. - publication_date: '2017-01-01' data_product_sector: Technology + publication_date: '2017-01-01' dp_variant: variant_name: Pure DP data_domain: '' # TODO @@ -36,7 +34,9 @@ deployment: justification: '' # TODO: Fill in correct value administrative: sources: https://docs-assets.developer.apple.com/ml-research/papers/learning-with-privacy-at-scale.pdf + status: Pending registry_authors: - Nicolas Berrios - Jack Fitzsimons - James Honaker + tier: 1 diff --git a/deployments/birth_dataset.yaml b/deployments/birth_dataset.yaml index ce352d16..fbea9ec4 100644 --- a/deployments/birth_dataset.yaml +++ b/deployments/birth_dataset.yaml @@ -1,6 +1,4 @@ url_slug: israel-s-national-registry-of-live-births-of-2014-israeli-ministry-of-health-2024 -status: Pending -tier: 1 deployment: basic: name: Israel's National Registry of Live Births of 2014 @@ -8,12 +6,12 @@ deployment: - data_curator_name: 'Israeli Ministry of Health' data_curator_type: 'Government' + description: Synthetic data covering live births in Israel during 2014. intended_use: Health Data data_product_type: Synthetic data data_product_region: Israel - description: Synthetic data covering live births in Israel during 2014. - publication_date: '2024-02-01' data_product_sector: Healthcare + publication_date: '2024-02-01' dp_variant: variant_name: Pure DP privacy_loss: @@ -28,7 +26,6 @@ deployment: is \\(\epsilon = 9.98\\)." deployment_model: model_name: Central - # model_name_details: TODO release_type: One release release_type_details: Static access_type: Non-interactive @@ -38,6 +35,8 @@ deployment: justification: '' # TODO: Fill in correct value administrative: sources: https://birth.dataset.pub + status: Pending registry_authors: - Shlomi Hod - James Honaker + tier: 1 diff --git a/deployments/broadband_coverage.yaml b/deployments/broadband_coverage.yaml index 44cb2162..2e82f941 100644 --- a/deployments/broadband_coverage.yaml +++ b/deployments/broadband_coverage.yaml @@ -1,6 +1,4 @@ url_slug: broadband-coverage-microsoft-2021 -status: Pending -tier: 3 deployment: basic: name: Broadband Coverage @@ -8,12 +6,12 @@ deployment: - data_curator_name: 'Microsoft' data_curator_type: 'Industry' + description: Publicly available U.S. Broadband Coverage dataset that reports broadband coverage percentages at a zip code-level. intended_use: Allow researchers and policy makers to understand problems related to broadband access and develop solutions to improve broadband access while simultaneously preserving the privacy of individual households. data_product_type: Dataset data_product_region: USA - description: Publicly available U.S. Broadband Coverage dataset that reports broadband coverage percentages at a zip code-level. - publication_date: '2021-04-01' data_product_sector: Technology # Telecommunication, Public Sector/Government? + publication_date: '2021-04-01' dp_variant: variant_name: Pure DP @@ -30,7 +28,6 @@ deployment: deployment_model: model_name: Central - # model_name_details: '' actors: | - Users whose devices connect to Microsoft services. They must trust Microsoft to handle their data and apply the DP protections as described. - Microsoft (Curator): collects the raw telemetry data, applies the Laplace mechanism to produce differentially private counts, and publishes the final data product. @@ -42,10 +39,15 @@ deployment: access_type: Non-interactive # access_type_details: '' - accounting: - post_processing: | - - Negative values are clamped to zero. - - The final privatized Broadband Coverage Estimation was calculated using noisy counts and the number of households in each ZIP code as reported in the Department of Housing and Urban Development’s publicly available dataset. + implementation: + preprocessing_and_hyperparameter_tuning: The 32,653 zip codes were predefined based on the average number of machines in the last years (not on the dataset being protected). + + mechanisms: | + - The Laplace mechanism with \\(\epsilon = 0.1\\) was applied to every count query (see section on Composition for query descriptions). + - All DP processing was done with the OpenDP SmartNoise library: https://github.com/microsoft/USBroadbandUsagePercentages + - No information was provided about protection against floating point errors. + - No information was provided about leakage due to idiosyncrasies of the computing platform. + - No information was provided about security measures in place to protect the underlying data. composition: | The following quantities are computed with DP (per ZIP code): @@ -61,15 +63,12 @@ deployment: - The total privacy loss corresponding to Microsoft services data is 0.1. By sequential composition, the total privacy loss is 0.2. (Note: The equation for Broadband Coverage Estimate does not use the "number of devices with speed < 25". In fact, it seems this number is never used.) - implementation: - preprocessing_and_hyperparameter_tuning: The 32,653 zip codes were predefined based on the average number of machines in the last years (not on the dataset being protected). - mechanisms: | - - The Laplace mechanism with \\(\epsilon = 0.1\\) was applied to every count query (see section on Composition for query descriptions). - - All DP processing was done with the OpenDP SmartNoise library: https://github.com/microsoft/USBroadbandUsagePercentages - - No information was provided about protection against floating point errors. - - No information was provided about leakage due to idiosyncrasies of the computing platform. - - No information was provided about security measures in place to protect the underlying data. + post_processing: | + - Negative values are clamped to zero. + - The final privatized Broadband Coverage Estimation was calculated using noisy counts and the number of households in each ZIP code as reported in the Department of Housing and Urban Development’s publicly available dataset. + + justification: | - The privacy budget \\(\epsilon = 0.1\\) used to obtain the counts is chosen to be comparable to other industry differentially private data releases. - The number of devices connected to the internet at broadband speed per each zip code is counted based on the Federal Communications Commission (FCC)’s definition of broadband that is 25mbps per download. @@ -79,7 +78,9 @@ deployment: sources: | - Paper: https://arxiv.org/abs/2103.14035 - Github: https://github.com/microsoft/USBroadbandUsagePercentages + status: Pending registry_authors: - Elena Ghazi - Nicolas Berrios - Jack Fitzsimons + tier: 3 diff --git a/deployments/census_demographic_and_housing.yaml b/deployments/census_demographic_and_housing.yaml index bbad258f..37a151dc 100644 --- a/deployments/census_demographic_and_housing.yaml +++ b/deployments/census_demographic_and_housing.yaml @@ -1,6 +1,4 @@ url_slug: disclosure-avoidance-system-for-demographic-and-housing-data-u-s-census-bureau-2023 -status: Pending -tier: 3 deployment: basic: name: Disclosure Avoidance System for Demographic and Housing Data @@ -8,12 +6,12 @@ deployment: - data_curator_name: 'U.S. Census Bureau' data_curator_type: 'Government' + description: Set of summary statistics (tables with counts of persons and housing units) published as the 2020 Census Demographic and Housing Characteristics File. intended_use: The Demographic and Housing data files provide detailed demographic and housing statistics of the United States, tabulated by geography, and are used for allocating government funding and by researchers such as demographers and economists. data_product_type: Summary statistics data_product_region: USA - description: Set of summary statistics (tables with counts of persons and housing units) published as the 2020 Census Demographic and Housing Characteristics File. - publication_date: '2023-05-25' data_product_sector: Government + publication_date: '2023-05-25' dp_variant: variant_name: Zero-concentrated DP @@ -44,7 +42,6 @@ deployment: deployment_model: model_name: Central - # model_name_details: '' actors: | - Curator: U.S. Census Bureau: collects, stores, and protects the confidential data - Data subjects: The respondents of the 2020 Census, comprising over 331 million people in the United States. Their trust in the curator's ability to protect their data is essential for the quality of the census. This trust is rationalized by the Bureau's legal mandate under Title 13 of the U.S. Code to protect respondent confidentiality, a requirement reinforced by a lifelong oath taken by every employee. @@ -55,24 +52,9 @@ deployment: release_type: One release # release_type_details: '' data_source_type: Static - # data_source_type_details: '' access_type: Non-interactive # access_type_details: '' - accounting: - post_processing: | - A series of post-processing functions are applied to make the data consistent and usable. The primary goal of post-processing is to produce a confidential, record-level Microdata Detail File (MDF) that can be used by the Census Bureau's standard tabulation systems. Key post-processing functions include: - - Ensuring Non-Negativity and Integrality: The system enforces that all final counts are non-negative integers, as required by the tabulation system. If adding noise results in a negative count, that value is adjusted. - - Enforcing Invariants and Constraints: Post-processing ensures that the final data adhere to all pre-defined invariants and other constraints. This includes holding state-level population totals to their exact enumerated values and ensuring that geographies with occupied group quarters facilities contain at least one person. - - Ensuring Consistency: The data are processed to be consistent both hierarchically and internally. This means that counts for smaller geographic units (like counties) sum to their parent geographies (the state), and that different characteristics within a table sum correctly to their totals. This consistency is achieved by solving a large-scale mathematical optimization problem. This is done in two primary steps for each level of the geographic hierarchy: - - A constrained weighted least squares optimization is performed to find a non-negative, real-valued (i.e., continuous) solution that is as close as possible to the noisy measurements while respecting the constraints. - - A controlled rounding problem is then solved to convert the real-valued solution into a final, non-negative integer solution. - - composition: | - Adaptive composition under zCDP is used (because of robust and mathematically simple properties): - - Under zCDP, the total privacy loss is calculated by summing the privacy-loss parameter \\(\rho\\) for each query that is applied to the data. - - For the TopDownAlgorithm, the total \\(\rho\\) is the sum of the \\(\rho\\) values allocated to all queries at every level of the geographic hierarchy (nation, state, county, tract, block group, and block). - implementation: preprocessing_and_hyperparameter_tuning: | - Preprocessing: the raw census responses undergo pre-processing steps to create the CEF. This involves: @@ -89,6 +71,19 @@ deployment: - Legal: Protection is legally mandated by Title 13 of the U.S. Code. - All Census Bureau employees take a lifelong oath to protect respondent information. + composition: | + Adaptive composition under zCDP is used (because of robust and mathematically simple properties): + - Under zCDP, the total privacy loss is calculated by summing the privacy-loss parameter \\(\rho\\) for each query that is applied to the data. + - For the TopDownAlgorithm, the total \\(\rho\\) is the sum of the \\(\rho\\) values allocated to all queries at every level of the geographic hierarchy (nation, state, county, tract, block group, and block). + + post_processing: | + A series of post-processing functions are applied to make the data consistent and usable. The primary goal of post-processing is to produce a confidential, record-level Microdata Detail File (MDF) that can be used by the Census Bureau's standard tabulation systems. Key post-processing functions include: + - Ensuring Non-Negativity and Integrality: The system enforces that all final counts are non-negative integers, as required by the tabulation system. If adding noise results in a negative count, that value is adjusted. + - Enforcing Invariants and Constraints: Post-processing ensures that the final data adhere to all pre-defined invariants and other constraints. This includes holding state-level population totals to their exact enumerated values and ensuring that geographies with occupied group quarters facilities contain at least one person. + - Ensuring Consistency: The data are processed to be consistent both hierarchically and internally. This means that counts for smaller geographic units (like counties) sum to their parent geographies (the state), and that different characteristics within a table sum correctly to their totals. This consistency is achieved by solving a large-scale mathematical optimization problem. This is done in two primary steps for each level of the geographic hierarchy: + - A constrained weighted least squares optimization is performed to find a non-negative, real-valued (i.e., continuous) solution that is as close as possible to the noisy measurements while respecting the constraints. + - A controlled rounding problem is then solved to convert the real-valued solution into a final, non-negative integer solution. + justification: | - Rationale for Adopting DP: Differential privacy was adopted after internal research confirmed that traditional disclosure avoidance methods, like data swapping, were vulnerable to modern database reconstruction and re-identification attacks. A 2018 simulation of such an attack on the 2010 Census data resulted in the confirmed re-identification of 52 million people (17% of the U.S. population). DP was chosen because it provides mathematically provable guarantees against a wide range of such privacy attacks. - Rationale for Specific Mechanisms: @@ -99,6 +94,7 @@ deployment: - The Census Bureau solicited and incorporated detailed feedback from a wide array of stakeholders, including the National Academy of Sciences, federal and state partners, academic researchers, and tribal leaders. - The final production parameters were set by the Data Stewardship Executive Policy Committee (DSEP) after a final review of the privacy guarantees and data accuracy demonstrated in the tuning experiments. + administrative: sources: | - 2020 Census Demographic and Housing Characteristics File (DHC) Technical Documentation: https://www2.census.gov/programs-surveys/decennial/2020/technical-documentation/complete-tech-docs/demographic-and-housing-characteristics-file-and-demographic-profile/2020census-demographic-and-housing-characteristics-file-and-demographic-profile-techdoc.pdf @@ -106,5 +102,7 @@ deployment: - Disclosure Avoidance for DHC and Guidance for Data Users: https://www2.census.gov/library/publications/decennial/2020/census-briefs/c2020br-08.pdf - Disclosure avoidance handbook for the 2020 Census: https://www2.census.gov/library/publications/decennial/2020/2020-census-disclosure-avoidance-handbook.pdf - The 2020 Census Disclosure Avoidance System TopDown Algorithm: https://arxiv.org/abs/2204.08986 + status: Pending registry_authors: - Curtis Mitchell + tier: 3 diff --git a/deployments/census_detailed_demographic_and_housing_a.yaml b/deployments/census_detailed_demographic_and_housing_a.yaml index c5273b51..c468bfc4 100644 --- a/deployments/census_detailed_demographic_and_housing_a.yaml +++ b/deployments/census_detailed_demographic_and_housing_a.yaml @@ -1,6 +1,4 @@ url_slug: disclosure-avoidance-system-for-detailed-demographic-and-housing-characteristics-file-a-u-s-census-bureau-2023 -status: Pending -tier: 3 deployment: basic: name: Disclosure Avoidance System for Detailed Demographic and Housing Characteristics File A @@ -8,12 +6,12 @@ deployment: - data_curator_name: 'U.S. Census Bureau' data_curator_type: 'Government' + description: Set of summary statistics (tables with population counts and sex-by-age statistics) published as the 2020 Census Detailed Demographic and Housing Characteristics File A. intended_use: The Detailed Demographic and Housing Characteristics data file A (Detailed DHC-A, or DDHC-A) provides population counts and sex-by-age statistics for approximately 1,500 detailed race and ethnic groups and detailed American Indian, Alaska Native, and Native Hawaiian (AIANNH) areas within the United States. These data are used for allocating government funding and by researchers such as demographers and economists. data_product_type: Summary statistics data_product_region: USA - description: Set of summary statistics (tables with population counts and sex-by-age statistics) published as the 2020 Census Detailed Demographic and Housing Characteristics File A. - publication_date: '2023-10-21' data_product_sector: Government + publication_date: '2023-10-21' dp_variant: variant_name: Zero-concentrated DP @@ -42,7 +40,6 @@ deployment: deployment_model: model_name: Central - # model_name_details: '' actors: | - Curator: U.S. Census Bureau: collects, stores, and protects the confidential data - Data subjects: The respondents of the 2020 Census, comprising over 331 million people in the United States. Their trust in the curator's ability to protect their data is essential for the quality of the census. This trust is rationalized by the Bureau's legal mandate under Title 13 of the U.S. Code to protect respondent confidentiality, a requirement reinforced by a lifelong oath taken by every employee. @@ -53,19 +50,9 @@ deployment: release_type: One release # release_type_details: '' data_source_type: Static - # data_source_type_details: '' access_type: Non-interactive # access_type_details: '' - accounting: - post_processing: | - The Detailed Demographic and Housing Characteristics File A is not consistent with the Redistricting Data (PL 94-171) Summary File, the Demographic and Housing Characteristics File (DHC), or the Demographic Profile. Key post-processing functions include: - - Inconsistencies From the Noise Necessary to Protect Confidentiality: Because of the way noise is applied to data to protect respondent confidentiality, aggregating counts can result in noisier (or less accurate) data. - - Inconsistent Data Because of Minimal Postprocessing Consistency Requirements: Noise is applied indepen- dently to each group at every geography level. This means that geographic hierarchies were not considered when applying noise to the data. - - Data Suppression: There are suppressed values in the Detailed DHC-A. Suppression was applied to counts in one of two ways: individual data cells and whole tables. - - An “X” appears in the data cell when an individual count was suppressed because of data reasonableness requirements. - - Whole tables were suppressed for detailed and regional groups because of population thresholds. These population thresholds ensure that published population counts reflect positive enumerated counts 99.99 percent of the time. - implementation: mechanisms: | - The SafeTab-P algorithm works by calculating statistics using the confidential 2020 Census data, then adds or subtracts a small amount of statistical noise to those statistics. The amount of noise added or subtracted from each statistic is determined by the privacy-loss budget (PLB) assigned to the calculation of that statistic. PLB allocations were determined to ensure that all tables included in the Detailed DHC-A meet preestablished margins of error (uncertainty) at least 95 percent of the time. @@ -74,6 +61,13 @@ deployment: - Security measures: - Legal: Protection is legally mandated by Title 13 of the U.S. Code. - All Census Bureau employees take a lifelong oath to protect respondent information. + post_processing: | + The Detailed Demographic and Housing Characteristics File A is not consistent with the Redistricting Data (PL 94-171) Summary File, the Demographic and Housing Characteristics File (DHC), or the Demographic Profile. Key post-processing functions include: + - Inconsistencies From the Noise Necessary to Protect Confidentiality: Because of the way noise is applied to data to protect respondent confidentiality, aggregating counts can result in noisier (or less accurate) data. + - Inconsistent Data Because of Minimal Postprocessing Consistency Requirements: Noise is applied indepen- dently to each group at every geography level. This means that geographic hierarchies were not considered when applying noise to the data. + - Data Suppression: There are suppressed values in the Detailed DHC-A. Suppression was applied to counts in one of two ways: individual data cells and whole tables. + - An “X” appears in the data cell when an individual count was suppressed because of data reasonableness requirements. + - Whole tables were suppressed for detailed and regional groups because of population thresholds. These population thresholds ensure that published population counts reflect positive enumerated counts 99.99 percent of the time. justification: | - Rationale for Adopting DP: Differential privacy was adopted after internal research confirmed that traditional disclosure avoidance methods, like data swapping, were vulnerable to modern database reconstruction and re-identification attacks. A 2018 simulation of such an attack on the 2010 Census data resulted in the confirmed re-identification of 52 million people (17% of the U.S. population). DP was chosen because it provides mathematically provable guarantees against a wide range of such privacy attacks. @@ -91,5 +85,7 @@ deployment: - Census Bureau Announces Release Date for 2020 Census Data Product on Race and Ethnicity: https://www.census.gov/newsroom/press-releases/2023/2020-census-detailed-dhc-a.html - Disclosure Avoidance Methods for the Detailed Demographic and Housing Characteristics File A (Detailed DHC-A): How SafeTab-P Works: https://www2.census.gov/library/publications/decennial/2020/census-briefs/c2020br-05.pdf - Disclosure avoidance handbook for the 2020 Census: https://www2.census.gov/library/publications/decennial/2020/2020-census-disclosure-avoidance-handbook.pdf + status: Pending registry_authors: - Curtis Mitchell + tier: 3 diff --git a/deployments/census_detailed_demographic_and_housing_b.yaml b/deployments/census_detailed_demographic_and_housing_b.yaml index 3cc72410..4ad5d5a7 100644 --- a/deployments/census_detailed_demographic_and_housing_b.yaml +++ b/deployments/census_detailed_demographic_and_housing_b.yaml @@ -1,6 +1,4 @@ url_slug: disclosure-avoidance-system-for-detailed-demographic-and-housing-characteristics-file-b-u-s-census-bureau-2024 -status: Pending -tier: 3 deployment: basic: name: Disclosure Avoidance System for Detailed Demographic and Housing Characteristics File B @@ -8,12 +6,12 @@ deployment: - data_curator_name: 'U.S. Census Bureau' data_curator_type: 'Government' + description: Set of summary statistics (tables with household type and tenure type statistics) published as the 2020 Census Detailed Demographic and Housing Characteristics File B. intended_use: The Detailed Demographic and Housing Characteristics data file B (Detailed DHC-B, or DDHC-B) provides household type (e.g., family or nonfamily) and tenure (e.g., whether the home is owned or rented) for approximately 1,500 detailed race and ethnic groups and detailed American Indian, Alaska Native, and Native Hawaiian (AIANNH) areas within the United States. These data are used for assessing and allocating funding for government programs and by researchers such as demographers and economists. data_product_type: Summary statistics data_product_region: USA - description: Set of summary statistics (tables with household type and tenure type statistics) published as the 2020 Census Detailed Demographic and Housing Characteristics File B. - publication_date: '2024-08-01' data_product_sector: Government + publication_date: '2024-08-01' dp_variant: variant_name: Zero-concentrated DP @@ -42,7 +40,6 @@ deployment: deployment_model: model_name: Central - # model_name_details: '' actors: | - Curator: U.S. Census Bureau: collects, stores, and protects the confidential data - Data subjects: The respondents of the 2020 Census, comprising over 331 million people in the United States. Their trust in the curator's ability to protect their data is essential for the quality of the census. This trust is rationalized by the Bureau's legal mandate under Title 13 of the U.S. Code to protect respondent confidentiality, a requirement reinforced by a lifelong oath taken by every employee. @@ -53,19 +50,9 @@ deployment: release_type: One release # release_type_details: '' data_source_type: Static - # data_source_type_details: '' access_type: Non-interactive # access_type_details: '' - accounting: - post_processing: | - The Detailed Demographic and Housing Characteristics File B is not consistent with the Redistricting Data (PL 94-171) Summary File, the Demographic and Housing Characteristics File (DHC), or the Demographic Profile. Key post-processing functions include: - - Inconsistencies From the Noise Necessary to Protect Confidentiality: Because of the way noise is applied to data to protect respondent confidentiality, aggregating counts can result in noisier (or less accurate) data. - - Inconsistent Data Because of Minimal Postprocessing Consistency Requirements: Noise is applied indepen- dently to each group at every geography level. This means that geographic hierarchies were not considered when applying noise to the data. - - Data Suppression: There are suppressed values in the Detailed DHC-B. Suppression was applied to counts in one of two ways: individual data cells and whole tables. - - An “X” appears in the data cell when an individual count was suppressed because of data reasonableness requirements. - - Whole tables were suppressed for detailed and regional groups because of population thresholds. These population thresholds ensure that published population counts reflect positive enumerated counts 99.99 percent of the time. - implementation: mechanisms: | - The SafeTab-H algorithm works by calculating statistics using the confidential 2020 Census data, then adds or subtracts a small amount of statistical noise to those statistics. The amount of noise added or subtracted from each statistic is determined by the privacy-loss budget (PLB) assigned to the calculation of that statistic. PLB allocations were determined to ensure that all tables included in the Detailed DHC-B meet preestablished margins of error (uncertainty) at least 95 percent of the time. @@ -74,6 +61,13 @@ deployment: - Security measures: - Legal: Protection is legally mandated by Title 13 of the U.S. Code. - All Census Bureau employees take a lifelong oath to protect respondent information. + post_processing: | + The Detailed Demographic and Housing Characteristics File B is not consistent with the Redistricting Data (PL 94-171) Summary File, the Demographic and Housing Characteristics File (DHC), or the Demographic Profile. Key post-processing functions include: + - Inconsistencies From the Noise Necessary to Protect Confidentiality: Because of the way noise is applied to data to protect respondent confidentiality, aggregating counts can result in noisier (or less accurate) data. + - Inconsistent Data Because of Minimal Postprocessing Consistency Requirements: Noise is applied indepen- dently to each group at every geography level. This means that geographic hierarchies were not considered when applying noise to the data. + - Data Suppression: There are suppressed values in the Detailed DHC-B. Suppression was applied to counts in one of two ways: individual data cells and whole tables. + - An “X” appears in the data cell when an individual count was suppressed because of data reasonableness requirements. + - Whole tables were suppressed for detailed and regional groups because of population thresholds. These population thresholds ensure that published population counts reflect positive enumerated counts 99.99 percent of the time. justification: | - Rationale for Adopting DP: Differential privacy was adopted after internal research confirmed that traditional disclosure avoidance methods, like data swapping, were vulnerable to modern database reconstruction and re-identification attacks. A 2018 simulation of such an attack on the 2010 Census data resulted in the confirmed re-identification of 52 million people (17% of the U.S. population). DP was chosen because it provides mathematically provable guarantees against a wide range of such privacy attacks. @@ -91,5 +85,7 @@ deployment: - Census Bureau to Hold Webinar on Release of 2020 Census Detailed Demographic and Housing Characteristics File B: https://www.census.gov/newsroom/press-releases/2024/webinar-2020-census-detailed-dhc-b.html - Disclosure Avoidance and the 2020 Census: How SafeTab-H Works: https://www2.census.gov/library/publications/decennial/2020/census-briefs/c2020br-11.pdf - Disclosure avoidance handbook for the 2020 Census: https://www2.census.gov/library/publications/decennial/2020/2020-census-disclosure-avoidance-handbook.pdf + status: Pending registry_authors: - Curtis Mitchell + tier: 3 diff --git a/deployments/census_redistricting_data.yaml b/deployments/census_redistricting_data.yaml index 077d172a..19f94443 100644 --- a/deployments/census_redistricting_data.yaml +++ b/deployments/census_redistricting_data.yaml @@ -1,6 +1,4 @@ url_slug: disclosure-avoidance-system-for-redistricting-data-u-s-census-bureau-2021 -status: Pending -tier: 3 deployment: basic: name: Disclosure Avoidance System for Redistricting Data @@ -8,12 +6,12 @@ deployment: - data_curator_name: 'U.S. Census Bureau' data_curator_type: 'Government' + description: Set of summary statistics (tables with counts of individuals, households, group quarters residents, or housing units with certain characteristics) published as the 2020 Census Redistricting Data (P.L. 94-171) Summary File. intended_use: The decennial census data are used for congressional apportionment, the redistricting of legislative bodies, the allocation of federal funds, public policy, and research. The Summary File is produced to satisfy the legal requirement to provide states with the population tabulations they need for legislative redistricting. data_product_type: Summary statistics data_product_region: USA - description: Set of summary statistics (tables with counts of individuals, households, group quarters residents, or housing units with certain characteristics) published as the 2020 Census Redistricting Data (P.L. 94-171) Summary File. - publication_date: '2021-08-01' data_product_sector: Government + publication_date: '2021-08-01' dp_variant: variant_name: Zero-concentrated DP @@ -44,7 +42,6 @@ deployment: deployment_model: model_name: Central - # model_name_details: '' actors: | - Curator: U.S. Census Bureau: collects, stores, and protects the confidential data - Data subjects: The respondents of the 2020 Census, comprising over 331 million people in the United States. Their trust in the curator's ability to protect their data is essential for the quality of the census. This trust is rationalized by the Bureau's legal mandate under Title 13 of the U.S. Code to protect respondent confidentiality, a requirement reinforced by a lifelong oath taken by every employee. @@ -55,24 +52,9 @@ deployment: release_type: One release # release_type_details: '' data_source_type: Static - # data_source_type_details: '' access_type: Non-interactive # access_type_details: '' - accounting: - post_processing: | - A series of post-processing functions are applied to make the data consistent and usable. The primary goal of post-processing is to produce a confidential, record-level Microdata Detail File (MDF) that can be used by the Census Bureau's standard tabulation systems. Key post-processing functions include: - - Ensuring Non-Negativity and Integrality: The system enforces that all final counts are non-negative integers, as required by the tabulation system. If adding noise results in a negative count, that value is adjusted. - - Enforcing Invariants and Constraints: Post-processing ensures that the final data adhere to all pre-defined invariants and other constraints. This includes holding state-level population totals to their exact enumerated values and ensuring that geographies with occupied group quarters facilities contain at least one person. - - Ensuring Consistency: The data are processed to be consistent both hierarchically and internally. This means that counts for smaller geographic units (like counties) sum to their parent geographies (the state), and that different characteristics within a table sum correctly to their totals. This consistency is achieved by solving a large-scale mathematical optimization problem. This is done in two primary steps for each level of the geographic hierarchy: - - A constrained weighted least squares optimization is performed to find a non-negative, real-valued (i.e., continuous) solution that is as close as possible to the noisy measurements while respecting the constraints. - - A controlled rounding problem is then solved to convert the real-valued solution into a final, non-negative integer solution. - - composition: | - Adaptive composition under zCDP is used (because of robust and mathematically simple properties): - - Under zCDP, the total privacy loss is calculated by summing the privacy-loss parameter \\(\rho\\) for each query that is applied to the data. - - For the TopDownAlgorith, the total \\(\rho\\) is the sum of the \\(\rho\\) values allocated to all queries at every level of the geographic hierarchy (nation, state, county, tract, block group, and block). - implementation: preprocessing_and_hyperparameter_tuning: | - Preprocessing: the raw census responses undergo pre-processing steps to create the CEF. This involves: @@ -89,6 +71,19 @@ deployment: - Legal: Protection is legally mandated by Title 13 of the U.S. Code. - All Census Bureau employees take a lifelong oath to protect respondent information. + composition: | + Adaptive composition under zCDP is used (because of robust and mathematically simple properties): + - Under zCDP, the total privacy loss is calculated by summing the privacy-loss parameter \\(\rho\\) for each query that is applied to the data. + - For the TopDownAlgorith, the total \\(\rho\\) is the sum of the \\(\rho\\) values allocated to all queries at every level of the geographic hierarchy (nation, state, county, tract, block group, and block). + + post_processing: | + A series of post-processing functions are applied to make the data consistent and usable. The primary goal of post-processing is to produce a confidential, record-level Microdata Detail File (MDF) that can be used by the Census Bureau's standard tabulation systems. Key post-processing functions include: + - Ensuring Non-Negativity and Integrality: The system enforces that all final counts are non-negative integers, as required by the tabulation system. If adding noise results in a negative count, that value is adjusted. + - Enforcing Invariants and Constraints: Post-processing ensures that the final data adhere to all pre-defined invariants and other constraints. This includes holding state-level population totals to their exact enumerated values and ensuring that geographies with occupied group quarters facilities contain at least one person. + - Ensuring Consistency: The data are processed to be consistent both hierarchically and internally. This means that counts for smaller geographic units (like counties) sum to their parent geographies (the state), and that different characteristics within a table sum correctly to their totals. This consistency is achieved by solving a large-scale mathematical optimization problem. This is done in two primary steps for each level of the geographic hierarchy: + - A constrained weighted least squares optimization is performed to find a non-negative, real-valued (i.e., continuous) solution that is as close as possible to the noisy measurements while respecting the constraints. + - A controlled rounding problem is then solved to convert the real-valued solution into a final, non-negative integer solution. + justification: | - Rationale for Adopting DP: Differential privacy was adopted after internal research confirmed that traditional disclosure avoidance methods, like data swapping, were vulnerable to modern database reconstruction and re-identification attacks. A 2018 simulation of such an attack on the 2010 Census data resulted in the confirmed re-identification of 52 million people (17% of the U.S. population). DP was chosen because it provides mathematically provable guarantees against a wide range of such privacy attacks. - Rationale for Specific Mechanisms: @@ -103,7 +98,9 @@ deployment: sources: | - Disclosure avoidance handbook for the 2020 Census: https://www2.census.gov/library/publications/decennial/2020/2020-census-disclosure-avoidance-handbook.pdf - The 2020 Census Disclosure Avoidance System TopDown Algorithm: https://arxiv.org/abs/2204.08986 + status: Pending registry_authors: - Elena Ghazi - Nicolas Berrios - Jack Fitzsimons + tier: 3 diff --git a/deployments/census_supplemental_demographic_and_housing.yaml b/deployments/census_supplemental_demographic_and_housing.yaml index 637be5be..e8d0b0b5 100644 --- a/deployments/census_supplemental_demographic_and_housing.yaml +++ b/deployments/census_supplemental_demographic_and_housing.yaml @@ -1,6 +1,4 @@ url_slug: disclosure-avoidance-system-for-supplemental-demographic-and-housing-characteristics-file-u-s-census-bureau-2024 -status: Pending -tier: 3 deployment: basic: name: Disclosure Avoidance System for Supplemental Demographic and Housing Characteristics File @@ -8,12 +6,12 @@ deployment: - data_curator_name: 'U.S. Census Bureau' data_curator_type: 'Government' + description: Set of summary statistics (tables with household type and tenure type statistics) published as the 2020 Census Supplemental Demographic and Housing Characteristics File. intended_use: The Supplemental Demographic and Housing Characteristics (Supplemental DHC, or S-DHC) data file provides combines the characteristics of households and the people living in them. These data are used for assessing and allocating funding for government programs and by researchers such as demographers and economists. data_product_type: Summary statistics data_product_region: USA - description: Set of summary statistics (tables with household type and tenure type statistics) published as the 2020 Census Supplemental Demographic and Housing Characteristics File. - publication_date: '2024-09-19' data_product_sector: Government + publication_date: '2024-09-19' dp_variant: variant_name: Zero-concentrated DP @@ -33,7 +31,6 @@ deployment: deployment_model: model_name: Central - # model_name_details: '' actors: | - Curator: U.S. Census Bureau: collects, stores, and protects the confidential data - Data subjects: The respondents of the 2020 Census, comprising over 331 million people in the United States. Their trust in the curator's ability to protect their data is essential for the quality of the census. This trust is rationalized by the Bureau's legal mandate under Title 13 of the U.S. Code to protect respondent confidentiality, a requirement reinforced by a lifelong oath taken by every employee. @@ -44,16 +41,9 @@ deployment: release_type: One release # release_type_details: '' data_source_type: Static - # data_source_type_details: '' access_type: Non-interactive # access_type_details: '' - accounting: - post_processing: | - The Supplemental Demographic and Housing Characteristics File is not consistent with the Demographic and Housing Characteristics File (DHC). Key post-processing functions include: - - Counts may not match across S-DHC tables. - - User-generated aggregations may not match the published counts. - - The underlying data are adjusted, through a process referred to as truncation, to not exceed household size maximum thresholds. implementation: mechanisms: | @@ -63,6 +53,11 @@ deployment: - Security measures: - Legal: Protection is legally mandated by Title 13 of the U.S. Code. - All Census Bureau employees take a lifelong oath to protect respondent information. + post_processing: | + The Supplemental Demographic and Housing Characteristics File is not consistent with the Demographic and Housing Characteristics File (DHC). Key post-processing functions include: + - Counts may not match across S-DHC tables. + - User-generated aggregations may not match the published counts. + - The underlying data are adjusted, through a process referred to as truncation, to not exceed household size maximum thresholds. justification: | - Rationale for Adopting DP: Differential privacy was adopted after internal research confirmed that traditional disclosure avoidance methods, like data swapping, were vulnerable to modern database reconstruction and re-identification attacks. A 2018 simulation of such an attack on the 2010 Census data resulted in the confirmed re-identification of 52 million people (17% of the U.S. population). DP was chosen because it provides mathematically provable guarantees against a wide range of such privacy attacks. @@ -80,5 +75,7 @@ deployment: - Census Bureau Releases Final 2020 Census Data Product: https://www.census.gov/newsroom/press-releases/2024/final-2020-census-data-product-s-dhc.html - Disclosure Avoidance and the Supplemental Demographic and Housing Characteristics File (S-DHC): How PHSafe Works: https://www2.census.gov/library/publications/decennial/2020/census-briefs/c2020br-12.pdf - Disclosure avoidance handbook for the 2020 Census: https://www2.census.gov/library/publications/decennial/2020/2020-census-disclosure-avoidance-handbook.pdf + status: Pending registry_authors: - Curtis Mitchell + tier: 3 diff --git a/deployments/county_business_patterns.yaml b/deployments/county_business_patterns.yaml index a61f3327..e7b9ca86 100644 --- a/deployments/county_business_patterns.yaml +++ b/deployments/county_business_patterns.yaml @@ -1,6 +1,4 @@ url_slug: census-county-business-patterns-u-s-census-bureau-2023 -status: Pending -tier: 1 deployment: basic: name: Census County Business Patterns @@ -8,12 +6,12 @@ deployment: - data_curator_name: 'U.S. Census Bureau' data_curator_type: 'Government' + description: Demonstration dataset releasing data on economic establishments. intended_use: Geographical Buisness Statistics data_product_type: Summary statistics data_product_region: USA - description: Demonstration dataset releasing data on economic establishments. - publication_date: '2023-01-01' data_product_sector: Government + publication_date: '2023-01-01' dp_variant: variant_name: Zero-concentrated DP data_domain: '' # TODO @@ -26,7 +24,6 @@ deployment: privacy_parameters_details: '"Noise added in the PRDP stage is randomly sampled from a discrete Gaussian distribution, and the total rho-zCDP privacy loss budget is 12.058."' deployment_model: model_name: Central - # model_name_details: TODO release_type: One release release_type_details: Static access_type: Non-interactive @@ -39,7 +36,9 @@ deployment: - https://www.census.gov/topics/business-economy/disclosure/about.html - https://www.census.gov/data/academy/webinars/2023/differential-privacy-webinar.html - https://www.census.gov/topics/business-economy/disclosure/data/tables.html + status: Pending registry_authors: - Nicolas Berrios - Jack Fitzsimons - James Honaker + tier: 1 diff --git a/deployments/covid19_notifications.yaml b/deployments/covid19_notifications.yaml index a53e20de..1f235bfa 100644 --- a/deployments/covid19_notifications.yaml +++ b/deployments/covid19_notifications.yaml @@ -1,6 +1,4 @@ url_slug: covid-19-exposure-notification-framework-google-apple-2021 -status: Pending -tier: 1 deployment: basic: name: COVID-19 Exposure Notification Framework @@ -11,12 +9,12 @@ deployment: - data_curator_name: 'Apple' data_curator_type: 'Industry' + description: Measurement system which provides summary statistics to public health authorities about how often COVID-19 exposure notifications were shown on users’ devices. intended_use: To help public health authorities adapt their response to the COVID-19 pandemic as it evolved. data_product_type: Summary statistics data_product_region: Global - description: Measurement system which provides summary statistics to public health authorities about how often COVID-19 exposure notifications were shown on users’ devices. - publication_date: '2021-01-01' data_product_sector: Healthcare + publication_date: '2021-01-01' dp_variant: variant_name: Approximate DP privacy_loss: @@ -30,7 +28,6 @@ deployment: - "The central differential privacy guarantees \\(\epsilon_c\\) that result from unlinkable aggregation of \\(\epsilon_0\\) = 8 metric report binary vectors, as used in ENPA." deployment_model: model_name: Varies - # model_name_details: TODO release_type: Many releases release_type_details: Varies access_type: Non-interactive @@ -41,7 +38,9 @@ deployment: administrative: sources: | - Paper: https://covid19-static.cdn-apple.com/applications/covid19/current/static/contact-tracing/pdf/ENPA_White_Paper.pdf + status: Pending registry_authors: - Priyanka Nanayakkara - Nicolas Berrios - Jack Fitzsimons + tier: 1 diff --git a/deployments/covid19_search_trends_symptoms.yaml b/deployments/covid19_search_trends_symptoms.yaml index 71a152cd..62c0c568 100644 --- a/deployments/covid19_search_trends_symptoms.yaml +++ b/deployments/covid19_search_trends_symptoms.yaml @@ -1,6 +1,4 @@ url_slug: covid-19-search-trends-symptoms-google-2020 -status: Pending -tier: 3 deployment: basic: name: COVID-19 Search Trends Symptoms @@ -8,13 +6,12 @@ deployment: - data_curator_name: 'Google' data_curator_type: 'Industry' - + description: Summary statistics, published as an aggregated and anonymized dataset. It provides daily or weekly time series for various geographic regions, showing the relative frequency of Google searches for approximately 400 predefined symptoms. These values are normalized by the total search activity in the corresponding region to show trends over time. intended_use: Help researchers, public health experts, and data analysts better understand the impact of COVID-19 via population-level symptom search trends. data_product_type: Summary statistics data_product_region: Global - description: Summary statistics, published as an aggregated and anonymized dataset. It provides daily or weekly time series for various geographic regions, showing the relative frequency of Google searches for approximately 400 predefined symptoms. These values are normalized by the total search activity in the corresponding region to show trends over time. - publication_date: '2020-09-02' data_product_sector: Healthcare + publication_date: '2020-09-02' dp_variant: variant_name: Pure DP @@ -35,7 +32,6 @@ deployment: deployment_model: model_name: Central - # model_name_details: '' actors: | - Google Users: Individuals performing Google searches. - Data Curator (Google): The central, trusted entity that collects, aggregates, anonymizes, and publishes the data. @@ -48,28 +44,6 @@ deployment: access_type: Non-interactive # access_type_details: '' - accounting: - post_processing: | - - Computing the Reported Data: Given a particular geographic region, symptom and time interval (either day or week), the normalized search count that is published in the released dataset is computed using \\(c \times \max {(A/B),0}\\), where \\(A\\) is the noisy symptom search count, \\(B\\) is the noisy normalization count, and $c$ is a fixed, region-specific scaling factor. For each geographic region, the scaling factor \\(c\\) is chosen in a way that maps the data in the initial release of the symptoms dataset to values between 0 and 100. It is computed once using data from February 2020 to July 2020, and kept fixed for future releases. - - Removing Unreliable Data: A metric is only kept and published if it has a 50% or greater chance of being within 25% of its raw value. This confidence interval is computed using the noisy symptom search count and noisy normalization count; therefore, no privacy budget is spent on this step. - - Deciding Between Daily and Weekly Granularity - - Country-level data is always published with daily granularity. - - For all other regions, the granularity is determined based on data from February 2020 to July 2020 and kept fixed afterwards: - - The regions are first ordered by their total search activity from highest to lowest based on noisy normalization counts. - - The system starts with the highest-activity regions, publishing their metrics daily. As it moves down the list to lower-activity regions, it performs a "majority vote" check. It looks at the last 20 regions it processed and checks how many of them had over 50% of their daily metrics dropped. If 11 or more of the last 20 regions were mostly empty (i.e., had over 50% of data dropped), it permanently switches to weekly granularity for all remaining regions down the list. - - - composition: | - This privacy budget \\(\epsilon\\) = 1.68 is composed as follows: - - Symptom Search Counts (Total \\(\epsilon\\) = 1.638): This portion accounts for 97.5% of the total budget. It is allocated across geographic levels: - - Level 0 (Country): \\(\epsilon_0\\) = 0.168 - - Level 1 (e.g., State): \\(\epsilon_1\\) = 0.37 - - Level 2 (e.g., County): \\(\epsilon_2\\) = 1.1 - - Normalization Counts (Total \\(\epsilon\\) = 0.042): This portion accounts for the remaining 2.5% of the total budget. It is allocated across geographic levels: - - Level 0 (Country): \\(\epsilon_0'\\) = 0.0023 - - Level 1 (e.g., State): \\(\epsilon_1'\\) = 0.0047 - - Level 2 (e.g., County): \\(\epsilon_2'\\) = 0.014 - implementation: preprocessing_and_hyperparameter_tuning: | - Individual search queries are mapped to one or more symptoms from a predefined list. @@ -82,6 +56,24 @@ deployment: - No information was provided about protection against floating point errors. - No information was provided about leakage due to idiosyncrasies of the computing platform. - No information was provided about security measures in place to protect the underlying data. + composition: | + This privacy budget \\(\epsilon\\) = 1.68 is composed as follows: + - Symptom Search Counts (Total \\(\epsilon\\) = 1.638): This portion accounts for 97.5% of the total budget. It is allocated across geographic levels: + - Level 0 (Country): \\(\epsilon_0\\) = 0.168 + - Level 1 (e.g., State): \\(\epsilon_1\\) = 0.37 + - Level 2 (e.g., County): \\(\epsilon_2\\) = 1.1 + - Normalization Counts (Total \\(\epsilon\\) = 0.042): This portion accounts for the remaining 2.5% of the total budget. It is allocated across geographic levels: + - Level 0 (Country): \\(\epsilon_0'\\) = 0.0023 + - Level 1 (e.g., State): \\(\epsilon_1'\\) = 0.0047 + - Level 2 (e.g., County): \\(\epsilon_2'\\) = 0.014 + post_processing: | + - Computing the Reported Data: Given a particular geographic region, symptom and time interval (either day or week), the normalized search count that is published in the released dataset is computed using \\(c \times \max {(A/B),0}\\), where \\(A\\) is the noisy symptom search count, \\(B\\) is the noisy normalization count, and $c$ is a fixed, region-specific scaling factor. For each geographic region, the scaling factor \\(c\\) is chosen in a way that maps the data in the initial release of the symptoms dataset to values between 0 and 100. It is computed once using data from February 2020 to July 2020, and kept fixed for future releases. + - Removing Unreliable Data: A metric is only kept and published if it has a 50% or greater chance of being within 25% of its raw value. This confidence interval is computed using the noisy symptom search count and noisy normalization count; therefore, no privacy budget is spent on this step. + - Deciding Between Daily and Weekly Granularity + - Country-level data is always published with daily granularity. + - For all other regions, the granularity is determined based on data from February 2020 to July 2020 and kept fixed afterwards: + - The regions are first ordered by their total search activity from highest to lowest based on noisy normalization counts. + - The system starts with the highest-activity regions, publishing their metrics daily. As it moves down the list to lower-activity regions, it performs a "majority vote" check. It looks at the last 20 regions it processed and checks how many of them had over 50% of their daily metrics dropped. If 11 or more of the last 20 regions were mostly empty (i.e., had over 50% of data dropped), it permanently switches to weekly granularity for all remaining regions down the list. justification: | - The decision to bound a user’s contribution to a maximum of three different symptom counts per day was based on the observation that approximately 75% of users search for three or fewer symptoms daily. @@ -92,7 +84,9 @@ deployment: - Documentation: https://storage.googleapis.com/gcp-public-data-symptom-search/COVID-19%20Search%20Trends%20symptoms%20dataset%20documentation%20.pdf?utm_source=chatgpt.com - Paper: https://arxiv.org/pdf/2009.01265 - Published data product: https://github.com/GoogleCloudPlatform/covid-19-open-data/blob/main/docs/table-search-trends.md + status: Pending registry_authors: - Elena Ghazi - Nicolas Berrios - Jack Fitzsimons + tier: 3 diff --git a/deployments/employment_outcomes.yaml b/deployments/employment_outcomes.yaml index b76f0a41..ac26c426 100644 --- a/deployments/employment_outcomes.yaml +++ b/deployments/employment_outcomes.yaml @@ -1,6 +1,4 @@ url_slug: post-secondary-employment-outcomes-u-s-census-bureau-2019 -status: Pending -tier: 2 deployment: basic: name: Post-Secondary Employment Outcomes @@ -8,12 +6,12 @@ deployment: - data_curator_name: 'U.S. Census Bureau' data_curator_type: 'Government' + description: Summary statistics that describe earning outcomes for college graduates at different levels, including but not limited to, degree level, degree field, and state of institution. intended_use: For college students and prospective college students to be informed about relationships between specific fields of study and earning outcomes. data_product_type: Summary statistics data_product_region: United States - description: Summary statistics that describe earning outcomes for college graduates at different levels, including but not limited to, degree level, degree field, and state of institution. - publication_date: '2019-01-01' data_product_sector: Education + publication_date: '2019-01-01' dp_variant: variant_name: Pure DP @@ -30,27 +28,23 @@ deployment: deployment_model: model_name: Central - # model_name_details: - # actors: release_type: One release data_source_type: Static access_type: Non-interactive - - # accounting: - # post_processing: - # composition: - - # implementation: + implementation: # preprocessing_and_hyperparameter_tuning: # mechanisms: # justification: + additional_information: | + - It appears that the data product is published over time, but it is not clear from the documentation alone whether the same DP system is used for these later publications. Hence, we describe this deployment as “one-release” and having “static” data to be conservative. + - Finally, the paper contains parameter ranges (e.g., for \\(epsilon\\)) that were tested in the experiments section, however we did not find parameter values that were actually used in deployment. + administrative: sources: | - Paper: https://journalprivacyconfidentiality.org/index.php/jpc/article/view/722/684 - Documentation: https://lehd.ces.census.gov/data/pseo_documentation.html - notes: | - - It appears that the data product is published over time, but it is not clear from the documentation alone whether the same DP system is used for these later publications. Hence, we describe this deployment as “one-release” and having “static” data to be conservative. - - Finally, the paper contains parameter ranges (e.g., for \\(epsilon\\)) that were tested in the experiments section, however we did not find parameter values that were actually used in deployment. + status: Pending registry_authors: - Priyanka Nanayakkara + tier: 2 diff --git a/deployments/google_mobility.yaml b/deployments/google_mobility.yaml index 9e041994..1a75768f 100644 --- a/deployments/google_mobility.yaml +++ b/deployments/google_mobility.yaml @@ -1,6 +1,4 @@ url_slug: community-mobility-place-visits-google-2020 -status: Pending -tier: 3 deployment: basic: name: Community Mobility Place Visits @@ -8,12 +6,12 @@ deployment: - data_curator_name: 'Google' data_curator_type: 'Industry' + description: Summary statistics, published as the COVID-19 Community Mobility Reports, that charts percentage changes in movement trends for various location categories relative to a historical baseline. These trends analyze the number of visits to public places like retail, parks, and workplaces, and also measure the change in duration of time spent in residences. intended_use: Provide public-health authorities insights into how mobility patterns changed in response to COVID-19 policies (e.g., work-from-home, shelter-in-place) to help with decision-making during the pandemic. data_product_type: Summary statistics data_product_region: Global - description: Summary statistics, published as the COVID-19 Community Mobility Reports, that charts percentage changes in movement trends for various location categories relative to a historical baseline. These trends analyze the number of visits to public places like retail, parks, and workplaces, and also measure the change in duration of time spent in residences. - publication_date: '2020-04-01' data_product_sector: Healthcare + publication_date: '2020-04-01' dp_variant: variant_name: Pure DP data_domain: | @@ -35,7 +33,6 @@ deployment: deployment_model: model_name: Central - # model_name_details: '' actors: | - Google users who have opted in to Location History. The model assumes that Google users trust Google to access their raw data and correctly apply the DP mechanisms before publication. - Curator (Google) @@ -49,14 +46,17 @@ deployment: data_source_type_details: Metrics are generated for each day, starting on 2020-01-01 and ending on 2022-10-15 access_type: Non-interactive # access_type_details: '' - accounting: - post_processing: | - - Percentage Change Calculation: the final published value is a percentage change from the baseline. This is calculated by taking the ratio between the noisy metric for a specific day and the computed baseline for that day of the week (the baseline metric uses a fixed 5-week period from January 3, 2020 to February 6, 2020, by taking the median of the 5 corresponding daily noisy metrics from within that 5-week range). - - Filtering Unreliable Metrics: - - Statistical Reliability: A metric is not published if there is a 5% or greater chance that the percentage change is wrong by more than ±10 absolute percentage points. This is determined by calculating confidence intervals for both the daily metric and its baseline. - - Geographic Size: All metrics for any geographic region smaller than 3km^2 are discarded. - - User Count: Metrics are also discarded if the noisy count of contributing users for a given region is smaller than 100. + implementation: + preprocessing_and_hyperparameter_tuning: 'User contributions for public places are bounded: If a user visits more than four (category, location) pairs on a given day at a specific geographic level, four are chosen at random, and the rest are discarded. Each user can also contribute at most once per category at a given location.' + mechanisms: | + - The Laplace mechanism was used for simple counts, like the number of daily visits to public places and workplaces. The DP Mean Mechanism: used to calculate the average time spent at residences. + - The mechanisms were implemented via Google's open-source C++ Differential Privacy Library: + - github.com/google/differential-privacy + - The provided DP bounded-mean GitHub link is broken. Possible path to new file: github.com/google/differential-privacy/blob/main/cc/algorithms/bounded-mean.h + - No information was provided about protection against floating point errors. + - No information was provided about leakage due to idiosyncrasies of the computing platform. + - No information was provided about security measures in place to protect the underlying data. composition: | Basic composition - Daily Visits in Public Places: counts unique user visits to categories like retail, parks, and transit stations: @@ -77,16 +77,12 @@ deployment: - \\(\epsilon\\) = 0.22 for county-level (granularity 2) - Composition: A user contributes at most once per geographic level. The total budget is the sum across the levels: 0.11*2 + 0.22 = 0.44 - implementation: - preprocessing_and_hyperparameter_tuning: 'User contributions for public places are bounded: If a user visits more than four (category, location) pairs on a given day at a specific geographic level, four are chosen at random, and the rest are discarded. Each user can also contribute at most once per category at a given location.' - mechanisms: | - - The Laplace mechanism was used for simple counts, like the number of daily visits to public places and workplaces. The DP Mean Mechanism: used to calculate the average time spent at residences. - - The mechanisms were implemented via Google's open-source C++ Differential Privacy Library: - - github.com/google/differential-privacy - - The provided DP bounded-mean GitHub link is broken. Possible path to new file: github.com/google/differential-privacy/blob/main/cc/algorithms/bounded-mean.h - - No information was provided about protection against floating point errors. - - No information was provided about leakage due to idiosyncrasies of the computing platform. - - No information was provided about security measures in place to protect the underlying data. + post_processing: | + - Percentage Change Calculation: the final published value is a percentage change from the baseline. This is calculated by taking the ratio between the noisy metric for a specific day and the computed baseline for that day of the week (the baseline metric uses a fixed 5-week period from January 3, 2020 to February 6, 2020, by taking the median of the 5 corresponding daily noisy metrics from within that 5-week range). + - Filtering Unreliable Metrics: + - Statistical Reliability: A metric is not published if there is a 5% or greater chance that the percentage change is wrong by more than ±10 absolute percentage points. This is determined by calculating confidence intervals for both the daily metric and its baseline. + - Geographic Size: All metrics for any geographic region smaller than 3km^2 are discarded. + - User Count: Metrics are also discarded if the noisy count of contributing users for a given region is smaller than 100. justification: | - Using scaling factors allows for continuous accuracy improvements with minimal impact on the overall privacy budget by grouping metrics and reusing previously generated data, rather than performing a full, costly re-computation. @@ -95,7 +91,9 @@ deployment: administrative: sources: https://arxiv.org/pdf/2004.04145 + status: Pending registry_authors: - Elena Ghazi - Nicolas Berrios - Jack Fitzsimons + tier: 3 diff --git a/deployments/healthkit.yaml b/deployments/healthkit.yaml index 1b669b03..0b7a6d81 100644 --- a/deployments/healthkit.yaml +++ b/deployments/healthkit.yaml @@ -1,6 +1,4 @@ url_slug: healthkit-usage-statistics-apple-2017 -status: Pending -tier: 1 deployment: basic: name: HealthKit Usage Statistics @@ -8,12 +6,12 @@ deployment: - data_curator_name: 'Apple' data_curator_type: 'Industry' + description: Summarize most popular health data types that Apple users record (e.g., heart rate, active calories). intended_use: To learn—internally within Apple—about popular health data types and help improve the Health app data_product_type: Summary statistics data_product_region: Global - description: Summarize most popular health data types that Apple users record (e.g., heart rate, active calories). - publication_date: '2017-01-01' data_product_sector: Healthcare + publication_date: '2017-01-01' dp_variant: variant_name: Pure DP data_domain: '' # TODO @@ -41,7 +39,9 @@ deployment: sources: | - Paper: https://docs-assets.developer.apple.com/ml-research/papers/learning-with-privacy-at-scale.pdf - Additional version: https://machinelearning.apple.com/research/learning-with-privacy-at-scale + status: Pending registry_authors: - Priyanka Nanayakkara - Nicolas Berrios - Jack Fitzsimons + tier: 1 diff --git a/deployments/korean_statistics_datahub.yaml b/deployments/korean_statistics_datahub.yaml index 92a4a5f5..754866f5 100644 --- a/deployments/korean_statistics_datahub.yaml +++ b/deployments/korean_statistics_datahub.yaml @@ -1,6 +1,4 @@ url_slug: korean-government-statistical-data-hub-platform-kostat-national-statistics-office-of-korea--2021 -status: Pending -tier: 1 deployment: basic: name: Korean Government Statistical Data Hub Platform @@ -8,23 +6,21 @@ deployment: - data_curator_name: 'KOSTAT (National Statistics Office of Korea)' data_curator_type: 'Government' - intended_use: Goverment Data Hub - data_product_type: Summary statistics # TODO: Is this correct? - data_product_region: Korea description: | "Statistics Korea is promoting the establishment of a public big data system that leverages cutting-edge privacy-preserving techniques to enable the safe linkage and use of scattered governmental data." - publication_date: '2021-01-01' + intended_use: Goverment Data Hub + data_product_type: Summary statistics # TODO: Is this correct? + data_product_region: Korea data_product_sector: Government - # dp_variant: TODO + publication_date: '2021-01-01' privacy_loss: privacy_unit: Aggregate-level # privacy_unit_details: TODO privacy_parameters: {} deployment_model: model_name: Central - # model_name_details: TODO release_type: One release # TODO: Is this correct? release_type_details: Not specified access_type: Non-interactive # TODO: Is this correct? @@ -34,7 +30,9 @@ deployment: justification: '' # TODO: Fill in correct value administrative: sources: https://unstats.un.org/wiki/display/UGTTOPPT/11.+Statistics+Korea%3A+Developing+a+privacy-preserving+Statistical+Data+Hub+Platform + status: Pending registry_authors: - Nicolas Berrios - Jack Fitzsimons - James Honaker + tier: 1 diff --git a/deployments/linkedin_hiring_reports.yaml b/deployments/linkedin_hiring_reports.yaml index b8c21ab5..9024afb6 100644 --- a/deployments/linkedin_hiring_reports.yaml +++ b/deployments/linkedin_hiring_reports.yaml @@ -1,6 +1,4 @@ url_slug: linkedin-hiring-reports-linkedin-2020 -status: Pending -tier: 1 deployment: basic: name: LinkedIn Hiring Reports @@ -8,15 +6,15 @@ deployment: - data_curator_name: 'LinkedIn' data_curator_type: 'Industry' - intended_use: Labor Market Reports - data_product_type: Summary statistics # TODO: Is this correct? - data_product_region: Not specified description: | "Therefore each report is (1.2, 10−10)-DP because the ratio that uses both counts accumulates the privacy loss. Therefore, for each report date, we guarantee the top-employer reports are together (4.8, 4 × 10−10)-DP." - publication_date: '2020-01-01' + intended_use: Labor Market Reports + data_product_type: Summary statistics # TODO: Is this correct? + data_product_region: Not specified data_product_sector: Technology + publication_date: '2020-01-01' dp_variant: variant_name: Approximate DP data_domain: '' # TODO @@ -32,7 +30,6 @@ deployment: delta: 4.0e-10 deployment_model: model_name: Central - # model_name_details: TODO release_type: One release # TODO: Is this correct? release_type_details: Monthly access_type: Non-interactive # TODO: Is this correct? @@ -42,7 +39,9 @@ deployment: justification: '' # TODO: Fill in correct value administrative: sources: https://arxiv.org/abs/2010.13981 + status: Pending registry_authors: - Nicolas Berrios - Jack Fitzsimons - James Honaker + tier: 1 diff --git a/deployments/lookup_hints.yaml b/deployments/lookup_hints.yaml index 9d17b6f5..e3b2db28 100644 --- a/deployments/lookup_hints.yaml +++ b/deployments/lookup_hints.yaml @@ -1,6 +1,4 @@ url_slug: lookup-hints-apple-2016 -status: Pending -tier: 1 deployment: basic: name: Lookup Hints @@ -8,16 +6,16 @@ deployment: - data_curator_name: 'Apple' data_curator_type: 'Industry' - intended_use: Web Browsing - data_product_type: Summary statistics # TODO: Is this correct? - data_product_region: Not specified description: | "The Apple differential privacy implementation incorporates the concept of a per- donation privacy budget (quantified by the parameter epsilon), and sets a strict limit on the number of contributions from a user in order to preserve their privacy... For Lookup Hints, Apple uses a privacy budget with epsilon of 4, and limits user contributions to two donations per day." - publication_date: '2016-01-01' # Not in source by referenced in Tang et al. https://arxiv.org/abs/1709.02753 + intended_use: Web Browsing + data_product_type: Summary statistics # TODO: Is this correct? + data_product_region: Not specified data_product_sector: Technology + publication_date: '2016-01-01' # Not in source by referenced in Tang et al. https://arxiv.org/abs/1709.02753 dp_variant: variant_name: Pure DP data_domain: '' # TODO @@ -39,7 +37,9 @@ deployment: justification: '' # TODO: Fill in correct value administrative: sources: https://www.apple.com/privacy/docs/Differential_Privacy_Overview.pdf + status: Pending registry_authors: - Nicolas Berrios - Jack Fitzsimons - James Honaker + tier: 1 diff --git a/deployments/mhusage.yaml b/deployments/mhusage.yaml index df7e2e6a..8dc9c866 100644 --- a/deployments/mhusage.yaml +++ b/deployments/mhusage.yaml @@ -1,6 +1,4 @@ url_slug: mhusage-transluce-hiding-nemo-2026 -status: Draft -tier: 3 deployment: basic: # Tier 1: @@ -9,17 +7,17 @@ deployment: - data_curator_name: 'Transluce & Hiding Nemo' data_curator_type: 'Nonprofit' - intended_use: Enable external researchers to study how AI assistants respond to users experiencing mental health crises. - data_product_type: Synthetic data - data_product_region: Global description: | A differentially private synthetic version of a dataset of 8,237 conversations with AI assistants from two LLM providers (Anthropic and OpenAI), each rated against a mental-health rubric. Each conversation in the original data is described by 204 columns: the LLM provider it came from, two numeric columns (number of messages and number of words), and 201 binary rubric features. The binary features comprise 11 applicability criteria (whether a conversation qualifies for a given review criterion), 11 corresponding bucket flags (whether it was included in that criterion's review bucket), and 179 further features covering crisis acuity, mental health presentation, conversation goals, engagement style, how the user positions the assistant, social situation, and related dimensions. The rubric features were assigned by LLM judges reading the transcripts, independently for each feature. The synthetic dataset has the same schema and semantics. The noisy marginals that the mechanism measured are also published, so that analysts can derive confidence intervals for the quantities that were measured directly. The model itself is also published, so additional records can be generated as needed. - publication_date: '2026-09-01' + intended_use: Enable external researchers to study how AI assistants respond to users experiencing mental health crises. + data_product_type: Synthetic data + data_product_region: Global data_product_sector: Healthcare + publication_date: '2026-09-01' # Tiers 2 and 3: dp_variant: @@ -55,17 +53,6 @@ deployment: access_type: Non-interactive access_type_details: Analysts receive the synthetic dataset and the noisy marginals as files and cannot query the underlying data. - accounting: - post_processing: | - The privacy-critical operations are limited to the computation of noisy marginals, and the selection of which noisy marginals to compute. Everything else is post-processing. - - - A mixture-of-products model with 2,400 components is fitted to the noisy marginals by gradient descent, weighting each marginal by the inverse of its noise scale. - - 8,239 records are drawn from the fitted model. Rather than an i.i.d. draw, rows are allocated evenly across mixture components and each column's expected counts within a component are rounded to integers and shuffled. This substantially reduces sampling noise, at the cost that the released rows are not an independent sample. - - The two numeric columns are decoded from their bins by drawing within each bin (log-uniform in interior bins, uniform in the bin anchored at zero, truncated Pareto in the open-ended top bin). - - Combinations of flags that the rubric's definitions make impossible are repaired: 38 rules covering exclusions, implications, one mutually exclusive group, and two disjunctions. Repairs touched 706 of 8,239 records and changed 918 of 1.68 million cells. The rules are derived from the rubric's definitions rather than from the data; the released dataset includes the pre-repair version as well, so that the effect of this step can be measured. - composition: | - Adaptive composition under zCDP. The complete schedule of per-query costs — 204 one-way marginals, then 4,574 fixed marginals, then 3,000 (selection, measurement) pairs — is declared in advance. Which two-way marginals get selected depends on earlier answers, but their budget does not. - implementation: preprocessing_and_hyperparameter_tuning: | Pre-processing consists only binning the two numeric columns at declared edges (fixed by the analyst). @@ -79,6 +66,15 @@ deployment: - Composition uses OpenDP's `make_adaptive_composition` with the fixed schedule described above. The estimator is a re-implementation of MBI's mixture-of-products model, with performance optimizations. It is written in JAX. + composition: | + Adaptive composition under zCDP. The complete schedule of per-query costs — 204 one-way marginals, then 4,574 fixed marginals, then 3,000 (selection, measurement) pairs — is declared in advance. Which two-way marginals get selected depends on earlier answers, but their budget does not. + post_processing: | + The privacy-critical operations are limited to the computation of noisy marginals, and the selection of which noisy marginals to compute. Everything else is post-processing. + + - A mixture-of-products model with 2,400 components is fitted to the noisy marginals by gradient descent, weighting each marginal by the inverse of its noise scale. + - 8,239 records are drawn from the fitted model. Rather than an i.i.d. draw, rows are allocated evenly across mixture components and each column's expected counts within a component are rounded to integers and shuffled. This substantially reduces sampling noise, at the cost that the released rows are not an independent sample. + - The two numeric columns are decoded from their bins by drawing within each bin (log-uniform in interior bins, uniform in the bin anchored at zero, truncated Pareto in the open-ended top bin). + - Combinations of flags that the rubric's definitions make impossible are repaired: 38 rules covering exclusions, implications, one mutually exclusive group, and two disjunctions. Repairs touched 706 of 8,239 records and changed 918 of 1.68 million cells. The rules are derived from the rubric's definitions rather than from the data; the released dataset includes the pre-repair version as well, so that the effect of this step can be measured. justification: | - The initial workload of always-measured marginals was chosen to preserve particularly important statistical properties of the original data. The budget split was chosen to roughly equalize the noise each family of marginals carries relative to what it measures. - The repair rules encode constraints that follow from the schema (for example, a conversation can only be in a criterion's bucket if it meets that criterion). @@ -88,5 +84,7 @@ deployment: sources: | - Description of the mechanism and its utility metrics: https://behaviors.transluce.org/mental-health?detail=page:app_H_privacy_statement&detail=page:app_F_prod_features - Published dataset: https://huggingface.co/datasets/Transluce/MHUsage + status: Draft registry_authors: - Damien Desfontaines + tier: 3 diff --git a/deployments/microsoft_ctdc_victim_perpertartor_synthetic_dataset_2023.yaml b/deployments/microsoft_ctdc_victim_perpertartor_synthetic_dataset_2023.yaml index 59592933..3518889a 100644 --- a/deployments/microsoft_ctdc_victim_perpertartor_synthetic_dataset_2023.yaml +++ b/deployments/microsoft_ctdc_victim_perpertartor_synthetic_dataset_2023.yaml @@ -1,6 +1,4 @@ url_slug: ctdc-global-victim-perpetrator-synthetic-dataset-microsoft-international-organization-for-migration-2023 -status: Draft -tier: 3 deployment: basic: # Tier 1: @@ -12,16 +10,16 @@ deployment: - data_curator_name: 'International Organization for Migration' data_curator_type: 'Nonprofit' - intended_use: Provides first-hand statistics on the relationships between victims and perpetrators using a synthetic dataset that preserves the statistical properties of the original victim data without representing actual victims. The dataset supports IOM and anti-trafficking actors in translating real-world case data into evidence that can inform counter-trafficking policies and build support systems for the estimated 50 million victims of forced labor globally. - - data_product_type: Synthetic data - data_product_region: Global description: | - Differentially private synthetic dataset describing victim-perpetrator relations in human trafficking. - The dataset includes IOM case data from over 17,000 victims and survivors of trafficking identified across 123 countries and territories, and their accounts of over 37,000 perpetrators who facilitated the trafficking process from 2005 to 2022. - Generated using Microsoft's Synthetic Data Showcase. - publication_date: '2023-03-01' + intended_use: Provides first-hand statistics on the relationships between victims and perpetrators using a synthetic dataset that preserves the statistical properties of the original victim data without representing actual victims. The dataset supports IOM and anti-trafficking actors in translating real-world case data into evidence that can inform counter-trafficking policies and build support systems for the estimated 50 million victims of forced labor globally. + + data_product_type: Synthetic data + data_product_region: Global data_product_sector: Government + publication_date: '2023-03-01' # Tiers 2 and 3: dp_variant: @@ -49,19 +47,12 @@ deployment: deployment_model: model_name: Central - # model_name_details: '' - # actors: '' release_type: One release # release_type_details: '' data_source_type: Static - # data_source_type_details: '' access_type: Interactive access_type_details: The global victim-perpetrator synthetic data dashboard is released so analysts can visualise descriptive trends and explore the relationship between variables. - # accounting: - # post_processing: '' # Tier 3 - # composition: '' # Tier 3 - implementation: # preprocessing_and_hyperparameter_tuning: '' # Tier 3 mechanisms: | @@ -86,5 +77,7 @@ deployment: - Data Codebook: https://www.ctdatacollaborative.org/sites/g/files/tmzbdl2011/files/2023-07/CTDCsyntheic_victim-perpetrator_codebook_v1.pdf - Product used to generate the dataset: https://github.com/microsoft/synthetic-data-showcase # notes: '' # other extra information + status: Draft registry_authors: - Michael Khavkin + tier: 3 diff --git a/deployments/mobility_trends_hurricane.yaml b/deployments/mobility_trends_hurricane.yaml index aa3cb79c..1e74f5ad 100644 --- a/deployments/mobility_trends_hurricane.yaml +++ b/deployments/mobility_trends_hurricane.yaml @@ -1,6 +1,4 @@ url_slug: mobility-trends-during-hurricane-spectus-2022 -status: Pending -tier: 1 deployment: basic: name: Mobility Trends During Hurricane @@ -8,12 +6,12 @@ deployment: - data_curator_name: 'Spectus' data_curator_type: 'Industry' + description: Evacuation metrics for residents of Harris County, Texas during 2017 hurricane. intended_use: Mobility Catastrophe Data data_product_type: Summary statistics data_product_region: USA - description: Evacuation metrics for residents of Harris County, Texas during 2017 hurricane. - publication_date: '2022-01-01' data_product_sector: Government # Could also be Technology, Healthcare, or new category such as Public health + publication_date: '2022-01-01' dp_variant: variant_name: Pure DP privacy_loss: @@ -26,7 +24,6 @@ deployment: is set as follows: • \\(\epsilon = 4\\) for CTU • \\(\epsilon = 2\\) for CE • \\(\epsilon = 1\\) for CED • \\(\epsilon = 3\\) for SD." deployment_model: model_name: Central - # model_name_details: TODO release_type: One release release_type_details: Not specified access_type: Non-interactive @@ -38,7 +35,9 @@ deployment: sources: | - https://spectus.ai/wp-content/uploads/2022/10/Spectus_DPWhitepaper_v01b.pdf - https://web.archive.org/web/20221209065442/https://spectus.ai/wp-content/uploads/2022/10/Spectus_DPWhitepaper_v01b.pdf + status: Pending registry_authors: - Nicolas Berrios - Jack Fitzsimons - James Honaker + tier: 1 diff --git a/deployments/movement_ranges_maps.yaml b/deployments/movement_ranges_maps.yaml index be006c37..a5edf134 100644 --- a/deployments/movement_ranges_maps.yaml +++ b/deployments/movement_ranges_maps.yaml @@ -1,6 +1,4 @@ url_slug: movement-ranges-map-meta-2020 -status: Pending -tier: 1 deployment: basic: name: Movement Ranges Map @@ -8,12 +6,12 @@ deployment: - data_curator_name: 'Meta' data_curator_type: 'Industry' + description: Aggregate statistics describing Facebook users’ movements during the COVID-19 pandemic compared to a time period before social distancing measures intended_use: Health Crisis Mobility Data data_product_type: Summary statistics data_product_region: Global - description: Aggregate statistics describing Facebook users’ movements during the COVID-19 pandemic compared to a time period before social distancing measures - publication_date: '2020-06-20' data_product_sector: Healthcare + publication_date: '2020-06-20' dp_variant: variant_name: Pure DP privacy_loss: @@ -24,7 +22,6 @@ deployment: privacy_parameters_details: '"Resulting in a total differential privacy epsilon value of 2.0."' deployment_model: model_name: Central - # model_name_details: TODO release_type: Many releases release_type_details: Static data_source_type: Dynamic @@ -37,8 +34,10 @@ deployment: sources: | - Blog post: https://research.facebook.com/blog/2020/06/protecting-privacy-in-facebook-mobility-data-during-the-covid-19-response/ - Downloadable data product: https://data.humdata.org/dataset/movement-range-maps + status: Pending registry_authors: - Priyanka Nanayakkara - Nicolas Berrios - Jack Fitzsimons - James Honaker + tier: 1 diff --git a/deployments/on_device_browser_rec.yaml b/deployments/on_device_browser_rec.yaml index ff48fadc..fe736da2 100644 --- a/deployments/on_device_browser_rec.yaml +++ b/deployments/on_device_browser_rec.yaml @@ -1,6 +1,4 @@ url_slug: on-device-browser-recommendations-brave-2021 -status: Pending -tier: 1 deployment: basic: name: On-device browser recommendations @@ -8,13 +6,13 @@ deployment: - data_curator_name: 'Brave' data_curator_type: 'Industry' + description: Brave is leveraging federated learning and differential + privacy to provide on-device browser recommendations. intended_use: Browsing Recommendations data_product_type: Summary statistics # TODO: Is this correct? data_product_region: Global - description: Brave is leveraging federated learning and differential - privacy to provide on-device browser recommendations. - publication_date: '2021-01-01' data_product_sector: Technology + publication_date: '2021-01-01' # dp_variant: TODO privacy_loss: privacy_unit: User-level @@ -22,7 +20,6 @@ deployment: privacy_parameters: {} deployment_model: model_name: Central - # model_name_details: TODO release_type: One release # TODO: Is this correct? release_type_details: Not specified access_type: Non-interactive # TODO: Is this correct? @@ -32,7 +29,9 @@ deployment: justification: '' # TODO: Fill in correct value administrative: sources: https://brave.com/blog/federated-learning/ + status: Pending registry_authors: - Nicolas Berrios - Jack Fitzsimons - James Honaker + tier: 1 diff --git a/deployments/private_third_party_audits.yaml b/deployments/private_third_party_audits.yaml index 751c5cbb..b67589b2 100644 --- a/deployments/private_third_party_audits.yaml +++ b/deployments/private_third_party_audits.yaml @@ -1,6 +1,4 @@ url_slug: private-third-party-audits-twitter-openmined-2022 -status: Pending -tier: 1 deployment: basic: name: Private Third Party Audits @@ -11,17 +9,17 @@ deployment: - data_curator_name: 'OpenMined' data_curator_type: 'Industry' - intended_use: Synthetic Data Releases for Audits - data_product_type: Summary statistics # TODO: Is this correct? - data_product_region: Global description: In September, 2022, in conjunction with the UN General Assembly and Christchurch Call leaders summit, New Zealand Prime Minister Jacinda Ardern and French President Emmanuel Macron announced the Christchurch Initiative on Algorithmic Outcomes, a partnership between New Zealand, the United States, Twitter, Microsoft and OpenMined to develop and test a differential privacy system to enable privacy preserving research across multiple online platforms. - publication_date: '2022-01-01' + intended_use: Synthetic Data Releases for Audits + data_product_type: Summary statistics # TODO: Is this correct? + data_product_region: Global data_product_sector: Technology + publication_date: '2022-01-01' # dp_variant: TODO privacy_loss: privacy_unit: Aggregate-level @@ -29,7 +27,6 @@ deployment: privacy_parameters: {} deployment_model: model_name: Central - # model_name_details: TODO release_type: One release # TODO: Is this correct? release_type_details: Static access_type: Non-interactive # TODO: Is this correct? @@ -39,6 +36,8 @@ deployment: justification: '' # TODO: Fill in correct value administrative: sources: https://unstats.un.org/wiki/display/UGTTOPPT/14.+Twitter+and+OpenMined%3A+Enabling+Third-party+Audits+and+Research+Reproducibility+over+Unreleased+Digital+Assets + status: Pending registry_authors: - Nicolas Berrios - Jack Fitzsimons + tier: 1 diff --git a/deployments/recurve_energy_dp.yaml b/deployments/recurve_energy_dp.yaml index 5cc3d9da..29f8c5e6 100644 --- a/deployments/recurve_energy_dp.yaml +++ b/deployments/recurve_energy_dp.yaml @@ -1,6 +1,4 @@ url_slug: energy-differential-privacy-for-ohmconnect-virtual-power-plant-measurement-recurve-2020 -status: Pending -tier: 3 deployment: basic: name: Energy Differential Privacy for OhmConnect Virtual Power Plant Measurement @@ -8,16 +6,16 @@ deployment: - data_curator_name: 'Recurve' data_curator_type: 'Industry' - intended_use: Measure the impact of OhmConnect's demand response program on electricity usage, while protecting customer privacy. - data_product_type: Summary statistics - data_product_region: California, USA description: | Aggregate summary statistics describing how much energy OhmConnect participants saved during a demand response event (during the August 14, 2020 California Blackout), relative to a matched, non-participant group. Specifically, the statistics are: - Average Load Shape: A private, hour-by-hour profile of the comparison group's average 'observed' and 'predicted' energy consumption over a 24-hour period. - Percent Load Change: The percentage difference between total predicted and total observed energy consumption for each group during the 3-hour event window. - Difference of Load Changes: The final net impact of the OhmConnect program, calculated by subtracting the comparison group's percent load change from the treatment group's percent load change. - publication_date: '2020-12-01' + intended_use: Measure the impact of OhmConnect's demand response program on electricity usage, while protecting customer privacy. + data_product_type: Summary statistics + data_product_region: California, USA data_product_sector: Energy + publication_date: '2020-12-01' dp_variant: variant_name: Approximate DP @@ -38,7 +36,6 @@ deployment: deployment_model: model_name: Central - # model_name_details: actors: | - Data Owners: MCE provided the sensitive data belonging to users not participating in OhmConnect's program. - Data Curator: Recurve @@ -48,30 +45,9 @@ deployment: release_type: One release # release_type_details: '' data_source_type: Static - # data_source_type_details: '' access_type: Non-interactive # access_type_details: '' - accounting: - post_processing: Calculating the final net impact by taking the private 'Percent Load Change' statistic computed with DP from the comparison group and subtracting it from the non-private 'Percent Load Change' of the treatment group to produce the final 'Difference of Load Changes'. - composition: | - Basic Composition: - - Consumption Histogram: - - A private histogram query was run on the comparison group's data to understand the distribution of hourly consumption values. This was used to generate a synthetic dataset to find the \\(epsilon\\) for the Sparse Vector Technique. - - \\(\epsilon\\) = 0.1 - - Sparse Vector Technique for Clamping Bound: - - The Sparse Vector Technique was used to privately select an upper bound for hourly energy consumption. - - \\(\epsilon\\) = 0.2 - - Comparison Group Load Shape: - - A 'vector-valued Gaussian Mechanism query' was executed to produce the private average load shape for the comparison group. This single query produced 48 data points (24 for observed hourly usage and 24 for predicted hourly usage). - - \\(\epsilon\\) = 4.0 and \\(\delta\\)=4.08e-8 - - Total Predicted and Observed Consumption Sums: - - Two sum queries using the Laplace Mechanism were used to calculate the inputs for the 'Percent Load Change' statistic. One query calculated the total predicted consumption and the other the total observed consumption during the 3-hour event window. - - \\(\epsilon\\) = 1.25 per query, therefore a total of \\(\epsilon \\) = 2.5. - - Total Privacy Impact Calculation: - - Pre-amplification: Under basic composition, the steps above yield \\(\epsilon=6.8\\) and \\(\delta\\)=4.08e-8. - - Because the comparison group was a sample of a larger population, the paper applies an 'amplification factor' of 0.124 to reduce the total privacy cost. The correct amplification formula yields \\(\epsilon = 4.72\\) and \\(\delta = 5.06e-9\\). - implementation: preprocessing_and_hyperparameter_tuning: | - The CalTRACK methodology was used to model and generate a 'predicted' (counterfactual) hourly energy usage for every customer in both the treatment (non-sensitive) and comparison (sensitive) groups. @@ -93,11 +69,32 @@ deployment: - Google's differential privacy Go library was used 'to determine the Analytical Gaussian sigma values and securely add noise', 'primarily to mitigate the floating-point attack against differential privacy'. - No information was provided about leakage due to idiosyncrasies of the computing platform. - No information was provided about security measures in place to protect the underlying data. + composition: | + Basic Composition: + - Consumption Histogram: + - A private histogram query was run on the comparison group's data to understand the distribution of hourly consumption values. This was used to generate a synthetic dataset to find the \\(epsilon\\) for the Sparse Vector Technique. + - \\(\epsilon\\) = 0.1 + - Sparse Vector Technique for Clamping Bound: + - The Sparse Vector Technique was used to privately select an upper bound for hourly energy consumption. + - \\(\epsilon\\) = 0.2 + - Comparison Group Load Shape: + - A 'vector-valued Gaussian Mechanism query' was executed to produce the private average load shape for the comparison group. This single query produced 48 data points (24 for observed hourly usage and 24 for predicted hourly usage). + - \\(\epsilon\\) = 4.0 and \\(\delta\\)=4.08e-8 + - Total Predicted and Observed Consumption Sums: + - Two sum queries using the Laplace Mechanism were used to calculate the inputs for the 'Percent Load Change' statistic. One query calculated the total predicted consumption and the other the total observed consumption during the 3-hour event window. + - \\(\epsilon\\) = 1.25 per query, therefore a total of \\(\epsilon \\) = 2.5. + - Total Privacy Impact Calculation: + - Pre-amplification: Under basic composition, the steps above yield \\(\epsilon=6.8\\) and \\(\delta\\)=4.08e-8. + - Because the comparison group was a sample of a larger population, the paper applies an 'amplification factor' of 0.124 to reduce the total privacy cost. The correct amplification formula yields \\(\epsilon = 4.72\\) and \\(\delta = 5.06e-9\\). + post_processing: Calculating the final net impact by taking the private 'Percent Load Change' statistic computed with DP from the comparison group and subtracting it from the non-private 'Percent Load Change' of the treatment group to produce the final 'Difference of Load Changes'. justification: | - The values of \\(epsilon\\) for each of the Average Load Shape and the Percent Load Change were chosen to meet specific levels of precision. - Google's differential privacy Go library was used to mitigate floating point attacks. - An 'isolation attack' was simulated to demonstrate that the noise added by a Gaussian mechanism with \\(\epsilon=0.843\\) 'limits an attacker to a confidence interval of around ±1,000 kWh for hourly consumption.' + additional_information: | + - The term 'Energy Differential Privacy', used in the deployment's name and throughout the paper, refers to the specific framework and open-source tools for applying differential privacy to energy data analysis. + administrative: sources: | @@ -105,7 +102,7 @@ deployment: - Revenue-Grade Analysis of the OhmConnect Virtual Power Plant During the California Blackouts (blog): https://www.recurve.com/blog/revenue-grade-analysis-of-the-ohmconnect-virtual-power-plant-during-the-california-blackouts - Differential Privacy for Expanding Access to Building Energy Data (paper that introduced Energy Differential Privacy): https://edp.recurve.com/uploads/8/6/5/0/8650231/aceee_paper_final.pdf - A list of real-world uses of differential privacy (blog that provides the correct privacy parameters): https://desfontain.es/blog/real-world-differential-privacy.html - notes: | - - The term 'Energy Differential Privacy', used in the deployment's name and throughout the paper, refers to the specific framework and open-source tools for applying differential privacy to energy data analysis. + status: Pending registry_authors: - Elena Ghazi + tier: 3 diff --git a/deployments/safari_energy_drain.yaml b/deployments/safari_energy_drain.yaml index c973e6e9..712021be 100644 --- a/deployments/safari_energy_drain.yaml +++ b/deployments/safari_energy_drain.yaml @@ -1,6 +1,4 @@ url_slug: safari-energy-draining-and-crashing-domains-apple-2017 -status: Pending -tier: 3 deployment: basic: name: Safari Energy Draining and Crashing Domains @@ -8,12 +6,12 @@ deployment: - data_curator_name: 'Apple' data_curator_type: 'Industry' + description: Identify sites with high memory or high CPU usage. intended_use: Browser Statistics data_product_type: Summary statistics data_product_region: Global - description: Identify sites with high memory or high CPU usage. - publication_date: '2017-01-01' data_product_sector: Technology + publication_date: '2017-01-01' dp_variant: variant_name: Pure DP @@ -38,26 +36,27 @@ deployment: release_type_details: Daily access_type: Non-interactive # pre-defined query - accounting: + implementation: + preprocessing_and_hyperparameter_tuning: 'No hyperperameters are required; no EDA is described' + mechanisms: Private Hadamard Count Mean Sketch (PHCMS) + composition: No information was provided about composition. post_processing: | - Server strips IP addresses and associated ids from each record. - Aggregation - private histograms are generated through sketch mechanism PHCMS - Threshold - only domain elements with counts above threshold T (level undocumented) are disclosed - composition: No information was provided about composition. - - implementation: - preprocessing_and_hyperparameter_tuning: 'No hyperperameters are required; no EDA is described' - mechanisms: Private Hadamard Count Mean Sketch (PHCMS) justification: '"Formal proof, described in Theorem 4.1, Theorem 4.3"' + additional_information: | + - There are multiple releases per device & per user -- statistics are reported automatically on IoS and MacOS devices, subject to daily aggregation. + - There is only one aggregation query applied per-event. + administrative: sources: https://docs-assets.developer.apple.com/ml-research/papers/learning-with-privacy-at-scale.pdf - notes: | - - There are multiple releases per device & per user -- statistics are reported automatically on IoS and MacOS devices, subject to daily aggregation. - - There is only one aggregation query applied per-event. + status: Pending registry_authors: - James Honaker - Micah Altman - Nicolas Berrios - Jack Fitzsimons + tier: 3 diff --git a/deployments/safety_classifier.yaml b/deployments/safety_classifier.yaml index cfd429ea..b0435f5d 100644 --- a/deployments/safety_classifier.yaml +++ b/deployments/safety_classifier.yaml @@ -1,6 +1,4 @@ url_slug: safety-classifier-google-2024 -status: Pending -tier: 2 deployment: basic: name: Safety Classifier @@ -8,12 +6,12 @@ deployment: - data_curator_name: 'Google' data_curator_type: 'Industry' + description: A safety classifier (machine learning model) that detects inappropriate content; trained on differentially private synthetic data which was generated based on internal (to Google) data intended_use: To classify safety of content shown to Google users on mobile devices and ensure that users do not see unsafe content. data_product_type: Machine learning model data_product_region: Global - description: A safety classifier (machine learning model) that detects inappropriate content; trained on differentially private synthetic data which was generated based on internal (to Google) data - publication_date: '2024-01-01' data_product_sector: Technology + publication_date: '2024-01-01' dp_variant: variant_name: Approximate DP @@ -30,15 +28,10 @@ deployment: deployment_model: model_name: Central - # actors: release_type: One release data_source_type: Static access_type: Non-interactive - # accounting: - # post_processing: - # composition: - # implementation: # preprocessing_and_hyperparameter_tuning: # mechanisms: @@ -47,5 +40,7 @@ deployment: sources: | - Blog post: https://research.google/blog/protecting-users-with-differentially-private-synthetic-training-data/ - Research paper: https://arxiv.org/pdf/2306.01684 + status: Pending registry_authors: - Priyanka Nanayakkara + tier: 2 diff --git a/deployments/sas_data_maker_vulnerable_persons.yaml b/deployments/sas_data_maker_vulnerable_persons.yaml index a8015b9a..e73accc2 100644 --- a/deployments/sas_data_maker_vulnerable_persons.yaml +++ b/deployments/sas_data_maker_vulnerable_persons.yaml @@ -1,6 +1,4 @@ url_slug: synthetic-data-to-test-third-party-solutions-in-financial-services-nationwide-building-society-hazy-now-sas-data-maker--2020 -status: Pending -tier: 2 # TODO: or possibly 3? deployment: basic: name: Synthetic data to test third-party solutions in financial services @@ -11,12 +9,12 @@ deployment: - data_curator_name: 'Hazy (now SAS Data Maker)' data_curator_type: 'Industry' + description: Synthetic data generated with Differential Privacy is used by Nationwide to evaluate a third-party solution for detecting and supporting vulnerable individuals in financial services. The data is produced on-premises using Hazy‘s (now SAS Data Maker) platform from over 50,000 customers, 100,000 accounts, and 50 million transactions across 18 months. It includes customer identifiers, account details, transaction amounts, and merchant information. intended_use: Enable Nationwide to generate and use synthetic data with Differential Privacy guarantees to evaluate a third-party solution for detecting and supporting vulnerable individuals in financial services. data_product_type: Synthetic data data_product_region: UK - description: Synthetic data generated with Differential Privacy is used by Nationwide to evaluate a third-party solution for detecting and supporting vulnerable individuals in financial services. The data is produced on-premises using Hazy‘s (now SAS Data Maker) platform from over 50,000 customers, 100,000 accounts, and 50 million transactions across 18 months. It includes customer identifiers, account details, transaction amounts, and merchant information. - publication_date: '2020-01-01' data_product_sector: Finance + publication_date: '2020-01-01' dp_variant: variant_name: Pure DP @@ -42,17 +40,6 @@ deployment: release_type_details: A single multi-table synthetic dataset, generated from 18 months of historical customer data, is released to the third-party company for testing purposes. access_type: Non-interactive - accounting: - post_processing: | - - generating random synthetic IDs/names after discarding real account/transaction/merchant IDs, and merchant names, using a similar format - - clipping negative values - - rounding (selected) numerical values to integers/desired precision - - removing implausible records - - ensuring logical consistency (e.g., income > spending) - composition: | - - Sequential composition: Two PrivBayes network models (with pure DP) are trained sequentially -- one on Customers table and one on Accounts table (conditioned on Customers). Since the datasets overlap, sequential composition was used, and the total privacy budget is \\(\epsilon\\) = 5.0 (2.5 + 2.5). - - Group privacy: To account for the one-to-many relationship between Customers and Accounts, the number of accounts per customer is bounded to 5 in the second PrivBayes network model, ensuring privacy loss accounting under group privacy. - implementation: preprocessing_and_hyperparameter_tuning: | - Preprocessing: | @@ -64,6 +51,16 @@ deployment: The PrivBayes network model constructs a Bayesian network to break down the high-dimensional joint distribution of the dataset into low-dimensional conditional distributions. It involves two main steps: - 1) uses the Exponential mechanism to add edges based on noisy mutual information scores, spending half of the privacy budget at this step - 2) applies the Laplace mechanism to generate noisy contingency tables (spending the other half of the privacy budget), which are normalized into consistent conditional distributions and used for synthetic data generation + composition: | + - Sequential composition: Two PrivBayes network models (with pure DP) are trained sequentially -- one on Customers table and one on Accounts table (conditioned on Customers). Since the datasets overlap, sequential composition was used, and the total privacy budget is \\(\epsilon\\) = 5.0 (2.5 + 2.5). + - Group privacy: To account for the one-to-many relationship between Customers and Accounts, the number of accounts per customer is bounded to 5 in the second PrivBayes network model, ensuring privacy loss accounting under group privacy. + post_processing: | + - generating random synthetic IDs/names after discarding real account/transaction/merchant IDs, and merchant names, using a similar format + - clipping negative values + - rounding (selected) numerical values to integers/desired precision + - removing implausible records + - ensuring logical consistency (e.g., income > spending) + justification: | - Synthetic data is used by Nationwide as it helps minimize the amount of personal data processed and reduces the risk of inappropriate access to customer data, while allowing for effectively testing the third-party product prior to procuring it. Nationwide ensures that real/customer data remains within its secure production environment by deploying the synthetic data solution from Hazy (now SAS Data Maker) on-premises. Moreover, a standardized contractual agreement with the third-party strictly prohibits any reidentification of real customers. - PrivBayes network is chosen to model the static tables (Customers and Accounts) because it offers a strong balance between utility, privacy, computational efficiency, scalability, and interpretability. @@ -75,5 +72,7 @@ deployment: - Usecase by Nationwide: https://medium.com/nationwide-technology/hazy-synthetic-data-to-fuel-rapid-innovation-fd24f2e21685 - PrivBayes paper: https://dl.acm.org/doi/10.1145/3134428 - DoppelGANger paper: https://arxiv.org/abs/1909.13403 + status: Pending registry_authors: - Georgi Ganev + tier: 2 # TODO: or possibly 3? diff --git a/deployments/shared_mobility_dataset.yaml b/deployments/shared_mobility_dataset.yaml index e25d2a0a..a6f36986 100644 --- a/deployments/shared_mobility_dataset.yaml +++ b/deployments/shared_mobility_dataset.yaml @@ -1,6 +1,4 @@ url_slug: shared-mobility-dataset-google-2022 -status: Pending -tier: 1 deployment: basic: name: Shared Mobility Dataset @@ -8,12 +6,12 @@ deployment: - data_curator_name: 'Google' data_curator_type: 'Industry' + description: Intra-urban metrics on trips. intended_use: Mobility Data data_product_type: Summary statistics data_product_region: Global - description: Intra-urban metrics on trips. - publication_date: '2022-01-01' data_product_sector: Technology + publication_date: '2022-01-01' dp_variant: variant_name: Approximate DP privacy_loss: @@ -31,7 +29,6 @@ deployment: of \\(\epsilon = 0.66\\) and \\(\delta = 2.1 × 10−29\\), which is very strong."' deployment_model: model_name: Central - # model_name_details: TODO release_type: One release release_type_details: Static access_type: Non-interactive @@ -41,7 +38,9 @@ deployment: justification: '"All trips were anonymized and aggregated by jointly applying differential privacy via the Laplace mechanism in combination with k-anonymity."' administrative: sources: https://www.nature.com/articles/s41467-019-12809-y + status: Pending registry_authors: - Nicolas Berrios - Jack Fitzsimons - James Honaker + tier: 1 diff --git a/deployments/spanish_language_next_word.yaml b/deployments/spanish_language_next_word.yaml index 25638077..2d45d424 100644 --- a/deployments/spanish_language_next_word.yaml +++ b/deployments/spanish_language_next_word.yaml @@ -1,6 +1,4 @@ url_slug: spanish-next-word-prediction-google-2022 -status: Pending -tier: 3 deployment: basic: name: Spanish Next Word Prediction @@ -8,12 +6,12 @@ deployment: - data_curator_name: 'Google' data_curator_type: 'Industry' + description: A machine learning model trained with federated learning for next‑word prediction on Spanish‑language Gboard. intended_use: Power next-word-prediction for Spanish-language Gboard users. data_product_type: Machine learning model data_product_region: Global - description: A machine learning model trained with federated learning for next‑word prediction on Spanish‑language Gboard. - publication_date: '2022-02-01' data_product_sector: Technology + publication_date: '2022-02-01' dp_variant: variant_name: Zero-concentrated DP @@ -29,7 +27,6 @@ deployment: deployment_model: model_name: Central - # model_name_details: actors: | - Users/Devices of Spanish-language Gboard users. They hold their own raw data and compute model updates locally. Hence, users do not need to trust the central server with their raw data, but must trust it to correctly implement the DP mechanism. - Server (Google): Receives model updates from devices, aggregates them, and adds the necessary noise to ensure DP by adding noise to one gradient and subtracting that same noise from a later gradient @@ -46,10 +43,6 @@ deployment: access_type: Non-interactive # access_type_details: '' - accounting: - post_processing: No details were provided about post-processing. - composition: The article does not specifically mention composition, only that the final privacy guarantee is rho = 0.81 zCDP. It mentions that training ran for 2000 rounds over six days, and devices participated in training at most once every 24 hours. - implementation: preprocessing_and_hyperparameter_tuning: No information is provided about pre-processing steps, EDA, or the process by which hyperparameters (privacy loss parameter or number of training rounds) were tuned. mechanisms: | @@ -58,6 +51,9 @@ deployment: - No information was provided about protection against floating point errors. - No information was provided about leakage due to idiosyncrasies of the computing platform. - The primary security measure protecting the underlying data is the federated learning architecture itself, as it keeps the data on each user's device, decoupling ML training from the need to store the data in the cloud. This approach limits access to data at all stages because only minimal updates for a specific model training task are transmitted from the device. + composition: The article does not specifically mention composition, only that the final privacy guarantee is rho = 0.81 zCDP. It mentions that training ran for 2000 rounds over six days, and devices participated in training at most once every 24 hours. + post_processing: No details were provided about post-processing. + justification: | - The main reason behind choosing federated learning was to train machine learning models while keeping the raw training data on each user's device. This approach provides data minimization advantages, and the DP guarantee protects all of the data on each device, not just individual training examples. - The project implemented user-level DP because example-level DP is not necessarily strong enough to ensure the users' data is not memorized when users can contribute multiple examples. @@ -69,7 +65,9 @@ deployment: sources: | - Federated Learning with Formal Differential Privacy Guarantees (Article published on February 28, 2022): https://research.google/blog/federated-learning-with-formal-differential-privacy-guarantees/ - Paper that introduces the DP variant of Follow-The-Regularized-Leader (DP-FTRL) used in the deployment: Practical and Private (Deep) Learning Without Sampling or Shuffling: https://arxiv.org/pdf/2103.00039 + status: Pending registry_authors: - Elena Ghazi - Nicolas Berrios - Jack Fitzsimons + tier: 3 diff --git a/deployments/synth_data_public_use.yaml b/deployments/synth_data_public_use.yaml index 6f7b966a..62addf29 100644 --- a/deployments/synth_data_public_use.yaml +++ b/deployments/synth_data_public_use.yaml @@ -1,6 +1,4 @@ url_slug: synthetic-data-for-public-use-office-of-national-statistics-uk-2020 -status: Pending -tier: 1 deployment: basic: name: Synthetic Data for Public Use @@ -8,14 +6,14 @@ deployment: - data_curator_name: 'Office of National Statistics UK' data_curator_type: 'Government' - intended_use: Synthetic data - data_product_type: Synthetic data - data_product_region: USA description: We recommend the use of generative adversarial networks (GANs) possibly in conjunction with privacy-preserving mechanisms such as differential privacy. - publication_date: '2020-01-01' + intended_use: Synthetic data + data_product_type: Synthetic data + data_product_region: USA data_product_sector: Government + publication_date: '2020-01-01' # dp_variant: TODO privacy_loss: privacy_unit: Aggregate Statistics @@ -23,7 +21,6 @@ deployment: privacy_parameters: {} deployment_model: model_name: Central - # model_name_details: TODO release_type: One release # TODO: Is this correct? release_type_details: Static access_type: Non-interactive # TODO: Is this correct? @@ -33,7 +30,9 @@ deployment: justification: '' # TODO: Fill in correct value administrative: sources: https://datasciencecampus.ons.gov.uk/projects/generative-adversarial-networks-gans-for-synthetic-dataset-generation-with-binary-classes/ + status: Pending registry_authors: - Nicolas Berrios - Jack Fitzsimons - James Honaker + tier: 1 diff --git a/deployments/telemetry_collection_windows.yaml b/deployments/telemetry_collection_windows.yaml index c7996014..20ded07f 100644 --- a/deployments/telemetry_collection_windows.yaml +++ b/deployments/telemetry_collection_windows.yaml @@ -1,6 +1,4 @@ url_slug: telemetry-collection-in-windows-microsoft-2017 -status: Pending -tier: 1 deployment: basic: name: Telemetry Collection in Windows @@ -8,12 +6,12 @@ deployment: - data_curator_name: 'Microsoft' data_curator_type: 'Industry' + description: Telemetry statistics about Microsoft users (specifically users of Windows Insiders in the Windows 10 Fall Creators Update), such as how many seconds they spent on a particular app in a six hour window. intended_use: For Microsoft to improve their products and services. data_product_type: Summary statistics data_product_region: Global - description: Telemetry statistics about Microsoft users (specifically users of Windows Insiders in the Windows 10 Fall Creators Update), such as how many seconds they spent on a particular app in a six hour window. - publication_date: '2017-12-01' data_product_sector: Technology + publication_date: '2017-12-01' dp_variant: variant_name: Pure DP privacy_loss: @@ -26,7 +24,6 @@ deployment: \\(\epsilon \prime \prime\\)-DP, where \\(\epsilon \prime \prime\\) = 1.672."' deployment_model: model_name: Local - # model_name_details: TODO release_type: Many releases release_type_details: 6 hours data_source_type: Dynamic @@ -38,8 +35,10 @@ deployment: administrative: sources: | - Paper: https://www.microsoft.com/en-us/research/publication/collecting-telemetry-data-privately/ + status: Pending registry_authors: - Priyanka Nanayakkara - Nicolas Berrios - Jack Fitzsimons - James Honaker + tier: 1 diff --git a/deployments/uber.yaml b/deployments/uber.yaml index 6b10b001..0315fe78 100644 --- a/deployments/uber.yaml +++ b/deployments/uber.yaml @@ -1,6 +1,4 @@ url_slug: flex-uber-2018 -status: Pending -tier: 2 deployment: basic: name: FLEX @@ -8,15 +6,15 @@ deployment: - data_curator_name: 'Uber' data_curator_type: 'Industry' + description: A flexible end-to-end system for computing differential privacy + using "elastic sensitivity" over heterogeneous existing database + environments, at scale. intended_use: To compute privacy-protected SQL queries using existing SQL databases, in real time, at millions-of-records scale data_product_type: SQL queries data_product_region: Global - description: A flexible end-to-end system for computing differential privacy - using "elastic sensitivity" over heterogeneous existing database - environments, at scale. - publication_date: '2018-01-01' data_product_sector: Technology + publication_date: '2018-01-01' dp_variant: variant_name: Approximate DP data_domain: Multi-table relational database (schema unspecifed) @@ -48,11 +46,6 @@ deployment: data_source_type_details: Database is updated dynamically through unspecified business processes access_type: Interactive access_type_details: 'Authorized internal users design, execute, and receive multiple queries over an extended period of time. Privacy parameters are determined by user, at request time.' - accounting: - post_processing: none described - composition: | - - deployment does not manage privacy budgets - - user can compute overall privacy loss using strong composition bounds on sequential queries implementation: preprocessing_and_hyperparameter_tuning: none described mechanisms: | @@ -66,6 +59,11 @@ deployment: - No information was provided about protection from timing attacks. - No information was provided about leakage due to idiosyncrasies of the computing platform. - Security measure to protect underlying data: none measured + composition: | + - deployment does not manage privacy budgets + - user can compute overall privacy loss using strong composition bounds on sequential queries + post_processing: none described + justification: | - Formal proof that elastic sensitivity is an upper bound of local sensistivity is presented. administrative: @@ -73,5 +71,7 @@ deployment: - Paper: https://vldb.org/pvldb/vol11/p526-johnson.pdf # https://doi.org/10.1145/3177732.3177733 is not presently working - Blog post: https://medium.com/uber-security-privacy/uber-open-source-differential-privacy-57f31e85c57a - Source repo: https://github.com/uber-archive/sql-differential-privacy + status: Pending registry_authors: - Micah Altman + tier: 2 diff --git a/deployments/user_url_privacy.yaml b/deployments/user_url_privacy.yaml index 27a9b7a9..32bd59f0 100644 --- a/deployments/user_url_privacy.yaml +++ b/deployments/user_url_privacy.yaml @@ -1,6 +1,4 @@ url_slug: user-url-privacy-dataset-meta-2020 -status: Pending -tier: 1 deployment: basic: name: User-URL Privacy Dataset @@ -8,12 +6,12 @@ deployment: - data_curator_name: 'Meta' data_curator_type: 'Industry' + description: '"A data set designed to allow researchers to study the distribution of URLs on Facebook and how users interacted with them."' intended_use: Web Browsing data_product_type: Dataset data_product_region: Global - description: '"A data set designed to allow researchers to study the distribution of URLs on Facebook and how users interacted with them."' - publication_date: '2020-06-06' data_product_sector: Technology + publication_date: '2020-06-06' dp_variant: variant_name: Zero-concentrated DP data_domain: User actions on a URL. @@ -36,7 +34,6 @@ deployment: \\(\rho_{user}\\) remains at 0.0052 for each individual action column and remains at 0.0728 across the entire dataset." deployment_model: model_name: Central - # model_name_details: TODO release_type: One release release_type_details: Static access_type: Non-interactive @@ -48,7 +45,9 @@ deployment: sources: | - Data: https://dataverse.harvard.edu/file.xhtml?persistentId=doi:10.7910/DVN/TDOAPG/DGSAMS&version=6.2 - FAQ: https://developers.facebook.com/docs/url-shares-dataset/support/faqs + status: Pending registry_authors: - Nicolas Berrios - Jack Fitzsimons - James Honaker + tier: 1 diff --git a/deployments/vaccine_search_insights.yaml b/deployments/vaccine_search_insights.yaml index bcf90c90..4f3af32a 100644 --- a/deployments/vaccine_search_insights.yaml +++ b/deployments/vaccine_search_insights.yaml @@ -1,6 +1,4 @@ url_slug: vaccine-search-insights-google-2021 -status: Pending -tier: 1 deployment: basic: name: Vaccine Search Insights @@ -8,12 +6,12 @@ deployment: - data_curator_name: 'Google' data_curator_type: 'Industry' + description: Dashboard and dataset of trends in searches on COVID-19 vaccination. intended_use: Health Data data_product_type: Summary statistics data_product_region: Global - description: Dashboard and dataset of trends in searches on COVID-19 vaccination. - publication_date: '2021-07-02' data_product_sector: Healthcare # Could also be Technology. Could also be new topic such as 'Public health'. + publication_date: '2021-07-02' dp_variant: variant_name: Approximate DP privacy_loss: @@ -28,7 +26,6 @@ deployment: privacy for \\(\epsilon\\) = 2.19 and \\(\delta\\) = 10−5." deployment_model: model_name: Central - # model_name_details: TODO release_type: One release # TODO: Is this correct? release_type_details: Daily access_type: Non-interactive # TODO: Is this correct? @@ -39,6 +36,8 @@ deployment: administrative: sources: | - Paper: https://arxiv.org/abs/2107.01179 + status: Pending registry_authors: - Nicolas Berrios - Jack Fitzsimons + tier: 1 diff --git a/deployments/wikimedia_current_usage_data.yaml b/deployments/wikimedia_current_usage_data.yaml index c8dfcbe2..c8d2091c 100644 --- a/deployments/wikimedia_current_usage_data.yaml +++ b/deployments/wikimedia_current_usage_data.yaml @@ -1,6 +1,4 @@ url_slug: current-pageviews-wikimedia-foundation-2023 -status: Pending -tier: 3 deployment: basic: name: Current Pageviews @@ -8,12 +6,12 @@ deployment: - data_curator_name: 'Wikimedia Foundation' data_curator_type: 'Nonprofit' + description: Daily statistics of Wikipedia pageview counts, broken down by country of origin starting February 2023. intended_use: Help Wikipedia editors prioritize their work and support academic research. data_product_type: Summary statistics data_product_region: Global - description: Daily statistics of Wikipedia pageview counts, broken down by country of origin starting February 2023. - publication_date: '2023-06-01' data_product_sector: Technology + publication_date: '2023-06-01' dp_variant: variant_name: Zero-concentrated DP @@ -46,10 +44,6 @@ deployment: access_type: Non-interactive # access_type_details: '' - accounting: - post_processing: 'Suppression of low counts: After adding DP noise, any count that falls below a pre-defined threshold \\(\tau\\) is removed from the final output dataset. For current data, \\(\tau\\) = 90.' - composition: 'There is no need for composition analysis: The data is partitioned by day, and there is no overlap between days, so the privacy loss does not accumulate.' - implementation: preprocessing_and_hyperparameter_tuning: | - Client-side filtering: Before pageview data is sent to the server, a client-side algorithm annotates each contribution with a boolean flag, which indicates whether the contribution should be included in the server side computation, the criteria being that only the first 10 unique pageviews per user, per day are included. The client uses a salted hash of the page ID to track which pages have already been counted, rather than storing raw identifiers, further reducing fingerprinting risk. This pre-processing step enables user-privacy without sending a user identifier to the server (which would be cookie-based and consistent across days). @@ -61,6 +55,9 @@ deployment: - No information was provided about protection against floating point errors. - No information was provided about leakage due to idiosyncrasies of the computing platform. - No information was provided about security measures in place to protect the underlying data. + composition: 'There is no need for composition analysis: The data is partitioned by day, and there is no overlap between days, so the privacy loss does not accumulate.' + post_processing: 'Suppression of low counts: After adding DP noise, any count that falls below a pre-defined threshold \\(\tau\\) is removed from the final output dataset. For current data, \\(\tau\\) = 90.' + justification: | - The Gaussian mechanism was chosen because its use of the \\(L_2\\) sensitivity allowed for less noise to be added. - The thresholds were chosen after extensive experimentation by measuring accuracy metrics (relative error distribution, drop rate, spurious rate) on the true data. Since these metrics could leak information, the team “kept fine-grained utility metrics confidential throughout the tuning process, minimizing data leakage”, and chose to “only publicly communicate approximate values of global utility metrics and the algorithmic parameters obtained from this tuning process”. @@ -74,7 +71,9 @@ deployment: - Client-side filtering code: https://gerrit.wikimedia.org/r/plugins/gitiles/operations/puppet/+/refs/heads/production/modules/varnish/templates/analytics.inc.vcl.erb#171 - Differential Privacy: https://gitlab.wikimedia.org/repos/security/differential-privacy - Pageviews Differential Privacy – Current – README: https://gitlab.wikimedia.org/htriedman/stat-spark3/-/tree/main/pageview_historical/notebooks + status: Pending registry_authors: - Elena Ghazi - Nicolas Berrios - Jack Fitzsimons + tier: 3 diff --git a/deployments/wikimedia_editor_activity_statistics_2023.yaml b/deployments/wikimedia_editor_activity_statistics_2023.yaml index 165d5789..347b5112 100644 --- a/deployments/wikimedia_editor_activity_statistics_2023.yaml +++ b/deployments/wikimedia_editor_activity_statistics_2023.yaml @@ -1,6 +1,4 @@ url_slug: wikipedia-geoeditors-activity-statistics-wikimedia-foundation-2023 -status: Draft -tier: 3 deployment: basic: # Tier 1: @@ -9,12 +7,12 @@ deployment: - data_curator_name: 'Wikimedia Foundation' data_curator_type: 'Nonprofit' + description: The Wikimedia Foundation has released differentially private statistics on editor activity (e.g., edit counts) by Wikipedia project and country from January 2018 to the present time. This has revolutionized the way the Wikimedia Foundation exposes information to the public about its projects, allowing it to release new statistics on a large scale. According to the Wikimedia Foundation, more than 135M statistics about Wikipedia page visits have been published, aggregating 325B page views in total. intended_use: Release daily Wikipedia page visits at a country level, based on current and historical Wikipedia page views. data_product_type: Summary statistics data_product_region: Global - description: The Wikimedia Foundation has released differentially private statistics on editor activity (e.g., edit counts) by Wikipedia project and country from January 2018 to the present time. This has revolutionized the way the Wikimedia Foundation exposes information to the public about its projects, allowing it to release new statistics on a large scale. According to the Wikimedia Foundation, more than 135M statistics about Wikipedia page visits have been published, aggregating 325B page views in total. - publication_date: '2023-01-01' data_product_sector: Technology + publication_date: '2023-01-01' # Tiers 2 and 3: dp_variant: @@ -33,19 +31,12 @@ deployment: deployment_model: model_name: Central - # model_name_details: '' - # actors: '' release_type: One release # release_type_details: '' data_source_type: Static - # data_source_type_details: '' access_type: Non-interactive # access_type_details: '' - # accounting: - # post_processing: '' # Tier 3 - # composition: '' # Tier 3 - implementation: # preprocessing_and_hyperparameter_tuning: '' # Tier 3 mechanisms: | @@ -67,5 +58,7 @@ deployment: - Online description page: https://analytics.wikimedia.org/published/datasets/geoeditors_monthly/00_README.html - Code repository: https://gitlab.wikimedia.org/repos/security/differential-privacy/-/blob/main/differential_privacy/geoeditors_monthly.py # notes: '' + status: Draft registry_authors: - Michael Khavkin + tier: 3 diff --git a/deployments/wikimedia_historical_usage_data.yaml b/deployments/wikimedia_historical_usage_data.yaml index 9859a81c..d48c43c8 100644 --- a/deployments/wikimedia_historical_usage_data.yaml +++ b/deployments/wikimedia_historical_usage_data.yaml @@ -1,6 +1,4 @@ url_slug: historical-pageviews-wikimedia-foundation-2023 -status: Pending -tier: 3 deployment: basic: name: Historical Pageviews @@ -8,12 +6,12 @@ deployment: - data_curator_name: 'Wikimedia Foundation' data_curator_type: 'Nonprofit' + description: Daily statistics of Wikipedia pageview counts, broken down by country of origin between February 2017 and February 2023. intended_use: Help Wikipedia editors prioritize their work and support academic research. data_product_type: Summary statistics data_product_region: Global - description: Daily statistics of Wikipedia pageview counts, broken down by country of origin between February 2017 and February 2023. - publication_date: '2023-06-01' data_product_sector: Technology + publication_date: '2023-06-01' dp_variant: variant_name: Pure DP @@ -43,7 +41,6 @@ deployment: deployment_model: model_name: Central - model_name_details: '' actors: | - End-user devices (clients): the individuals who browse Wikipedia. - Trusted Curator (The Wikimedia Foundation): Receives client-annotated pageviews, performs server-side differential privacy (adds noise, suppresses counts), and publishes the final data. @@ -56,10 +53,6 @@ deployment: access_type: Non-interactive # access_type_details: '' - accounting: - post_processing: 'Suppression of low counts: After adding DP noise, any count that falls below a pre-defined threshold tau is removed from the final output dataset. For 2017-2023 historical data, \\(\tau=450\\). For 2015-2017 data, \\(\tau=2500\\).' - composition: 'The results are published once with \\(\epsilon=1\\), and there is no additional privacy consumption.' - implementation: preprocessing_and_hyperparameter_tuning: | - The server determines the list of groups for which to release statistics by identifying all pages that have more than a global pageview threshold (which was chosen based on the true data with no DP) and create the cross-product of those pages with a pre-defined list of countries. @@ -73,10 +66,16 @@ deployment: - Raw pageview logs contain no persistent user identifiers and are only kept in “current” form for 90 days; older data is aggregated hourly, reducing sensitivity. - A salted hash function is used on the client side to protect the cookie that tracks visited pages, providing protection against an attacker who might obtain access to it. - All raw data handling is governed by the Wikimedia Foundation’s Privacy Policy and Data Retention Guidelines. Because of the Wikimedia Foundation’s commitment to minimal data retention, this data is only kept in this form for 90 days before being aggregated. + composition: 'The results are published once with \\(\epsilon=1\\), and there is no additional privacy consumption.' + post_processing: 'Suppression of low counts: After adding DP noise, any count that falls below a pre-defined threshold tau is removed from the final output dataset. For 2017-2023 historical data, \\(\tau=450\\). For 2015-2017 data, \\(\tau=2500\\).' + justification: | - The Laplace mechanism was chosen (and not Gaussian as in the current pageviews deployment) because the team only had a bound on the \\(L_1\\) sensitivity of the dataset. - The thresholds were chosen after extensive experimentation by measuring accuracy metrics (relative error distribution, drop rate, spurious rate) on the true data. Since these metrics could leak information, the team “kept fine-grained utility metrics confidential throughout the tuning process, minimizing data leakage”, and chose to “only publicly communicate approximate values of global utility metrics and the algorithmic parameters obtained from this tuning process”. - After an in-depth comparison of available open-source tools, the Wikimedia Foundation decided to use Tumult Analytics, framework chosen for its robustness, production-readiness, compatibility with Wikimedia’s compute infrastructure, and support for advanced features like zCDP-based privacy accounting, and started a collaboration with Tumult Labs. + additional_information: | + - The paper does not explicitly state what constitutes adjacent datasets, and it is not obvious, because the maximum contribution per user was estimated to be m. Therefore, the reported input metric and maximum input distance may need additional follow-up to verify its accuracy. + administrative: sources: | @@ -85,9 +84,9 @@ deployment: - Differential Privacy: https://gitlab.wikimedia.org/repos/security/differential-privacy - Pageviews Differential Privacy – Current – README: https://gitlab.wikimedia.org/htriedman/stat-spark3/-/tree/main/pageview_historical/notebooks - Pageviews Differential Privacy – Historical – README: https://analytics.wikimedia.org/published/datasets/country_project_page_historical/00_README.html - notes: | - - The paper does not explicitly state what constitutes adjacent datasets, and it is not obvious, because the maximum contribution per user was estimated to be m. Therefore, the reported input metric and maximum input distance may need additional follow-up to verify its accuracy. + status: Pending registry_authors: - Elena Ghazi - Nicolas Berrios - Jack Fitzsimons + tier: 3 diff --git a/deployments/wikimedia_russian_editor_activity_statistics_2023.yaml b/deployments/wikimedia_russian_editor_activity_statistics_2023.yaml index 14b6ba90..5410a327 100644 --- a/deployments/wikimedia_russian_editor_activity_statistics_2023.yaml +++ b/deployments/wikimedia_russian_editor_activity_statistics_2023.yaml @@ -1,6 +1,4 @@ url_slug: wikipedia-russian-editor-activity-statistics-wikimedia-foundation-2023 -status: Draft -tier: 3 deployment: basic: # Tier 1: @@ -9,12 +7,12 @@ deployment: - data_curator_name: 'Wikimedia Foundation' data_curator_type: 'Nonprofit' + description: The Wikimedia Foundation has released a one-time dataset of the number of Wikipedia editors editing from Russia from 2022 to 2023, requested by the Russian Wikimedia community to support analysis of the editor landscape in Russia and better understand the editor community in the country. intended_use: The Wikimedia Foundation has released one year of data quantifying the number of Wikipedia editors editing from Russia to help better understand the editor community in Russia. data_product_type: Summary statistics data_product_region: Russia - description: The Wikimedia Foundation has released a one-time dataset of the number of Wikipedia editors editing from Russia from 2022 to 2023, requested by the Russian Wikimedia community to support analysis of the editor landscape in Russia and better understand the editor community in the country. - publication_date: '2023-01-01' data_product_sector: Technology + publication_date: '2023-01-01' # Tiers 2 and 3: dp_variant: @@ -33,19 +31,12 @@ deployment: deployment_model: model_name: Central - # model_name_details: '' - # actors: '' release_type: One release # release_type_details: '' data_source_type: Static - # data_source_type_details: '' access_type: Non-interactive # access_type_details: '' - # accounting: - # post_processing: '' # Tier 3 - # composition: '' # Tier 3 - implementation: # preprocessing_and_hyperparameter_tuning: '' # Tier 3 mechanisms: | @@ -67,5 +58,7 @@ deployment: - Code or table building: https://gerrit.wikimedia.org/r/plugins/gitiles/analytics/refinery/%2B/refs/heads/master/hql/geoeditors/geoeditors_monthly.hql - Code for private data release: https://gitlab.wikimedia.org/htriedman/stat-spark3/-/blob/main/geoeditors/notebooks/04_russia_release.ipynb # notes: '' # other extra information + status: Draft registry_authors: - Michael Khavkin + tier: 3 diff --git a/schemas/deployments-schema.yaml b/schemas/deployments-schema.yaml index ed258854..1866a481 100644 --- a/schemas/deployments-schema.yaml +++ b/schemas/deployments-schema.yaml @@ -9,44 +9,17 @@ type: object additionalProperties: False required: - url_slug - - status - - tier - deployment properties: url_slug: description: Used to create a stable URL. type: string - status: - description: Approval status of this record. - type: string - enum: - - "Converted" # Automatically converted from earlier collection of "cases". - - "Draft" # Newly authored record. - - "Pending" # Waiting for review by the board. - - "Changes Required" # Board requires changes. - - "Approved" # Yay! - - "Approved (Update Requested)" # Small updates requested, but still can be displayed as "Approved". - - "Approved (Pending)" # Small updates made. - tier: - description: The completeness of the description. For higher tiers, more fields are filled in, but this is guidance, rather than a requirement. - description_long: | - Deployments are registered at one of three **tiers**. Each increasing tier requires more information to be disclosed about the deployment, allowing for a tradeoff between ease of registration and greater transparency. Below are descriptions of each tier. - - **Tier 1 (\*):** Tier 1 entries provide basic information about each deployment, such as the curator, intended use of the data product, a description of the data product, and publication date. These entries may also include more information (e.g., in the form of external links) where available. - - **Tier 2 (\*\*):** Tier 2 entries provide an intermediate level of information about each deployment. They include the information required for Tier 1 entries, plus information about the variant of differential privacy, privacy unit and privacy loss parameters, and deployment model. - - **Tier 3 (\*\*\*\):** Tier 3 entries provide an advanced level of information about each deployment, including information required for Tier 1 and Tier 2 entries plus additional information. This additional information includes extra details related to the variant of DP (such as the data domain and unprotected quantities) and a more detailed description of the privacy unit. It also includes information on privacy accounting (on post-processing and composition) and implementation & security, including rationale for implementation decisions. - type: integer - enum: - - 1 - - 2 - - 3 deployment: type: object additionalProperties: False required: - basic + - administrative properties: basic: type: object @@ -75,6 +48,8 @@ properties: properties: data_curator_name: description: The name of the entity publishing the data product. + description_long: | + The **curator** refers to the entity who published the data product. Examples of curators include specific government agencies and companies. type: string data_curator_type: description: Category that best describes the entity publishing the data product. @@ -85,6 +60,10 @@ properties: - Nonprofit # Add to this list as needed. minItems: 1 + description: + tier: 1 + description: Brief description of the data product. + type: string intended_use: tier: 1 description: Intended use(s) of the data product. @@ -107,17 +86,6 @@ properties: tier: 1 description: Free text. What region does the data describe, and/or, what region's laws apply to this data product. type: string - description: - tier: 1 - description: Brief description of the data product. - type: string - publication_date: - tier: 1 - description: When the data product was published, in YYYY-MM-DD format. Day and month can be set to "01" if unknown. In cases of many releases, the publication date can be expressed as the date of first publication. - description_long: | - The publication **date** refers to when the data product was published or otherwise created. In cases of data products that are created under continual release, the date listed is the date of first publication. - type: string - format: date data_product_sector: tier: 1 description: The industry or domain described by the data product. @@ -130,6 +98,13 @@ properties: - Energy - Finance # Add to this list as needed. + publication_date: + tier: 1 + description: When the data product was published, in YYYY-MM-DD format. Day and month can be set to "01" if unknown. In cases of many releases, the publication date can be expressed as the date of first publication. + description_long: | + The publication **date** refers to when the data product was published or otherwise created. In cases of data products that are created under continual release, the date listed is the date of first publication. + type: string + format: date # Tiers 2 and 3 dp_variant: type: object @@ -184,6 +159,12 @@ properties: tier: 3 description: A precise specification of what constitutes adjacent datasets (e.g., in terms of your dataset schema. Function that computes the distance between between any two datasets in the data domain (based partly on HoDP, pg 55) Maximum distance, computed by the input metric, between any two datasets in the data domain; pairs of datasets whose distance is within this bound are called “adjacent datasets”. type: string + number_of_privacy_units: + tier: 3 + description: | + Number of privacy units (e.g., number of users, households, sets of all events associated with individual users in a day). + In cases where the dataset size is not public, an approximate number should be reported here. + type: integer privacy_parameters: tier: 2 description: Intensity of protection, as characterized by values set for parameters like epsilon, delta, or rho. Which parameters are specified will vary according to the DP variant. @@ -248,7 +229,9 @@ properties: - Federated - Varies # Present in converted records model_name_details: + tier: 3 type: string + description: TODO; Not included in whitepaper, but not sure where existing data should be moved. trust_assumptions: tier: 3 type: string @@ -256,8 +239,10 @@ properties: Describe relevant actors (anyone who may access the data product or underlying data) and trust assumptions for each. Include rationale for these assumptions where possible. Consider, for example, what kinds of outside information these actors may have and why. actors: tier: 3 - description: Who are the relevant actors in the deployment? This includes anyone who may see the data product, even partially, and including adversaries. What are their trust assumptions and what is the rationale for these trust assumptions? - type: string + description: | + Describe relevant actors (anyone who may access the data product or underlying data) and trust assumptions for each. + Include rationale for these assumptions where possible. + Consider, for example, what kinds of outside information these actors may have and why. release_type: tier: 2 description: | @@ -286,8 +271,9 @@ properties: - Static - Dynamic data_source_type_details: - tier: 3 - description: If more description is useful. + tier: 2 + type: string + description: TODO; Not included in whitepaper, but not sure where existing data should be moved. access_type: tier: 2 description: | @@ -302,32 +288,6 @@ properties: tier: 3 description: Tier 3 entries that are interactive should also describe how the privacy loss budget is apportioned to and across analysts, and whether non-collusion between analysts is assumed. Under non-interactive deployments, people cannot interactively query the underlying data. Instead, they must interact with the published data product as is. type: string - accounting: - description_long: | - **Privacy-loss accounting**, i.e., how privacy loss is accounted for over multiple queries to the underlying data, is described by composition and post-processing: - type: object - additionalProperties: False - properties: - post_processing: - tier: 3 - description: Functions applied to the data product after being protected under DP - description_long: | - Once differential privacy has been applied to generate a data product, functions applied to the data product thereafter, termed post-processing, do not incur additional privacy loss. - The impact of these functions may be important to know about for downstream applications using the data product, so they are reported for Tier 3 entries. - For example, imagine a data curator releasing a data product that includes counts of people in different geographic regions. - A privacy-protected count may be negative, but the curator may not want to publish a seemingly nonsensical count and may turn these negative counts into zeros. - type: string - composition: - tier: 3 - description: How privacy loss is accounted across multiple differentially private queries, like sequential or parallel composition - description_long: | - Data products are often generated via multiple queries to the underlying data. - Each query to the data requires specifying values for the privacy parameters. - How privacy parameter values accumulate under differential privacy is described by different forms of composition. - Examples of composition include sequential composition and parallel composition. - Under sequential composition, assuming pure DP, the sum of the privacy parameter value (epsilon) per query is the total privacy parameter value. - Parallel composition, again assuming pure DP, says that the total privacy parameter value for a set of queries applied on disjoint subsets of the data is the maximum privacy parameter value used for one of the queries. - type: string implementation: description_long: | **Implementation** details are provided for Tier 3 entries. They include the following information: @@ -355,6 +315,26 @@ properties: Differential privacy can be achieved using several mechanisms, i.e., algorithms that take the data as input and output a privacy-protected data product. Tier 3 entries describe these mechanisms and how they were implemented (e.g., using a library). These entries also describe other security-related implementation details, like what measures were taken to protect against known vulnerabilities, like floating point and timing channel attacks (e.g., see (Jin, McMurtry, Rubinstein, Ohrimenko 2022). If available, code for implementation should be linked in this section. type: string + composition: + tier: 3 + description: How privacy loss is accounted across multiple differentially private queries, like sequential or parallel composition + description_long: | + Data products are often generated via multiple queries to the underlying data. + Each query to the data requires specifying values for the privacy parameters. + How privacy parameter values accumulate under differential privacy is described by different forms of composition. + Examples of composition include sequential composition and parallel composition. + Under sequential composition, assuming pure DP, the sum of the privacy parameter value (epsilon) per query is the total privacy parameter value. + Parallel composition, again assuming pure DP, says that the total privacy parameter value for a set of queries applied on disjoint subsets of the data is the maximum privacy parameter value used for one of the queries. + type: string + post_processing: + tier: 3 + description: Functions applied to the data product after being protected under DP + description_long: | + Once differential privacy has been applied to generate a data product, functions applied to the data product thereafter, termed post-processing, do not incur additional privacy loss. + The impact of these functions may be important to know about for downstream applications using the data product, so they are reported for Tier 3 entries. + For example, imagine a data curator releasing a data product that includes counts of people in different geographic regions. + A privacy-protected count may be negative, but the curator may not want to publish a seemingly nonsensical count and may turn these negative counts into zeros. + type: string justification: tier: 3 description: | @@ -364,11 +344,22 @@ properties: description_long: | Rationale for how decisions around implementation were made by the curator or others. In short, this information should answer the following questions laid out by Dwork Mulligan Kohli 2019: “What were the assumptions, modelling decisions, thresholds, and subjective decisions made in determining the implementation choices above? Why is the approach a thorough test of the stated assumptions? Was the process validated and verified? If so, how? type: string + utility: + tier: 3 + description: | + Details about utility of the data product, if available. There is not a well-agreed-upon set of utility metrics, so we do not prescribe metrics that should be reported here. Instead, we recommend the advisory board determine how utility should be expressed. + additional_information: + tier: 1 + description: | + Additional details about the deployment, which could optionally include complete details of the implementation pipeline and performance data. + May also include links to white papers, blog posts, or other resources describing the deployment, links to the data product if publicly available, and any other miscellaneous notes that do not fit into any of the above sections. administrative: type: object additionalProperties: False required: + - status - registry_authors + - tier properties: sources: description_long: | @@ -376,12 +367,35 @@ properties: This section may also include links to the data product itself, if publicly available. tier: 1 type: string - notes: - tier: 1 + status: + description: Approval status of this record. type: string + enum: + - "Converted" # Automatically converted from earlier collection of "cases". + - "Draft" # Newly authored record. + - "Pending" # Waiting for review by the board. + - "Changes Required" # Board requires changes. + - "Approved" # Yay! + - "Approved (Update Requested)" # Small updates requested, but still can be displayed as "Approved". + - "Approved (Pending)" # Small updates made. registry_authors: description: The author(s) of this record. This refers to who filled out the entry, which might be different from who made the deployment. type: array items: type: string minItems: 1 + tier: + description: The completeness of the description. For higher tiers, more fields are filled in, but this is guidance, rather than a requirement. + description_long: | + Deployments are registered at one of three **tiers**. Each increasing tier requires more information to be disclosed about the deployment, allowing for a tradeoff between ease of registration and greater transparency. Below are descriptions of each tier. + + **Tier 1 (\*):** Tier 1 entries provide basic information about each deployment, such as the curator, intended use of the data product, a description of the data product, and publication date. These entries may also include more information (e.g., in the form of external links) where available. + + **Tier 2 (\*\*):** Tier 2 entries provide an intermediate level of information about each deployment. They include the information required for Tier 1 entries, plus information about the variant of differential privacy, privacy unit and privacy loss parameters, and deployment model. + + **Tier 3 (\*\*\*\):** Tier 3 entries provide an advanced level of information about each deployment, including information required for Tier 1 and Tier 2 entries plus additional information. This additional information includes extra details related to the variant of DP (such as the data domain and unprotected quantities) and a more detailed description of the privacy unit. It also includes information on privacy accounting (on post-processing and composition) and implementation & security, including rationale for implementation decisions. + type: integer + enum: + - 1 + - 2 + - 3 diff --git a/scripts/dump.py b/scripts/dump.py new file mode 100644 index 00000000..cd0f3e2c --- /dev/null +++ b/scripts/dump.py @@ -0,0 +1,21 @@ +from pathlib import Path +from yaml import load, Loader +import re + + +def _get_path_nodes(path, schema): + yield (path, schema) + if "properties" in schema: + for subpath, subschema in schema["properties"].items(): + for path_node in _get_path_nodes(f"{path}/{subpath}", subschema): + yield path_node + + +if __name__ == "__main__": + root = Path(__file__).parent.parent + schema = load((root / "schemas/deployments-schema.yaml").open(), Loader=Loader) + + path_nodes = list(_get_path_nodes("", schema)) + paths = [re.sub(r"[^/]+/", " ", path[1:]) for path, node in path_nodes] + + print("\n".join(paths)) diff --git a/tests/bad_deployments/check_quoting_instead_of_use_to_preserve_whitespace_in_these_collapse_hurts_markdown_n.yaml b/tests/bad_deployments/check_quoting_instead_of_use_to_preserve_whitespace_in_these_collapse_hurts_markdown_n.yaml index 22b9ef12..4967b252 100644 --- a/tests/bad_deployments/check_quoting_instead_of_use_to_preserve_whitespace_in_these_collapse_hurts_markdown_n.yaml +++ b/tests/bad_deployments/check_quoting_instead_of_use_to_preserve_whitespace_in_these_collapse_hurts_markdown_n.yaml @@ -1,6 +1,4 @@ url_slug: bad -status: Draft -tier: 1 deployment: basic: name: ... @@ -18,3 +16,5 @@ deployment: administrative: registry_authors: - ... + status: Draft + tier: 1 diff --git a/tests/bad_deployments/check_schema_additional_properties_are_not_allowed_surprise_was_unexpected.yaml b/tests/bad_deployments/check_schema_additional_properties_are_not_allowed_surprise_was_unexpected.yaml index 143062fd..e351ed68 100644 --- a/tests/bad_deployments/check_schema_additional_properties_are_not_allowed_surprise_was_unexpected.yaml +++ b/tests/bad_deployments/check_schema_additional_properties_are_not_allowed_surprise_was_unexpected.yaml @@ -1,7 +1,5 @@ url_slug: bad surprise: ... -status: Draft -tier: 1 deployment: basic: name: ... @@ -17,3 +15,5 @@ deployment: administrative: registry_authors: - ... + status: Draft + tier: 1 diff --git a/tests/bad_deployments/check_schema_status_is_a_required_property.yaml b/tests/bad_deployments/check_schema_administrative_status_is_a_required_property.yaml similarity index 96% rename from tests/bad_deployments/check_schema_status_is_a_required_property.yaml rename to tests/bad_deployments/check_schema_administrative_status_is_a_required_property.yaml index 7017cf0f..f7fde02b 100644 --- a/tests/bad_deployments/check_schema_status_is_a_required_property.yaml +++ b/tests/bad_deployments/check_schema_administrative_status_is_a_required_property.yaml @@ -1,5 +1,4 @@ url_slug: bad -tier: 1 deployment: basic: name: ... @@ -13,5 +12,6 @@ deployment: data_product_region: ... publication_date: "2026-01-01" administrative: + tier: 1 registry_authors: - ... diff --git a/tests/bad_deployments/check_schema_deployment_basic_publication_date_1_1_2026_is_not_a_date.yaml b/tests/bad_deployments/check_schema_publication_date_1_1_2026_is_not_a_date.yaml similarity index 92% rename from tests/bad_deployments/check_schema_deployment_basic_publication_date_1_1_2026_is_not_a_date.yaml rename to tests/bad_deployments/check_schema_publication_date_1_1_2026_is_not_a_date.yaml index e0fe00bc..cc2041ac 100644 --- a/tests/bad_deployments/check_schema_deployment_basic_publication_date_1_1_2026_is_not_a_date.yaml +++ b/tests/bad_deployments/check_schema_publication_date_1_1_2026_is_not_a_date.yaml @@ -1,6 +1,4 @@ url_slug: bad -status: Draft -tier: 1 deployment: basic: name: ... @@ -16,3 +14,5 @@ deployment: administrative: registry_authors: - ... + status: Draft + tier: 1 diff --git a/tests/bad_deployments/check_schema_deployment_basic_publication_date_datetime_date_2026_1_1_is_not_of_type_string.yaml b/tests/bad_deployments/check_schema_publication_date_date_2026_1_1_is_not_of_type_string.yaml similarity index 92% rename from tests/bad_deployments/check_schema_deployment_basic_publication_date_datetime_date_2026_1_1_is_not_of_type_string.yaml rename to tests/bad_deployments/check_schema_publication_date_date_2026_1_1_is_not_of_type_string.yaml index 464ff244..42814e04 100644 --- a/tests/bad_deployments/check_schema_deployment_basic_publication_date_datetime_date_2026_1_1_is_not_of_type_string.yaml +++ b/tests/bad_deployments/check_schema_publication_date_date_2026_1_1_is_not_of_type_string.yaml @@ -1,6 +1,4 @@ url_slug: bad -status: Draft -tier: 1 deployment: basic: name: ... @@ -16,3 +14,5 @@ deployment: administrative: registry_authors: - ... + status: Draft + tier: 1 diff --git a/tests/bad_deployments/check_schema_tier_0_is_not_one_of_1_2_3.yaml b/tests/bad_deployments/check_schema_tier_0_is_not_one_of_1_2_3.yaml index 976f7190..0a028f86 100644 --- a/tests/bad_deployments/check_schema_tier_0_is_not_one_of_1_2_3.yaml +++ b/tests/bad_deployments/check_schema_tier_0_is_not_one_of_1_2_3.yaml @@ -1,6 +1,5 @@ url_slug: bad -status: Draft -tier: 0 + deployment: basic: name: ... @@ -14,5 +13,7 @@ deployment: data_product_region: ... publication_date: "2026-01-01" administrative: + status: Draft + tier: 0 registry_authors: - ... diff --git a/tests/good_deployments/google_tier_1.yaml b/tests/good_deployments/google_tier_1.yaml index f3d41163..0e814ed2 100644 --- a/tests/good_deployments/google_tier_1.yaml +++ b/tests/good_deployments/google_tier_1.yaml @@ -1,6 +1,4 @@ url_slug: google_tier_1 -status: Draft -tier: 1 deployment: basic: name: Placeholder @@ -14,5 +12,7 @@ deployment: data_product_region: Global publication_date: "2020-09-01" administrative: + status: Draft registry_authors: - Elena Ghazi + tier: 1 diff --git a/tests/good_deployments/google_tier_2.yaml b/tests/good_deployments/google_tier_2.yaml index a2d28726..fe9cca9c 100644 --- a/tests/good_deployments/google_tier_2.yaml +++ b/tests/good_deployments/google_tier_2.yaml @@ -1,6 +1,4 @@ url_slug: google_tier_2 -status: Draft -tier: 2 deployment: basic: name: Placeholder @@ -27,5 +25,7 @@ deployment: access_type: Non-interactive data_source_type: Static administrative: + status: Draft registry_authors: - Elena Ghazi + tier: 2 diff --git a/tests/good_deployments/google_tier_3.yaml b/tests/good_deployments/google_tier_3.yaml index bc241ea8..fd98abf9 100644 --- a/tests/good_deployments/google_tier_3.yaml +++ b/tests/good_deployments/google_tier_3.yaml @@ -1,6 +1,4 @@ url_slug: google_tier_3 -status: Draft -tier: 3 deployment: basic: name: Placeholder @@ -31,8 +29,9 @@ deployment: access_type: Non-interactive access_type_details: PLACEHOLDER FOR TEST data_source_type: Static - - accounting: + implementation: + preprocessing_and_hyperparameter_tuning: Not available + mechanisms: Laplace mechanism for both symptom search and normalization counts via Google’s open-source DP library post_processing: | A region-specific constant c is chosen once based on the February-July 2020 window to ensure we end up in a [0,100] percent scale. @@ -41,11 +40,9 @@ deployment: Deciding whether to release daily or weekly granularity is made once on the initial window for top-level geopolitical subdivisions (e.g., US states) or higher-resolution granularity (e.g., US counties). Release per country is always daily. composition: | Basic sequential composition (counting symptoms + normalization) - - implementation: - preprocessing_and_hyperparameter_tuning: Not available - mechanisms: Laplace mechanism for both symptom search and normalization counts via Google’s open-source DP library justification: Not available administrative: + status: Draft registry_authors: - Elena Ghazi + tier: 3 diff --git a/tests/good_deployments/template.yaml b/tests/good_deployments/template.yaml index e2a90be9..84da6806 100644 --- a/tests/good_deployments/template.yaml +++ b/tests/good_deployments/template.yaml @@ -1,6 +1,4 @@ url_slug: template -status: Draft -tier: 3 deployment: basic: # Tier 1: @@ -9,12 +7,12 @@ deployment: - data_curator_name: 'U.S. Census Bureau' # Fill in correct value. data_curator_type: 'Government' # Fill in correct value. + description: '' intended_use: '' data_product_type: 'Summary statistics' # Fill in correct value. data_product_region: '' - description: '' - publication_date: '1900-01-01' # Fill in correct value. data_product_sector: 'Technology' # Fill in correct value. + publication_date: '1900-01-01' # Fill in correct value. # Tiers 2 and 3: dp_variant: @@ -25,6 +23,7 @@ deployment: privacy_loss: privacy_unit: '' privacy_unit_details: '' + number_of_privacy_units: 1 privacy_parameters: # Fill in correct value, and remove others epsilon: 1.0 delta: 1.0e-10 @@ -43,17 +42,18 @@ deployment: access_type: Non-interactive # Fill in correct value. access_type_details: '' - accounting: - post_processing: '' # Tier 3 - composition: '' # Tier 3 - implementation: preprocessing_and_hyperparameter_tuning: '' # Tier 3 mechanisms: '' # Tier 3 + composition: '' # Tier 3 + post_processing: '' # Tier 3 justification: '' # Tier 3 + utility: '' + additional_information: '' administrative: sources: '' # white papers - notes: '' # other extra information + status: Draft registry_authors: - '' # Fill in correct value. + tier: 3 diff --git a/tests/test_schema.py b/tests/test_schema.py index fbffed20..0478526e 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -36,7 +36,6 @@ def test_node_has_details(path, node): "/deployment/privacy_loss/privacy_parameters/rho", "/deployment/privacy_loss/privacy_parameters/delta", "/deployment/deployment_model", - "/deployment/deployment_model/model_name_details", "/deployment/accounting", "/deployment/implementation", "/deployment/administrative", @@ -71,21 +70,27 @@ def test_node_has_details_long(path, node): "/deployment/privacy_loss/privacy_parameters/delta", "/deployment/privacy_loss/privacy_parameters/rho", "/deployment/privacy_loss/privacy_parameters_details", + "/deployment/privacy_loss/number_of_privacy_units", "/deployment/deployment_model", - "/deployment/deployment_model/model_name_details", "/deployment/deployment_model/trust_assumptions", "/deployment/deployment_model/actors", "/deployment/deployment_model/release_type_details", - "/deployment/deployment_model/data_source_type_details", "/deployment/deployment_model/access_type_details", + "/deployment/deployment_model/data_source_type_details", + "/deployment/deployment_model/model_name_details", + "/deployment/implementation/additional_information", + "/deployment/implementation/utility", "/deployment/administrative", "/deployment/administrative/notes", "/deployment/administrative/registry_authors", + "/deployment/administrative/status", ] if path in skip_list: assert "description_long" not in node.keys() pytest.skip("TODO: More description_long would be nice to have") - assert "description_long" in node.keys() + assert ( + "description_long" in node.keys() + ), f"Add 'description_long' to {path}, or add path to skip_list" @pytest.mark.parametrize(("path", "node"), path_nodes, ids=paths) @@ -96,14 +101,16 @@ def test_node_has_tier(path, node): return # Tier is not needed at the top level. if path.startswith("/deployment/privacy_loss/privacy_parameters/"): return # Tier not needed on individual parameters. - if path in [ - "/deployment/deployment_model/model_name_details", + skip_list = [ + "/deployment/administrative/status", "/deployment/deployment_model/release_type_details", "/deployment/administrative/registry_authors", - ]: + "/deployment/administrative/tier", + ] + if path in skip_list: assert "tier" not in node.keys() pytest.skip("TODO: More tiers would be nice to have") - assert "tier" in node.keys() + assert "tier" in node.keys(), f"Add 'tier' to {path}, or add path to skip_list" @pytest.mark.parametrize(("path", "node"), path_nodes, ids=paths) @@ -115,4 +122,4 @@ def test_object_has_additional_properties_false(path, node): def test_template_is_complete(): template_paths = [path for path, node in path_templates] - assert template_paths == paths + assert paths == template_paths diff --git a/tests/test_yaml.py b/tests/test_yaml.py index f07c3fac..bf7e64b5 100644 --- a/tests/test_yaml.py +++ b/tests/test_yaml.py @@ -49,6 +49,11 @@ def test_good_yaml(yaml_path): ) def test_bad_yaml(bad_yaml_path): errors = check(bad_yaml_path) - assert errors, "Expected errors" - clean_error = re.sub(r"\W+", " ", str(errors)).strip().replace(" ", "_").lower() - assert clean_error == bad_yaml_path.stem, errors + assert errors, "Expected errors, but there aren't any" + errors_stem = errors_to_stem(errors) + assert errors_stem == bad_yaml_path.stem, errors + + +def errors_to_stem(errors): + short_paths = re.sub(r"\S+\.", ".", str(errors)) + return re.sub(r"\W+", " ", short_paths).strip().replace(" ", "_").lower()