diff --git a/deployments/apple_iconic_scenes_photo_selections_2023.yaml b/deployments/apple_iconic_scenes_photo_selections_2023.yaml index a6aa569a..99427079 100644 --- a/deployments/apple_iconic_scenes_photo_selections_2023.yaml +++ b/deployments/apple_iconic_scenes_photo_selections_2023.yaml @@ -46,8 +46,13 @@ deployment: - Apple is committed to balancing privacy and utility: "We combined local noise addition with a technique called secure aggregation to address these concerns of balancing privacy with utility" administrative: - sources: | - - Blog post: https://machinelearning.apple.com/research/scenes-differential-privacy + evidence_sources: + apple_iconic: + evidence_source_type: Formal publication by the deploying organization + citation: + title: Learning Iconic Scenes with Differential Privacy + url: https://machinelearning.apple.com/research/scenes-differential-privacy + year: 2023 # notes: '' # other extra information status: Draft registry_authors: diff --git a/deployments/apple_popular_emojis.yaml b/deployments/apple_popular_emojis.yaml index 047dd48a..64bf5297 100644 --- a/deployments/apple_popular_emojis.yaml +++ b/deployments/apple_popular_emojis.yaml @@ -72,8 +72,13 @@ deployment: - The paper covers collecting aggregate statistics for different use cases: trending new words, popular emojis, popular HealthKit data types, Safari media-playback preferences, and high energy/memory domains. We only treated the popular emojis use case, treating it as one deployment. administrative: - sources: | - - Paper: https://docs-assets.developer.apple.com/ml-research/papers/learning-with-privacy-at-scale.pdf + evidence_sources: + apple_emoji: + evidence_source_type: Formal publication by the deploying organization + citation: + title: Learning with Privacy at Scale + url: https://docs-assets.developer.apple.com/ml-research/papers/learning-with-privacy-at-scale.pdf + author: Differential Privacy Team, Apple status: Pending registry_authors: - Elena Ghazi diff --git a/deployments/assistive_ai.yaml b/deployments/assistive_ai.yaml index db507119..1f792660 100644 --- a/deployments/assistive_ai.yaml +++ b/deployments/assistive_ai.yaml @@ -34,9 +34,21 @@ deployment: - Restricted to some languages (EN, ES, FR, PR, IT) administrative: - sources: | - - Blog post: https://www.microsoft.com/en-us/research/articles/assistive-ai-makes-replying-easier-2/ - - Paper: https://arxiv.org/pdf/2002.09745 + evidence_sources: + microsoft_2020: + evidence_source_type: Formal publication by the deploying organization + citation: + year: 2020 + author: Microsoft Search, Assistant and Intelligence + title: Assistive AI Makes Replying Easier + url: https://www.microsoft.com/en-us/research/articles/assistive-ai-makes-replying-easier-2/ + microsoft_2022: + evidence_source_type: Formal publication by the deploying organization + citation: + year: 2022 + title: Differentially Private Set Union + url: https://arxiv.org/pdf/2002.09745 + author: 'Sivakanth Gopi, Pankaj Gulhane, Janardhan Kulkarni, Judy Hanwen Shen, Milad Shokouhi, and Sergey Yekhanin' status: Pending registry_authors: - Priyanka Nanayakkara diff --git a/deployments/audience_engagement.yaml b/deployments/audience_engagement.yaml index 0f2d8335..ba0a9442 100644 --- a/deployments/audience_engagement.yaml +++ b/deployments/audience_engagement.yaml @@ -38,8 +38,15 @@ deployment: - The paper states that “data is purged within 30 days of a user leaving the ecosystem because it has a 30 day retention.” We take this to mean that the data are refreshed on a monthly basis, i.e., that leaving the ecosystem means leaving a single user session on LinkedIn as opposed to account deletion. administrative: - sources: | - - Paper: https://arxiv.org/pdf/2002.05839 + evidence_sources: + linkedin_2020: + evidence_source_type: Formal publication by the deploying organization + citation: + title: 'LinkedIn’s Audience Engagements API: A Privacy Preserving Data Analytics System at Scale' + author: Ryan Rogers, Subbu Subramaniam, Sean Peng, David Durfee, Seunghyun Lee, Santosh Kumar Kancha, Shraddha Sahay, and Parvez Ahammad + year: 2020 + month: November + url: https://arxiv.org/pdf/2002.05839 status: Pending registry_authors: - Priyanka Nanayakkara diff --git a/deployments/autoplay_intent.yaml b/deployments/autoplay_intent.yaml index ec99b582..d335540a 100644 --- a/deployments/autoplay_intent.yaml +++ b/deployments/autoplay_intent.yaml @@ -33,7 +33,13 @@ deployment: mechanisms: Private Count Mean Sketch justification: '' # TODO: Fill in correct value administrative: - sources: https://docs-assets.developer.apple.com/ml-research/papers/learning-with-privacy-at-scale.pdf + evidence_sources: + apple: + evidence_source_type: Formal publication by the deploying organization + citation: + title: Learning with Privacy at Scale + author: Differential Privacy Team, Apple + url: https://docs-assets.developer.apple.com/ml-research/papers/learning-with-privacy-at-scale.pdf status: Pending registry_authors: - Nicolas Berrios diff --git a/deployments/birth_dataset.yaml b/deployments/birth_dataset.yaml index fbea9ec4..17d74ef0 100644 --- a/deployments/birth_dataset.yaml +++ b/deployments/birth_dataset.yaml @@ -34,7 +34,15 @@ deployment: mechanisms: PrivBayes with Private Selection (Universal Microdata Scheme) justification: '' # TODO: Fill in correct value administrative: - sources: https://birth.dataset.pub + evidence_sources: + # TODO: Consider adding other, more official, sources. + # https://data.gov.il/dataset/birth-data/resource/b1cf2a1f-b923-40ae-a3dc-75930c3b3064/download/readme-2014-english.pdf + # https://arxiv.org/abs/2405.00267 + birth_dataset: + evidence_source_type: Other public source + citation: + title: Release of Israel's Live Birth Dataset + url: https://birth.dataset.pub status: Pending registry_authors: - Shlomi Hod diff --git a/deployments/broadband_coverage.yaml b/deployments/broadband_coverage.yaml index 2e82f941..136f1988 100644 --- a/deployments/broadband_coverage.yaml +++ b/deployments/broadband_coverage.yaml @@ -75,9 +75,17 @@ deployment: - "All differential privacy processing was done with the OpenDP SmartNoise library. The SmartNoise library includes a comprehensive set of differential privacy mechanisms, algorithms, and validator. The library is open source, and is maintained and vetted by OpenDP." - A Monte Carlo simulation process to estimate the error introduced by DP was chosen, the rationale being that this empirical method is better for estimating the combined error from several private sources and does not result in additional privacy losses. administrative: - sources: | - - Paper: https://arxiv.org/abs/2103.14035 - - Github: https://github.com/microsoft/USBroadbandUsagePercentages + evidence_sources: + broadband_paper: + citation: + title: 'U.S. Broadband Coverage Data Set: A Differentially Private Data Release' + url: https://arxiv.org/abs/2103.14035 + year: 2021 + author: Mayana Pereira, Allen Kim, Joshua Allen, Kevin White, Juan Lavista Ferres, Rahul Dodhia + broadband_github: + citation: + title: United States Broadband Usage Percentages Dataset + url: https://github.com/microsoft/USBroadbandUsagePercentages status: Pending registry_authors: - Elena Ghazi diff --git a/deployments/census_demographic_and_housing.yaml b/deployments/census_demographic_and_housing.yaml index 37a151dc..a61279ce 100644 --- a/deployments/census_demographic_and_housing.yaml +++ b/deployments/census_demographic_and_housing.yaml @@ -96,12 +96,36 @@ deployment: administrative: - sources: | - - 2020 Census Demographic and Housing Characteristics File (DHC) Technical Documentation: https://www2.census.gov/programs-surveys/decennial/2020/technical-documentation/complete-tech-docs/demographic-and-housing-characteristics-file-and-demographic-profile/2020census-demographic-and-housing-characteristics-file-and-demographic-profile-techdoc.pdf - - Census Bureau Releases New 2020 Census Data on Age, Sex, Race, Hispanic Origin, Households and Housing: https://www.census.gov/newsroom/press-releases/2023/2020-census-demographic-profile-and-dhc.html - - Disclosure Avoidance for DHC and Guidance for Data Users: https://www2.census.gov/library/publications/decennial/2020/census-briefs/c2020br-08.pdf - - Disclosure avoidance handbook for the 2020 Census: https://www2.census.gov/library/publications/decennial/2020/2020-census-disclosure-avoidance-handbook.pdf - - The 2020 Census Disclosure Avoidance System TopDown Algorithm: https://arxiv.org/abs/2204.08986 + evidence_sources: + census_dhc: + evidence_source_type: Formal publication by the deploying organization + citation: + title: 2020 Census Demographic and Housing Characteristics File (DHC) Technical Documentation + url: https://www2.census.gov/programs-surveys/decennial/2020/technical-documentation/complete-tech-docs/demographic-and-housing-characteristics-file-and-demographic-profile/2020census-demographic-and-housing-characteristics-file-and-demographic-profile-techdoc.pdf + author: U.S. Census Bureau + census_press_release: + evidence_source_type: Formal publication by the deploying organization # TODO: Would a different type be better? + citation: + title: Census Bureau Releases New 2020 Census Data on Age, Sex, Race, Hispanic Origin, Households and Housing + url: https://www.census.gov/newsroom/press-releases/2023/2020-census-demographic-profile-and-dhc.html + year: 2023 + month: May + census_guidance: + evidence_source_type: Formal publication by the deploying organization + citation: + title: Disclosure Avoidance for DHC and Guidance for Data Users + url: https://www2.census.gov/library/publications/decennial/2020/census-briefs/c2020br-08.pdf + census_handbook: + evidence_source_type: Formal publication by the deploying organization + citation: + title: Disclosure avoidance handbook for the 2020 Census + url: https://www2.census.gov/library/publications/decennial/2020/2020-census-disclosure-avoidance-handbook.pdf + census_algorithm: + evidence_source_type: Formal publication by the deploying organization + citation: + title: The 2020 Census Disclosure Avoidance System TopDown Algorithm + url: https://arxiv.org/abs/2204.08986 + author: John M. Abowd, Robert Ashmead, Ryan Cumings-Menon, Simson Garfinkel, Micah Heineck, Christine Heiss, Robert Johns, Daniel Kifer, Philip Leclerc, Ashwin Machanavajjhala, Brett Moran, William Sexton, Matthew Spence, Pavel Zhuravlev status: Pending registry_authors: - Curtis Mitchell diff --git a/deployments/census_detailed_demographic_and_housing_a.yaml b/deployments/census_detailed_demographic_and_housing_a.yaml index c468bfc4..c2fadc85 100644 --- a/deployments/census_detailed_demographic_and_housing_a.yaml +++ b/deployments/census_detailed_demographic_and_housing_a.yaml @@ -80,11 +80,27 @@ deployment: - The final production parameters were set by the Data Stewardship Executive Policy Committee (DSEP) after a final review of the privacy guarantees and data accuracy demonstrated in the tuning experiments. administrative: - sources: | - - 2020 Census Detailed Demographic and Housing Characteristics File A (Detailed DHC-A) Technical Documentation: https://www2.census.gov/programs-surveys/decennial/2020/technical-documentation/complete-tech-docs/detailed-demographic-and-housing-characteristics-file-a/2020census-detailed-dhc-a-techdoc.pdf - - Census Bureau Announces Release Date for 2020 Census Data Product on Race and Ethnicity: https://www.census.gov/newsroom/press-releases/2023/2020-census-detailed-dhc-a.html - - Disclosure Avoidance Methods for the Detailed Demographic and Housing Characteristics File A (Detailed DHC-A): How SafeTab-P Works: https://www2.census.gov/library/publications/decennial/2020/census-briefs/c2020br-05.pdf - - Disclosure avoidance handbook for the 2020 Census: https://www2.census.gov/library/publications/decennial/2020/2020-census-disclosure-avoidance-handbook.pdf + evidence_sources: + census_a_dhc: + evidence_source_type: Formal publication by the deploying organization + citation: + title: 2020 Census Detailed Demographic and Housing Characteristics File A (Detailed DHC-A) Technical Documentation + url: https://www2.census.gov/programs-surveys/decennial/2020/technical-documentation/complete-tech-docs/detailed-demographic-and-housing-characteristics-file-a/2020census-detailed-dhc-a-techdoc.pdf + census_a_press_release: + evidence_source_type: Formal publication by the deploying organization + citation: + title: Census Bureau Announces Release Date for 2020 Census Data Product on Race and Ethnicity + url: https://www.census.gov/newsroom/press-releases/2023/2020-census-detailed-dhc-a.html + census_a_methods: + evidence_source_type: Formal publication by the deploying organization + citation: + title: 'Disclosure Avoidance Methods for the Detailed Demographic and Housing Characteristics File A (Detailed DHC-A): How SafeTab-P Works' + url: https://www2.census.gov/library/publications/decennial/2020/census-briefs/c2020br-05.pdf + census_a_handbook: + evidence_source_type: Formal publication by the deploying organization + citation: + title: Disclosure avoidance handbook for the 2020 Census + url: https://www2.census.gov/library/publications/decennial/2020/2020-census-disclosure-avoidance-handbook.pdf status: Pending registry_authors: - Curtis Mitchell diff --git a/deployments/census_detailed_demographic_and_housing_b.yaml b/deployments/census_detailed_demographic_and_housing_b.yaml index 4ad5d5a7..9ff606c0 100644 --- a/deployments/census_detailed_demographic_and_housing_b.yaml +++ b/deployments/census_detailed_demographic_and_housing_b.yaml @@ -80,11 +80,27 @@ deployment: - The final production parameters were set by the Data Stewardship Executive Policy Committee (DSEP) after a final review of the privacy guarantees and data accuracy demonstrated in the tuning experiments. administrative: - sources: | - - 2020 Census Detailed Demographic and Housing Characteristics File B (Detailed DHC-B) Technical Documentation: https://www2.census.gov/programs-surveys/decennial/2020/technical-documentation/complete-tech-docs/detailed-demographic-and-housing-characteristics-file-b/2020census-detailed-dhc-b-techdoc.pdf - - Census Bureau to Hold Webinar on Release of 2020 Census Detailed Demographic and Housing Characteristics File B: https://www.census.gov/newsroom/press-releases/2024/webinar-2020-census-detailed-dhc-b.html - - Disclosure Avoidance and the 2020 Census: How SafeTab-H Works: https://www2.census.gov/library/publications/decennial/2020/census-briefs/c2020br-11.pdf - - Disclosure avoidance handbook for the 2020 Census: https://www2.census.gov/library/publications/decennial/2020/2020-census-disclosure-avoidance-handbook.pdf + evidence_sources: + census_b_dhc: + evidence_source_type: Formal publication by the deploying organization + citation: + title: 2020 Census Detailed Demographic and Housing Characteristics File B (Detailed DHC-B) Technical Documentation + url: https://www2.census.gov/programs-surveys/decennial/2020/technical-documentation/complete-tech-docs/detailed-demographic-and-housing-characteristics-file-b/2020census-detailed-dhc-b-techdoc.pdf + census_b_press_release: + evidence_source_type: Formal publication by the deploying organization + citation: + title: Census Bureau to Hold Webinar on Release of 2020 Census Detailed Demographic and Housing Characteristics File B + url: https://www.census.gov/newsroom/press-releases/2024/webinar-2020-census-detailed-dhc-b.html + census_b_safetab: + evidence_source_type: Formal publication by the deploying organization + citation: + title: 'Disclosure Avoidance and the 2020 Census: How SafeTab-H Works' + url: https://www2.census.gov/library/publications/decennial/2020/census-briefs/c2020br-11.pdf + census_b_handbook: + evidence_source_type: Formal publication by the deploying organization + citation: + title: Disclosure avoidance handbook for the 2020 Census + url: https://www2.census.gov/library/publications/decennial/2020/2020-census-disclosure-avoidance-handbook.pdf status: Pending registry_authors: - Curtis Mitchell diff --git a/deployments/census_redistricting_data.yaml b/deployments/census_redistricting_data.yaml index 19f94443..26c57764 100644 --- a/deployments/census_redistricting_data.yaml +++ b/deployments/census_redistricting_data.yaml @@ -95,9 +95,18 @@ deployment: - The final production parameters were set by the Data Stewardship Executive Policy Committee (DSEP) after a final review of the privacy guarantees and data accuracy demonstrated in the tuning experiments. administrative: - sources: | - - Disclosure avoidance handbook for the 2020 Census: https://www2.census.gov/library/publications/decennial/2020/2020-census-disclosure-avoidance-handbook.pdf - - The 2020 Census Disclosure Avoidance System TopDown Algorithm: https://arxiv.org/abs/2204.08986 + evidence_sources: + census_redistricting_handbook: + evidence_source_type: Formal publication by the deploying organization + citation: + title: Disclosure avoidance handbook for the 2020 Census + url: https://www2.census.gov/library/publications/decennial/2020/2020-census-disclosure-avoidance-handbook.pdf + census_redistricting_algorithm: + evidence_source_type: Formal publication by the deploying organization + citation: + title: The 2020 Census Disclosure Avoidance System TopDown Algorithm + url: https://arxiv.org/abs/2204.08986 + author: ohn M. Abowd, Robert Ashmead, Ryan Cumings-Menon, Simson Garfinkel, Micah Heineck, Christine Heiss, Robert Johns, Daniel Kifer, Philip Leclerc, Ashwin Machanavajjhala, Brett Moran, William Sexton, Matthew Spence, Pavel Zhuravlev status: Pending registry_authors: - Elena Ghazi diff --git a/deployments/census_supplemental_demographic_and_housing.yaml b/deployments/census_supplemental_demographic_and_housing.yaml index e8d0b0b5..eac53e75 100644 --- a/deployments/census_supplemental_demographic_and_housing.yaml +++ b/deployments/census_supplemental_demographic_and_housing.yaml @@ -70,11 +70,27 @@ deployment: - The final production parameters were set by the Data Stewardship Executive Policy Committee (DSEP) after a final review of the privacy guarantees and data accuracy demonstrated in the tuning experiments. administrative: - sources: | - - 2020 Supplemental Demographic and Housing Characteristics File (S-DHC) Technical Documentation: https://www2.census.gov/programs-surveys/decennial/2020/technical-documentation/complete-tech-docs/supplemental-demographic-and-housing-characteristics-file/2020census-supplemental-dhc-techdoc.pdf - - Census Bureau Releases Final 2020 Census Data Product: https://www.census.gov/newsroom/press-releases/2024/final-2020-census-data-product-s-dhc.html - - Disclosure Avoidance and the Supplemental Demographic and Housing Characteristics File (S-DHC): How PHSafe Works: https://www2.census.gov/library/publications/decennial/2020/census-briefs/c2020br-12.pdf - - Disclosure avoidance handbook for the 2020 Census: https://www2.census.gov/library/publications/decennial/2020/2020-census-disclosure-avoidance-handbook.pdf + evidence_sources: + census_supplemental_dhc: + evidence_source_type: Formal publication by the deploying organization + citation: + title: 2020 Supplemental Demographic and Housing Characteristics File (S-DHC) Technical Documentation + url: https://www2.census.gov/programs-surveys/decennial/2020/technical-documentation/complete-tech-docs/supplemental-demographic-and-housing-characteristics-file/2020census-supplemental-dhc-techdoc.pdf + census_supplemental_press_release: + evidence_source_type: Formal publication by the deploying organization + citation: + title: Census Bureau Releases Final 2020 Census Data Product + url: https://www.census.gov/newsroom/press-releases/2024/final-2020-census-data-product-s-dhc.html + census_supplemental_how_it_works: + evidence_source_type: Formal publication by the deploying organization + citation: + title: 'Disclosure Avoidance and the Supplemental Demographic and Housing Characteristics File (S-DHC): How PHSafe Works' + url: https://www2.census.gov/library/publications/decennial/2020/census-briefs/c2020br-12.pdf + census_supplemental_handbook: + evidence_source_type: Formal publication by the deploying organization + citation: + title: Disclosure avoidance handbook for the 2020 Census + url: https://www2.census.gov/library/publications/decennial/2020/2020-census-disclosure-avoidance-handbook.pdf status: Pending registry_authors: - Curtis Mitchell diff --git a/deployments/county_business_patterns.yaml b/deployments/county_business_patterns.yaml index e7b9ca86..c91eb18e 100644 --- a/deployments/county_business_patterns.yaml +++ b/deployments/county_business_patterns.yaml @@ -32,10 +32,22 @@ deployment: mechanisms: Gaussian justification: '' # TODO: Fill in correct value administrative: - sources: | - - https://www.census.gov/topics/business-economy/disclosure/about.html - - https://www.census.gov/data/academy/webinars/2023/differential-privacy-webinar.html - - https://www.census.gov/topics/business-economy/disclosure/data/tables.html + evidence_sources: + about: + evidence_source_type: Formal publication by the deploying organization + citation: + url: https://www.census.gov/topics/business-economy/disclosure/about.html + title: About Data Privacy on Business & the Economy + webinar: + evidence_source_type: Formal publication by the deploying organization + citation: + url: https://www.census.gov/data/academy/webinars/2023/differential-privacy-webinar.html + title: Researching Formal Privacy for the Census Bureau's County Business Patterns Program + tables: + evidence_source_type: Formal publication by the deploying organization + citation: + url: https://www.census.gov/topics/business-economy/disclosure/data/tables.html + title: Data Privacy Research on Businesses & the Economy Tables status: Pending registry_authors: - Nicolas Berrios diff --git a/deployments/covid19_notifications.yaml b/deployments/covid19_notifications.yaml index 1f235bfa..19306a87 100644 --- a/deployments/covid19_notifications.yaml +++ b/deployments/covid19_notifications.yaml @@ -36,8 +36,15 @@ deployment: mechanisms: Randomized Response justification: '' # TODO: Fill in correct value administrative: - sources: | - - Paper: https://covid19-static.cdn-apple.com/applications/covid19/current/static/contact-tracing/pdf/ENPA_White_Paper.pdf + evidence_sources: + white_paper: + evidence_source_type: Formal publication by the deploying organization + citation: + url: https://covid19-static.cdn-apple.com/applications/covid19/current/static/contact-tracing/pdf/ENPA_White_Paper.pdf + title: Exposure Notification Privacy-preserving Analytics (ENPA) White Paper + author: Apple and Google + year: 2021 + month: April status: Pending registry_authors: - Priyanka Nanayakkara diff --git a/deployments/covid19_search_trends_symptoms.yaml b/deployments/covid19_search_trends_symptoms.yaml index 62c0c568..81d1a24b 100644 --- a/deployments/covid19_search_trends_symptoms.yaml +++ b/deployments/covid19_search_trends_symptoms.yaml @@ -80,10 +80,35 @@ deployment: - For each geographic region and symptom, data is released as either daily or weekly aggregates, depending on data quality. While the system provides daily aggregates whenever possible, it switches to weekly aggregates if privacy protections significantly affect the data's accuracy. Weekly aggregates are more robust because they are based on more data, which reduces the relative error from the added privacy noise. This choice was determined once in a differentially private manner using data from February to July 2020. After being set, the temporal granularity for a given region and symptom remains fixed for the entire duration of the dataset's release. administrative: - sources: | - - Documentation: https://storage.googleapis.com/gcp-public-data-symptom-search/COVID-19%20Search%20Trends%20symptoms%20dataset%20documentation%20.pdf?utm_source=chatgpt.com - - Paper: https://arxiv.org/pdf/2009.01265 - - Published data product: https://github.com/GoogleCloudPlatform/covid-19-open-data/blob/main/docs/table-search-trends.md + evidence_sources: + convid19_documentation: + evidence_source_type: Formal publication by the deploying organization + citation: + title: COVID-19 Search Trends symptoms dataset + url: https://storage.googleapis.com/gcp-public-data-symptom-search/COVID-19%20Search%20Trends%20symptoms%20dataset%20documentation%20.pdf + year: 2021 + month: 2 + convid19_paper: + evidence_source_type: Formal publication by the deploying organization + citation: + title: 'Google COVID-19 Search Trends Symptoms Dataset: Anonymization Process Description (version 1.0)' + author: | + Shailesh Bavadekar, Andrew Dai, John Davis, Damien Desfontaines, Ilya Eckstein, Katie + Everett, Alex Fabrikant, Gerardo Flores, Evgeniy Gabrilovich, Krishna Gadepalli, Shane + Glass, Rayman Huang, Chaitanya Kamath, Dennis Kraft, Akim Kumok, Hinali Marfatia, + Yael Mayer, Benjamin Miller, Adam Pearce, Irippuge Milinda Perera, Venky + Ramachandran, Karthik Raman, Thomas Roessler, Izhak Shafran, Tomer Shekel, + Charlotte Stanton, Jacob Stimes, Mimi Sun, Gregory Wellenius, and Masrour Zoghi + year: 2020 + month: September + url: https://arxiv.org/pdf/2009.01265 + convid19_data_product: + evidence_source_type: Formal publication by the deploying organization + citation: + title: COVID-19 Search Trends symptoms dataset + url: https://github.com/GoogleCloudPlatform/covid-19-open-data/blob/main/docs/table-search-trends.md + year: 2021 + month: 2 status: Pending registry_authors: - Elena Ghazi diff --git a/deployments/employment_outcomes.yaml b/deployments/employment_outcomes.yaml index ac26c426..59c2da9d 100644 --- a/deployments/employment_outcomes.yaml +++ b/deployments/employment_outcomes.yaml @@ -41,9 +41,22 @@ deployment: - Finally, the paper contains parameter ranges (e.g., for \\(epsilon\\)) that were tested in the experiments section, however we did not find parameter values that were actually used in deployment. administrative: - sources: | - - Paper: https://journalprivacyconfidentiality.org/index.php/jpc/article/view/722/684 - - Documentation: https://lehd.ces.census.gov/data/pseo_documentation.html + evidence_sources: + employment_paper: + evidence_source_type: Formal publication by the deploying organization + citation: + url: https://journalprivacyconfidentiality.org/index.php/jpc/article/view/722/684 + title: 'Releasing Earnings Distributions Using Differential Privacy: Disclosure Avoidance System For Post-Secondary Employment Outcomes (PSEO)' + author: 'Andrew Foote, Ashwin Machanavajjhala, and Kevin McKinney' + journal: Journal of Privacy and Confidentiality + volume: 9 + number: 2 + year: 2019 + employment_documentation: + evidence_source_type: Formal publication by the deploying organization + citation: + url: https://lehd.ces.census.gov/data/pseo_documentation.html + title: Post-Secondary Employment Outcomes (PSEO) Help and Documentation status: Pending registry_authors: - Priyanka Nanayakkara diff --git a/deployments/google_mobility.yaml b/deployments/google_mobility.yaml index 1a75768f..92ea5a80 100644 --- a/deployments/google_mobility.yaml +++ b/deployments/google_mobility.yaml @@ -90,7 +90,18 @@ deployment: - The implementation decision to bound each user's contribution to a maximum of four (category, location) pairs per day is justified by an analysis of user behavior. This threshold was chosen because it "does not significantly affect data accuracy," given that 99% of users at the U.S. county level contribute to three or fewer pairs on average. administrative: - sources: https://arxiv.org/pdf/2004.04145 + evidence_sources: + mobility_paper: + evidence_source_type: Formal publication by the deploying organization + citation: + title: 'Google COVID-19 Community Mobility Reports: Anonymization Process Description (version 1.1)' + url: https://arxiv.org/pdf/2004.04145 + author: | + Ahmet Aktay, Shailesh Bavadekar, Gwen Cossoul, John Davis, Damien Desfontaines, Alex + Fabrikant, Evgeniy Gabrilovich, Krishna Gadepalli, Bryant Gipson, Miguel Guevara, + Chaitanya Kamath, Mansi Kansal, Ali Lange, Chinmoy Mandayam, Andrew Oplinger, + Christopher Pluntke, Thomas Roessler, Arran Schlosberg, Tomer Shekel, Swapnil Vispute, + Mia Vu, Gregory Wellenius, Brian Williams, and Royce J Wilson status: Pending registry_authors: - Elena Ghazi diff --git a/deployments/healthkit.yaml b/deployments/healthkit.yaml index 0b7a6d81..1a5cd1f6 100644 --- a/deployments/healthkit.yaml +++ b/deployments/healthkit.yaml @@ -36,9 +36,19 @@ deployment: mechanisms: Private Count Mean Sketch justification: '' # TODO: Fill in correct value administrative: - sources: | - - Paper: https://docs-assets.developer.apple.com/ml-research/papers/learning-with-privacy-at-scale.pdf - - Additional version: https://machinelearning.apple.com/research/learning-with-privacy-at-scale + evidence_sources: + healthkit_paper: + evidence_source_type: Formal publication by the deploying organization + citation: + title: Learning with Privacy at Scale + url: https://docs-assets.developer.apple.com/ml-research/papers/learning-with-privacy-at-scale.pdf + author: Differential Privacy Team, Apple + healthkit_additional: + evidence_source_type: Formal publication by the deploying organization + citation: + title: Learning with Privacy at Scale + url: https://machinelearning.apple.com/research/learning-with-privacy-at-scale + author: Differential Privacy Team status: Pending registry_authors: - Priyanka Nanayakkara diff --git a/deployments/korean_statistics_datahub.yaml b/deployments/korean_statistics_datahub.yaml index 754866f5..8a88e8dd 100644 --- a/deployments/korean_statistics_datahub.yaml +++ b/deployments/korean_statistics_datahub.yaml @@ -29,7 +29,13 @@ deployment: mechanisms: Unknown justification: '' # TODO: Fill in correct value administrative: - sources: https://unstats.un.org/wiki/display/UGTTOPPT/11.+Statistics+Korea%3A+Developing+a+privacy-preserving+Statistical+Data+Hub+Platform + evidence_sources: + datahub_paper: + evidence_source_type: Formal publication by the deploying organization + citation: + url: https://unstats.un.org/wiki/display/UGTTOPPT/11.+Statistics+Korea%3A+Developing+a+privacy-preserving+Statistical+Data+Hub+Platform + title: 'Statistics Korea: Developing a privacy-preserving Statistical Data Hub Platform' + author: David Buckley status: Pending registry_authors: - Nicolas Berrios diff --git a/deployments/linkedin_hiring_reports.yaml b/deployments/linkedin_hiring_reports.yaml index 9024afb6..93f2a998 100644 --- a/deployments/linkedin_hiring_reports.yaml +++ b/deployments/linkedin_hiring_reports.yaml @@ -38,7 +38,15 @@ deployment: mechanisms: Laplace, Gumbel justification: '' # TODO: Fill in correct value administrative: - sources: https://arxiv.org/abs/2010.13981 + evidence_sources: + linkedin_paper: + evidence_source_type: Formal publication by the deploying organization + citation: + title: A Members First Approach to Enabling LinkedIn's Labor Market Insights at Scale + url: https://arxiv.org/abs/2010.13981 + author: Ryan Rogers, Adrian Rivera Cardoso, Koray Mancuhan, Akash Kaura, Nikhil Gahlawat, Neha Jain, Paul Ko, Parvez Ahammad + year: 2020 + month: October status: Pending registry_authors: - Nicolas Berrios diff --git a/deployments/lookup_hints.yaml b/deployments/lookup_hints.yaml index e3b2db28..124df2ae 100644 --- a/deployments/lookup_hints.yaml +++ b/deployments/lookup_hints.yaml @@ -36,7 +36,12 @@ deployment: mechanisms: Count Mean Sketch or Hadamard Count Mean Sketch justification: '' # TODO: Fill in correct value administrative: - sources: https://www.apple.com/privacy/docs/Differential_Privacy_Overview.pdf + evidence_sources: + hints_paper: + evidence_source_type: Formal publication by the deploying organization + citation: + title: Differential Privacy + url: https://www.apple.com/privacy/docs/Differential_Privacy_Overview.pdf status: Pending registry_authors: - Nicolas Berrios diff --git a/deployments/mhusage.yaml b/deployments/mhusage.yaml index 8dc9c866..d37213e1 100644 --- a/deployments/mhusage.yaml +++ b/deployments/mhusage.yaml @@ -81,9 +81,18 @@ deployment: - OpenDP was used to mitigate the risk of floating-point vulnerabilities or errors in the privacy budget accounting. Using user-defined transformations was necessary for performance reasons. administrative: - sources: | - - Description of the mechanism and its utility metrics: https://behaviors.transluce.org/mental-health?detail=page:app_H_privacy_statement&detail=page:app_F_prod_features - - Published dataset: https://huggingface.co/datasets/Transluce/MHUsage + evidence_sources: + transluce_report: + citation: + title: 'Mental Health Behavior Report, Appendix F: Comparing real and simulated users' + author: Daniel D. Johnson, Robert Friel, Cassidy Laidlaw, Luke Hewitt, Nari Johnson, Conrad Stosz, Sarah Schwettmann + url: https://behaviors.transluce.org/mental-health?detail=page:app_H_privacy_statement&detail=page:app_F_prod_features + year: 2026 + month: August + transluce_data: + citation: + title: MHUsage + url: https://huggingface.co/datasets/Transluce/MHUsage status: Draft registry_authors: - Damien Desfontaines diff --git a/deployments/microsoft_ctdc_victim_perpertartor_synthetic_dataset_2023.yaml b/deployments/microsoft_ctdc_victim_perpertartor_synthetic_dataset_2023.yaml index 3518889a..5687e359 100644 --- a/deployments/microsoft_ctdc_victim_perpertartor_synthetic_dataset_2023.yaml +++ b/deployments/microsoft_ctdc_victim_perpertartor_synthetic_dataset_2023.yaml @@ -71,12 +71,31 @@ deployment: - Also, the CTDC states: "Both datasets preserve privacy by design". administrative: - sources: | - - Microsoft Research Blog: https://www.microsoft.com/en-us/research/blog/iom-and-microsoft-release-first-ever-differentially-private-synthetic-dataset-to-counter-human-trafficking/ - - Dataset: https://www.ctdatacollaborative.org/global-victim-perpetrator-synthetic-dataset#no-back - - Data Codebook: https://www.ctdatacollaborative.org/sites/g/files/tmzbdl2011/files/2023-07/CTDCsyntheic_victim-perpetrator_codebook_v1.pdf - - Product used to generate the dataset: https://github.com/microsoft/synthetic-data-showcase - # notes: '' # other extra information + evidence_sources: + ctdc_blog: + evidence_source_type: Formal publication by the deploying organization + citation: + title: IOM and Microsoft release first-ever differentially private synthetic dataset to counter human trafficking + url: https://www.microsoft.com/en-us/research/blog/iom-and-microsoft-release-first-ever-differentially-private-synthetic-dataset-to-counter-human-trafficking/ + year: 2022 + month: December + ctdc_dataset: + evidence_source_type: Formal publication by the deploying organization + citation: + title: The Global Victim-Perpetrator Synthetic Dataset + url: https://www.ctdatacollaborative.org/global-victim-perpetrator-synthetic-dataset + ctdc_codebook: + evidence_source_type: Formal publication by the deploying organization + citation: + title: 'Counter Trafficking Data Collaborative (CTDC): The Global Victim-Perpetrator Synthetic Data Codebook' + url: https://www.ctdatacollaborative.org/sites/g/files/tmzbdl2011/files/2023-07/CTDCsyntheic_victim-perpetrator_codebook_v1.pdf + year: 2022 + month: December + ctdc_sourcecode: + evidence_source_type: Formal publication by the deploying organization + citation: + title: Synthetic data showcase + url: https://github.com/microsoft/synthetic-data-showcase status: Draft registry_authors: - Michael Khavkin diff --git a/deployments/mobility_trends_hurricane.yaml b/deployments/mobility_trends_hurricane.yaml index 1e74f5ad..c4870deb 100644 --- a/deployments/mobility_trends_hurricane.yaml +++ b/deployments/mobility_trends_hurricane.yaml @@ -32,9 +32,19 @@ deployment: mechanisms: Laplace justification: '' # TODO: Fill in correct value administrative: - sources: | - - https://spectus.ai/wp-content/uploads/2022/10/Spectus_DPWhitepaper_v01b.pdf - - https://web.archive.org/web/20221209065442/https://spectus.ai/wp-content/uploads/2022/10/Spectus_DPWhitepaper_v01b.pdf + evidence_sources: + 404_whitepaper: + evidence_source_type: Formal publication by the deploying organization + citation: + title: Differential Privacy + url: https://spectus.ai/wp-content/uploads/2022/10/Spectus_DPWhitepaper_v01b.pdf + year: 2022 + archive_whitepaper: + evidence_source_type: Formal publication by the deploying organization # Accurate? + citation: + title: Differential Privacy + url: https://web.archive.org/web/20221209065442/https://spectus.ai/wp-content/uploads/2022/10/Spectus_DPWhitepaper_v01b.pdf + year: 2022 status: Pending registry_authors: - Nicolas Berrios diff --git a/deployments/movement_ranges_maps.yaml b/deployments/movement_ranges_maps.yaml index a5edf134..0c47486f 100644 --- a/deployments/movement_ranges_maps.yaml +++ b/deployments/movement_ranges_maps.yaml @@ -31,9 +31,20 @@ deployment: mechanisms: Laplace justification: '' # TODO: Fill in correct value administrative: - sources: | - - Blog post: https://research.facebook.com/blog/2020/06/protecting-privacy-in-facebook-mobility-data-during-the-covid-19-response/ - - Downloadable data product: https://data.humdata.org/dataset/movement-range-maps + evidence_sources: + movement_blog: + evidence_source_type: Formal publication by the deploying organization + citation: + title: Protecting privacy in Facebook mobility data during the COVID-19 response + url: https://research.facebook.com/blog/2020/06/protecting-privacy-in-facebook-mobility-data-during-the-covid-19-response/ + author: Amaç Herdağdelen, Alex Dow, Bogdan State, Payman Mohassel, Alex Pompe + year: 2020 + month: June + movement_data: + evidence_source_type: Formal publication by the deploying organization + citation: + title: Movement Range Maps + url: https://data.humdata.org/dataset/movement-range-maps status: Pending registry_authors: - Priyanka Nanayakkara diff --git a/deployments/on_device_browser_rec.yaml b/deployments/on_device_browser_rec.yaml index fe736da2..118fab91 100644 --- a/deployments/on_device_browser_rec.yaml +++ b/deployments/on_device_browser_rec.yaml @@ -28,7 +28,14 @@ deployment: mechanisms: DP SGD* justification: '' # TODO: Fill in correct value administrative: - sources: https://brave.com/blog/federated-learning/ + evidence_sources: + on_device_rec_paper: + evidence_source_type: Formal publication by the deploying organization + citation: + title: Using Federated Learning to Improve Brave’s On-Device Recommendations While Protecting Your Privacy + url: https://brave.com/blog/federated-learning/ + year: 2021 + month: 6 status: Pending registry_authors: - Nicolas Berrios diff --git a/deployments/private_third_party_audits.yaml b/deployments/private_third_party_audits.yaml index b67589b2..8ec7f4c5 100644 --- a/deployments/private_third_party_audits.yaml +++ b/deployments/private_third_party_audits.yaml @@ -35,7 +35,13 @@ deployment: mechanisms: Not Specificed, Synthetic Data justification: '' # TODO: Fill in correct value administrative: - sources: https://unstats.un.org/wiki/display/UGTTOPPT/14.+Twitter+and+OpenMined%3A+Enabling+Third-party+Audits+and+Research+Reproducibility+over+Unreleased+Digital+Assets + evidence_sources: + case_study: + evidence_source_type: Formal publication by the deploying organization + citation: + title: 'Twitter and OpenMined: Enabling Third-party Audits and Research Reproducibility over Unreleased Digital Assets' + url: https://unstats.un.org/wiki/spaces/UGTTOPPT/pages/152797274/14.+Twitter+and+OpenMined+Enabling+Third-party+Audits+and+Research+Reproducibility+over+Unreleased+Digital+Assets + author: David Buckley status: Pending registry_authors: - Nicolas Berrios diff --git a/deployments/recurve_energy_dp.yaml b/deployments/recurve_energy_dp.yaml index 29f8c5e6..b48efbdb 100644 --- a/deployments/recurve_energy_dp.yaml +++ b/deployments/recurve_energy_dp.yaml @@ -97,11 +97,27 @@ deployment: administrative: - sources: | - - Applying Energy Differential Privacy To Enable Measurement of the OhmConnect Virtual Power Plant (paper): https://assets.website-files.com/5cb0a177570549b5f11b9550/5ffddb83b5ea5d67f5c43661_Quantifying%20The%20OhmConnect%20Virtual%20Power%20Plant%20During%20the%20California%20Blackouts.pdf - - Revenue-Grade Analysis of the OhmConnect Virtual Power Plant During the California Blackouts (blog): https://www.recurve.com/blog/revenue-grade-analysis-of-the-ohmconnect-virtual-power-plant-during-the-california-blackouts - - Differential Privacy for Expanding Access to Building Energy Data (paper that introduced Energy Differential Privacy): https://edp.recurve.com/uploads/8/6/5/0/8650231/aceee_paper_final.pdf - - A list of real-world uses of differential privacy (blog that provides the correct privacy parameters): https://desfontain.es/blog/real-world-differential-privacy.html + evidence_sources: + recurve_paper: + evidence_source_type: Formal publication by the deploying organization + citation: + title: Applying Energy Differential Privacy To Enable Measurement of the OhmConnect Virtual Power Plant + url: https://assets.website-files.com/5cb0a177570549b5f11b9550/5ffddb83b5ea5d67f5c43661_Quantifying%20The%20OhmConnect%20Virtual%20Power%20Plant%20During%20the%20California%20Blackouts.pdf + recurve_blog: + evidence_source_type: Formal publication by the deploying organization + citation: + title: Revenue-Grade Analysis of the OhmConnect Virtual Power Plant During the California Blackouts + url: https://www.recurve.com/blog/revenue-grade-analysis-of-the-ohmconnect-virtual-power-plant-during-the-california-blackouts + recurve_intro_paper: + evidence_source_type: Formal publication by the deploying organization + citation: + title: Differential Privacy for Expanding Access to Building Energy Data + url: https://edp.recurve.com/uploads/8/6/5/0/8650231/aceee_paper_final.pdf + recurve_correction_blog: + evidence_source_type: Other public source + citation: + title: A list of real-world uses of differential privacy + url: https://desfontain.es/blog/real-world-differential-privacy.html status: Pending registry_authors: - Elena Ghazi diff --git a/deployments/safari_energy_drain.yaml b/deployments/safari_energy_drain.yaml index 712021be..52102f1a 100644 --- a/deployments/safari_energy_drain.yaml +++ b/deployments/safari_energy_drain.yaml @@ -50,9 +50,15 @@ deployment: - There are multiple releases per device & per user -- statistics are reported automatically on IoS and MacOS devices, subject to daily aggregation. - There is only one aggregation query applied per-event. - administrative: - sources: https://docs-assets.developer.apple.com/ml-research/papers/learning-with-privacy-at-scale.pdf + evidence_sources: + safari_paper: + evidence_source_type: Formal publication by the deploying organization + citation: + title: Learning with Privacy at Scale + author: Differential Privacy Team, Apple + url: https://docs-assets.developer.apple.com/ml-research/papers/learning-with-privacy-at-scale.pdf + status: Pending registry_authors: - James Honaker diff --git a/deployments/safety_classifier.yaml b/deployments/safety_classifier.yaml index b0435f5d..4270282c 100644 --- a/deployments/safety_classifier.yaml +++ b/deployments/safety_classifier.yaml @@ -37,9 +37,23 @@ deployment: # mechanisms: # justification: administrative: - sources: | - - Blog post: https://research.google/blog/protecting-users-with-differentially-private-synthetic-training-data/ - - Research paper: https://arxiv.org/pdf/2306.01684 + evidence_sources: + safety_blog: + evidence_source_type: Formal publication by the deploying organization + citation: + title: Protecting users with differentially private synthetic training data + url: https://research.google/blog/protecting-users-with-differentially-private-synthetic-training-data/ + author: Alexey Kurakin, and Natalia Ponomareva + year: 2024 + month: May + safety_paper: + evidence_source_type: Formal publication by the deploying organization + citation: + title: Harnessing large-language models to generate private synthetic text + url: https://arxiv.org/abs/2306.01684 + author: Alexey Kurakin, Natalia Ponomareva, Umar Syed, Liam MacDermed, Andreas Terzis + # There are multiple versions: Report year/month for v1 or v2? + status: Pending registry_authors: - Priyanka Nanayakkara diff --git a/deployments/sas_data_maker_vulnerable_persons.yaml b/deployments/sas_data_maker_vulnerable_persons.yaml index e73accc2..899ac8e5 100644 --- a/deployments/sas_data_maker_vulnerable_persons.yaml +++ b/deployments/sas_data_maker_vulnerable_persons.yaml @@ -67,11 +67,38 @@ deployment: - Autoregressive model is chosen to model the Transactions table because it handles time-series data effectively. However, DP is not applied to this model since DP-SGD requires adding too much noise, reducing utility to an unacceptably low level. administrative: - sources: | - - Usecase by ICO: https://ico.org.uk/for-organisations/uk-gdpr-guidance-and-resources/data-sharing/privacy-enhancing-technologies/case-studies/synthetic-data-to-test-the-effectiveness-of-a-vulnerable-persons-detection-system-in-financial-services/ - - Usecase by Nationwide: https://medium.com/nationwide-technology/hazy-synthetic-data-to-fuel-rapid-innovation-fd24f2e21685 - - PrivBayes paper: https://dl.acm.org/doi/10.1145/3134428 - - DoppelGANger paper: https://arxiv.org/abs/1909.13403 + evidence_sources: + ico_usecase: + evidence_source_type: Formal publication by the deploying organization + citation: + url: https://ico.org.uk/for-organisations/uk-gdpr-guidance-and-resources/data-sharing/privacy-enhancing-technologies/case-studies/synthetic-data-to-test-the-effectiveness-of-a-vulnerable-persons-detection-system-in-financial-services/ + title: Synthetic data to test the effectiveness of a vulnerable person's detection system in financial services + nationwide_usecase: + evidence_source_type: Formal publication by the deploying organization + citation: + url: https://medium.com/nationwide-technology/hazy-synthetic-data-to-fuel-rapid-innovation-fd24f2e21685 + title: 'Hazy: Synthetic Data to fuel Rapid Innovation' + author: Alexander Mikhalev + privbayes_paper: + evidence_source_type: Formal publication by the deploying organization + citation: + url: https://dl.acm.org/doi/10.1145/3134428 + title: 'PrivBayes: Private Data Release via Bayesian Networks' + author: Jun Zhang, Graham Cormode, Cecilia M. Procopiuc, Divesh Srivastava, Xiaokui Xiao + year: 2017 + month: October + volume: 42 + number: 4 + doi: https://doi.org/10.1145/313442 + journal: ACM Transactions on Database Systems + doppelganger_paper: + evidence_source_type: Formal publication by the deploying organization + citation: + url: https://arxiv.org/abs/1909.13403 + title: 'Using GANs for Sharing Networked Time Series Data: Challenges, Initial Promise, and Open Questions' + author: Zinan Lin, Alankar Jain, Chen Wang, Giulia Fanti, Vyas Sekar + doi: https://doi.org/10.48550/arXiv.1909.13403 + status: Pending registry_authors: - Georgi Ganev diff --git a/deployments/shared_mobility_dataset.yaml b/deployments/shared_mobility_dataset.yaml index a6f36986..1c59c4d6 100644 --- a/deployments/shared_mobility_dataset.yaml +++ b/deployments/shared_mobility_dataset.yaml @@ -37,7 +37,20 @@ deployment: mechanisms: Laplace Mechanism justification: '"All trips were anonymized and aggregated by jointly applying differential privacy via the Laplace mechanism in combination with k-anonymity."' administrative: - sources: https://www.nature.com/articles/s41467-019-12809-y + evidence_sources: + hierarchical_paper: + evidence_source_type: Formal publication by the deploying organization + citation: + title: Hierarchical organization of urban mobility and its connection with city livability + url: https://www.nature.com/articles/s41467-019-12809-y + author: | + Aleix Bassolas, Hugo Barbosa-Filho, Brian Dickinson, Xerxes Dotiwalla, + Paul Eastham, Riccardo Gallotti, Gourab Ghoshal, Bryant Gipson, Surendra A. Hazarie, + Henry Kautz, Onur Kucuktunc, Allison Lieber, Adam Sadilek & José J. Ramasco + year: 2019 + month: October + journal: Nature Communications + volume: 10 status: Pending registry_authors: - Nicolas Berrios diff --git a/deployments/spanish_language_next_word.yaml b/deployments/spanish_language_next_word.yaml index 2d45d424..7b206a21 100644 --- a/deployments/spanish_language_next_word.yaml +++ b/deployments/spanish_language_next_word.yaml @@ -62,9 +62,25 @@ deployment: - DP-FTRL was chosen to address this challenge, given the observation that training convergence depends on accuracy of cumulative sums of gradients rather than individual ones, and that it is possible to provide accurate estimates of cumulative sums with a strong DP guarantee by using negatively correlated noise, since some of the privacy noise cancels out from step to step, allowing the model's learning trajectory to stay closer to the true gradient descent steps and achieve better accuracy for a given level of privacy. administrative: - sources: | - - Federated Learning with Formal Differential Privacy Guarantees (Article published on February 28, 2022): https://research.google/blog/federated-learning-with-formal-differential-privacy-guarantees/ - - Paper that introduces the DP variant of Follow-The-Regularized-Leader (DP-FTRL) used in the deployment: Practical and Private (Deep) Learning Without Sampling or Shuffling: https://arxiv.org/pdf/2103.00039 + evidence_sources: + spanish_blog: + evidence_source_type: Formal publication by the deploying organization + citation: + title: Federated Learning with Formal Differential Privacy Guarantees + url: https://research.google/blog/federated-learning-with-formal-differential-privacy-guarantees/ + author: Brendan McMahan and Abhradeep Thakurta + year: 2022 + month: February + spanish_paper: + # Paper that introduces the DP variant of Follow-The-Regularized-Leader (DP-FTRL) used in the deployment + evidence_source_type: Formal publication by the deploying organization + citation: + title: Practical and Private (Deep) Learning Without Sampling or Shuffling + url: https://arxiv.org/pdf/2103.00039 + author: Peter Kairouz, Brendan McMahan, Shuang Song, Om Thakkar, Abhradeep Thakurta, Zheng Xu + year: 2021 + month: December + status: Pending registry_authors: - Elena Ghazi diff --git a/deployments/synth_data_public_use.yaml b/deployments/synth_data_public_use.yaml index 62addf29..75fca2d8 100644 --- a/deployments/synth_data_public_use.yaml +++ b/deployments/synth_data_public_use.yaml @@ -29,7 +29,13 @@ deployment: mechanisms: DP GAN* justification: '' # TODO: Fill in correct value administrative: - sources: https://datasciencecampus.ons.gov.uk/projects/generative-adversarial-networks-gans-for-synthetic-dataset-generation-with-binary-classes/ + evidence_sources: + gans_paper: + citation: + title: Generative adversarial networks (GANs) for synthetic dataset generation with binary classes + url: https://datasciencecampus.ons.gov.uk/projects/generative-adversarial-networks-gans-for-synthetic-dataset-generation-with-binary-classes/ + year: 2019 + month: February status: Pending registry_authors: - Nicolas Berrios diff --git a/deployments/telemetry_collection_windows.yaml b/deployments/telemetry_collection_windows.yaml index 20ded07f..5a6dd04c 100644 --- a/deployments/telemetry_collection_windows.yaml +++ b/deployments/telemetry_collection_windows.yaml @@ -33,8 +33,15 @@ deployment: mechanisms: Laplace, Randomized Rounding justification: '' # TODO: Fill in correct value administrative: - sources: | - - Paper: https://www.microsoft.com/en-us/research/publication/collecting-telemetry-data-privately/ + evidence_sources: + telemetry_paper: + evidence_source_type: Formal publication by the deploying organization + citation: + title: Collecting Telemetry Data Privately + url: https://www.microsoft.com/en-us/research/publication/collecting-telemetry-data-privately/ + author: Bolin Ding, Janardhan (Jana) Kulkarni, Sergey Yekhanin + year: 2017 + month: December status: Pending registry_authors: - Priyanka Nanayakkara diff --git a/deployments/uber.yaml b/deployments/uber.yaml index 0315fe78..5f9e9199 100644 --- a/deployments/uber.yaml +++ b/deployments/uber.yaml @@ -67,10 +67,28 @@ deployment: justification: | - Formal proof that elastic sensitivity is an upper bound of local sensistivity is presented. administrative: - sources: | - - Paper: https://vldb.org/pvldb/vol11/p526-johnson.pdf # https://doi.org/10.1145/3177732.3177733 is not presently working - - Blog post: https://medium.com/uber-security-privacy/uber-open-source-differential-privacy-57f31e85c57a - - Source repo: https://github.com/uber-archive/sql-differential-privacy + evidence_sources: + uber_paper: + evidence_source_type: Formal publication by the deploying organization + citation: + title: Towards Practical Differential Privacy for SQL Queries + url: https://vldb.org/pvldb/vol11/p526-johnson.pdf # https://doi.org/10.1145/3177732.3177733 is not presently working + author: Noah Johnson, Joseph P. Near, Dawn Song + journal: Proceedings of the VLDB Endowment + volume: 11 + uber_blog: + evidence_source_type: Formal publication by the deploying organization + citation: + title: Updates to Uber’s Open Source Project for Differential Privacy + url: https://medium.com/uber-security-privacy/uber-open-source-differential-privacy-57f31e85c57a + author: Menotti Minutillo + year: 2018 + month: January + uber_source: + evidence_source_type: Formal publication by the deploying organization + citation: + title: sql-differential-privacy + url: https://github.com/uber-archive/sql-differential-privacy status: Pending registry_authors: - Micah Altman diff --git a/deployments/user_url_privacy.yaml b/deployments/user_url_privacy.yaml index 32bd59f0..78f4c08b 100644 --- a/deployments/user_url_privacy.yaml +++ b/deployments/user_url_privacy.yaml @@ -42,9 +42,22 @@ deployment: mechanisms: Laplace, Modified Report Noisy Max justification: '' # TODO: Fill in correct value administrative: - sources: | - - Data: https://dataverse.harvard.edu/file.xhtml?persistentId=doi:10.7910/DVN/TDOAPG/DGSAMS&version=6.2 - - FAQ: https://developers.facebook.com/docs/url-shares-dataset/support/faqs + evidence_sources: + data: + evidence_source_type: Formal publication by the deploying organization + citation: + title: Facebook Privacy-Protected Full URLs Data Set + url: https://dataverse.harvard.edu/file.xhtml?persistentId=doi:10.7910/DVN/TDOAPG/DGSAMS&version=6.2 + author: | + Messing, Solomon; DeGregorio, Christina; Hillenbrand, Bennett; King, Gary; Mahanti, Saurav; + Mukerjee, Zagreb; Nayak, Chaya; Persily, Nate; State, Bogdan; Wilkins, Arjun + year: 2020 + month: June + faq: + evidence_source_type: Formal publication by the deploying organization + citation: + title: URL Shares FAQ + url: https://developers.facebook.com/docs/url-shares-dataset/support/faqs status: Pending registry_authors: - Nicolas Berrios diff --git a/deployments/vaccine_search_insights.yaml b/deployments/vaccine_search_insights.yaml index 4f3af32a..119f38c5 100644 --- a/deployments/vaccine_search_insights.yaml +++ b/deployments/vaccine_search_insights.yaml @@ -34,8 +34,13 @@ deployment: mechanisms: Gaussian justification: '' # TODO: Fill in correct value administrative: - sources: | - - Paper: https://arxiv.org/abs/2107.01179 + evidence_sources: + vaccine_paper: + evidence_source_type: Formal publication by the deploying organization + citation: + title: 'Google COVID-19 Vaccination Search Insights: Anonymization Process Description' + url: https://arxiv.org/abs/2107.01179 + author: Shailesh Bavadekar, Adam Boulanger, John Davis, Damien Desfontaines, Evgeniy Gabrilovich, Krishna Gadepalli, Badih Ghazi, Tague Griffith, Jai Gupta, Chaitanya Kamath, Dennis Kraft, Ravi Kumar, Akim Kumok, Yael Mayer, Pasin Manurangsi, Arti Patankar, Irippuge Milinda Perera, Chris Scott, Tomer Shekel, Benjamin Miller, Karen Smith, Charlotte Stanton, Mimi Sun, Mark Young, Gregory Wellenius status: Pending registry_authors: - Nicolas Berrios diff --git a/deployments/wikimedia_current_usage_data.yaml b/deployments/wikimedia_current_usage_data.yaml index c8d2091c..2c57c11e 100644 --- a/deployments/wikimedia_current_usage_data.yaml +++ b/deployments/wikimedia_current_usage_data.yaml @@ -65,12 +65,20 @@ deployment: - Choosing \\(\rho\\) =0.015 is considered to be conservative as it provides much stronger guarantee than \\((\epsilon,\delta)\\)-DP with \\(\epsilon=1\\) and \\(\delta = 10^{-7}\\) administrative: - sources: | - - Paper: https://arxiv.org/pdf/2308.16298 - - Several relevant links are provided in the paper’s references, among which: - - Client-side filtering code: https://gerrit.wikimedia.org/r/plugins/gitiles/operations/puppet/+/refs/heads/production/modules/varnish/templates/analytics.inc.vcl.erb#171 - - Differential Privacy: https://gitlab.wikimedia.org/repos/security/differential-privacy - - Pageviews Differential Privacy – Current – README: https://gitlab.wikimedia.org/htriedman/stat-spark3/-/tree/main/pageview_historical/notebooks + evidence_sources: + wikimedia_paper: + evidence_source_type: Formal publication by the deploying organization + citation: + title: Publishing Wikipedia usage data with strong privacy guarantees + url: https://arxiv.org/pdf/2308.16298 + author: | + Temilola Adeleye, Skye Berghel, Damien Desfontaines, Michael Hay, + Isaac Johnson, Cléo Lemoisson, Ashwin Machanavajjhala, Tom Magerlein, + Gabriele Modena, David Pujol, Daniel Simmons-Marengo, and Hal Triedman + # - Several relevant links are provided in the paper’s references, among which: + # - Client-side filtering code: https://gerrit.wikimedia.org/r/plugins/gitiles/operations/puppet/+/refs/heads/production/modules/varnish/templates/analytics.inc.vcl.erb#171 + # - Differential Privacy: https://gitlab.wikimedia.org/repos/security/differential-privacy + # - Pageviews Differential Privacy – Current – README: https://gitlab.wikimedia.org/htriedman/stat-spark3/-/tree/main/pageview_historical/notebooks status: Pending registry_authors: - Elena Ghazi diff --git a/deployments/wikimedia_editor_activity_statistics_2023.yaml b/deployments/wikimedia_editor_activity_statistics_2023.yaml index 347b5112..fe620cfa 100644 --- a/deployments/wikimedia_editor_activity_statistics_2023.yaml +++ b/deployments/wikimedia_editor_activity_statistics_2023.yaml @@ -54,10 +54,16 @@ deployment: - In their report, the Wikimedia Foundation states: "This dataset was optimized to perform well across several metrics: median relative error, relative error below 50% and relative error below 90%, spurious rate, and drop rate." administrative: - sources: | - - Online description page: https://analytics.wikimedia.org/published/datasets/geoeditors_monthly/00_README.html - - Code repository: https://gitlab.wikimedia.org/repos/security/differential-privacy/-/blob/main/differential_privacy/geoeditors_monthly.py - # notes: '' + evidence_sources: + readme: + evidence_source_type: Other public source + citation: + title: Geoeditors Differential Privacy — Monthly + url: https://analytics.wikimedia.org/published/datasets/geoeditors_monthly/00_README.html + code: + evidence_source_type: Other public source + citation: + url: https://gitlab.wikimedia.org/repos/security/differential-privacy/-/blob/main/differential_privacy/geoeditors_monthly.py status: Draft registry_authors: - Michael Khavkin diff --git a/deployments/wikimedia_historical_usage_data.yaml b/deployments/wikimedia_historical_usage_data.yaml index d48c43c8..09b59c5a 100644 --- a/deployments/wikimedia_historical_usage_data.yaml +++ b/deployments/wikimedia_historical_usage_data.yaml @@ -78,12 +78,20 @@ deployment: administrative: - sources: | - - Paper: https://arxiv.org/pdf/2308.16298 - - Several relevant links are provided in the paper’s references, among which: - - Differential Privacy: https://gitlab.wikimedia.org/repos/security/differential-privacy - - Pageviews Differential Privacy – Current – README: https://gitlab.wikimedia.org/htriedman/stat-spark3/-/tree/main/pageview_historical/notebooks - - Pageviews Differential Privacy – Historical – README: https://analytics.wikimedia.org/published/datasets/country_project_page_historical/00_README.html + evidence_sources: + paper: + evidence_source_type: Formal publication by the deploying organization + citation: + title: Publishing Wikipedia usage data with strong privacy guarantees + url: https://arxiv.org/pdf/2308.16298 + author: | + Temilola Adeleye, Skye Berghel, Damien Desfontaines, Michael Hay, + Isaac Johnson, Cléo Lemoisson, Ashwin Machanavajjhala, Tom Magerlein, + Gabriele Modena, David Pujol, Daniel Simmons-Marengo, and Hal Triedman + # - Several relevant links are provided in the paper’s references, among which: + # - Differential Privacy: https://gitlab.wikimedia.org/repos/security/differential-privacy + # - Pageviews Differential Privacy – Current – README: https://gitlab.wikimedia.org/htriedman/stat-spark3/-/tree/main/pageview_historical/notebooks + # - Pageviews Differential Privacy – Historical – README: https://analytics.wikimedia.org/published/datasets/country_project_page_historical/00_README.html status: Pending registry_authors: - Elena Ghazi diff --git a/deployments/wikimedia_russian_editor_activity_statistics_2023.yaml b/deployments/wikimedia_russian_editor_activity_statistics_2023.yaml index 5410a327..4001db1f 100644 --- a/deployments/wikimedia_russian_editor_activity_statistics_2023.yaml +++ b/deployments/wikimedia_russian_editor_activity_statistics_2023.yaml @@ -53,11 +53,14 @@ deployment: - In their report, they also state that "we have taken extra privacy precautions (specifically, using differential privacy) with this data release." administrative: - sources: | - - Online description page: https://wikitech.wikimedia.org/wiki/Russian_editor_information_(2022-23) - - Code or table building: https://gerrit.wikimedia.org/r/plugins/gitiles/analytics/refinery/%2B/refs/heads/master/hql/geoeditors/geoeditors_monthly.hql - - Code for private data release: https://gitlab.wikimedia.org/htriedman/stat-spark3/-/blob/main/geoeditors/notebooks/04_russia_release.ipynb - # notes: '' # other extra information + evidence_sources: + page: + evidence_source_type: Authorized organizational attestation + citation: + title: Russian editor information (2022-23) + url: https://wikitech.wikimedia.org/wiki/Russian_editor_information_(2022-23) + # - Code or table building: https://gerrit.wikimedia.org/r/plugins/gitiles/analytics/refinery/%2B/refs/heads/master/hql/geoeditors/geoeditors_monthly.hql + # - Code for private data release: https://gitlab.wikimedia.org/htriedman/stat-spark3/-/blob/main/geoeditors/notebooks/04_russia_release.ipynb status: Draft registry_authors: - Michael Khavkin diff --git a/schemas/deployments-schema.yaml b/schemas/deployments-schema.yaml index 1866a481..baf13d65 100644 --- a/schemas/deployments-schema.yaml +++ b/schemas/deployments-schema.yaml @@ -361,12 +361,25 @@ properties: - registry_authors - tier properties: - sources: + evidence_sources: description_long: | This section includes links to white papers, blogposts, or other sources that describe the deployment. This section may also include links to the data product itself, if publicly available. tier: 1 - type: string + type: object + patternProperties: + ^\w+$: + type: object + additionalProperties: False + properties: + evidence_source_type: + type: string + enum: + - Peer-reviewed publication + - Formal publication by the deploying organization + - Authorized organizational attestation + - Other public source + citation: { $ref: "#/$defs/citation" } status: description: Approval status of this record. type: string @@ -399,3 +412,44 @@ properties: - 1 - 2 - 3 +$defs: + string_or_number: + # We want to avoid spurious validation errors + # just because someone enters an unquoted number in yaml. + # Downstream string concatenation will work either way. + anyOf: + - type: string + - type: number + citation: + # Based on the fields available in bibtex. + type: object + additionalProperties: False + properties: + address: { type: string } + annote: { type: string } + author: { type: string } + booktitle: { type: string } + chapter: { $ref: "#/$defs/string_or_number" } + # crossref (https://bibtex.eu/fields/ only) + doi: { type: string } + edition: { $ref: "#/$defs/string_or_number" } + editor: { type: string } + email: { type: string } + howpublished: { type: string } + institution: { type: string } + issn: { $ref: "#/$defs/string_or_number" } + isbn: { $ref: "#/$defs/string_or_number" } + journal: { type: string } + month: { $ref: "#/$defs/string_or_number" } + note: { type: string } + number: { $ref: "#/$defs/string_or_number" } + organization: { type: string } + pages: { $ref: "#/$defs/string_or_number" } + publisher: { type: string } + school: { type: string } + type: { type: string } + series: { type: string } + title: { type: string } + url: { type: string } + volume: { $ref: "#/$defs/string_or_number" } + year: { $ref: "#/$defs/string_or_number" } diff --git a/scripts/check.py b/scripts/check.py index 585c37fe..57bbab7e 100755 --- a/scripts/check.py +++ b/scripts/check.py @@ -99,7 +99,7 @@ def check_latex_escapes(yaml_path): checks = {name for name in globals().keys() if name.startswith("check_")} -def check(yaml_path: Path, only=checks): +def check_local(yaml_path: Path, only=checks): detail_checks = [ function for name, function in globals().items() @@ -116,6 +116,23 @@ def check(yaml_path: Path, only=checks): return errors +def check_global(yaml_paths: list[Path]): + errors = {} + evidence_source_keys = set() + for yaml_path in yaml_paths: + deployment = load(yaml_path.open(), Loader=Loader) + new_keys = set( + deployment["deployment"]["administrative"]["evidence_sources"].keys() + ) + + if intersection := new_keys & evidence_source_keys: + errors[ + yaml_path.name + ] = f"Keys not globally unique: {', '.join(intersection)}" + evidence_source_keys |= new_keys + return errors + + if __name__ == "__main__": import argparse @@ -149,12 +166,20 @@ def check(yaml_path: Path, only=checks): print("No files selected") exit(1) errors = {} + + global_error = check_global(yaml_paths) + if global_error: + errors["global"] = global_error + for yaml_path in yaml_paths: print(f"Validating {yaml_path.name}...") - error = check(yaml_path, only=args.only or (checks - set(args.skip))) - assert isinstance(error, dict), f"Expected list, not {error}" - if error: - errors[yaml_path.name] = error + local_error = check_local( + yaml_path, only=args.only or (checks - set(args.skip)) + ) + assert isinstance(local_error, dict), f"Expected dict, not {local_error}" + if local_error: + errors[yaml_path.name] = local_error + if errors: print(dump(errors)) exit(1) diff --git a/tests/good_deployments/template.yaml b/tests/good_deployments/template.yaml index 84da6806..5e90eeca 100644 --- a/tests/good_deployments/template.yaml +++ b/tests/good_deployments/template.yaml @@ -52,7 +52,37 @@ deployment: additional_information: '' administrative: - sources: '' # white papers + evidence_sources: + example_source_1: + evidence_source_type: "Peer-reviewed publication" + citation: + address: '' + annote: '' + author: '' + booktitle: '' + chapter: '' + doi: '' + edition: '' + editor: '' + email: '' + howpublished: '' + institution: '' + issn: '' + isbn: '' + journal: '' + month: '' + note: '' + number: '' + organization: '' + pages: '' + publisher: '' + school: '' + type: '' + series: '' + title: '' + url: '' + volume: '' + year: '' status: Draft registry_authors: - '' # Fill in correct value. diff --git a/tests/test_schema.py b/tests/test_schema.py index 0478526e..14c79dbd 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -40,6 +40,7 @@ def test_node_has_details(path, node): "/deployment/implementation", "/deployment/administrative", "/deployment/administrative/sources", + "/deployment/administrative/evidence_sources", "/deployment/administrative/notes", ]: assert "description" not in node.keys() @@ -121,5 +122,7 @@ def test_object_has_additional_properties_false(path, node): def test_template_is_complete(): - template_paths = [path for path, node in path_templates] + template_paths = [ + path for path, node in path_templates if "evidence_sources/" not in path + ] assert paths == template_paths diff --git a/tests/test_ui_hints.py b/tests/test_ui_hints.py index 99a203bc..89626c05 100644 --- a/tests/test_ui_hints.py +++ b/tests/test_ui_hints.py @@ -68,7 +68,9 @@ def test_deployments_in_order(yaml_path): # UI code will be simpler if it can rely on deployment YAML # always being in the prescribed order. deployment = load(yaml_path.open(), Loader=Loader) - deployment_paths = list(walk_dict(deployment)) + deployment_paths = [ + path for path in walk_dict(deployment) if "evidence_sources/" not in path + ] misordered = out_of_order(deployment_paths, paths) assert ( not misordered diff --git a/tests/test_yaml.py b/tests/test_yaml.py index bf7e64b5..c1a9a030 100644 --- a/tests/test_yaml.py +++ b/tests/test_yaml.py @@ -1,4 +1,4 @@ -from check import check, checks +from check import check_local, checks from update_slug import update_slug import pytest @@ -30,7 +30,7 @@ def test_updated_slugs(yaml_path): "yaml_path", root.glob("deployments/*.yaml"), ids=lambda path: path.name ) def test_real_yaml(yaml_path): - errors = check(yaml_path, only=checks - {"check_urls"}) + errors = check_local(yaml_path, only=checks - {"check_urls"}) assert not errors @@ -38,7 +38,7 @@ def test_real_yaml(yaml_path): "yaml_path", root.glob("tests/good_deployments/*.yaml"), ids=lambda path: path.name ) def test_good_yaml(yaml_path): - errors = check(yaml_path) + errors = check_local(yaml_path) assert not errors @@ -48,7 +48,7 @@ def test_good_yaml(yaml_path): ids=lambda path: path.name, ) def test_bad_yaml(bad_yaml_path): - errors = check(bad_yaml_path) + errors = check_local(bad_yaml_path) assert errors, "Expected errors, but there aren't any" errors_stem = errors_to_stem(errors) assert errors_stem == bad_yaml_path.stem, errors