diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml new file mode 100644 index 0000000..4f93e5c --- /dev/null +++ b/.github/workflows/CI.yml @@ -0,0 +1,54 @@ +name: CI +on: + push: + branches: + - main + tags: ['*'] + pull_request: + workflow_dispatch: +concurrency: + # Skip intermediate builds: always. + # Cancel intermediate builds: only if it is a pull request build. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} +jobs: + test: + name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + permissions: # needed to allow julia-actions/cache to proactively delete old caches that it has created + actions: write + contents: read + strategy: + fail-fast: false + matrix: + version: + - '1.12' + os: + - ubuntu-latest + arch: + - x64 + steps: + - uses: actions/checkout@v6 + - uses: r-lib/actions/setup-r@v2 + with: + use-public-rspm: true + r-version: 'release' + - name: Set LD_LIBRARY_PATH for R + run: echo "LD_LIBRARY_PATH=$(R RHOME)/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + if: matrix.os == 'ubuntu-latest' + - name: Install rio from GitHub + run: Rscript -e 'install.packages(c("remotes", "haven")); remotes::install_github("gesistsa/rio")' + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + - name: Install all rio format handlers + run: Rscript -e 'rio::install_formats()' + - name: Install cloc + run: sudo apt-get update && sudo apt-get install -y cloc + - uses: julia-actions/setup-julia@v2 + with: + version: ${{ matrix.version }} + arch: ${{ matrix.arch }} + - uses: julia-actions/cache@v2 + - uses: julia-actions/julia-buildpkg@v1 + - uses: julia-actions/julia-runtest@v1 diff --git a/.gitignore b/.gitignore index 3a18368..8b13789 100644 --- a/.gitignore +++ b/.gitignore @@ -1,154 +1 @@ -# this is from https://github.com/AEADataEditor/replication-template-development/blob/development/.gitignore -# Data file extensions -# They are duplicated in upper case and in some cases mixed case to cover all bases -# If you want to make the parsing overall case-insensitive, you can use the following: -# git config core.ignorecase false -# Version: 2025-06-06 -*.gpkg -*.GPKG -*.bak -*.BAK -*.tmp -*.TMP -*.aux -*.AUX -*.mat -*.MAT -*.sav -*.SAV -*.sas7bdat -*.SAS7BDAT -*.dta -*.DTA -*.csv -*.CSV -*.tsv -*.TSV -*.RData -*.RDATA -*.rdata -*.Rdata -*.Rds -*.RDs -*.RDS -*.rds -*.RDA -*.Rda -*.raw -*.Raw -*.RAW -*.dbf -*.DBF -*.prj -*.PRJ -*.shp -*.SHP -*.shx -*.SHX -*.xml -*.XML -*.xpt -*.XPT -*.cpg -*.CPG -*.sbn -*.SBN -*.sbx -*.SBX -*.mpk -*.MPK -*.tfw -*.TFW -*.pkl -*.PKL -*.pbf -*.PBF -*.jld2 -*.JLD2 -*.dat -*.DAT -*.Dat -*.feather -*.pickle -*.tab -*.TAB -# Sometimes, these are data files, too -*.txt -*.TXT -*.asc -*.ASC -# Usually, we include images, but TIFF files can be huge -*.tif* -*.TIF* -*.ovr -*.OVR -*.lyrx -*.lyrx -# Archives -*.rar -*.RAR -*.zip -*.ZIP -*.gz -*.tgz -*.xz -*.7z -*.tar -# Databases -**/.*gdb/ -# Random binary formats -*.bin -*.BIN -*.exe -*.EXE -*.dll -*.DLL -*.so -*.SO -# MacOS specific files and directories -*.DS_Store -*.DS_store -*.ds_store -__MACOSX/ -*.dmg -*.DMG -# Stata-specific temp files -*.stswp -# build-specific directories -.rproj* -.Rhistory -.Rproj.user -# test -# .vscode -*.npy -__pycache__ -libraries -# Exclusions we need for our work -!generated/ -!generated/** -!images/ -!images/** -!src/ -!src/** -!test/ -!test/** - -!R*.txt -!R*.TXT -!pii_stata_output.csv -!package-output-map.xlsx - -# Exclusions we need -TEMPLATE.html -TEMPLATE_files/ -build/requirements.txt -codeocean-*-live -tmp -cache -# some files which should never be included -*.lic -*.LIC -# Ignore all files in any directory named "data" -**/data/* -/.quarto/ diff --git a/Manifest.toml b/Manifest.toml new file mode 100644 index 0000000..91f90be --- /dev/null +++ b/Manifest.toml @@ -0,0 +1,813 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.12.4" +manifest_format = "2.0" +project_hash = "f66c37198ac13584fdbf884df619aab71244ce39" + +[[deps.AbstractTrees]] +git-tree-sha1 = "2d9c9a55f9c93e8887ad391fbae72f8ef55e1177" +uuid = "1520ce14-60c1-5f80-bbc7-55ef81b5835c" +version = "0.4.5" + +[[deps.AdobeGlyphList]] +deps = ["DelimitedFiles", "Pkg"] +git-tree-sha1 = "d90a9c83fe96e3e93c66e1d3202877f0e92fcdfe" +uuid = "715cd884-8f22-55db-b077-449d1f7eef73" +version = "0.1.2" + +[[deps.AliasTables]] +deps = ["PtrArrays", "Random"] +git-tree-sha1 = "9876e1e164b144ca45e9e3198d0b689cadfed9ff" +uuid = "66dad0bd-aa9a-41b7-9441-69ab47430ed8" +version = "1.1.3" + +[[deps.ArgTools]] +uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" +version = "1.1.2" + +[[deps.Artifacts]] +uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" +version = "1.11.0" + +[[deps.Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" +version = "1.11.0" + +[[deps.BinDeps]] +deps = ["Libdl", "Pkg", "SHA", "URIParser", "Unicode"] +git-tree-sha1 = "1289b57e8cf019aede076edab0587eb9644175bd" +uuid = "9e28174c-4ba2-5203-b857-d8d62c4213ee" +version = "1.0.2" + +[[deps.BitFlags]] +git-tree-sha1 = "0691e34b3bb8be9307330f88d1a3c3f25466c24d" +uuid = "d1d4a3ce-64b1-5f1a-9ba4-7e7e69966f35" +version = "0.1.9" + +[[deps.CSV]] +deps = ["CodecZlib", "Dates", "FilePathsBase", "InlineStrings", "Mmap", "Parsers", "PooledArrays", "PrecompileTools", "SentinelArrays", "Tables", "Unicode", "WeakRefStrings", "WorkerUtilities"] +git-tree-sha1 = "deddd8725e5e1cc49ee205a1964256043720a6c3" +uuid = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" +version = "0.10.15" + +[[deps.CategoricalArrays]] +deps = ["Compat", "DataAPI", "Future", "Missings", "Printf", "Requires", "Statistics", "Unicode"] +git-tree-sha1 = "73acb4ed51b1855e1b5ce5c610334363a98d13f1" +uuid = "324d7699-5711-5eae-9e2f-1d82baa6b597" +version = "1.0.2" + + [deps.CategoricalArrays.extensions] + CategoricalArraysArrowExt = "Arrow" + CategoricalArraysJSONExt = "JSON" + CategoricalArraysRecipesBaseExt = "RecipesBase" + CategoricalArraysSentinelArraysExt = "SentinelArrays" + CategoricalArraysStatsBaseExt = "StatsBase" + CategoricalArraysStructTypesExt = "StructTypes" + + [deps.CategoricalArrays.weakdeps] + Arrow = "69666777-d1a9-59fb-9406-91d4454c9d45" + JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" + RecipesBase = "3cdcf5f2-1ef4-517c-9805-6587b60abb01" + SentinelArrays = "91c51154-3ec4-41a3-a24f-3f23e20d615c" + StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" + StructTypes = "856f2bd8-1eba-4b0a-8007-ebc267875bd4" + +[[deps.CodeTracking]] +deps = ["InteractiveUtils", "UUIDs"] +git-tree-sha1 = "b7231a755812695b8046e8471ddc34c8268cbad5" +uuid = "da1fd8a2-8d9e-5ec2-8556-3022fb5608a2" +version = "3.0.0" + +[[deps.CodecZlib]] +deps = ["TranscodingStreams", "Zlib_jll"] +git-tree-sha1 = "962834c22b66e32aa10f7611c08c8ca4e20749a9" +uuid = "944b1d66-785c-5afd-91f1-9de20f533193" +version = "0.7.8" + +[[deps.Compat]] +deps = ["TOML", "UUIDs"] +git-tree-sha1 = "9d8a54ce4b17aa5bdce0ea5c34bc5e7c340d16ad" +uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" +version = "4.18.1" +weakdeps = ["Dates", "LinearAlgebra"] + + [deps.Compat.extensions] + CompatLinearAlgebraExt = "LinearAlgebra" + +[[deps.Compiler]] +git-tree-sha1 = "382d79bfe72a406294faca39ef0c3cef6e6ce1f1" +uuid = "807dbc54-b67e-4c79-8afb-eafe4df6f2e1" +version = "0.1.1" + +[[deps.CompilerSupportLibraries_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" +version = "1.3.0+1" + +[[deps.ConcurrentUtilities]] +deps = ["Serialization", "Sockets"] +git-tree-sha1 = "d9d26935a0bcffc87d2613ce14c527c99fc543fd" +uuid = "f0e56b4a-5159-44fe-b623-3e5288b988bb" +version = "2.5.0" + +[[deps.Conda]] +deps = ["Downloads", "JSON", "VersionParsing"] +git-tree-sha1 = "8f06b0cfa4c514c7b9546756dbae91fcfbc92dc9" +uuid = "8f4d0f93-b110-5947-807f-2305c1781a2d" +version = "1.10.3" + +[[deps.Crayons]] +git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15" +uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f" +version = "4.1.1" + +[[deps.DataAPI]] +git-tree-sha1 = "abe83f3a2f1b857aac70ef8b269080af17764bbe" +uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" +version = "1.16.0" + +[[deps.DataFrames]] +deps = ["Compat", "DataAPI", "DataStructures", "Future", "InlineStrings", "InvertedIndices", "IteratorInterfaceExtensions", "LinearAlgebra", "Markdown", "Missings", "PooledArrays", "PrecompileTools", "PrettyTables", "Printf", "Random", "Reexport", "SentinelArrays", "SortingAlgorithms", "Statistics", "TableTraits", "Tables", "Unicode"] +git-tree-sha1 = "d8928e9169ff76c6281f39a659f9bca3a573f24c" +uuid = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" +version = "1.8.1" + +[[deps.DataStructures]] +deps = ["OrderedCollections"] +git-tree-sha1 = "e357641bb3e0638d353c4b29ea0e40ea644066a6" +uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" +version = "0.19.3" + +[[deps.DataValueInterfaces]] +git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6" +uuid = "e2d170a0-9d28-54be-80f0-106bbe20a464" +version = "1.0.0" + +[[deps.Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" +version = "1.11.0" + +[[deps.DelimitedFiles]] +deps = ["Mmap"] +git-tree-sha1 = "9e2f36d3c96a820c678f2f1f1782582fcf685bae" +uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab" +version = "1.9.1" + +[[deps.DocStringExtensions]] +git-tree-sha1 = "7442a5dfe1ebb773c29cc2962a8980f47221d76c" +uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" +version = "0.9.5" + +[[deps.Downloads]] +deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] +uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +version = "1.7.0" + +[[deps.ExceptionUnwrapping]] +deps = ["Test"] +git-tree-sha1 = "d36f682e590a83d63d1c7dbd287573764682d12a" +uuid = "460bff9d-24e4-43bc-9d9f-a8973cb893f4" +version = "0.1.11" + +[[deps.FilePathsBase]] +deps = ["Compat", "Dates"] +git-tree-sha1 = "3bab2c5aa25e7840a4b065805c0cdfc01f3068d2" +uuid = "48062228-2e41-5def-b9a4-89aafe57970f" +version = "0.9.24" +weakdeps = ["Mmap", "Test"] + + [deps.FilePathsBase.extensions] + FilePathsBaseMmapExt = "Mmap" + FilePathsBaseTestExt = "Test" + +[[deps.FileWatching]] +uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" +version = "1.11.0" + +[[deps.Future]] +deps = ["Random"] +uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" +version = "1.11.0" + +[[deps.Glob]] +git-tree-sha1 = "83cb0092e2792b9e3a865b6655e88f5b862607e2" +uuid = "c27321d9-0574-5035-807b-f59d2c89b15c" +version = "1.4.0" + +[[deps.HTTP]] +deps = ["Base64", "CodecZlib", "ConcurrentUtilities", "Dates", "ExceptionUnwrapping", "Logging", "LoggingExtras", "MbedTLS", "NetworkOptions", "OpenSSL", "PrecompileTools", "Random", "SimpleBufferStream", "Sockets", "URIs", "UUIDs"] +git-tree-sha1 = "5e6fe50ae7f23d171f44e311c2960294aaa0beb5" +uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3" +version = "1.10.19" + +[[deps.HypergeometricFunctions]] +deps = ["LinearAlgebra", "OpenLibm_jll", "SpecialFunctions"] +git-tree-sha1 = "68c173f4f449de5b438ee67ed0c9c748dc31a2ec" +uuid = "34004b35-14d8-5ef3-9330-4cdb6864b03a" +version = "0.3.28" + +[[deps.Infiltrator]] +deps = ["InteractiveUtils", "Markdown", "REPL", "UUIDs"] +git-tree-sha1 = "d07c6718c1ce7324341947f8f608f5d69795dde5" +uuid = "5903a43b-9cc3-4c30-8d17-598619ec4e9b" +version = "1.9.7" + +[[deps.InlineStrings]] +git-tree-sha1 = "8f3d257792a522b4601c24a577954b0a8cd7334d" +uuid = "842dd82b-1e85-43dc-bf29-5d0ee9dffc48" +version = "1.4.5" + + [deps.InlineStrings.extensions] + ArrowTypesExt = "ArrowTypes" + ParsersExt = "Parsers" + + [deps.InlineStrings.weakdeps] + ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd" + Parsers = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" + +[[deps.IntegerMathUtils]] +git-tree-sha1 = "4c1acff2dc6b6967e7e750633c50bc3b8d83e617" +uuid = "18e54dd8-cb9d-406c-a71d-865a43cbb235" +version = "0.1.3" + +[[deps.InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" +version = "1.11.0" + +[[deps.InvertedIndices]] +git-tree-sha1 = "6da3c4316095de0f5ee2ebd875df8721e7e0bdbe" +uuid = "41ab1584-1d38-5bbf-9106-f11c6c58b48f" +version = "1.3.1" + +[[deps.IrrationalConstants]] +git-tree-sha1 = "b2d91fe939cae05960e760110b328288867b5758" +uuid = "92d709cd-6900-40b7-9082-c6be49f344b6" +version = "0.2.6" + +[[deps.IteratorInterfaceExtensions]] +git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856" +uuid = "82899510-4779-5014-852e-03e436cf321d" +version = "1.0.0" + +[[deps.JLLWrappers]] +deps = ["Artifacts", "Preferences"] +git-tree-sha1 = "0533e564aae234aff59ab625543145446d8b6ec2" +uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" +version = "1.7.1" + +[[deps.JPEtools]] +deps = ["CSV", "DataFrames", "Dates", "Glob", "HTTP", "Infiltrator", "JSON3", "PDFIO", "Pkg", "Printf", "Revise", "SHA", "TestItemRunner", "TestItems"] +path = "." +uuid = "3207fb7f-a2d6-47fa-abcb-ee7fd07ee121" +version = "1.0.0-DEV" + +[[deps.JSON]] +deps = ["Dates", "Logging", "Parsers", "PrecompileTools", "StructUtils", "UUIDs", "Unicode"] +git-tree-sha1 = "b3ad4a0255688dcb895a52fafbaae3023b588a90" +uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +version = "1.4.0" + + [deps.JSON.extensions] + JSONArrowExt = ["ArrowTypes"] + + [deps.JSON.weakdeps] + ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd" + +[[deps.JSON3]] +deps = ["Dates", "Mmap", "Parsers", "PrecompileTools", "StructTypes", "UUIDs"] +git-tree-sha1 = "411eccfe8aba0814ffa0fdf4860913ed09c34975" +uuid = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" +version = "1.14.3" + + [deps.JSON3.extensions] + JSON3ArrowExt = ["ArrowTypes"] + + [deps.JSON3.weakdeps] + ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd" + +[[deps.JuliaInterpreter]] +deps = ["CodeTracking", "InteractiveUtils", "Random", "UUIDs"] +git-tree-sha1 = "80580012d4ed5a3e8b18c7cd86cebe4b816d17a6" +uuid = "aa1ae85d-cabe-5617-a682-6adf51b2e16a" +version = "0.10.9" + +[[deps.JuliaSyntaxHighlighting]] +deps = ["StyledStrings"] +uuid = "ac6e5ff7-fb65-4e79-a425-ec3bc9c03011" +version = "1.12.0" + +[[deps.LaTeXStrings]] +git-tree-sha1 = "dda21b8cbd6a6c40d9d02a73230f9d70fed6918c" +uuid = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" +version = "1.4.0" + +[[deps.LabelNumerals]] +deps = ["RomanNumerals"] +git-tree-sha1 = "d7d8d199b61941d4f0ba666ef5eb017dab5d79f4" +uuid = "2e475f56-4567-5c41-94ff-3f69a29378f9" +version = "0.1.1" + +[[deps.LibCURL]] +deps = ["LibCURL_jll", "MozillaCACerts_jll"] +uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" +version = "0.6.4" + +[[deps.LibCURL_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "OpenSSL_jll", "Zlib_jll", "nghttp2_jll"] +uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" +version = "8.15.0+0" + +[[deps.LibGit2]] +deps = ["LibGit2_jll", "NetworkOptions", "Printf", "SHA"] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" +version = "1.11.0" + +[[deps.LibGit2_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "OpenSSL_jll"] +uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" +version = "1.9.0+0" + +[[deps.LibSSH2_jll]] +deps = ["Artifacts", "Libdl", "OpenSSL_jll"] +uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" +version = "1.11.3+1" + +[[deps.Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" +version = "1.11.0" + +[[deps.LinearAlgebra]] +deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] +uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +version = "1.12.0" + +[[deps.LogExpFunctions]] +deps = ["DocStringExtensions", "IrrationalConstants", "LinearAlgebra"] +git-tree-sha1 = "13ca9e2586b89836fd20cccf56e57e2b9ae7f38f" +uuid = "2ab3a3ac-af41-5b50-aa03-7779005ae688" +version = "0.3.29" + + [deps.LogExpFunctions.extensions] + LogExpFunctionsChainRulesCoreExt = "ChainRulesCore" + LogExpFunctionsChangesOfVariablesExt = "ChangesOfVariables" + LogExpFunctionsInverseFunctionsExt = "InverseFunctions" + + [deps.LogExpFunctions.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + ChangesOfVariables = "9e997f8a-9a97-42d5-a9f1-ce6bfc15e2c0" + InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112" + +[[deps.Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" +version = "1.11.0" + +[[deps.LoggingExtras]] +deps = ["Dates", "Logging"] +git-tree-sha1 = "f00544d95982ea270145636c181ceda21c4e2575" +uuid = "e6f89c97-d47a-5376-807f-9c37f3926c36" +version = "1.2.0" + +[[deps.LoweredCodeUtils]] +deps = ["CodeTracking", "Compiler", "JuliaInterpreter"] +git-tree-sha1 = "65ae3db6ab0e5b1b5f217043c558d9d1d33cc88d" +uuid = "6f1432cf-f94c-5a45-995e-cdbf5db27b0b" +version = "3.5.0" + +[[deps.Markdown]] +deps = ["Base64", "JuliaSyntaxHighlighting", "StyledStrings"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" +version = "1.11.0" + +[[deps.MbedTLS]] +deps = ["Dates", "MbedTLS_jll", "MozillaCACerts_jll", "NetworkOptions", "Random", "Sockets"] +git-tree-sha1 = "c067a280ddc25f196b5e7df3877c6b226d390aaf" +uuid = "739be429-bea8-5141-9913-cc70e7f3736d" +version = "1.1.9" + +[[deps.MbedTLS_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "ff69a2b1330bcb730b9ac1ab7dd680176f5896b8" +uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" +version = "2.28.1010+0" + +[[deps.Missings]] +deps = ["DataAPI"] +git-tree-sha1 = "ec4f7fbeab05d7747bdf98eb74d130a2a2ed298d" +uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28" +version = "1.2.0" + +[[deps.Mmap]] +uuid = "a63ad114-7e13-5084-954f-fe012c677804" +version = "1.11.0" + +[[deps.MozillaCACerts_jll]] +uuid = "14a3606d-f60d-562e-9121-12d972cd8159" +version = "2025.11.4" + +[[deps.NetworkOptions]] +uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" +version = "1.3.0" + +[[deps.OpenBLAS_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] +uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" +version = "0.3.29+0" + +[[deps.OpenLibm_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "05823500-19ac-5b8b-9628-191a04bc5112" +version = "0.8.7+0" + +[[deps.OpenSSL]] +deps = ["BitFlags", "Dates", "MozillaCACerts_jll", "NetworkOptions", "OpenSSL_jll", "Sockets"] +git-tree-sha1 = "1d1aaa7d449b58415f97d2839c318b70ffb525a0" +uuid = "4d8831e6-92b7-49fb-bdf8-b643e874388c" +version = "1.6.1" + +[[deps.OpenSSL_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" +version = "3.5.4+0" + +[[deps.OpenSpecFun_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl"] +git-tree-sha1 = "1346c9208249809840c91b26703912dff463d335" +uuid = "efe28fd5-8261-553b-a9e1-b2916fc3738e" +version = "0.5.6+0" + +[[deps.OrderedCollections]] +git-tree-sha1 = "05868e21324cede2207c6f0f466b4bfef6d5e7ee" +uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +version = "1.8.1" + +[[deps.PDFIO]] +deps = ["AbstractTrees", "AdobeGlyphList", "BinDeps", "Dates", "DelimitedFiles", "LabelNumerals", "Libdl", "LinearAlgebra", "OpenSSL_jll", "Pkg", "Printf", "Rectangle", "RomanNumerals", "Zlib_jll"] +git-tree-sha1 = "b08b9887ea42989124ccc1b84a642221b4d81096" +uuid = "4d0d745f-9d9a-592e-8d18-1ad8a0f42b92" +version = "0.1.15" + +[[deps.PIIScanner]] +deps = ["CSV", "DataFrames", "Dates", "Infiltrator", "RCall", "Test", "TestItemRunner"] +git-tree-sha1 = "ada3560eed10c1c8151bf2fd190cf271ff9e3a7b" +repo-rev = "main" +repo-url = "https://github.com/floswald/PIIScanner.jl" +uuid = "cf352d04-0051-48bd-b141-6a91af5c6085" +version = "1.0.0-DEV" + +[[deps.Parsers]] +deps = ["Dates", "PrecompileTools", "UUIDs"] +git-tree-sha1 = "7d2f8f21da5db6a806faf7b9b292296da42b2810" +uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" +version = "2.8.3" + +[[deps.Pkg]] +deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "Random", "SHA", "TOML", "Tar", "UUIDs", "p7zip_jll"] +uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +version = "1.12.1" +weakdeps = ["REPL"] + + [deps.Pkg.extensions] + REPLExt = "REPL" + +[[deps.PooledArrays]] +deps = ["DataAPI", "Future"] +git-tree-sha1 = "36d8b4b899628fb92c2749eb488d884a926614d3" +uuid = "2dfb63ee-cc39-5dd5-95bd-886bf059d720" +version = "1.4.3" + +[[deps.PrecompileTools]] +deps = ["Preferences"] +git-tree-sha1 = "07a921781cab75691315adc645096ed5e370cb77" +uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a" +version = "1.3.3" + +[[deps.Preferences]] +deps = ["TOML"] +git-tree-sha1 = "522f093a29b31a93e34eaea17ba055d850edea28" +uuid = "21216c6a-2e73-6563-6e65-726566657250" +version = "1.5.1" + +[[deps.PrettyTables]] +deps = ["Crayons", "LaTeXStrings", "Markdown", "PrecompileTools", "Printf", "REPL", "Reexport", "StringManipulation", "Tables"] +git-tree-sha1 = "c5a07210bd060d6a8491b0ccdee2fa0235fc00bf" +uuid = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" +version = "3.1.2" + +[[deps.Primes]] +deps = ["IntegerMathUtils"] +git-tree-sha1 = "25cdd1d20cd005b52fc12cb6be3f75faaf59bb9b" +uuid = "27ebfcd6-29c5-5fa9-bf4b-fb8fc14df3ae" +version = "0.5.7" + +[[deps.Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" +version = "1.11.0" + +[[deps.PtrArrays]] +git-tree-sha1 = "1d36ef11a9aaf1e8b74dacc6a731dd1de8fd493d" +uuid = "43287f4e-b6f4-7ad1-bb20-aadabca52c3d" +version = "1.3.0" + +[[deps.RCall]] +deps = ["CategoricalArrays", "Conda", "DataFrames", "DataStructures", "Dates", "Libdl", "Preferences", "REPL", "Random", "StatsModels", "WinReg"] +git-tree-sha1 = "1a878b6369dcff2b1df9d18511f35942bb322aa3" +uuid = "6f49c342-dc21-5d91-9882-a32aef131414" +version = "0.14.11" + + [deps.RCall.extensions] + RCallAxisArraysExt = ["AxisArrays"] + + [deps.RCall.weakdeps] + AxisArrays = "39de3d68-74b9-583c-8d2d-e117c070f3a9" + +[[deps.REPL]] +deps = ["InteractiveUtils", "JuliaSyntaxHighlighting", "Markdown", "Sockets", "StyledStrings", "Unicode"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" +version = "1.11.0" + +[[deps.Random]] +deps = ["SHA"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +version = "1.11.0" + +[[deps.Rectangle]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "656434329bee23e402f1db2f86c5362bf859de8c" +uuid = "9a9db56c-8f71-5460-add5-cb450131785e" +version = "0.1.3" + +[[deps.Reexport]] +git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b" +uuid = "189a3867-3050-52da-a836-e630ba90ab69" +version = "1.2.2" + +[[deps.Requires]] +deps = ["UUIDs"] +git-tree-sha1 = "62389eeff14780bfe55195b7204c0d8738436d64" +uuid = "ae029012-a4dd-5104-9daa-d747884805df" +version = "1.3.1" + +[[deps.Revise]] +deps = ["CodeTracking", "FileWatching", "InteractiveUtils", "JuliaInterpreter", "LibGit2", "LoweredCodeUtils", "OrderedCollections", "Preferences", "REPL", "UUIDs"] +git-tree-sha1 = "dfd6fab9aa325b382773b8930998ce84c2918e5e" +uuid = "295af30f-e4ad-537b-8983-00126c2a3abe" +version = "3.13.1" + + [deps.Revise.extensions] + DistributedExt = "Distributed" + + [deps.Revise.weakdeps] + Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" + +[[deps.Rmath]] +deps = ["Random", "Rmath_jll"] +git-tree-sha1 = "5b3d50eb374cea306873b371d3f8d3915a018f0b" +uuid = "79098fc4-a85e-5d69-aa6a-4863f24498fa" +version = "0.9.0" + +[[deps.Rmath_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "58cdd8fb2201a6267e1db87ff148dd6c1dbd8ad8" +uuid = "f50d1b31-88e8-58de-be2c-1cc44531875f" +version = "0.5.1+0" + +[[deps.RomanNumerals]] +deps = ["Primes"] +git-tree-sha1 = "e2f9f2357f68b95d5b9a130ba54508423602f287" +uuid = "37834d88-8936-577c-80c9-1066ecf66832" +version = "0.3.3" + +[[deps.SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" + +[[deps.SentinelArrays]] +deps = ["Dates", "Random"] +git-tree-sha1 = "ebe7e59b37c400f694f52b58c93d26201387da70" +uuid = "91c51154-3ec4-41a3-a24f-3f23e20d615c" +version = "1.4.9" + +[[deps.Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" +version = "1.11.0" + +[[deps.ShiftedArrays]] +git-tree-sha1 = "503688b59397b3307443af35cd953a13e8005c16" +uuid = "1277b4bf-5013-50f5-be3d-901d8477a67a" +version = "2.0.0" + +[[deps.SimpleBufferStream]] +git-tree-sha1 = "f305871d2f381d21527c770d4788c06c097c9bc1" +uuid = "777ac1f9-54b0-4bf8-805c-2214025038e7" +version = "1.2.0" + +[[deps.Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" +version = "1.11.0" + +[[deps.SortingAlgorithms]] +deps = ["DataStructures"] +git-tree-sha1 = "64d974c2e6fdf07f8155b5b2ca2ffa9069b608d9" +uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c" +version = "1.2.2" + +[[deps.SparseArrays]] +deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"] +uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" +version = "1.12.0" + +[[deps.SpecialFunctions]] +deps = ["IrrationalConstants", "LogExpFunctions", "OpenLibm_jll", "OpenSpecFun_jll"] +git-tree-sha1 = "f2685b435df2613e25fc10ad8c26dddb8640f547" +uuid = "276daf66-3868-5448-9aa4-cd146d93841b" +version = "2.6.1" + + [deps.SpecialFunctions.extensions] + SpecialFunctionsChainRulesCoreExt = "ChainRulesCore" + + [deps.SpecialFunctions.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + +[[deps.Statistics]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "ae3bb1eb3bba077cd276bc5cfc337cc65c3075c0" +uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +version = "1.11.1" +weakdeps = ["SparseArrays"] + + [deps.Statistics.extensions] + SparseArraysExt = ["SparseArrays"] + +[[deps.StatsAPI]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "178ed29fd5b2a2cfc3bd31c13375ae925623ff36" +uuid = "82ae8749-77ed-4fe6-ae5f-f523153014b0" +version = "1.8.0" + +[[deps.StatsBase]] +deps = ["AliasTables", "DataAPI", "DataStructures", "IrrationalConstants", "LinearAlgebra", "LogExpFunctions", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics", "StatsAPI"] +git-tree-sha1 = "aceda6f4e598d331548e04cc6b2124a6148138e3" +uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" +version = "0.34.10" + +[[deps.StatsFuns]] +deps = ["HypergeometricFunctions", "IrrationalConstants", "LogExpFunctions", "Reexport", "Rmath", "SpecialFunctions"] +git-tree-sha1 = "91f091a8716a6bb38417a6e6f274602a19aaa685" +uuid = "4c63d2b9-4356-54db-8cca-17b64c39e42c" +version = "1.5.2" + + [deps.StatsFuns.extensions] + StatsFunsChainRulesCoreExt = "ChainRulesCore" + StatsFunsInverseFunctionsExt = "InverseFunctions" + + [deps.StatsFuns.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112" + +[[deps.StatsModels]] +deps = ["DataAPI", "DataStructures", "LinearAlgebra", "Printf", "REPL", "ShiftedArrays", "SparseArrays", "StatsAPI", "StatsBase", "StatsFuns", "Tables"] +git-tree-sha1 = "b12d37d25a2378f01abba02591cfd39a6cc4936f" +uuid = "3eaba693-59b7-5ba5-a881-562e759f1c8d" +version = "0.7.8" + +[[deps.StringManipulation]] +deps = ["PrecompileTools"] +git-tree-sha1 = "a3c1536470bf8c5e02096ad4853606d7c8f62721" +uuid = "892a3eda-7b42-436c-8928-eab12a02cf0e" +version = "0.4.2" + +[[deps.StructTypes]] +deps = ["Dates", "UUIDs"] +git-tree-sha1 = "159331b30e94d7b11379037feeb9b690950cace8" +uuid = "856f2bd8-1eba-4b0a-8007-ebc267875bd4" +version = "1.11.0" + +[[deps.StructUtils]] +deps = ["Dates", "UUIDs"] +git-tree-sha1 = "9297459be9e338e546f5c4bedb59b3b5674da7f1" +uuid = "ec057cc2-7a8d-4b58-b3b3-92acb9f63b42" +version = "2.6.2" + + [deps.StructUtils.extensions] + StructUtilsMeasurementsExt = ["Measurements"] + StructUtilsTablesExt = ["Tables"] + + [deps.StructUtils.weakdeps] + Measurements = "eff96d63-e80a-5855-80a2-b1b0885c5ab7" + Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" + +[[deps.StyledStrings]] +uuid = "f489334b-da3d-4c2e-b8f0-e476e12c162b" +version = "1.11.0" + +[[deps.SuiteSparse_jll]] +deps = ["Artifacts", "Libdl", "libblastrampoline_jll"] +uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c" +version = "7.8.3+2" + +[[deps.TOML]] +deps = ["Dates"] +uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +version = "1.0.3" + +[[deps.TableTraits]] +deps = ["IteratorInterfaceExtensions"] +git-tree-sha1 = "c06b2f539df1c6efa794486abfb6ed2022561a39" +uuid = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c" +version = "1.0.1" + +[[deps.Tables]] +deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "OrderedCollections", "TableTraits"] +git-tree-sha1 = "f2c1efbc8f3a609aadf318094f8fc5204bdaf344" +uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" +version = "1.12.1" + +[[deps.Tar]] +deps = ["ArgTools", "SHA"] +uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" +version = "1.10.0" + +[[deps.Test]] +deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +version = "1.11.0" + +[[deps.TestItemRunner]] +deps = ["Pkg", "TOML", "Test", "TestItems", "UUIDs"] +git-tree-sha1 = "76f275a3f3d83ece88ec69d73058048de5acb1dc" +uuid = "f8b46487-2199-4994-9208-9a1283c18c0a" +version = "1.1.4" + +[[deps.TestItems]] +git-tree-sha1 = "42fd9023fef18b9b78c8343a4e2f3813ffbcefcb" +uuid = "1c621080-faea-4a02-84b6-bbd5e436b8fe" +version = "1.0.0" + +[[deps.TranscodingStreams]] +git-tree-sha1 = "0c45878dcfdcfa8480052b6ab162cdd138781742" +uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" +version = "0.11.3" + +[[deps.URIParser]] +deps = ["Unicode"] +git-tree-sha1 = "53a9f49546b8d2dd2e688d216421d050c9a31d0d" +uuid = "30578b45-9adc-5946-b283-645ec420af67" +version = "0.4.1" + +[[deps.URIs]] +git-tree-sha1 = "bef26fb046d031353ef97a82e3fdb6afe7f21b1a" +uuid = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" +version = "1.6.1" + +[[deps.UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" +version = "1.11.0" + +[[deps.Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" +version = "1.11.0" + +[[deps.VersionParsing]] +git-tree-sha1 = "58d6e80b4ee071f5efd07fda82cb9fbe17200868" +uuid = "81def892-9a0e-5fdd-b105-ffc91e053289" +version = "1.3.0" + +[[deps.WeakRefStrings]] +deps = ["DataAPI", "InlineStrings", "Parsers"] +git-tree-sha1 = "b1be2855ed9ed8eac54e5caff2afcdb442d52c23" +uuid = "ea10d353-3f73-51f8-a26c-33c1cb351aa5" +version = "1.4.2" + +[[deps.WinReg]] +git-tree-sha1 = "cd910906b099402bcc50b3eafa9634244e5ec83b" +uuid = "1b915085-20d7-51cf-bf83-8f477d6f5128" +version = "1.0.0" + +[[deps.WorkerUtilities]] +git-tree-sha1 = "cd1659ba0d57b71a464a29e64dbc67cfe83d54e7" +uuid = "76eceee3-57b5-4d4a-8e66-0e911cebbf60" +version = "1.6.1" + +[[deps.Zlib_jll]] +deps = ["Libdl"] +uuid = "83775a58-1f1d-513f-b197-d71354ab007a" +version = "1.3.1+2" + +[[deps.libblastrampoline_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" +version = "5.15.0+0" + +[[deps.nghttp2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" +version = "1.64.0+1" + +[[deps.p7zip_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] +uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" +version = "17.7.0+0" diff --git a/Project.toml b/Project.toml index c1771d4..46f86f4 100644 --- a/Project.toml +++ b/Project.toml @@ -1,38 +1,29 @@ name = "JPEtools" uuid = "3207fb7f-a2d6-47fa-abcb-ee7fd07ee121" -authors = ["Florian Oswald and contributors"] version = "1.0.0-DEV" +authors = ["Florian Oswald and contributors"] [deps] CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" -DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" -Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6" -FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" Glob = "c27321d9-0574-5035-807b-f59d2c89b15c" HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" Infiltrator = "5903a43b-9cc3-4c30-8d17-598619ec4e9b" JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" -Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" -Mustache = "ffc61752-8dc7-55ee-8c37-f3e9cdd09e70" PDFIO = "4d0d745f-9d9a-592e-8d18-1ad8a0f42b92" -PackageAnalyzer = "e713c705-17e4-4cec-abe0-95bf5bf3e10c" +PIIScanner = "cf352d04-0051-48bd-b141-6a91af5c6085" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" -ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca" -RCall = "6f49c342-dc21-5d91-9882-a32aef131414" Revise = "295af30f-e4ad-537b-8983-00126c2a3abe" SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce" -Term = "22787eb5-b846-44ae-b979-8e399b8463ab" TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a" TestItems = "1c621080-faea-4a02-84b6-bbd5e436b8fe" [compat] CSV = "0.10.15" -Mustache = "1.0.21" PDFIO = "0.1.15" -ProgressMeter = "1.10.4" +PIIScanner = "1.0.0" [extras] Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/src/JPEtools.jl b/src/JPEtools.jl index 0a58f32..7eda91e 100644 --- a/src/JPEtools.jl +++ b/src/JPEtools.jl @@ -1,27 +1,19 @@ module JPEtools -using Markdown using Dates using SHA -# using Glob -# using PackageAnalyzer using HTTP using JSON3 -using DataStructures -using Dates using DataFrames -using RCall -using Downloads using Pkg using Printf using Glob using Infiltrator using TestItems -using Term.Prompts using PDFIO -using ProgressMeter using CSV +using PIIScanner # dv token diff --git a/src/actions.jl b/src/actions.jl index 360b9bc..afe5572 100644 --- a/src/actions.jl +++ b/src/actions.jl @@ -12,8 +12,6 @@ # git commit -m "${{ env.CI_COMMIT_MESSAGE }}" # git push -myfun() = abspath(@__DIR__,"..") - "recursively search a directory tree for match" function rdir(dir::AbstractString, pat::Glob.FilenameMatch) result = String[] @@ -48,9 +46,9 @@ function classify_files(pkg_path::String,kind::String,fp::String; relpath = fals # write to `generated` mkpath(fp) - sensitivenames = [] - nonsensitivenames = [] - files = [] + sensitivenames = String[] + nonsensitivenames = String[] + files = String[] if kind == "code" extensions = ["ado","do","r","rmd","qmd","ox","m","py","nb","ipynb","sas","jl","f","f90","c","c++","sh","toml","yaml","yml","fs","fsx","tex","typst"] @@ -142,10 +140,11 @@ end # PII checker strings -pii_strings_names() = join(["name", "fname", "lname", "first_name", "last_name"],"|") -pii_strings_dates() = join(["birth", "birthday", "bday", "dob"],"|") -pii_strings_locations() = join([ - "district", "city", "country", "subcountry", "parish", "loc", "street", "village", "community", "address", "gps", "degree", "minute", "second", "lat", "lon", "coord", "location", "house", "compound", "panchayat", "territory", "municipality", "precinct", "block", "zipcode", "zip"],"|") +word_boundary(x) = "\\b$x\\b" +pii_strings_names() = join(word_boundary.(["name", "fname", "lname", "firstname","lastname","first_name", "last_name"]),"|") +pii_strings_dates() = join(word_boundary.(["birth", "birthday", "bday", "dob"]),"|") +pii_strings_locations() = join(word_boundary.([ + "district", "city", "country", "subcountry", "parish", "loc", "street", "village", "community", "address", "gps", "degree", "minute", "second", "lat", "lon", "coord", "location", "house", "compound", "panchayat", "territory", "municipality", "precinct", "block", "zipcode", "zip"]),"|") pii_strings_other() = join([ "school", "social", "network", "census", "gender", "sex", "fax", "email", "url", "child", "beneficiary", "mother", "wife", "father", "husband", "phone", "spouse", "daughter", "son"], "|") @@ -158,10 +157,6 @@ function check_file_paths(filepath) has_windows = false has_unix = false has_drive = false - pii_names = String[] - pii_dates = String[] - pii_locations = String[] - pii_other = String[] try open(filepath, "r") do io @@ -184,18 +179,6 @@ function check_file_paths(filepath) if contains(line, hardcode_regex()) push!(hardcodes, @sprintf("Line %d, : %s",i,strip(line))) end - if contains(line, Regex("$(pii_strings_other())","i")) - push!(pii_other,@sprintf("Line %d, : %s",i,strip(line))) - end - if contains(line, Regex("$(pii_strings_names())","i")) - push!(pii_names,@sprintf("Line %d, : %s",i,strip(line))) - end - if contains(line, Regex("$(pii_strings_dates())","i")) - push!(pii_dates,@sprintf("Line %d, : %s",i,strip(line))) - end - if contains(line, Regex("$(pii_strings_locations())","i")) - push!(pii_locations,@sprintf("Line %d, : %s",i,strip(line))) - end end end catch e @@ -212,17 +195,7 @@ function check_file_paths(filepath) "none" end - # collate all piis - piis = [pii_names...,pii_dates...,pii_locations...,pii_other...] - - # pii = Dict( - # :names => pii_names, - # :dates => pii_dates, - # :locations => pii_locations, - # :other => pii_other - # ) - - return (lines, classification, has_drive, hardcodes,piis) + return (lines, classification, has_drive, hardcodes) end @@ -248,16 +221,13 @@ function file_paths(files::Array,fp::String) for file in files file = strip(file) - lines, classification, has_drive, hardcoded, pii = check_file_paths(file) + lines, classification, has_drive, hardcoded = check_file_paths(file) if !isempty(lines) results[file] = lines end if !isempty(hardcoded) hardcodes[file] = hardcoded end - if !isempty(pii) - piis[file] = pii - end if classification != "none" stats[classification] += 1 end @@ -333,22 +303,22 @@ function file_paths(files::Array,fp::String) end end - open(joinpath(fp,"report-pii.md"),"w") do io - println(io, "## Potential Personal Identifiable Information (PII)\n") - - if isempty(piis) - println(io, "✅ No PII found.") - else - println(io,"\nWe found the following instances of potentially personally identifying information. This may be completely legitimate but might be worth checking. *As a reminder, privacy legislation in many countries (e.g. GDPR in EU) prohibits the dissemination of personal identifiable information without prior (and documented) consent of individuals.* If indeed you want to publish such information with your replication package, you should probably have obtained IRB approval for this - please check!\n") - for (file, lines) in piis - println(io, "**$(path_splitter(file,splitat))**\n") - for l in lines - println(io, "- ", l) - end - println(io) - end - end - end + # open(joinpath(fp,"report-pii.md"),"w") do io + # println(io, "## Potential Personal Identifiable Information (PII)\n") + + # if isempty(piis) + # println(io, "✅ No PII found.") + # else + # println(io,"\nWe found the following instances of potentially personally identifying information. This may be completely legitimate but might be worth checking. *As a reminder, privacy legislation in many countries (e.g. GDPR in EU) prohibits the dissemination of personal identifiable information without prior (and documented) consent of individuals.* If indeed you want to publish such information with your replication package, you should probably have obtained IRB approval for this - please check!\n") + # for (file, lines) in piis + # println(io, "**$(path_splitter(file,splitat))**\n") + # for l in lines + # println(io, "- ", l) + # end + # println(io) + # end + # end + # end nothing end @@ -570,15 +540,6 @@ function make_test_paths() x end -function delete_package() - pkg = joinpath(root(),"replication-package") - - a = ask(DefaultPrompt(["y", "no"], 1, "This will delete the replication package. Sure?")) - if a == "y" - rm(pkg,recursive = true) - end -end - function delete_generated() pkg = joinpath(root(),"generated") rm(pkg,recursive = true) @@ -737,6 +698,11 @@ function precheck_package(pkg_loc::String) @info "Parse code files and search for filepaths" file_paths(codefiles,out) + @info "look for PII" + data_matches = PIIScanner.scan_data_files(datafiles) + code_matches = PIIScanner.scan_code_files(codefiles) + PIIScanner.write_pii_report(data_matches,code_matches,out,splitat = dirname(pkg_loc)) + # parse README @info "Read the README file" read_README( pkg_loc, out ) diff --git a/src/prechecks.jl b/src/prechecks.jl deleted file mode 100644 index 30041ee..0000000 --- a/src/prechecks.jl +++ /dev/null @@ -1,110 +0,0 @@ - -# Step 1: Get email with paper, appendix, DOI, and slugs -function get_email_details() - println("Fetching email details...") - # Implement logic to fetch email details - return Dict("paper" => "example_paper", "doi" => "10.1234/example") -end - - - -# Step 2: Download package under DOI -function download_package(doi::String, token::String) - if is_dv - - else - - end - println("Downloading package for DOI: $doi with token: $token") - # Implement logic to download package using the token - return "/path/to/downloaded/package.zip" -end - - -# Step 5: Unzip if necessary -function unzip_package(package_path::String, output_dir::String) - println("Unzipping package...") - # Implement logic to unzip the package -end - -# Step 6: Create folder `generated` -function create_generated_folder(base_path::String) - generated_path = joinpath(base_path, "generated") - if !isdir(generated_path) - mkdir(generated_path) - end - println("Created folder: $generated_path") - return generated_path -end - -# Step 7: File sizes and MD5 checksums - - -# Step 8: Analyze code (cloc, libraries, etc.) -function analyze_code(folder_path::String) - println("Analyzing code...") - # Example: Count lines of code using cloc - cloc_result = run(`cloc --md --out=./generated/cloc.md --sum-one --quiet --ignored=./generated/cloc-ignored.md $(folder_path)`) - # Implement additional analysis for R, Stata, Python libraries, etc. -end - -# Step 9: Write to markdown table -function write_to_markdown_table(data::Any, output_file::String) - println("Writing data to markdown table...") - # Implement logic to write data to markdown -end - -# Step 10: Generate template report -function generate_template_report(output_path::String) - println("Generating template report...") - # Implement logic to generate a report -end - -# Step 11: Commit everything that is not data -function commit_changes(repo_path::String) - println("Committing changes...") - # Implement logic to commit changes -end - -# Step 12: Push to GitHub -function push_to_github(repo_path::String) - println("Pushing to GitHub...") - # Implement logic to push to GitHub -end - -# Step 13: Notify replicator -function notify_replicator() - println("Notifying replicator...") - # Implement logic to notify replicator -end - -# Step 14: Log in local database that package has been dispatched -function log_dispatch(details::Dict) - println("Logging dispatch in database...") - # Implement logic to log dispatch -end - -# Main workflow -function main() - token = ENV["JPE_DV"] # Read the API token from the environment variable - details = get_email_details() - package_path = download_package(details["doi"], token) - log_database("pre-processing", details) - create_repo("/path/to/template", "new_repo") - unzip_package(package_path, "/path/to/output") - generated_folder = create_generated_folder("/path/to/output") - generate_file_sizes_md5(generated_folder, joinpath(generated_folder, "file_sizes.md")) - analyze_code("/path/to/output") - generate_template_report("/path/to/report") - write_gitignore(package_path) - commit_changes("/path/to/repo") - push_to_github("/path/to/repo") - notify_replicator() - log_dispatch(details) -end - -# Run the workflow -# main() - - - diff --git a/test/deps.jl b/test/deps.jl index 0e4ba2c..c8e323d 100644 --- a/test/deps.jl +++ b/test/deps.jl @@ -8,3 +8,48 @@ function make_file_list() push!(x,raw"D:\\Your\\dummy\\path/with/mixed/separators") x end + +function make_test_package() + pkg_root = mktempdir() + pkg_loc = joinpath(pkg_root,"replication-package") + pkg_data = joinpath(pkg_loc,"data") + pkg_code = joinpath(pkg_loc,"code") + mkpath(pkg_data) + mkpath(pkg_code) + + # make a readme + open(joinpath(pkg_loc,"README.md"), "w") do io + println(io, + """ + # [TESTING] Replication Package + + This readme is for my test replication package. + + There is not much to see here. + + Good night, and good luck. + + """ + ) + end + + # create a dataset + N = 1000 + d = DataFrame(i = 1:N, y = rand(N), telephone = rand(N), lat = rand(N), income = rand(N)) + CSV.write(joinpath(pkg_data,"data.csv"),d) + + # some code + test_file = joinpath(pkg_code, "test_script.R") + open(test_file, "w") do io + println(io, "# Analysis script") + println(io, "data <- read.csv('file.csv')") + println(io, "df\$first_name <- clean_text(df\$name)") + println(io, "model <- lm(y ~ age + email + income)") + println(io, "summary(model)") + end + + return pkg_loc + +end + + diff --git a/test/runtests.jl b/test/runtests.jl index 2abc023..17b3512 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,7 +1,8 @@ using JPEtools using TestItemRunner +include(joinpath(@__DIR__,"deps.jl")) -@run_package_tests filter=ti->!(:skipci in ti.tags) +@run_package_tests diff --git a/test/test_paths.jl b/test/test_paths.jl index aa5ef69..f61a9dd 100644 --- a/test/test_paths.jl +++ b/test/test_paths.jl @@ -34,11 +34,6 @@ end end -@testitem "test test" begin - @test true -end - - @testitem "test detect_paths_kind()" begin tl = JPEtools.make_test_paths() results = [(false,true,false),(false,true,false),(false,true,false),(true,false,true),(true,false,false),(true,true,true)] @@ -48,11 +43,23 @@ end end end +@testitem "precheck dummy package" begin + using DataFrames, CSV + include(joinpath(@__DIR__,"deps.jl")) + + loc = make_test_package() + + JPEtools.precheck_package(loc) + + @test isdir(joinpath(loc,"..","generated")) + @test isfile(joinpath(loc,"..","generated","data-files.md")) + @test isfile(joinpath(loc,"..","generated","report-pii.md")) + + pii = readlines(joinpath(loc,"..","generated","report-pii.md")) + # get the 16th line of this report + pii_data = strip.(split(pii[16], "|")) + @test pii_data[3] == "`test_script.R`" + @test pii_data[5] == "first_name, name, email" + +end -@testitem "test check_file_paths()" tags=[:skipci] begin - JPEtools.PKG_ROOT = JPEtools.test_package_path("ECTA") - tl = JPEtools.classify_files(JPEtools.PKG_ROOT,"code") - @test JPEtools.check_file_paths(tl[2])[3] - @test JPEtools.check_file_paths(tl[2])[2] == "windows" - @test !JPEtools.check_file_paths(tl[1])[3] -end \ No newline at end of file