diff --git a/.bandit.yml b/.bandit.yml deleted file mode 100644 index ab3cb21..0000000 --- a/.bandit.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- -# Configuration file for the Bandit python security scanner -# https://bandit.readthedocs.io/en/latest/config.html - -# Tests are first included by `tests`, and then excluded by `skips`. -# If `tests` is empty, all tests are considered included. - -tests: -# - B101 -# - B102 - -skips: -# - B101 # skip "assert used" check since assertions are required in pytests diff --git a/.flake8 b/.flake8 index 92ff826..e9271ff 100644 --- a/.flake8 +++ b/.flake8 @@ -1,25 +1,40 @@ [flake8] max-line-length = 80 + # Select (turn on) -# * Complexity violations reported by mccabe (C) - -# http://flake8.pycqa.org/en/latest/user/error-codes.html#error-violation-codes -# * Documentation conventions compliance reported by pydocstyle (D) - -# http://www.pydocstyle.org/en/stable/error_codes.html -# * Default errors and warnings reported by pycodestyle (E and W) - +# * C: Complexity violations reported by mccabe - +# https://flake8.pycqa.org/en/latest/user/error-codes.html#error-violation-codes +# * C4: Default errors and warnings reported by flake8-comprehensions - +# https://github.com/adamchainz/flake8-comprehensions#rules +# * D: Documentation conventions compliance reported by pydocstyle - +# https://github.com/PyCQA/pydocstyle/blob/master/docs/error_codes.rst +# * DUO: Default errors and warnings reported by dlint - +# https://github.com/dlint-py/dlint/tree/master/docs +# * E: Default errors reported by pycodestyle - # https://pycodestyle.readthedocs.io/en/latest/intro.html#error-codes -# * Default errors reported by pyflakes (F) - -# http://flake8.pycqa.org/en/latest/glossary.html#term-pyflakes -# * Default warnings reported by flake8-bugbear (B) - +# * F: Default errors reported by pyflakes - +# https://flake8.pycqa.org/en/latest/glossary.html#term-pyflakes +# * N: Default errors and warnings reported by pep8-naming - +# https://github.com/PyCQA/pep8-naming#error-codes +# * NQA: Default errors and warnings reported by flake8-noqa - +# https://github.com/plinss/flake8-noqa#error-codes +# * W: Default warnings reported by pycodestyle - +# https://pycodestyle.readthedocs.io/en/latest/intro.html#error-codes +# * B: Default warnings reported by flake8-bugbear - # https://github.com/PyCQA/flake8-bugbear#list-of-warnings -# * The B950 flake8-bugbear opinionated warning - +# * B950: Bugbear opinionated warning for line too long - # https://github.com/PyCQA/flake8-bugbear#opinionated-warnings -select = C,D,E,F,W,B,B950 -# Ignore flake8's default warning about maximum line length, which has -# a hard stop at the configured value. Instead we use -# flake8-bugbear's B950, which allows up to 10% overage. -# -# Also ignore flake8's warning about line breaks before binary -# operators. It no longer agrees with PEP8. See, for example, here: -# https://github.com/ambv/black/issues/21. Guido agrees here: -# https://github.com/python/peps/commit/c59c4376ad233a62ca4b3a6060c81368bd21e85b. -ignore = E501,W503 +select = C,C4,D,DUO,E,F,N,NQA,W,B,B950 + +# Ignore +# * E203: pycodestyle's default warning about whitespace before ':' because Black enforces +# an equal amount of whitespace around slice operators (':'). +# * E501: pycodestyle's default warning about maximum line length, which has a hard stop +# at the configured value. Instead we use flake8-bugbear's B950, which +# allows up to 10% overage. +# * W503: pycodestyle's warning about line breaks before binary operators. It no longer +# agrees with PEP8. See, for example, here: +# https://github.com/ambv/black/issues/21 +# Guido agrees here: +# https://github.com/python/peps/commit/c59c4376ad233a62ca4b3a6060c81368bd21e85b +ignore = E203,E501,W503 diff --git a/.github/labeler.yml b/.github/labeler.yml index 05478bd..b720437 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -61,7 +61,6 @@ test: - any-glob-to-any-file: # Add any test-related files or paths. - .ansible-lint - - .bandit.yml - .flake8 - .isort.cfg - .mdl_config.yaml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fddf200..b28c58b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -149,12 +149,12 @@ jobs: - uses: hashicorp/setup-packer@v3 with: version: ${{ steps.setup-env.outputs.packer-version }} - - uses: hashicorp/setup-terraform@v3 + - uses: hashicorp/setup-terraform@v4 with: terraform_version: ${{ steps.setup-env.outputs.terraform-version }} - name: Install go-critic env: - PACKAGE_URL: github.com/go-critic/go-critic/cmd/gocritic + PACKAGE_URL: github.com/go-critic/go-critic/cmd/go-critic PACKAGE_VERSION: ${{ steps.setup-env.outputs.go-critic-version }} run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} - name: Install goimports diff --git a/.github/workflows/sync-labels.yml b/.github/workflows/sync-labels.yml index f60bc84..a8d01be 100644 --- a/.github/workflows/sync-labels.yml +++ b/.github/workflows/sync-labels.yml @@ -87,7 +87,7 @@ jobs: - uses: actions/checkout@v6 - name: Sync repository labels if: success() - uses: crazy-max/ghaction-github-labeler@v5 + uses: crazy-max/ghaction-github-labeler@v6 with: # This is a hideous ternary equivalent so we only do a dry run unless # this workflow is triggered by the develop branch. diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f62b0ce..cc92622 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -47,7 +47,7 @@ repos: # Text file hooks - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.47.0 + rev: v0.48.0 hooks: - id: markdownlint args: @@ -65,7 +65,7 @@ repos: # GitHub Actions hooks - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.36.2 + rev: 0.37.0 hooks: - id: check-github-actions - id: check-github-workflows @@ -107,7 +107,7 @@ repos: # Shell script hooks - repo: https://github.com/scop/pre-commit-shfmt - rev: v3.12.0-2 + rev: v3.13.0-1 hooks: - id: shfmt args: @@ -131,13 +131,11 @@ repos: # Python hooks - repo: https://github.com/PyCQA/bandit - rev: 1.9.3 + rev: 1.9.4 hooks: - id: bandit - args: - - --config=.bandit.yml - repo: https://github.com/psf/black-pre-commit-mirror - rev: 26.1.0 + rev: 26.3.1 hooks: - id: black - repo: https://github.com/PyCQA/flake8 @@ -145,9 +143,14 @@ repos: hooks: - id: flake8 additional_dependencies: + - dlint==0.16.0 + - flake8-bugbear==25.11.29 + - flake8-comprehensions==3.17.0 - flake8-docstrings==1.7.0 + - flake8-noqa==1.5.0 + - pep8-naming==0.15.1 - repo: https://github.com/PyCQA/isort - rev: 8.0.0 + rev: 8.0.1 hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy @@ -162,6 +165,22 @@ repos: hooks: - id: pip-audit args: + # We have to ignore this vulnerability for now since an + # update for pygments has not yet been released. + # + # In any event, this vulnerability is unlikely to cause us + # any problems since we don't feed any regexes to pygments + # directly. pygments is pulled in as a dependency of + # pytest. + # + # See also: + # - https://nvd.nist.gov/vuln/detail/CVE-2026-4539 + # - https://github.com/pygments/pygments/issues/3058 + # + # TODO: Remove this when it becomes possible. See + # cisagov/skeleton-generic#257 for more details. + - --ignore-vuln + - CVE-2026-4539 # Add any pip requirements files to scan - --requirement - requirements-dev.txt @@ -182,6 +201,9 @@ repos: # Ansible hooks - repo: https://github.com/ansible/ansible-lint + # We need to stay on this version because we are still using Python 3.13 in + # our GitHub Actions configuration. Later versions require Python 3.14 for + # the hook to run. rev: v26.1.1 hooks: - id: ansible-lint @@ -212,6 +234,15 @@ repos: hooks: - id: terraform_fmt - id: terraform_validate + # This needs to run after the terraform_validate hook so that any Terraform + # configurations are initialized. + - id: terraform_providers_lock + args: + - --args=-platform=darwin_amd64 + - --args=-platform=darwin_arm64 + - --args=-platform=linux_amd64 + - --args=-platform=linux_arm64 + - --hook-config=--mode=always-regenerate-lockfile # Docker hooks - repo: https://github.com/IamTheFij/docker-pre-commit diff --git a/project_setup/scripts/iam-to-travis b/project_setup/scripts/iam-to-travis deleted file mode 100755 index e1233a3..0000000 --- a/project_setup/scripts/iam-to-travis +++ /dev/null @@ -1,131 +0,0 @@ -#!/usr/bin/env python - -"""Extract AWS credentials from terraform state, encrypt, and format for Travis. - -This command must be executed in the directory containing the .terraform state -within the a GitHub project. - -Usage: - iam-to-travis [--log-level=LEVEL] [--indent=SPACES] [--width=WIDTH] - iam-to-travis (-h | --help) - -Options: - -h --help Show this message. - -i --indent=SPACES Number of spaces to indent yaml block. Minimum 2. - [default: 6] - --log-level=LEVEL If specified, then the log level will be set to - the specified value. Valid values are "debug", "info", - "warning", "error", and "critical". [default: warning] - -w --width=WIDTH Maximum width of yaml block. Minimum 16. [default: 80] -""" - -# Standard Python Libraries -import json -import logging -import subprocess # nosec -import sys - -# Third-Party Libraries -import docopt - - -def creds_from_child(child_module): - """Search for IAM access keys in child resources. - - Returns (key_id, secret) if found, (None, None) otherwise. - """ - for resource in child_module["resources"]: - if resource["address"] == "aws_iam_access_key.key": - key_id = resource["values"]["id"] - secret = resource["values"]["secret"] - return key_id, secret - return None, None - - -def creds_from_terraform(): - """Retrieve IAM credentials from terraform state. - - Returns (key_id, secret) if found, (None, None) otherwise. - """ - c = subprocess.run( # nosec - "terraform show --json", shell=True, stdout=subprocess.PIPE # nosec - ) - j = json.loads(c.stdout) - - if not j.get("values"): - return None, None - - for child_module in j["values"]["root_module"]["child_modules"]: - key_id, secret = creds_from_child(child_module) - if key_id: - return key_id, secret - else: - return None, None - - -def wrap_for_yml(s, indent=6, width=75): - """Wrap a string in yamly way.""" - result = [] - width = width - 1 - while True: - result.append(s[:width]) - s = s[width:] - if not s: - break - s = " " * indent + s - return "\\\n".join(result) - - -def encrypt_for_travis(variable_name, value, indent, width): - """Encrypt a value for a variable and print it as yaml.""" - logging.debug(f"Encrypting {variable_name}.") - command = f'travis encrypt --com --no-interactive "{variable_name}={value}"' - c = subprocess.run(command, shell=True, stdout=subprocess.PIPE) # nosec - s = f"{' ' * (indent - 2)}- secure: {c.stdout.decode('utf-8')}" - print(f"{' ' * (indent - 2)}# {variable_name}") - print(wrap_for_yml(s, indent, width)) - - -def main(): - """Set up logging and call the requested commands.""" - args = docopt.docopt(__doc__, version="0.0.1") - - # Set up logging - log_level = args["--log-level"] - try: - logging.basicConfig( - format="%(asctime)-15s %(levelname)s %(message)s", level=log_level.upper() - ) - except ValueError: - logging.critical( - f'"{log_level}" is not a valid logging level. Possible values ' - "are debug, info, warning, and error." - ) - return 1 - - indent = int(args["--indent"]) - width = int(args["--width"]) - - if width < 16: - logging.error("Width must be 16 or greater.") - sys.exit(-1) - - if indent < 2 or indent > width - 10: - logging.error("Indent must be greater than 2, and less than (width - 10).") - sys.exit(-1) - - logging.info("Searching Terraform state for IAM credentials.") - key_id, secret = creds_from_terraform() - if key_id is None: - logging.error("Credentials not found in terraform state.") - logging.error("Is there a .terraform state directory here?") - sys.exit(-1) - - encrypt_for_travis("AWS_ACCESS_KEY_ID", key_id, indent, width) - encrypt_for_travis("AWS_SECRET_ACCESS_KEY", secret, indent, width) - - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/project_setup/scripts/skeleton b/project_setup/scripts/skeleton deleted file mode 100755 index 85700f4..0000000 --- a/project_setup/scripts/skeleton +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env python - -"""Helper tool to start a new github project from a skeleton github repository. - -Usage: - skeleton (-h | --help) - skeleton list [--org=] - skeleton clone [options] - -Options: - -c --change-dir= Create clone in this directory. - -h --help Show this message. - -o --org= Organization to search [default: cisagov]. -""" - -# Standard Python Libraries -import os -from pathlib import Path -import subprocess # nosec -import sys - -# Third-Party Libraries -import docopt -from github import Github -import yaml - -LINEAGE_CONFIG = Path(".github/lineage.yml") -LINEAGE_CONFIG_VERSION = "1" -VERSION = "0.0.1" - - -def run(cmd, comment): - """Run a command and display its output and return code.""" - print("―" * 80) - if comment: - print(f"💬 {comment}") - print(f"➤ {cmd}") - proc = subprocess.run(cmd, shell=True) # nosec - if proc.returncode == 0: - print("✅ success") - else: - print(f"❌ ERROR! return code: {proc.returncode}") - sys.exit(proc.returncode) - - -def print_available_skeletons(org): - """Print a list of skeleton repos available for cloning.""" - g = Github() - skel_repos = g.search_repositories(query=f"org:{org} topic:skeleton archived:false") - print(f"Available skeletons in {org}:\n") - for repo in skel_repos: - print(f"{repo.name}\n\t{repo.description}\n") - - -def clone_repo(parent_repo, new_repo, org, dir=None): - """Clone a repository to a new name and prepare it for publication.""" - if dir: - os.chdir(dir) - run( - f"git clone --origin {parent_repo} git@github.com:{org}/{parent_repo}.git {new_repo}", - "Clone an existing remote repository to the new name locally.", - ) - os.chdir(new_repo) - run( - f"git remote set-url --push {parent_repo} no_push", - "Disable pushing to the upstream (parent) repository.", - ) - run( - f"git remote add origin git@github.com:{org}/{new_repo}.git", - "Add a new remote origin for the this repository.", - ) - run("git tag -d $(git tag -l)", f"Delete all local git tags from {parent_repo}") - run( - rf"find . \( ! -regex '.*/\.git/.*' \) -type f -exec " - rf"perl -pi -e s/{parent_repo}/{new_repo}/g {{}} \;", - "Search and replace repository name in source files.", - ) - lineage = { - "version": LINEAGE_CONFIG_VERSION, - "lineage": { - "skeleton": {"remote-url": f"https://github.com/{org}/{parent_repo}.git"} - }, - } - with LINEAGE_CONFIG.open("w") as f: - yaml.dump(lineage, stream=f, explicit_start=True) - run("git add --verbose .", "Stage modified files.") - run( - 'git commit --message "Rename repository references after clone."', - "Commit staged files to the new repository.", - ) - print("―" * 80) - print(f""" -The repository "{parent_repo}" has been cloned and renamed to "{new_repo}". -Use the following commands to push the new repository to github: - cd {os.path.join(dir, new_repo) if dir else new_repo} - git push --set-upstream origin develop - """) - - -def main(): - """Parse arguments and perform requested actions.""" - args = docopt.docopt(__doc__, version=VERSION) - - org = args["--org"] - - if args["list"]: - print_available_skeletons(org) - elif args["clone"]: - parent_repo = args[""] - new_repo = args[""] - dir = args["--change-dir"] - clone_repo(parent_repo, new_repo, org, dir) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/project_setup/scripts/terraform-to-secrets b/project_setup/scripts/terraform-to-secrets index 0d555f3..12695b5 100755 --- a/project_setup/scripts/terraform-to-secrets +++ b/project_setup/scripts/terraform-to-secrets @@ -109,7 +109,7 @@ def find_tagged_secret( # its value is not None. Both of these cases can occur. tags: dict[str, str] if "tags" not in resource_data or resource_data.get("tags") is None: - tags = dict() + tags = {} else: tags = resource_data["tags"] @@ -150,7 +150,7 @@ def find_outputs( and resource.get("type") == "terraform_remote_state" ): continue - if resource.get("values", dict()).get("outputs", dict()): + if resource.get("values", {}).get("outputs", {}): yield resource["values"]["outputs"] @@ -269,7 +269,10 @@ def set_secret( """Create a secret in a repository or environment.""" if github_env: logging.info(f"Creating secret {secret_name} in environment {github_env}") - api_url = f"https://api.github.com/repos/{repo_name}/environments/{github_env}/secrets/{secret_name}" + api_url = ( + f"https://api.github.com/repos/{repo_name}/environments/" + f"{github_env}/secrets/{secret_name}" + ) else: logging.info(f"Creating repository secret {secret_name}") api_url = ( @@ -314,7 +317,7 @@ def get_users(terraform_state: dict) -> dict[str, tuple[str, str]]: aws_user: str | None = None aws_key_id: str | None = None aws_secret: str | None = None - user_creds: dict[str, tuple[str, str]] = dict() + user_creds: dict[str, tuple[str, str]] = {} logging.info("Searching Terraform state for IAM credentials.") for aws_user, aws_key_id, aws_secret in parse_creds(terraform_state): @@ -330,7 +333,7 @@ def get_resource_secrets( terraform_state: dict, include_remote_state: bool ) -> dict[str, str]: """Collect secrets from tagged Terraform resources.""" - secrets: dict[str, str] = dict() + secrets: dict[str, str] = {} logging.info("Searching Terraform state for tagged resources.") for secret_name, secret_value in parse_tagged_resources( terraform_state, include_remote_state @@ -347,7 +350,7 @@ def get_resource_secrets( def create_user_secrets(user_creds: dict[str, tuple[str, str]]) -> dict[str, str]: """Create secrets for user key IDs and key values.""" - secrets: dict[str, str] = dict() + secrets: dict[str, str] = {} for user_name, creds in user_creds.items(): # If there is more than one user add the name as a suffix if len(user_creds) > 1: diff --git a/setup.py b/setup.py index bb6342d..6c02b9b 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ def get_version(version_file): setup( name="project_setup", # Versions should comply with PEP440 - version="1.0.0", + version="1.1.0", description="Documentation for Github projects in the cisagov organization.", long_description=readme(), long_description_content_type="text/markdown", @@ -107,9 +107,7 @@ def get_version(version_file): }, scripts=[ "project_setup/scripts/ansible-roles", - "project_setup/scripts/iam-to-travis", "project_setup/scripts/terraform-to-secrets", - "project_setup/scripts/skeleton", "project_setup/scripts/ssm-param", ], entry_points={}, diff --git a/version.txt b/version.txt index 3eefcb9..9084fa2 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.0.0 +1.1.0