From 3144baaf1747a25626d4e97078bf7443c0730f2a Mon Sep 17 00:00:00 2001 From: NilashishC Date: Thu, 17 Apr 2025 20:32:48 +0530 Subject: [PATCH 1/3] Add GHA workflows Signed-off-by: NilashishC --- .githooks/README.md | 29 ++++++++ .githooks/pre-commit | 50 +++++++++++++ .githooks/pre-push | 46 ++++++++++++ .github/CODE_OF_CONDUCT.md | 3 + .github/PULL_REQUEST_TEMPLATE.md | 60 ++++++++++++++++ .github/workflows/collection.yml | 99 +++++++++++++++++++++++++ .github/workflows/linting.yml | 41 +++++++++++ .gitignore | 115 ++++++++++++++++++++++++++++++ Makefile | 64 +++++++++++++++++ requirements/requirements_dev.txt | 20 ++++++ tools/ansible/ansible.cfg | 2 + tox.ini | 30 ++++++++ 12 files changed, 559 insertions(+) create mode 100644 .githooks/README.md create mode 100755 .githooks/pre-commit create mode 100755 .githooks/pre-push create mode 100644 .github/CODE_OF_CONDUCT.md create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 .github/workflows/collection.yml create mode 100644 .github/workflows/linting.yml create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 requirements/requirements_dev.txt create mode 100644 tools/ansible/ansible.cfg create mode 100644 tox.ini diff --git a/.githooks/README.md b/.githooks/README.md new file mode 100644 index 00000000..d38befd7 --- /dev/null +++ b/.githooks/README.md @@ -0,0 +1,29 @@ +# .githooks + +This folder contains executable files that will be invoked by git at certain git operations. + +By default git hooks are located i `.git/hooks` folder at the root of the repository. Since the default folder is hidden by most IDEs, this repository reconfigures git's hook location in order to make the hooks visible and easier to maintain. + +## Configuration + +Normal development flows (see file [../README.md](../README.md) ) will call git to change the location of git hooks. + +To make this change manually you can invoke following command from the root of the repository: + +```sh +make git_hooks_config +``` + +## Git hooks implementation + +Git hooks are simply executable files that follow the rules below: + +* have executable permissions +* file name must correspond to git hook name with no extension(no `.sh` or `.py`) (see documentation section below) + +Return code other than zero(0) will cause the git operation that triggerred the hook to fail, while zero(0) return code indicates success and git opertaion will succeed. + +## Documentation + +Git hooks documentation: + diff --git a/.githooks/pre-commit b/.githooks/pre-commit new file mode 100755 index 00000000..4f234402 --- /dev/null +++ b/.githooks/pre-commit @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +if [ -z $GW_IGNORE_SYNTAX ] ; then + python_files_changed=$(git diff --cached --name-only --diff-filter=AM | grep -E '\.py$' | xargs) + echo $python_files_changed + if [ "x$python_files_changed" != "x" ] ; then + FAILED=0 + for target in check_black check_flake8 ; do + CHECK_SYNTAX_FILES="${python_files_changed}" make ${target} + if [ $? != 0 ] ; then + FAILED=1 + fi + echo "" + done + # We can't run isort on just a file name because it works differently + make check_isort + if [ $? != 0 ] ; then + FAILED=1 + fi + if [ $FAILED == 1 ] ; then + exit 1 + fi + fi +fi + +if [ -z $GW_IGNORE_USER ] ; then + FAILED=0 + export CHANGED_FILES=$(git diff --cached --name-only --diff-filter=AM) + echo "Running user pre commit for ${CHANGED_FILES}" + if [ -d ./pre-commit-user ] ; then + for SCRIPT in `find ./pre-commit-user -type f` ; do + if [ -x $SCRIPT ] ; then + echo "Running user pre-commit hook $SCRIPT" + $SCRIPT + if [ $? != 0 ] ; then + echo "User test $SCRIPT failed" + FAILED=1 + fi + else + echo "FIle ${SCRIPT} is not executable" + fi + done + fi + if [ $FAILED == 1 ] ; then + echo "One or more user tests failed, see messages above" + exit 1 + fi +else + echo "Ignoring user commit scripts due to GW_IGNORE_ERROR" +fi diff --git a/.githooks/pre-push b/.githooks/pre-push new file mode 100755 index 00000000..e4e86351 --- /dev/null +++ b/.githooks/pre-push @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +# Name of default branch from which feature branches are created and to which PRs will be merged back to +DEFAULT_BRANCH="devel" +# Regexp to match jira number AAP-NNNNN or magic string "NO_JIRA" +NO_JIRA_MARKER="NO_JIRA" +JIRA_REGEXP="(aap-[0-9]+|${NO_JIRA_MARKER})" + +# Fetch current branch name and list of commits since diverging from default branch +CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD) +CURRENT_COMMITS=$(git --no-pager log --format=%s --reverse ${DEFAULT_BRANCH}..) + +# Extract Jira number or magic marker from branch and commit messages(filtered for unique values) +BRANCH_JIRA=$(grep -i -o -E "${JIRA_REGEXP}" <<< ${CURRENT_BRANCH}) +COMMIT_JIRAS=$(grep -i -o -E "${JIRA_REGEXP}" <<< ${CURRENT_COMMITS} | uniq ) +# Count all Jira numbers and those matching Jira from branch name +COMMIT_JIRA_COUNT=$(grep -c . <<< ${COMMIT_JIRAS}) +MATCHING_JIRAS_COUNT=$(grep -ic -E "${BRANCH_JIRA}" <<< ${COMMIT_JIRAS}) + +echo "JIRA number from branch name: ${BRANCH_JIRA}" +echo "JIRA numbers from commits:" +echo "${COMMIT_JIRAS}" +echo "Number of JIRA numbers from commits matching JIRA number from branch name: ${MATCHING_JIRAS_COUNT}" + +# if no Jira or no magic marker found in branch name, fail +echo "Checking branch name..." +if [ "${BRANCH_JIRA}" = "" ]; then + echo "Fail: Branch name does not contain a JIRA number or a ${NO_JIRA_MARKER} marker." + exit 1 +# if branch does not have the magic marker, check the commits as well +elif [ "${BRANCH_JIRA}" != "${NO_JIRA_MARKER}" ]; then + echo "Checking commit messages..." + # if there is no Jira number or magic marker, fail + if [ ${COMMIT_JIRA_COUNT} -eq 0 ]; then + echo "Fail: No commit message contains a JIRA number or a ${NO_JIRA_MARKER} marker." + exit 1 + # if no Jira numbers or magic marker match the Jira number from branch name, inform the user + # this case might be happening when code is being back-ported under different Jira number in branch name + elif [ ${MATCHING_JIRAS_COUNT} -eq 0 ]; then + echo "Warning: No Jira numbers or ${NO_JIRA_MARKER} marker in commit messages match Jira number from branch name." + else + echo "OK. Found Jira numbers(or ${NO_JIRA_MARKER} marker) in commit messages that match Jira number in branch name." + fi +else + echo "OK. Skipping checks of commit messages, branch name includes ${NO_JIRA_MARKER}." +fi diff --git a/.github/CODE_OF_CONDUCT.md b/.github/CODE_OF_CONDUCT.md new file mode 100644 index 00000000..0164155b --- /dev/null +++ b/.github/CODE_OF_CONDUCT.md @@ -0,0 +1,3 @@ +# Community Code of Conduct + +Please see the official [Ansible Community Code of Conduct](https://docs.ansible.com/ansible/latest/community/code_of_conduct.html). diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000..2dcdac85 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,60 @@ +## Description + +- What is being changed? +- Why is this change needed? +- How does this change address the issue? + +## Type of Change + +- [ ] Bug fix (non-breaking change which fixes an issue) +- [ ] New feature (non-breaking change which adds functionality) +- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) +- [ ] Documentation update +- [ ] Test update +- [ ] Refactoring (no functional changes) +- [ ] Development environment change +- [ ] Configuration change + +## Self-Review Checklist + +- [ ] I have performed a self-review of my code +- [ ] I have added relevant comments to complex code sections +- [ ] I have updated documentation where needed +- [ ] I have considered the security impact of these changes +- [ ] I have considered performance implications +- [ ] I have thought about error handling and edge cases +- [ ] I have tested the changes in my local environment + +## Testing Instructions + + +### Prerequisites + + +### Steps to Test +1. +2. +3. + +### Expected Results + + +## Additional Context + + +### Required Actions + + +- [ ] Requires documentation updates + +- [ ] Requires downstream repository changes + +- [ ] Requires infrastructure/deployment changes + +- [ ] Requires coordination with other teams + +- [ ] Blocked by PR/MR: #XXX + + +### Screenshots/Logs + diff --git a/.github/workflows/collection.yml b/.github/workflows/collection.yml new file mode 100644 index 00000000..13fb007d --- /dev/null +++ b/.github/workflows/collection.yml @@ -0,0 +1,99 @@ +--- +name: platform collection tests +on: + push: + pull_request: + +jobs: + sanity: + name: platform collection sanity + runs-on: ubuntu-latest + env: + HEADLESS: "yes" + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + ansible: + - stable-2.16 + - stable-2.17 + - stable-2.18 + - stable-2.19 + steps: + - uses: actions/checkout@v3 + + - name: Perform sanity testing + uses: ansible-community/ansible-test-gh-action@release/v1 + with: + ansible-core-version: ${{ matrix.ansible }} + collection-root: . + testing-type: sanity + target-python-version: 3.11 + + # TO-DO + # - name: Upload gateway jUnit test reports to the unified dashboard + # if: >- + # !cancelled() + # && steps.make-run.outputs.test-result-files != '' + # && github.event_name == 'push' + # && env.UPSTREAM_REPOSITORY_ID == github.repository_id + # && github.ref_name == github.event.repository.default_branch + # run: | + # for junit_file in $(echo '${{ steps.make-run.outputs.test-result-files }}' | sed 's/,/ /') + # do + # curl \ + # -v \ + # --user "${{ vars.PDE_ORG_RESULTS_AGGREGATOR_UPLOAD_USER }}:${{ secrets.PDE_ORG_RESULTS_UPLOAD_PASSWORD }}" \ + # --form "xunit_xml=@${junit_file}" \ + # --form "component_name=gateway" \ + # --form "git_commit_sha=${{ github.sha }}" \ + # --form "git_repository_url=https://github.com/${{ github.repository }}" \ + # "${{ vars.PDE_ORG_RESULTS_AGGREGATOR_UPLOAD_URL }}/api/results/upload/" + # done + + docs: + name: Check module doc strings + runs-on: ubuntu-latest + env: + HEADLESS: "yes" + steps: + - name: Install python 3.11 + uses: actions/setup-python@v4 + with: + python-version: 3.11 + + - name: Install requirements + run: pip3.11 install --upgrade ansible + + - uses: actions/checkout@v3 + + - name: Run ansible-doc + run: make collection-docs + + - name: Get ansible-doc version + run: ansible-doc --version + if: failure() + + lint: + name: Lint module + runs-on: ubuntu-latest + env: + HEADLESS: "yes" + steps: + - name: Install python 3.11 + uses: actions/setup-python@v4 + with: + python-version: 3.11 + + - name: Install requirements + run: pip3.11 install --upgrade ansible-lint + + - uses: actions/checkout@v3 + + - name: Run ansible-lint + run: make collection-lint + + - name: Get ansible-lint version + run: ansible-lint --version + if: failure() +... diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml new file mode 100644 index 00000000..289a8739 --- /dev/null +++ b/.github/workflows/linting.yml @@ -0,0 +1,41 @@ +--- +name: Linting +env: + LC_ALL: "C.UTF-8" # prevent ERROR: Ansible could not initialize the preferred locale: unsupported locale setting +on: + pull_request: + push: +jobs: + common-tests: + name: ${{ matrix.tests.name }} + runs-on: ubuntu-latest + permissions: + packages: write + contents: read + strategy: + fail-fast: false + matrix: + tests: + - name: api-flake8 + command: check_flake8 + - name: api-black + command: check_black + - name: api-isort + command: check_isort + steps: + - name: Install make + run: sudo apt install make + + - uses: actions/checkout@v2 + + - name: Install python 3.11 + uses: actions/setup-python@v4 + with: + python-version: 3.11 + + - name: Install requirements + run: pip3.11 install -r requirements/requirements_dev.txt + + - name: Run check ${{ matrix.tests.name }} + run: make ${{ matrix.tests.command }} +... diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..763752d5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,115 @@ +# A collection directory, resulting from the use of the pytest-ansible-units plugin +collections/ + + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +# ide +*.code-workspace +*.vscode/ +.DS_Store + +changelogs/.plugin-cache.yaml diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..0421688c --- /dev/null +++ b/Makefile @@ -0,0 +1,64 @@ +SHELL=/bin/bash + +# Prefer python 3.11 but take python3 if 3.11 is not installed +PYTHON := $(notdir $(shell for i in python3.11 python3; do command -v $$i; done|sed 1q)) +RM ?= /bin/rm +ANSIBLE_CONFIG ?= tools/ansible/ansible.cfg +export ANSIBLE_CONFIG + +.PHONY: PYTHON_VERSION clean git_hooks_config \ + collection-install collection-test collection-docs \ + collection-lint collection-sanity collection-test-completeness \ + collection-test-integration-check + +## Set the local git configuration(specific to this repo) to look for hooks in .githooks folder +git_hooks_config: + git config --local core.hooksPath .githooks + +## Zero out all of the temp and build files +clean: + @-find . -type f -regex ".*\.py[co]$$" -print0 | xargs -0 $(RM) -f + @-find . -type d -name "__pycache__" -print0 \ + -o -type d -name ".pytest_cache" -print0 | xargs -0 $(RM) -rf + +## Run black syntax check +check_black: + tox -e black -- --check $(CHECK_SYNTAX_FILES) + +## Run flake8 syntax check +check_flake8: + tox -e flake8 -- $(CHECK_SYNTAX_FILES) + +## Run isort syntax check +check_isort: + tox -e isort -- --check $(CHECK_SYNTAX_FILES) + +## Install the collection locally on your machine +collection-install: + ansible-galaxy collection install . --force + +## Run the collection sanity tests +collection-sanity: collection-install + cd /tmp/collections/ansible_collections/ansible/platform && \ + ansible-test sanity + +## Run the collections docs check +collection-docs: collection-install + @RC=0 ; \ + for file_name in $$(ls plugins/modules/*.py) ; do \ + module=$$(echo $${file_name} | sed 's:^.*/::' | sed 's:\..*::') ; \ + ansible-doc -M plugins/modules $${module} 1> /dev/null ; \ + RC=$$(( RC + $$? )) ; \ + done ; \ + for file_name in $$(ls plugins/lookup/*.py) ; do \ + module=$$(echo $${file_name} | sed 's:^.*/::' | sed 's:\..*::') ; \ + ansible-doc -M plugins/lookup -t lookup $${module} 1> /dev/null ; \ + RC=$$(( RC + $$? )) ; \ + done ; \ + if [[ $$RC -eq 0 ]] ; then echo "Doc Passed" ; else echo "Docs Failed" ; fi ; \ + exit $$RC + +## Run the collection lint check +collection-lint: collection-install + # ansible-lint gets its settings from ansible_platform_collection/.ansible-lint + ansible-lint --profile=production diff --git a/requirements/requirements_dev.txt b/requirements/requirements_dev.txt new file mode 100644 index 00000000..f9a5f2c2 --- /dev/null +++ b/requirements/requirements_dev.txt @@ -0,0 +1,20 @@ +ansible # Needed by the dev environment to run playbooks +black==25.1.0 # Linting tool, if changed update pyproject.toml as well +build +debugpy # Used for VSCode debugging +django-debug-toolbar +flake8==7.1.1 # Linting tool, if changed update pyproject.toml as well +Flake8-pyproject==1.2.3 # Linting tool, if changed update pyproject.toml as well +ipython # Used in shell_plus +isort==6.0.0 # Linting tool, if changed update pyproject.toml as well +remote-pdb # The debugger setup with Python's breakpoint() +tox # Used for unit tests +tox-docker # For running postgres during tests +colorama # Allows addition of color to logs +logutils +psycopg[binary] +requests +setuptools>=70.0.0 # Needed for the plumbing of the side cars, pinned for https://github.com/ansible/aap-gateway/security/dependabot/25 +setuptools_scm # Needed for proper version string on /api/gateway/v1/ping +docker # Needed for the plumbing of the side cars via Ansible +PyYAML # Collection tests diff --git a/tools/ansible/ansible.cfg b/tools/ansible/ansible.cfg new file mode 100644 index 00000000..71142516 --- /dev/null +++ b/tools/ansible/ansible.cfg @@ -0,0 +1,2 @@ +[defaults] +collections_path = /tmp/collections diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..571bf3fa --- /dev/null +++ b/tox.ini @@ -0,0 +1,30 @@ +[black] +line-length = 160 +fast = true +skip-string-normalization = true +force-exclude = + ( + .*/migrations/ + | aap-dev/* + ) + +[isort] +profile = black +line_length = 160 +extend_skip = + aap_gateway_api/migrations + django-ansible-base + aap-dev + services + +[flake8] +max-line-length = 160 +extend-ignore = E203 +exclude = + aap_gateway_api/migrations/* + .tox + django-ansible-base + aap-dev/* + services/* +per-file-ignores = + plugins/modules/*:E402 From 904eb88c086366d88b728bcea49433b55377c276 Mon Sep 17 00:00:00 2001 From: NilashishC Date: Thu, 17 Apr 2025 21:15:57 +0530 Subject: [PATCH 2/3] Remove pull request template since it is already set at Org level Signed-off-by: NilashishC --- .github/PULL_REQUEST_TEMPLATE.md | 60 -------------------------------- 1 file changed, 60 deletions(-) delete mode 100644 .github/PULL_REQUEST_TEMPLATE.md diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md deleted file mode 100644 index 2dcdac85..00000000 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ /dev/null @@ -1,60 +0,0 @@ -## Description - -- What is being changed? -- Why is this change needed? -- How does this change address the issue? - -## Type of Change - -- [ ] Bug fix (non-breaking change which fixes an issue) -- [ ] New feature (non-breaking change which adds functionality) -- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) -- [ ] Documentation update -- [ ] Test update -- [ ] Refactoring (no functional changes) -- [ ] Development environment change -- [ ] Configuration change - -## Self-Review Checklist - -- [ ] I have performed a self-review of my code -- [ ] I have added relevant comments to complex code sections -- [ ] I have updated documentation where needed -- [ ] I have considered the security impact of these changes -- [ ] I have considered performance implications -- [ ] I have thought about error handling and edge cases -- [ ] I have tested the changes in my local environment - -## Testing Instructions - - -### Prerequisites - - -### Steps to Test -1. -2. -3. - -### Expected Results - - -## Additional Context - - -### Required Actions - - -- [ ] Requires documentation updates - -- [ ] Requires downstream repository changes - -- [ ] Requires infrastructure/deployment changes - -- [ ] Requires coordination with other teams - -- [ ] Blocked by PR/MR: #XXX - - -### Screenshots/Logs - From 5fa70561dbf6867fff51fcec6404430570d0fe5d Mon Sep 17 00:00:00 2001 From: NilashishC Date: Thu, 17 Apr 2025 21:17:35 +0530 Subject: [PATCH 3/3] Correctly set environment list Signed-off-by: NilashishC --- tox.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tox.ini b/tox.ini index 571bf3fa..8003f35e 100644 --- a/tox.ini +++ b/tox.ini @@ -1,3 +1,6 @@ +[tox] +envlist = flake8, black, isort + [black] line-length = 160 fast = true