diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 004f4dc5..7b1a5f73 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -15,6 +15,10 @@ updates: directory: "/" # Location of package manifests schedule: interval: "weekly" + groups: + github-actions: + patterns: + - "*" target-branch: "develop" - package-ecosystem: "docker" # See documentation for possible values diff --git a/.github/workflows/branch-cicd.yaml b/.github/workflows/branch-cicd.yaml index ed099e77..2184a76f 100644 --- a/.github/workflows/branch-cicd.yaml +++ b/.github/workflows/branch-cicd.yaml @@ -35,19 +35,19 @@ jobs: strategy: matrix: - java-version: [17] + java-version: [21] steps: - name: 💳 Checkout - uses: actions/checkout@v5 + uses: actions/checkout@v7 with: lfs: true fetch-depth: 0 token: ${{secrets.ADMIN_GITHUB_TOKEN || github.token}} - name: 💵 Maven Cache - uses: actions/cache@v4 + uses: actions/cache@v6 with: path: ~/.m2/repository # The "key" used to indicate a set of cached files is the operating system runner @@ -75,51 +75,33 @@ jobs: run: echo "jar_file=$(find ./service/target/ -maxdepth 1 -regextype posix-extended -regex '.*/registry-api-service-[0-9]+\.[0-9]+\.[0-9]+(-SNAPSHOT)?\.jar')" >> $GITHUB_OUTPUT - name: 🎰 QEMU Multiple Machine Emulation - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@v4 - name: 🚢 Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: 🧱 Image Construction and Publication - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 with: context: ./ file: ./docker/Dockerfile build-args: api_jar=${{steps.jarrer.outputs.jar_file}} push: false load: true - tags: nasapds/registry-api-service:latest - - - name: ∫ Integration tests … hold onto your hats, pardners - run: | - git clone --quiet https://github.com/NASA-PDS/registry.git - cd registry/docker/certs - ./generate-certs.sh - cd .. - docker image inspect nasapds/registry-api-service:latest >/dev/null - docker compose \ - --ansi never --profile int-registry-batch-loader --project-name registry \ - up --quiet-pull --detach - # --abort-on-container-exit - #echo "===== Docker logs =====" - #docker compose logs --no-color - docker compose \ - --ansi never --profile int-registry-batch-loader --project-name registry \ - run --rm --no-TTY reg-api-integration-test-with-wait + tags: nasapds/registry-api-service:develop - - name: Set up Python 3 - uses: actions/setup-python@v6 - with: - python-version: '3.13' + name: Install jq + uses: dcarbone/install-jq-action@v4 - - name: ∫ Test PDS Deep Archive compatibility + name: ∫ Integration and deep archive tests … hold onto your hats, pardners run: | - git clone --quiet https://github.com/NASA-PDS/deep-archive.git - cd deep-archive - pip install . - pds-deep-registry-archive -u http://localhost:8080 -s PDS_ENG urn:nasa:pds:insight_rad::2.1 --debug + if [[ "${{ github.actor }}" == "dependabot[bot]" ]]; then + .github/workflows/integration_tests.sh + else + .github/workflows/integration_tests.sh --verify + fi ... diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 7e2dccec..535d21e6 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -23,7 +23,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@v7 with: # We must fetch at least the immediate parents so that if this is # a pull request then we can checkout the head. @@ -80,7 +80,7 @@ jobs: - name: Upload CodeQL Artifacts - uses: actions/upload-artifact@v5 + uses: actions/upload-artifact@v7 with: name: codeql-artifacts path: ${{ env.RESULTS_DIR }} @@ -93,7 +93,7 @@ jobs: steps: - name: 💳 Checkout - uses: actions/checkout@v5 + uses: actions/checkout@v7 with: lfs: true fetch-depth: 0 @@ -108,7 +108,7 @@ jobs: - name: Upload SLOC - uses: actions/upload-artifact@v5 + uses: actions/upload-artifact@v7 with: name: sloc-count path: ${{ github.workspace }}/cloc.md diff --git a/.github/workflows/integration_tests.sh b/.github/workflows/integration_tests.sh new file mode 100755 index 00000000..f92e7649 --- /dev/null +++ b/.github/workflows/integration_tests.sh @@ -0,0 +1,234 @@ +#! /usr/bin/env bash +# +# requires: docker, git, mvn, and shellcheck +# +# docker - builds the registry api image, uses compose to run a host of services +# git clones registry repo +# jq - bash JSON tool +# mvn - to build the jar file for the current source registry api source code +# "shellcheck" - linter to keep this script clean +# + +build() { + mvn --quiet clean package + jar_file="$(find ./service/target/ -maxdepth 1 -name 'registry-api-service-*.jar')" + echo "jar file: $jar_file" + [ -s "$jar_file" ] || { echo "jar file not found or empty"; return 1; } + docker build --build-arg api_jar="$jar_file" -t nasapds/registry-api-service:latest -f docker/Dockerfile . +} + +clean() { + # shellcheck disable=SC2086 # for correct docker interpretation + docker compose \ + --ansi never \ + --profile int-registry-batch-loader \ + --project-name registry \ + down ${IT_CLEANSE:---rmi all} +} + +deep_archive() { + cd "$tdir" || return 1 + python3.12 -m venv "$tdir"/da + # shellcheck disable=SC1091 # cannot find dynamically created script + source "$tdir"/da/bin/activate + git clone --quiet https://github.com/NASA-PDS/deep-archive.git + cd deep-archive || return 1 + pip install . + if [ "$(python3 -c "import sys; print(sys.version_info.minor)")" -gt 12 ] + then + echo "Python 3.$MINOR_VERSION detected. Upgrading zope.interface..." + pip install --upgrade "zope.interface>=8.0.0" + fi + pds-deep-registry-archive -u http://localhost:8080 -s PDS_ENG urn:nasa:pds:insight_rad::2.1 --debug +} + +double_check_logfile() { + echo "everything looked ok, so double check postman logs" + [ -s "$1" ] || { echo "$1 is an empty file"; return 1; } + grep -Eq "[[:space:]]*#[[:space:]]+failure[[:space:]]+detail" "$1" \ + && { echo "postman log file reported failures" ; return 2; } + return 0 +} + +record() { + cat > last_integration_test.json </dev/null + echo "launch services" + docker compose \ + --ansi never \ + --profile int-registry-batch-loader \ + --project-name registry \ + up --detach --quiet-pull || { + echo "--- docker compose ps ---" + docker compose --ansi never --project-name registry ps -a + if $verbose; then + echo "--- docker compose logs ---" + docker compose --ansi never --project-name registry logs + fi + return 5 + } + echo "launch tests" + if docker compose \ + --ansi never \ + --profile int-registry-batch-loader \ + --project-name registry \ + run --rm --no-TTY reg-api-integration-test \ + 2>&1 | tee "$rdir/integration_test_results.txt" + then + deep_archive + status=$? + else + status=1 + fi + echo "run status: ${status}" + cd "$ddir" || return 1 + echo "--- docker compose ps ---" + docker compose --ansi never --project-name registry ps -a + if $verbose; then + echo "--- docker compose logs ---" + docker compose \ + --ansi never \ + --profile int-registry-batch-loader \ + --project-name registry \ + logs + fi + clean + # shellcheck disable=SC2086 # because we need to return an int + return $status +} + +verbose=false +verify=false +for arg in "$@"; do + case "$arg" in + --verbose) verbose=true ;; + --verify) verify=true ;; + *) + echo "Error: Invalid argument '$arg'" + echo "Usage: $0 [--verify] [--verbose]" + exit 1 ;; + esac +done + +bdir=$(dirname "$(realpath "$0")") +rdir=$(realpath "$bdir/../..") +cd "$rdir" || exit 1 +api_gitrev=$(git describe --always --abbrev=40 --dirty='+' --exclude '*') +branchname=$(git branch --show-current) +branchname=${branchname/issue/api} +branchname=${branchname/_/-} +tdir=$(mktemp -d) +echo "temporary directory: $tdir" +# The EXIT pseudo-signal covers normal exits, errors, and interruptions (Ctrl+C) +trap 'rm -rf "$tdir"' EXIT +export tdir +cd "$tdir" || exit 1 +git clone --quiet https://github.com/NASA-PDS/registry.git +cd registry || exit 1 +if git show-ref --verify --quiet refs/remotes/origin/"$branchname" +then + git switch "$branchname" +fi +echo "registry being used" +git status +reg_gitrev=$(git describe --always --abbrev=40 --dirty='+' --exclude '*') +if $verify; then + echo "Running in VERIFY mode..." + status=failure + cd "$tdir" || exit 1 + record "$api_gitrev" "$reg_gitrev" "$status" + cd "$rdir" || exit 1 + test_key=$(jq -r '.api_gitrev' "$bdir"/last_integration_test.json | sed 's/+$//') + files=$(git diff --name-only -r "$test_key") + # shellcheck disable=SC2046 # because comparing integers + if [ $(echo "$files" | wc -l) -eq 1 ] + then + if [ "$files" == ".github/workflows/last_integration_test.json" ] + then + if [ -s "$files" ] + then + # do a one line diff from last test run + # look at additions or subtractions + # ignore --- and +++ because those are the filenames + # ignore the api_gitrev because that must be different + # count all other changes + # if there are none, then status is meaningful + # shellcheck disable=SC2126 # because simpler to understand + if [ $(git diff -U0 -r "$test_key" | \ + grep "^[+-]" | \ + grep -v "^---" | \ + grep -v "^+++" | \ + grep -v "api_gitrev" | \ + wc -l) == 0 ] + then + status=$(jq -r '.status' "$bdir"/last_integration_test.json) + echo "Found the I&T test to be: ${status}" + else + git diff -r "$test_key" + fi + else + echo "Reporting file is empty" + fi + else + echo "the file changed was not for I&T: $files" + fi + else + echo "commit contains edits beyond those of last_integration_test.json" + echo "files changed: $files" + fi + if [ "$status" == "failure" ] + then + echo + echo "If you are reading this in the github actions log, then it seems" + echo "this test cannot verify that this registry-api repository branch" + echo "has been successfully tested. The first step at resolving this" + echo "message is to run the script .github/workflows/integration_tests.sh" + echo "locally. If it is successful, then commit all changes and push." + echo "Otherwise, fix any problems demonstrated from running the tests," + echo "then commit and push all changes when the script is successful." + echo "Once commited, run this script again to generate the single file" + echo "last_integration_test.json, commit it, and push it." + echo + echo "Note: there are timing tests that can cause temporary failures." + echo " If those failures occur, just run the script again until" + echo " a success is achived." + echo + echo "Note: to determine if the latest commit will pass, run the script" + echo " with 'integration_tests.sh --verify'" + else + echo "Verified tests completed and successful" + fi +else + cd "$rdir" || exit 1 + clean || exit 2 + build || exit 3 + cd "$tdir"/registry || exit 1 + ( set -o pipefail ; run 2>&1 | tee "$rdir"/integration_tests.rpt.txt ) \ + && status=success || status=failure + if [ "$status" == "success" ] + then + double_check_logfile "$rdir"/integration_tests.rpt.txt \ + || status=failure + else + echo "docker run or deep archive did not return success" + fi + cd "$bdir" || exit 1 + record "$api_gitrev" "$reg_gitrev" "$status" + [ "$status" == "success" ] && rm "$rdir"/integration_tests.rpt.txt +fi + +echo "Status: $status" +[ "$status" == "success" ] && exit 0 || exit 1 diff --git a/.github/workflows/last_integration_test.json b/.github/workflows/last_integration_test.json new file mode 100644 index 00000000..ef58c8ab --- /dev/null +++ b/.github/workflows/last_integration_test.json @@ -0,0 +1,5 @@ +{ + "api_gitrev": "b0711d49e711ff012dca26ff455682b29a5ca99c+", + "reg_gitrev": "dda5e706096b4e50575006f85f82f3c111d05b1f", + "status": "success" +} diff --git a/.github/workflows/secrets-detection.yaml b/.github/workflows/secrets-detection.yaml index 65c29546..cc73adb1 100644 --- a/.github/workflows/secrets-detection.yaml +++ b/.github/workflows/secrets-detection.yaml @@ -13,7 +13,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v5 + uses: actions/checkout@v7 - name: Install necessary packages run: | diff --git a/.github/workflows/stable-cicd.yaml b/.github/workflows/stable-cicd.yaml index fc5bca5b..9c12f21a 100644 --- a/.github/workflows/stable-cicd.yaml +++ b/.github/workflows/stable-cicd.yaml @@ -50,14 +50,14 @@ jobs: steps: - name: 💳 Checkout - uses: actions/checkout@v5 + uses: actions/checkout@v7 with: lfs: true token: ${{secrets.ADMIN_GITHUB_TOKEN}} fetch-depth: 0 - name: 💵 Maven Cache - uses: actions/cache@v4 + uses: actions/cache@v6 with: path: ~/.m2/repository # The "key" used to indicate a set of cached files is the operating system runner @@ -89,19 +89,19 @@ jobs: echo "image_tag=$(echo ${{github.ref}} | awk -F/ '{print $NF}')" >> $GITHUB_OUTPUT - name: 💳 Docker Hub Identification - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: username: ${{secrets.DOCKERHUB_USERNAME}} password: ${{secrets.DOCKERHUB_TOKEN}} - name: 🎰 QEMU Multiple Machine Emulation - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@v4 - name: 🚢 Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: 🧱 Image Construction and Publication - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 with: context: ./ file: ./docker/Dockerfile diff --git a/.github/workflows/unstable-cicd.yaml b/.github/workflows/unstable-cicd.yaml index 8753bc7b..a1a3cfdc 100644 --- a/.github/workflows/unstable-cicd.yaml +++ b/.github/workflows/unstable-cicd.yaml @@ -50,14 +50,14 @@ jobs: steps: - name: 💳 Checkout - uses: actions/checkout@v5 + uses: actions/checkout@v7 with: lfs: true fetch-depth: 0 token: ${{secrets.ADMIN_GITHUB_TOKEN}} - name: 💵 Maven Cache - uses: actions/cache@v4 + uses: actions/cache@v6 with: path: ~/.m2/repository # The "key" used to indicate a set of cached files is the operating system runner @@ -73,7 +73,7 @@ jobs: uses: NASA-PDS/roundup-action@stable with: assembly: unstable - packages: openjdk17-jdk + packages: openjdk21 maven-doc-phases: package env: central_portal_username: ${{secrets.CENTRAL_REPOSITORY_USERNAME}} @@ -85,28 +85,43 @@ jobs: name: 🫙 Jar File Determination id: jarrer run: echo "jar_file=$(find ./service/target/ -maxdepth 1 -regextype posix-extended -regex '.*/registry-api-service-[0-9]+\.[0-9]+\.[0-9]+(-SNAPSHOT)?\.jar')" >> $GITHUB_OUTPUT + - + name: 🎰 QEMU Multiple Machine Emulation + uses: docker/setup-qemu-action@v4 + - + name: 🚢 Docker Buildx + uses: docker/setup-buildx-action@v4 + # we want to publish the docker image ""locally" to GHCR as the AWS Pull Through Cache requires authentication + # and we don't have an organization account to manage read-only logins for the cache + - + name: 💳 GHCR Identification + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{github.actor}} + password: ${{secrets.GITHUB_TOKEN}} + # also push the image to DockerHub because we keep providing that distribution channel. - name: 💳 Docker Hub Identification - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: username: ${{secrets.DOCKERHUB_USERNAME}} password: ${{secrets.DOCKERHUB_TOKEN}} - - name: 🎰 QEMU Multiple Machine Emulation - uses: docker/setup-qemu-action@v3 - - - name: 🚢 Docker Buildx - uses: docker/setup-buildx-action@v3 + name: Set lowercase repo name + run: echo "LOWER_GHCR_REPO=$(echo '${{ github.repository }}' | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV - name: 🧱 Image Construction and Publication - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 with: context: ./ file: ./docker/Dockerfile build-args: api_jar=${{steps.jarrer.outputs.jar_file}} platforms: linux/amd64,linux/arm64 push: true - tags: ${{secrets.DOCKERHUB_USERNAME}}/registry-api-service:develop + tags: | + ghcr.io/${{env.LOWER_GHCR_REPO}}:develop + ${{secrets.DOCKERHUB_USERNAME}}/registry-api-service:develop - name: ∫ Integration tests … hold onto your hats, pardners run: | diff --git a/.gitignore b/.gitignore index c9fffc88..8cfbf204 100644 --- a/.gitignore +++ b/.gitignore @@ -65,6 +65,9 @@ src/test/temp/ # terraform .terraform/ +terraform/*.tfvars +!terraform/*.tfvars.example +!.terraform.lock.hcl # other stuff *.xpr @@ -87,3 +90,6 @@ application-*.properties # macOS specific stuff .DS_Store + +# reports to help separate testing from actions due to limited resources +*.rpt.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index 18e900b2..ce631540 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,15 +1,34 @@ # Changelog -## [«unknown»](https://github.com/NASA-PDS/registry-api/tree/«unknown») (2025-11-06) +## [«unknown»](https://github.com/NASA-PDS/registry-api/tree/«unknown») (2026-07-15) [Full Changelog](https://github.com/NASA-PDS/registry-api/compare/v1.6.2...«unknown») +**Requirements:** + +- As a user, I want the exists operator to match OpenSearch's native behavior for all fields [\#712](https://github.com/NASA-PDS/registry-api/issues/712) +- As a user, I want to search by a full/unique hierarchical path for a specific attribute [\#611](https://github.com/NASA-PDS/registry-api/issues/611) +- As a user, I want to query for documents where a specific search field exists in the document [\#406](https://github.com/NASA-PDS/registry-api/issues/406) + +**Improvements:** + +- As a user, I want the exists operator to be prepended to the query [\#727](https://github.com/NASA-PDS/registry-api/issues/727) +- Update registry API `/members/members` algorithm per deprecation of `parent_bundle_identifier` metadata non-aggregate products [\#699](https://github.com/NASA-PDS/registry-api/issues/699) + **Defects:** +- When a request /classes/{product class} does not match an existing product class, I want a 404 error. [\#767](https://github.com/NASA-PDS/registry-api/issues/767) [[s.medium](https://github.com/NASA-PDS/registry-api/labels/s.medium)] +- Investigate and fix skipped `product/{id}/member*` integration tests [\#748](https://github.com/NASA-PDS/registry-api/issues/748) [[s.medium](https://github.com/NASA-PDS/registry-api/labels/s.medium)] +- Integration tests in unstable build suite do not pass when run locally [\#745](https://github.com/NASA-PDS/registry-api/issues/745) [[s.medium](https://github.com/NASA-PDS/registry-api/labels/s.medium)] +- Unstable build does not complete on develop branch due to GitHub Actions runner timeout [\#744](https://github.com/NASA-PDS/registry-api/issues/744) [[s.medium](https://github.com/NASA-PDS/registry-api/labels/s.medium)] +- A query to pds.nasa.gov does not respond the same as a query to pds.mcp.nasa.gov [\#742](https://github.com/NASA-PDS/registry-api/issues/742) [[s.medium](https://github.com/NASA-PDS/registry-api/labels/s.medium)] +- API in production is unstable and returns 500 errors [\#716](https://github.com/NASA-PDS/registry-api/issues/716) [[s.critical](https://github.com/NASA-PDS/registry-api/labels/s.critical)] +- Inconsistent support for `application/vnd.nasa.pds.pds4+json` response format [\#705](https://github.com/NASA-PDS/registry-api/issues/705) [[s.high](https://github.com/NASA-PDS/registry-api/labels/s.high)] - API search results using "search-after" returns empty \[data\] block even though I can find the product by lidvid [\#677](https://github.com/NASA-PDS/registry-api/issues/677) [[s.high](https://github.com/NASA-PDS/registry-api/labels/s.high)] **Other closed issues:** +- B13.1 Registry + API [\#724](https://github.com/NASA-PDS/registry-api/issues/724) - Registry API Test Suite is failing [\#680](https://github.com/NASA-PDS/registry-api/issues/680) [[s.critical](https://github.com/NASA-PDS/registry-api/labels/s.critical)] - Manage errors as recommended in the spring mvc framework [\#286](https://github.com/NASA-PDS/registry-api/issues/286) @@ -74,11 +93,8 @@ - As a user, I want to apply an additional query filter \(`q=`\) to the `/classes/{class}` result set [\#493](https://github.com/NASA-PDS/registry-api/issues/493) - As a user, I want to apply an additional query filter \(`q=`\) to the `/products/{identifier}/member-of/member-of` result set [\#492](https://github.com/NASA-PDS/registry-api/issues/492) - As a user, I want to apply an additional query filter \(`q=`\) to the `/products/{identifier}/member-of` result set [\#491](https://github.com/NASA-PDS/registry-api/issues/491) -- As a user, I want to apply an additional query filter \(`q=`\) to members of the members of an aggregate product \(`/products/{identifier}/members/members`\) [\#490](https://github.com/NASA-PDS/registry-api/issues/490) - As a user, by default, I want to search for the latest versions of all products on the `/classes/{class}` endpoint unless explicitly requested [\#488](https://github.com/NASA-PDS/registry-api/issues/488) -- As a user, by default, I want to search only for the latest versions of all products on the `/products/{identifier}/member-of/member-of` endpoint [\#487](https://github.com/NASA-PDS/registry-api/issues/487) - As a user, by default, I want to search for only the latest versions of all products on the `/products/{identifier}/member-of` endpoint [\#486](https://github.com/NASA-PDS/registry-api/issues/486) -- As a user, by default, I want to search for only the latest versions of all products on the `/products/{identifier}/members/members` endpoint [\#485](https://github.com/NASA-PDS/registry-api/issues/485) - As a user, by default, I want to search for only the latest versions of all products on the `/products/{identifier}/members` endpoint [\#484](https://github.com/NASA-PDS/registry-api/issues/484) - As a user, I want to filter the products by any available PDS4 property using a combination of comparison, logical, and precedence grouping operators [\#469](https://github.com/NASA-PDS/registry-api/issues/469) - As a user, I want to get all product versions associated to one lid [\#436](https://github.com/NASA-PDS/registry-api/issues/436) @@ -138,7 +154,6 @@ **Requirements:** - As a user, I want my API request to execute successfully even when the registry contains corrupted documents [\#361](https://github.com/NASA-PDS/registry-api/issues/361) -- As a PDS operator, I want to know the health of the registry API service [\#336](https://github.com/NASA-PDS/registry-api/issues/336) **Defects:** diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 296d8288..61c4984a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -63,8 +63,24 @@ There are a few steps required to prepare for merging code back into the main br 1. Create a pull request if have not done this already. 1. Address all automated messages. -1. Run all regression checks to make sure changes have re-introduced already fixed bugs. +1. **Add required integration tests** (see [Integration Testing Requirements](#integration-testing-requirements) below). +1. Run all regression checks to make sure changes have not re-introduced already fixed bugs. 1. Move from draft to ready for review if in draft mode. 1. Request review. +## Integration Testing Requirements + +**IMPORTANT**: Each new feature, requirement, or bug fix must include at least one integration test added to the Postman collection. + +Integration tests are maintained in the [`registry` repository](https://github.com/NASA-PDS/registry) and must be updated as part of your contribution. For detailed instructions on creating and submitting integration tests, see: + +**[Integration Testing Guide](https://nasa-pds.github.io/registry/developer/integration-testing.html)** + +The guide covers: +- When tests are required +- Step-by-step process for adding tests to Postman +- TestRail integration (for internal developers) +- Running and validating tests locally +- Submitting test updates via pull request + diff --git a/README.md b/README.md index 745a43a3..bdb35cf0 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ Follow instructions in README.txt in the decompressed folder To build and run the application you need: -- jdk 17 +- jdk 25 - maven Additionally, harvested data will only be picked up correctly by the API if all of the following are true: @@ -113,23 +113,19 @@ The integration tests will be automatically applied. Check the results, update/c ## Tests -**Important note:** As a developer you are asked to complete the postman test suite according to the new feature you are developing. Do a pull request in the `registry` project to submit the updates. +### Testing Requirements -Integration test are maintained in postman. +**IMPORTANT:** As a developer, you are **required** to add integration tests to the Postman test suite for: +- Each new feature or requirement +- Each bug fix +- Any changes to existing API behavior -### Edit/Run of the integration tests in postman GUI +### Integration Testing Guide -Install the postman desktop, from https://www.postman.com/downloads/ +Integration tests are maintained in the `registry` repository as Postman collections. For complete instructions on creating, running, and submitting integration tests, see: -Download and open the test suite found in https://github.com/NASA-PDS/registry/tree/main/docker/postman +**[Integration Testing Guide](https://nasa-pds.github.io/registry/developer/integration-testing.html)** -### Run the integration tests in command line - -In the `registry` project. - -Launch the test in command line: - - npm install newman - newman run docker/postman/postman_collection.json --env-var baseUrl=http://localhost:8080 +All test updates must be submitted as pull requests to the [`registry` repository](https://github.com/NASA-PDS/registry). diff --git a/docker/Dockerfile b/docker/Dockerfile index 220a79a3..59c85c45 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -41,8 +41,8 @@ # Normally we'd prefer Alpine Linux, but JDK 11 isn't available with it, so # we go with a slim Debian. Debian's good too. -#FROM tomcat:9.0.58-jdk17-openjdk-slim -FROM tomcat:10.1.0-jdk17-openjdk-slim +FROM tomcat:jre25-temurin-noble + # API JAR file # ------------ @@ -92,4 +92,4 @@ CMD java $JAVA_OPTS -jar /usr/local/registry-api-service/registry-api-service.ja LABEL "org.label-schema.name" "PDS Registry API" LABEL "org.label-schema.description" "Planetary Data System's Application Programmer's Interface for the Registry" -LABEL "org.label-schema.url" "https://github.com/NASA-PDS/registry-api" \ No newline at end of file +LABEL "org.label-schema.url" "https://github.com/NASA-PDS/registry-api" diff --git a/docker/Dockerfile.local b/docker/Dockerfile.local deleted file mode 100644 index be5a76f5..00000000 --- a/docker/Dockerfile.local +++ /dev/null @@ -1,37 +0,0 @@ -FROM ubuntu:24.04 - - # Get arguments from the build command line - ARG version - ENV VERSION=$version - - # Build up the OS - RUN export DEBIAN_FRONTEND=noninteractive && \ - apt-get update && \ - apt-get install -y curl \ - libtcnative-1 \ - maven \ - openjdk-17-jdk-headless \ - tar - - # Make room for the app - RUN mkdir -p /usr/local/registry-${VERSION} - - # Copy the data into the building container - COPY LICENSE.md /usr/local/registry-${VERSION}/ - COPY pom.xml /usr/local/registry-${VERSION}/ - COPY SECURITY.md /usr/local/registry-${VERSION}/ - COPY lexer /usr/local/registry-${VERSION}/lexer - COPY model /usr/local/registry-${VERSION}/model - COPY service /usr/local/registry-${VERSION}/service - - # Resources shared with the rest of the world - EXPOSE 8080 - - # Build the application and deploy it inside the container - RUN set -x && \ - cd /usr/local/registry-${VERSION} && \ - mvn clean install - - # Run the sevice by default - WORKDIR /usr/local/registry-${VERSION}/service - CMD ["mvn", "spring-boot:run"] diff --git a/docker/README.md b/docker/README.md index 54163268..b95a11f9 100644 --- a/docker/README.md +++ b/docker/README.md @@ -46,12 +46,3 @@ For example on AWS, with OpenSearch serverless as a back-end: SPRING_BOOT_APP_ARGS=--openSearch.host= --openSearch.CCSEnabled=true --openSearch.username="" --openSearch.disciplineNodes=atm-delta,en-delta --registry.service.version=1.5.0-SNAPSHOT SERVER_PORT=80 - - - - - - -## 📍 Dockerfile.local - -You can ignore `Dockerfile.local` unless you're @al-niessner. diff --git a/lexer/pom.xml b/lexer/pom.xml index 94a4b143..15149cee 100644 --- a/lexer/pom.xml +++ b/lexer/pom.xml @@ -57,7 +57,7 @@ POSSIBILITY OF SUCH DAMAGE. org.apache.commons commons-lang3 - 3.4 + 3.18.0 diff --git a/lexer/src/main/antlr4/gov/nasa/pds/api/registry/lexer/Search.g4 b/lexer/src/main/antlr4/gov/nasa/pds/api/registry/lexer/Search.g4 index dbccf1f7..3502488e 100644 --- a/lexer/src/main/antlr4/gov/nasa/pds/api/registry/lexer/Search.g4 +++ b/lexer/src/main/antlr4/gov/nasa/pds/api/registry/lexer/Search.g4 @@ -1,13 +1,15 @@ grammar Search; -query : queryTerm EOF ; -queryTerm : comparison | likeComparison | group ; +query : queryTerm EOF ; +queryTerm : comparison | likeComparison | existence | group ; +fields : FIELDNAME | ALL LPAREN FIELDNAME RPAREN | ANY LPAREN FIELDNAME RPAREN ; group : NOT? LPAREN expression RPAREN ; +existence : EXISTS fields; expression : andStatement | orStatement | queryTerm ; andStatement : queryTerm (AND queryTerm)+ ; orStatement : queryTerm (OR queryTerm)+ ; -comparison : FIELD operator ( NUMBER | STRINGVAL ) ; -likeComparison : FIELD LIKE STRINGVAL ; +comparison : fields operator ( NUMBER | STRINGVAL ) ; +likeComparison : fields LIKE STRINGVAL ; operator : EQ | NE | GT | GE | LT | LE ; NOT : 'NOT' | 'not' ; @@ -19,15 +21,18 @@ GE : G E ; LT : L T ; LE : L E ; +EXISTS: E X I S T S; LIKE: L I K E; LPAREN : '(' ; RPAREN : ')' ; +ALL : A L L ; AND : A N D ; +ANY : A N Y ; OR : O R ; -FIELD : [A-Za-z_] [A-Za-z0-9_.:/]* ; +FIELDNAME : [A-Za-z_*] [A-Za-z0-9_.:/*]* ; STRINGVAL : '"' ~["\r\n]* '"' ; NUMBER : ('-')? [0-9]+ ('.' [0-9]*)? ; @@ -60,4 +65,4 @@ fragment V : [vV]; fragment W : [wW]; fragment X : [xX]; fragment Y : [yY]; -fragment Z : [zZ]; \ No newline at end of file +fragment Z : [zZ]; diff --git a/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/MockedListener.java b/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/MockedListener.java index 8d10f4fc..bea1625a 100644 --- a/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/MockedListener.java +++ b/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/MockedListener.java @@ -1,5 +1,6 @@ package api.pds.nasa.gov.api_search_query_lexer; +import java.util.ArrayList; import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.tree.ErrorNode; import org.antlr.v4.runtime.tree.ParseTreeListener; @@ -8,8 +9,10 @@ import gov.nasa.pds.api.registry.lexer.SearchParser.AndStatementContext; import gov.nasa.pds.api.registry.lexer.SearchParser.ComparisonContext; import gov.nasa.pds.api.registry.lexer.SearchParser.ExpressionContext; +import gov.nasa.pds.api.registry.lexer.SearchParser.FieldsContext; import gov.nasa.pds.api.registry.lexer.SearchParser.GroupContext; import gov.nasa.pds.api.registry.lexer.SearchParser.LikeComparisonContext; +import gov.nasa.pds.api.registry.lexer.SearchParser.ExistenceContext; import gov.nasa.pds.api.registry.lexer.SearchParser.OperatorContext; import gov.nasa.pds.api.registry.lexer.SearchParser.OrStatementContext; import gov.nasa.pds.api.registry.lexer.SearchParser.QueryContext; @@ -17,98 +20,96 @@ public class MockedListener implements ParseTreeListener, SearchListener { - - TerminalNode field = null, number = null, strval = null; + ArrayList fields = new ArrayList(); + TerminalNode number = null, strval = null; boolean isNot = false; @Override public void enterQuery(QueryContext ctx) { - // TODO Auto-generated method stub + // Nothing useful to do in this mocked version } @Override public void exitQuery(QueryContext ctx) { - // TODO Auto-generated method stub + // Nothing useful to do in this mocked version } @Override public void enterQueryTerm(QueryTermContext ctx) { - // TODO Auto-generated method stub + // Nothing useful to do in this mocked version } @Override public void exitQueryTerm(QueryTermContext ctx) { - // TODO Auto-generated method stub + // Nothing useful to do in this mocked version } @Override public void enterGroup(GroupContext ctx) { - // TODO Auto-generated method stub + // Nothing useful to do in this mocked version } @Override public void exitGroup(GroupContext ctx) { - // TODO Auto-generated method stub + // Nothing useful to do in this mocked version } @Override public void enterExpression(ExpressionContext ctx) { - // TODO Auto-generated method stub + // Nothing useful to do in this mocked version } @Override public void exitExpression(ExpressionContext ctx) { - // TODO Auto-generated method stub + // Nothing useful to do in this mocked version } @Override public void enterAndStatement(AndStatementContext ctx) { - // TODO Auto-generated method stub + // Nothing useful to do in this mocked version } @Override public void exitAndStatement(AndStatementContext ctx) { - // TODO Auto-generated method stub + // Nothing useful to do in this mocked version } @Override public void enterOrStatement(OrStatementContext ctx) { - // TODO Auto-generated method stub + // Nothing useful to do in this mocked version } @Override public void exitOrStatement(OrStatementContext ctx) { - // TODO Auto-generated method stub + // Nothing useful to do in this mocked version } @Override public void enterComparison(ComparisonContext ctx) { - this.field = ctx.FIELD(); this.number = ctx.NUMBER(); this.strval = ctx.STRINGVAL(); } @Override public void exitComparison(ComparisonContext ctx) { - // TODO Auto-generated method stub + // Nothing useful to do in this mocked version } @Override public void enterLikeComparison(LikeComparisonContext ctx) { - this.field = ctx.FIELD(); this.strval = ctx.STRINGVAL(); String op = ctx.getChild(1).getText(); @@ -118,44 +119,81 @@ public void enterLikeComparison(LikeComparisonContext ctx) { @Override public void exitLikeComparison(LikeComparisonContext ctx) { - // TODO Auto-generated method stub + // Nothing useful to do in this mocked version } @Override public void enterOperator(OperatorContext ctx) { - // TODO Auto-generated method stub + // Nothing useful to do in this mocked version } @Override public void exitOperator(OperatorContext ctx) { - // TODO Auto-generated method stub + // Nothing useful to do in this mocked version } @Override public void visitTerminal(TerminalNode node) { - // TODO Auto-generated method stub + // Nothing useful to do in this mocked version } @Override public void visitErrorNode(ErrorNode node) { - // TODO Auto-generated method stub + // Nothing useful to do in this mocked version } @Override public void enterEveryRule(ParserRuleContext ctx) { - // TODO Auto-generated method stub + // Nothing useful to do in this mocked version } @Override public void exitEveryRule(ParserRuleContext ctx) { - // TODO Auto-generated method stub + // Nothing useful to do in this mocked version + + } + + + @Override + public void enterExistence(ExistenceContext ctx) { + // Nothing useful to do in this mocked version + + } + @Override + public void exitExistence(ExistenceContext ctx) { + // Nothing useful to do in this mocked version + } + + @Override + public void enterFields(FieldsContext ctx) { + // Nothing useful to do in this mocked version + } + + @Override + public void exitFields(FieldsContext ctx) { + boolean any = ctx.ALL() == null; + String fieldname = ""; + if (ctx.FIELDNAME() != null) { + fieldname = ctx.FIELDNAME().getText(); + } + if (ctx.ALL() != null ) { + fieldname = ctx.ALL().getText(); + } + if (ctx.ANY() != null) { + fieldname = ctx.ANY().getText(); + } + if (fieldname.contains("*")) { + fields.add(fieldname.replace(".", "\\.").replace("*", ".*")); + } else { + fields.add(fieldname); + } } } diff --git a/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/TestParsing.java b/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/TestParsing.java index 0ce5c8a3..4c0a6742 100644 --- a/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/TestParsing.java +++ b/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/TestParsing.java @@ -5,11 +5,8 @@ import org.antlr.v4.runtime.CharStreams; import org.antlr.v4.runtime.CodePointCharStream; import org.antlr.v4.runtime.CommonTokenStream; -import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.misc.ParseCancellationException; -import org.antlr.v4.runtime.tree.ErrorNode; import org.antlr.v4.runtime.tree.ParseTree; -import org.antlr.v4.runtime.tree.ParseTreeListener; import org.antlr.v4.runtime.tree.ParseTreeWalker; import org.junit.jupiter.api.Test; import gov.nasa.pds.api.registry.lexer.SearchLexer; @@ -39,8 +36,6 @@ public void testMaliciousQuery() { par.setErrorHandler(new BailErrorStrategy()); ParseTree tree = par.query(); }, "Expected code to throw, but it didn't"); - - } @@ -57,8 +52,8 @@ public void testNumber() { MockedListener listener = new MockedListener(); walker.walk(listener, tree); - Assertions.assertNotNull(listener.field); - Assertions.assertEquals(listener.field.getSymbol().getText(), "lid"); + Assertions.assertEquals(listener.fields.size(), 1); + Assertions.assertEquals(listener.fields.get(0), "lid"); Assertions.assertNotEquals(listener.number, null); Assertions.assertEquals(listener.number.getSymbol().getText(), "1234"); @@ -77,8 +72,8 @@ public void testStringVal() { MockedListener listener = new MockedListener(); walker.walk(listener, tree); - Assertions.assertNotNull(listener.field); - Assertions.assertEquals(listener.field.getSymbol().getText(), "lid"); + Assertions.assertEquals(listener.fields.size(), 1); + Assertions.assertEquals(listener.fields.get(0), "lid"); Assertions.assertNotNull(listener.strval); Assertions.assertEquals(listener.strval.getSymbol().getText(), "\"*text*\""); @@ -97,8 +92,8 @@ public void testLike() { MockedListener listener = new MockedListener(); walker.walk(listener, tree); - Assertions.assertNotNull(listener.field); - Assertions.assertEquals(listener.field.getText(), "lid"); + Assertions.assertNotNull(listener.fields); + Assertions.assertEquals(listener.fields.get(0), "lid"); Assertions.assertNotNull(listener.strval); Assertions.assertEquals(listener.strval.getText(), "\"*text*\""); @@ -121,6 +116,79 @@ void testTemporalRange() { // TODO: Parse } -} + @Test + void testFieldExistence() { + String queryString = "exists apple"; + CodePointCharStream input = CharStreams.fromString(queryString); + SearchLexer lex = new SearchLexer(input); + CommonTokenStream tokens = new CommonTokenStream(lex); + SearchParser par = new SearchParser(tokens); + ParseTree tree = par.query(); + ParseTreeWalker walker = new ParseTreeWalker(); + MockedListener listener = new MockedListener(); + walker.walk(listener, tree); + + Assertions.assertEquals(1, listener.fields.size()); + Assertions.assertNull(listener.strval); + Assertions.assertEquals("apple", listener.fields.get(0)); + } + @Test + void testParenFieldExistence() { + String queryString = "(exists apple)"; + CodePointCharStream input = CharStreams.fromString(queryString); + SearchLexer lex = new SearchLexer(input); + CommonTokenStream tokens = new CommonTokenStream(lex); + SearchParser par = new SearchParser(tokens); + ParseTree tree = par.query(); + + ParseTreeWalker walker = new ParseTreeWalker(); + MockedListener listener = new MockedListener(); + walker.walk(listener, tree); + + Assertions.assertEquals(1, listener.fields.size()); + Assertions.assertNull(listener.strval, "strval should be null not: " + listener.strval); + Assertions.assertEquals("apple", listener.fields.get(0)); + } + + @Test + void testWildExistence() { + String queryString = "exists *.apple"; + CodePointCharStream input = CharStreams.fromString(queryString); + SearchLexer lex = new SearchLexer(input); + CommonTokenStream tokens = new CommonTokenStream(lex); + SearchParser par = new SearchParser(tokens); + ParseTree tree = par.query(); + + ParseTreeWalker walker = new ParseTreeWalker(); + MockedListener listener = new MockedListener(); + walker.walk(listener, tree); + + Assertions.assertEquals(1, listener.fields.size()); + Assertions.assertNull(listener.strval, "strval should be null not: " + listener.strval); + Assertions.assertEquals(".*\\.apple", listener.fields.get(0)); + } + + + @Test + void testParenWildExistence() { + String queryString = "(exists *apple)"; + CodePointCharStream input = CharStreams.fromString(queryString); + SearchLexer lex = new SearchLexer(input); + CommonTokenStream tokens = new CommonTokenStream(lex); + SearchParser par = new SearchParser(tokens); + ParseTree tree = par.query(); + + ParseTreeWalker walker = new ParseTreeWalker(); + MockedListener listener = new MockedListener(); + walker.walk(listener, tree); + + Assertions.assertEquals(listener.fields.size(), 1); + Assertions.assertNull(listener.strval); + Assertions.assertEquals(".*apple", listener.fields.get(0)); + } + + + +} diff --git a/model/pom.xml b/model/pom.xml index de09232a..8624ade7 100644 --- a/model/pom.xml +++ b/model/pom.xml @@ -138,7 +138,7 @@ jakarta.servlet jakarta.servlet-api - 6.0.0 + 6.1.0 provided @@ -180,14 +180,14 @@ com.fasterxml.jackson.dataformat jackson-dataformat-xml - 2.18.2 + 2.19.1 com.github.joschi.jackson jackson-datatype-threetenbp - 2.12.5 + 2.18.2 @@ -200,7 +200,7 @@ joda-time joda-time - 2.13.0 + 2.14.2 @@ -223,7 +223,7 @@ io.swagger.core.v3 swagger-models - 2.2.8 + 2.2.49 diff --git a/model/swagger.yml b/model/swagger.yml index 2b9fe25e..a8fb6c89 100644 --- a/model/swagger.yml +++ b/model/swagger.yml @@ -2,10 +2,12 @@ openapi: 3.0.0 info: description: | Registry API enabling advanced search on PDS data and metadata. The API provides end-points to search for bundles, collections and any PDS products with advanced search queries. It also enables to browse the archive hierarchically downward (e.g. collection/s products) or upward (e.g. bundles containing a product). - version: 1.3.0 + Property values are cast to string in responses due to limitations of JSON typing, and should be interpreted by the client/user according to the data dictionary. + As a result of the string cast, some missing values might be found as "null". title: PDS Registry Search API termsOfService: 'http://pds.nasa.gov' contact: + name: "Contact PDS Engineering Node Support" email: pds-operator@jpl.nasa.gov license: name: Apache 2.0 @@ -319,6 +321,14 @@ paths: - 2. product references summary: | returns all of the members of the members of the given lid/lidvid + deprecated: true + description: | + ⚠️ This endpoint is deprecated and does not work anymore. It will be removed in a future release.\n\n + + Please call `/{id}/members` instead, as follows:\n + 1. Get the collection members of the bundle {id} with a first call.\n + 2. Use the collection ids found and get their products by calling the `/{coll_id}/members` for each.\n + operationId: product-members-members responses: '200': @@ -664,7 +674,7 @@ components: schema: $ref: '#/components/schemas/errorMessage' Plural: - description: Successful request + description: Successful request. content: "*": schema: diff --git a/pom.xml b/pom.xml index 523043d7..1c8091aa 100644 --- a/pom.xml +++ b/pom.xml @@ -50,7 +50,7 @@ Go through this file line-by-line and replace the template values with your own. Registry API UTF-8 17 - 6.2.2 + 6.2.18 gov.nasa.pds @@ -304,7 +304,7 @@ Go through this file line-by-line and replace the template values with your own. org.apache.maven.plugins maven-gpg-plugin - 3.0.1 + 3.2.8 sign-artifacts diff --git a/service/README.md b/service/README.md index 1bcf5d35..ca69f082 100644 --- a/service/README.md +++ b/service/README.md @@ -9,7 +9,7 @@ For more information, please visit https://nasa-pds.github.io/registry-api-servi ## Prerequisites -This software requires open jdk 17. +This software requires open jdk 25. ## Administrator @@ -32,7 +32,16 @@ Note, the registry index in elasticSearch is hard-coded. It need to be `registry mvn clean mvn install + cd service mvn spring-boot:run + + The API will now be accessible on (by default) https://localhost:8080 + + With a specific configuration profile you can run the application with a specific configuration. Define a dedicated application.properties, for example application-dev.properties that does not need to be committed on git. Launch it as follows: + + mvn -Dspring-boot.run.profiles=dev spring-boot:run + + 👉 **Note:** in order to run in this way, you will need to modify the `spring-boot-starter-thymeleaf` dependency by pinning it to version `1.5.1.RELEASE` and excluding the `logback-classic` artifact in the `pom.xml` file as follows: diff --git a/service/pom.xml b/service/pom.xml index 4ccbb754..40de6608 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -30,6 +30,8 @@ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --> + 4.0.0 @@ -42,197 +44,114 @@ registry-api-service Registry API Service - Registry API Service contributing to the PDS Federated Search API - 17 + 21 UTF-8 - 3.4.1 - 2.18.2 + + 3.5.12 + + 6.2.14 + 2.19.1 2.31.54 - - - - src/main/resources - true - - - - - org.springframework.boot - spring-boot-maven-plugin - ${spring-boot-version} - - - - repackage - - - - - gov.nasa.pds.api.registry.SpringBootMain - nasapds/registry-api-service - JAR - - - 17 - -XX:MaxDirectMemorySize=1G - - - - - - - org.apache.maven.plugins - maven-assembly-plugin - 3.1.1 - - - bin-release - package - - single - - - true - - src/main/assembly/tar-assembly.xml - src/main/assembly/zip-assembly.xml - - - jar-with-dependencies - - - - - - posix - - - - org.apache.maven.plugins - maven-compiler-plugin - - - com.iluwatar.urm - urm-maven-plugin - 2.0.0 - - ${project.basedir}/target - - gov.nasa.pds.api.registry - - - - true - false - mermaid - - jar-with-dependencies - - - - - process-classes - - map - - - - - - - + + + + + + org.springframework + spring-framework-bom + ${spring-framework.version} + pom + import + + + + org.springframework.boot + spring-boot-dependencies + ${spring-boot-version} + pom + import + + + + com.fasterxml.jackson + jackson-bom + ${jackson-version} + pom + import + + + + software.amazon.awssdk + bom + 2.31.54 + pom + import + + + - + org.springframework.boot spring-boot-starter-web - - org.springframework.boot - spring-boot-starter-actuator + org.springframework.boot + spring-boot-starter-actuator - - org.springframework.data - spring-data-commons - - - - org.springdoc - springdoc-openapi-starter-webmvc-ui - 2.8.4 - - - - - org.springdoc - springdoc-openapi-starter-common - 2.8.4 - - - org.springframework.boot spring-boot-starter-thymeleaf - - - - io.swagger.core.v3 - swagger-core - 2.2.28 - - - - org.springframework.boot spring-boot-autoconfigure - - - + + + org.springframework.data + spring-data-commons + - - - jakarta.validation - jakarta.validation-api - 3.0.2 - - - - - jakarta.annotation - jakarta.annotation-api - 3.0.0 - + + + org.springdoc + springdoc-openapi-starter-webmvc-ui + 2.8.4 + + + org.springdoc + springdoc-openapi-starter-common + 2.8.4 + + + io.swagger.core.v3 + swagger-core + 2.2.28 + + + + jakarta.validation + jakarta.validation-api + + + jakarta.annotation + jakarta.annotation-api + + com.fasterxml.jackson.jaxrs jackson-jaxrs-base - ${jackson-version} com.fasterxml.jackson.core jackson-core - ${jackson-version} com.fasterxml.jackson.core @@ -241,34 +160,27 @@ com.fasterxml.jackson.core jackson-databind - ${jackson-version} com.fasterxml.jackson.jaxrs jackson-jaxrs-json-provider - ${jackson-version} - com.fasterxml.jackson.dataformat jackson-dataformat-xml - ${jackson-version} - + com.github.joschi.jackson jackson-datatype-threetenbp - 2.12.5 + 2.18.2 - - joda-time joda-time - 2.13.0 + 2.14.2 - com.sun.xml.bind jaxb-core @@ -284,227 +196,146 @@ javassist 3.30.2-GA - - + - org.threeten - threetenbp - 1.4.4 + org.apache.httpcomponents.client5 + httpclient5 - + + org.springframework.boot + spring-boot-starter-aop + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.platform + junit-platform-launcher + test + + + org.opensearch.client + spring-data-opensearch-starter + 1.4.0 + + gov.nasa.pds.registry-api registry-api-model ${project.version} - gov.nasa.pds.registry-api registry-api-lexer ${project.version} - - - org.antlr - antlr4-runtime - 4.11.1 - - - - - org.opensearch.client - opensearch-java - - 2.24.0 - - - - - org.apache.httpcomponents.client5 - httpclient5 - - 5.4.1 - - - - - - org.apache.httpcomponents.core5 - httpcore5 - 5.3.3 - - - - - - org.apache.httpcomponents.core5 - httpcore5-h2 - 5.3.3 - - - - - - - - - software.amazon.awssdk - apache-client - ${awssdk-version} - - - - - software.amazon.awssdk - checksums - ${awssdk-version} - - - - - software.amazon.awssdk - regions - ${awssdk-version} - - - - - - - - software.amazon.awssdk - auth - ${awssdk-version} - - - + - software.amazon.awssdk - secretsmanager - ${awssdk-version} + org.opensearch.client + opensearch-java + 3.8.0 - - - - software.amazon.awssdk - sdk-core - ${awssdk-version} - - - - - org.opensearch.client opensearch-rest-client 2.18.0 - - + org.opensearch.client opensearch-rest-high-level-client - 1.2.4 + 3.6.0 - - - - + - org.apache.httpcomponents - httpclient - 4.5.13 + software.amazon.awssdk + apache-client - - - org.apache.commons - commons-collections4 - 4.2 + software.amazon.awssdk + auth - - - - org.springframework.boot - spring-boot-starter-test - - - junit - junit - - + software.amazon.awssdk + aws-core - - - org.junit.jupiter - junit-jupiter-engine - 5.7.0 - test + software.amazon.awssdk + opensearch - - - org.mockito - mockito-core - 3.6.28 - test + software.amazon.awssdk + regions - - - org.springframework.boot - spring-boot-starter-validation + software.amazon.awssdk + sdk-core - - - - + + software.amazon.awssdk + secretsmanager + + com.google.guava guava 33.4.8-jre - - - - jakarta.servlet - jakarta.servlet-api - 6.0.0 - provided - - - - - org.springframework - spring-aspects - 6.2.5 + + + org.apache.commons + commons-collections4 + 4.5.0 + + + + org.antlr + antlr4-runtime + 4.13.2 + + + + org.owasp.encoder + encoder + 1.4.0 - - - - - + + + + src/main/resources + true + + + + org.springframework.boot - spring-boot-dependencies + spring-boot-maven-plugin ${spring-boot-version} - pom - import - - - - + + + + repackage + + + + + gov.nasa.pds.api.registry.SpringBootMain + nasapds/registry-api-service + + + 21 + -XX:MaxDirectMemorySize=1G + + + + + + diff --git a/service/src/main/java/gov/nasa/pds/api/registry/SpringBootMain.java b/service/src/main/java/gov/nasa/pds/api/registry/SpringBootMain.java index a2bba5fa..1baad868 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/SpringBootMain.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/SpringBootMain.java @@ -1,12 +1,16 @@ package gov.nasa.pds.api.registry; import java.lang.IllegalArgumentException; +import org.opensearch.spring.boot.autoconfigure.OpenSearchRestHighLevelClientAutoConfiguration; +import org.opensearch.spring.boot.autoconfigure.data.OpenSearchDataAutoConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.CommandLineRunner; import org.springframework.boot.ExitCodeGenerator; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.data.elasticsearch.ElasticsearchDataAutoConfiguration; +import org.springframework.boot.autoconfigure.data.elasticsearch.ElasticsearchRepositoriesAutoConfiguration; import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.context.annotation.AnnotationConfigApplicationContext; import org.springframework.context.annotation.ComponentScan; @@ -16,12 +20,27 @@ // add archive status filter // add other resolver endpoints -@SpringBootApplication +@SpringBootApplication(exclude = { + // 1. Prevents the 'elasticsearchTemplate' creation error you are seeing + ElasticsearchDataAutoConfiguration.class, + // 2. Prevents conflict with OpenSearch repositories + ElasticsearchRepositoriesAutoConfiguration.class, + // 3. Resolves the 'opensearchClient' name collision + OpenSearchRestHighLevelClientAutoConfiguration.class, + // 4. Prevents the elasticsearchTemplate alias conflict in 2.x + OpenSearchDataAutoConfiguration.class + +}) @OpenAPIDefinition @EnableScheduling -@ComponentScan(basePackages = {"gov.nasa.pds.api.registry.configuration", - "gov.nasa.pds.api.registry.controllers", "gov.nasa.pds.api.registry.model", - "gov.nasa.pds.api.registry.search", "gov.nasa.pds.api.registry.util", "javax.servlet.http"}) +@ComponentScan(basePackages = { + "gov.nasa.pds.api.registry.configuration", + "gov.nasa.pds.api.registry.controllers", + "gov.nasa.pds.api.registry.model", + "gov.nasa.pds.api.registry.search", + "gov.nasa.pds.api.registry.util" + // jakarta.servlet.http is loaded and configured automatically with springboot 3 + }) public class SpringBootMain implements CommandLineRunner { private static final Logger log = LoggerFactory.getLogger(SpringBootMain.class); diff --git a/service/src/main/java/gov/nasa/pds/api/registry/configuration/OpenApiConfiguration.java b/service/src/main/java/gov/nasa/pds/api/registry/configuration/OpenApiConfiguration.java index 1fa153d3..8003f7f8 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/configuration/OpenApiConfiguration.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/configuration/OpenApiConfiguration.java @@ -22,9 +22,10 @@ public class OpenApiConfiguration { @Bean public OpenAPI customOpenAPI() { OpenAPI customOpenAPI = new OpenAPI() - .info(new Info().title("PDS Registry Search API") - .description( - "RestFul web API provided to search all classes of products in the PDS registries.") + .info(new Info().title("PDS Registry Search API").description( + "Registry API enabling advanced search on PDS data and metadata. The API provides end-points to search for bundles, collections and any PDS products with advanced search queries. It also enables to browse the archive hierarchically downward (e.g. collection/s products) or upward (e.g. bundles containing a product).\n" + + " Property values are cast to string in responses due to limitations of JSON typing, and should be interpreted by the client/user according to the data dictionary.\n" + + " As a result of the string cast, some missing values might be found as \"null\".") .version(this.version) .contact(new Contact().name("Contact PDS Engineering Node Support") .email("pds_operator@jpl.nasa.gov")) diff --git a/service/src/main/java/gov/nasa/pds/api/registry/controllers/HealthController.java b/service/src/main/java/gov/nasa/pds/api/registry/controllers/HealthController.java index 9ddd127b..d9406bc6 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/controllers/HealthController.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/controllers/HealthController.java @@ -11,9 +11,8 @@ public class HealthController implements HealthApi { @Override public ResponseEntity> health() { - // To Be Completed - return new ResponseEntity<>(HttpStatus.OK); - + Map response = Map.of("status", "ok"); + return new ResponseEntity<>(response, HttpStatus.OK); } } diff --git a/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java b/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java index 562fb1ea..bfa5357c 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java @@ -3,7 +3,10 @@ import java.lang.reflect.InvocationTargetException; import java.io.IOException; import java.util.*; +import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.Stream; + import gov.nasa.pds.api.base.ClassesApi; import gov.nasa.pds.api.base.PropertiesApi; import gov.nasa.pds.api.registry.model.exceptions.*; @@ -50,6 +53,8 @@ // corresponding controllers public class ProductsController implements ProductsApi, ClassesApi, PropertiesApi { + private static final String OPS_PROVENANCE_OPS_ANCESTOR_REFS = "ops:Provenance/ops:ancestor_refs"; + @Override // TODO: Remove this when the common controller code is refactored out - it is only necessary // because additional @@ -275,7 +280,7 @@ private HashMap getLidVid(PdsProductIdentifier identifier, throw new NotFoundException("No product found with identifier " + identifier.toString()); } HashMap product = searchResponse.hits().hits().get(0).source(); - ProductsController.log.debug("Found product with lid=" + product.get("lid")); + ProductsController.log.debug("Found product with lid={}", product.get("lid")); return product; } @@ -301,8 +306,8 @@ private HashMap getLatestLidVid(PdsProductIdentifier identifier, } HashMap product = searchResponse.hits().hits().get(0).source(); - ProductsController.log.debug("Found product with lid=" + product.get("lid")); - return (HashMap) searchResponse.hits().hits().get(0).source(); + ProductsController.log.debug("Found product with lid={}", product.get("lid")); + return searchResponse.hits().hits().get(0).source(); } @@ -378,8 +383,7 @@ private PdsLidVid resolveLatestLidvid(PdsProductIdentifier identifier) * @throws AcceptFormatNotSupportedException */ private PdsLidVid resolveIdentifierToLidvid(PdsProductIdentifier identifier) - throws NotFoundException, IOException, AcceptFormatNotSupportedException, UnhandledException, - OpenSearchException { + throws NotFoundException, IOException, OpenSearchException { return identifier.isLidvid() ? (PdsLidVid) identifier : resolveLatestLidvid(identifier); } @@ -399,10 +403,10 @@ public ResponseEntity productMembers(String identifier, List use new RegistrySearchRequestBuilder(this.connectionContext); if (productClass.isBundle()) { - searchRequestBuilder.matchMembersOfBundle(lidvid); + searchRequestBuilder.matchMembers(lidvid); searchRequestBuilder.onlyCollections(); } else if (productClass.isCollection()) { - searchRequestBuilder.matchMembersOfCollection(lidvid); + searchRequestBuilder.matchMembers(lidvid); searchRequestBuilder.onlyBasicProducts(); } else { throw new BadRequestException( @@ -423,32 +427,20 @@ public ResponseEntity productMembers(String identifier, List use public ResponseEntity productMembersMembers(String identifier, List userRequestedFields, Integer limit, String q, List sort, List searchAfter, List facetFields, Integer facetLimit) - throws NotFoundException, UnhandledException, SortSearchAfterMismatchException, - BadRequestException, AcceptFormatNotSupportedException, UnparsableQParamException { + throws DeprecatedEndPointException { - try { - PdsProductIdentifier pdsIdentifier = PdsProductIdentifier.fromString(identifier); - PdsProductClasses productClass = resolveProductClass(pdsIdentifier); - PdsLidVid lidvid = resolveIdentifierToLidvid(pdsIdentifier); + String message = + """ + This endpoint is deprecated and does not work anymore. It will be removed in a future release. - RegistrySearchRequestBuilder searchRequestBuilder = - new RegistrySearchRequestBuilder(this.connectionContext); + Please call `/{id}/members` instead, as follows: + 1. Get the collection members of the bundle {id} with a first call. + 2. Use the collection ids found and get their products by calling the `/{coll_id}/members` for each. + """; - if (productClass.isBundle()) { - searchRequestBuilder.matchMembersOfBundle(lidvid); - searchRequestBuilder.onlyBasicProducts(); - } else { - throw new BadRequestException( - "productMembers endpoint is only valid for products with Product_Class '" - + PdsProductClasses.Product_Bundle + "' (got '" + productClass + "')"); - } + throw new DeprecatedEndPointException(message); - return searchAndTransform(userRequestedFields, List.of(), limit, q, sort, searchAfter, - facetFields, facetLimit, searchRequestBuilder); - } catch (IOException | OpenSearchException e) { - throw new UnhandledException(e); - } } /** @@ -458,13 +450,27 @@ public ResponseEntity productMembersMembers(String identifier, * * @param identifier the LID/LIDVID for which to retrieve documents * @param fieldName the name of the document _source property/field from which to extract results - * @return a deduplicated list of the aggregated property/field contents, converted to - * PdsProductLidvids + * @return a list of the aggregated property/field contents, converted to PdsProductLidvids * @throws AcceptFormatNotSupportedException */ private List resolveLidVidsFromProductField(PdsProductIdentifier identifier, String fieldName) throws OpenSearchException, IOException, NotFoundException, UnhandledException { + return resolveLidVidsFromProductField(identifier, fieldName, 0); + } + + /** + * Internal implementation with recursion depth protection against the unlikely event that a LID + * value ever turns up erroneously in the lidvid property. + */ + private List resolveLidVidsFromProductField(PdsProductIdentifier identifier, + String fieldName, int recursionDepth) + throws OpenSearchException, IOException, NotFoundException, UnhandledException { + + if (recursionDepth > 1) { + throw new UnhandledException( + "Recursion depth exceeded in resolveLidVidsFromProductField. Maximum depth is 1."); + } RegistrySearchRequestBuilder searchRequestBuilder = new RegistrySearchRequestBuilder(this.connectionContext); @@ -488,9 +494,32 @@ private List resolveLidVidsFromProductField(PdsProductIdentifier iden throw new NotFoundException("No product found with identifier " + identifier); } - return searchResponse.hits().hits().stream() - .map(hit -> (List) hit.source().get(fieldName)).filter(Objects::nonNull) - .flatMap(Collection::stream).map(PdsLidVid::fromString).toList(); + return searchResponse.hits().hits().stream().map(hit -> hit.source().get(fieldName)) + .filter(Objects::nonNull) + // the following map() is necessary to support non-array fields like 'lidvid' by normalising + // them to multi-element collections + .map(el -> el instanceof Collection ? el : List.of(el)).map(x -> (List) x) + .flatMap(Collection::stream).flatMap(idString -> { + try { + PdsProductIdentifier parsedId = PdsProductIdentifier.fromString(idString); + + if (parsedId != null && parsedId.isLidvid()) { + return Stream.of((PdsLidVid) parsedId); + } else if (parsedId != null && parsedId.isLid()) { + // Recurse to resolve LID to LIDVIDs + return resolveLidVidsFromProductField(parsedId, "lidvid", recursionDepth + 1) + .stream(); + } else { + throw new UnhandledException( + "Parsed identifier is neither LID nor LIDVID: " + idString); + } + } catch (NotFoundException e) { + log.warn("Product not found for identifier {}: {}", idString, e.getMessage()); + return Stream.empty(); + } catch (IOException | UnhandledException | OpenSearchException e) { + throw new RuntimeException(e); + } + }).distinct().toList(); } @@ -508,11 +537,9 @@ public ResponseEntity productMemberOf(String identifier, List us List parentIds; if (productClass.isCollection()) { - parentIds = - resolveLidVidsFromProductField(lidvid, "ops:Provenance/ops:parent_bundle_identifier"); + parentIds = resolveLidVidsFromProductField(lidvid, OPS_PROVENANCE_OPS_ANCESTOR_REFS); } else if (productClass.isBasicProduct()) { - parentIds = resolveLidVidsFromProductField(lidvid, - "ops:Provenance/ops:parent_collection_identifier"); + parentIds = resolveLidVidsFromProductField(lidvid, OPS_PROVENANCE_OPS_ANCESTOR_REFS); } else { throw new BadRequestException( "productMembersOf endpoint is not valid for products with Product_Class '" @@ -531,6 +558,16 @@ public ResponseEntity productMemberOf(String identifier, List us } } + + private Stream safeResolveLidVidsFromAncestor(PdsLidVid obj) { + try { + return resolveLidVidsFromProductField(obj, OPS_PROVENANCE_OPS_ANCESTOR_REFS).stream(); + } catch (OpenSearchException | IOException | NotFoundException | UnhandledException e) { + throw new RuntimeException(e); + } + } + + @Override public ResponseEntity productMemberOfOf(String identifier, List userRequestedFields, Integer limit, String q, List sort, @@ -543,10 +580,14 @@ public ResponseEntity productMemberOfOf(String identifier, PdsProductClasses productClass = resolveProductClass(pdsIdentifier); PdsLidVid lidvid = resolveIdentifierToLidvid(pdsIdentifier); - List parentIds; + List greatParentIds; if (productClass.isBasicProduct()) { - parentIds = - resolveLidVidsFromProductField(lidvid, "ops:Provenance/ops:parent_bundle_identifier"); + + greatParentIds = safeResolveLidVidsFromAncestor(lidvid) + .flatMap(this::safeResolveLidVidsFromAncestor).toList(); + + + } else { throw new BadRequestException( "productMembersOf endpoint is not valid for products with Product_Class '" @@ -556,7 +597,7 @@ public ResponseEntity productMemberOfOf(String identifier, RegistrySearchRequestBuilder searchRequestBuilder = new RegistrySearchRequestBuilder(this.connectionContext).matchFieldAnyOfIdentifiers("_id", - parentIds); + greatParentIds); return searchAndTransform(userRequestedFields, List.of(), limit, q, sort, searchAfter, facetFields, facetLimit, searchRequestBuilder); @@ -576,7 +617,7 @@ public ResponseEntity classList(String propertyClass, List userR try { pdsProductClass = PdsProductClasses.fromSwaggerName(propertyClass); } catch (IllegalArgumentException err) { - throw new BadRequestException(err.getMessage()); + throw new NotFoundException(err.getMessage()); } RegistrySearchRequestBuilder searchRequestBuilder = @@ -598,7 +639,7 @@ public ResponseEntity> classes() throws Exception { /** * Resolve the appropriate enumerated user type hint from an OpenSearch Property */ - protected PropertiesListInner.TypeEnum _resolvePropertyToEnumType(Property property) { + protected static PropertiesListInner.TypeEnum resolvePropertyToEnumType(Property property) { if (property.isBoolean()) { return PropertiesListInner.TypeEnum.BOOLEAN; } else if (property.isKeyword() || property.isText()) { @@ -616,12 +657,17 @@ protected PropertiesListInner.TypeEnum _resolvePropertyToEnumType(Property prope @Override public ResponseEntity> productPropertiesList() throws Exception { + return ProductsController.productPropertiesList(this.connectionContext); + } + + public static ResponseEntity> productPropertiesList( + ConnectionContext connectionContext) throws OpenSearchException, IOException { - List indexNames = this.connectionContext.getRegistryIndices(); + List indexNames = connectionContext.getRegistryIndices(); GetMappingRequest getMappingRequest = new GetMappingRequest.Builder().index(indexNames).build(); - OpenSearchIndicesClient indicesClient = this.openSearchClient.indices(); + OpenSearchIndicesClient indicesClient = connectionContext.getOpenSearchClient().indices(); GetMappingResponse getMappingResponse = indicesClient.getMapping(getMappingRequest); @@ -633,8 +679,7 @@ public ResponseEntity> productPropertiesList() throws for (Map.Entry property : indexProperties) { String jsonPropertyName = PdsProperty.toJsonPropertyString(property.getKey()); Property openPropertyName = property.getValue(); - PropertiesListInner.TypeEnum propertyEnumType = - _resolvePropertyToEnumType(openPropertyName); + PropertiesListInner.TypeEnum propertyEnumType = resolvePropertyToEnumType(openPropertyName); // No consistency-checking between duplicates, for now. TODO: add error log for mismatching // duplicates diff --git a/service/src/main/java/gov/nasa/pds/api/registry/controllers/RegistryApiResponseEntityExceptionHandler.java b/service/src/main/java/gov/nasa/pds/api/registry/controllers/RegistryApiResponseEntityExceptionHandler.java index c9bd1768..e4d537ad 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/controllers/RegistryApiResponseEntityExceptionHandler.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/controllers/RegistryApiResponseEntityExceptionHandler.java @@ -4,7 +4,7 @@ import java.util.Set; import gov.nasa.pds.api.registry.model.exceptions.*; import gov.nasa.pds.api.registry.model.transformers.ResponseTransformerRegistry; - +import org.owasp.encoder.Encode; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; @@ -17,7 +17,6 @@ @ControllerAdvice public class RegistryApiResponseEntityExceptionHandler extends ResponseEntityExceptionHandler { - private String errorDisclaimerHeader = "An error occured.\n"; private String errorDisclaimerFooter = "For assistance, forward this error message to pds-operator@jpl.nasa.gov"; @@ -34,7 +33,7 @@ private ResponseEntity genericExceptionHandler(RegistryApiException ex, String bodyOfResponse = status.toString() + "\n Request " + requestDescription + " failed with message:\n" + errorDescription + "(ref:" + errorIdentifier + ")\n" + errorDisclaimerFooter; - return handleExceptionInternal(ex, bodyOfResponse, new HttpHeaders(), status, request); + return handleExceptionInternal(ex, Encode.forHtml(bodyOfResponse), new HttpHeaders(), status, request); } @@ -92,5 +91,10 @@ public ResponseEntity unknownQueryParameter(UnauthorizedForwardedHostExc return genericExceptionHandler(ex, request, "", HttpStatus.BAD_REQUEST); } + @ExceptionHandler(value = {DeprecatedEndPointException.class}) + public ResponseEntity deprecatedEndPoint(DeprecatedEndPointException ex, + WebRequest request) { + return genericExceptionHandler(ex, request, "", HttpStatus.GONE); + } } diff --git a/service/src/main/java/gov/nasa/pds/api/registry/controllers/SecurityValidationFilter.java b/service/src/main/java/gov/nasa/pds/api/registry/controllers/SecurityValidationFilter.java index 925363d5..1b6396e5 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/controllers/SecurityValidationFilter.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/controllers/SecurityValidationFilter.java @@ -5,12 +5,12 @@ import jakarta.servlet.http.HttpServletResponse; import gov.nasa.pds.api.registry.model.exceptions.UnauthorizedForwardedHostException; import gov.nasa.pds.api.registry.model.exceptions.UnknownQueryParameterException; -import io.micrometer.core.instrument.util.StringEscapeUtils; import java.util.Arrays; import java.util.Enumeration; import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.owasp.encoder.Encode; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import org.springframework.web.servlet.HandlerInterceptor; @@ -44,7 +44,7 @@ public boolean preHandle(HttpServletRequest request, HttpServletResponse respons if (!ALLOWED_QUERY_PARAMETERS.contains(paramName)) { throw new UnknownQueryParameterException( "Query parameter not enumerated in SecurityValidationFilter.ALLOWED_QUERY_PARAMETERS: " - + paramName); + + Encode.forHtml(paramName)); } } diff --git a/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java b/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java index f5fb4fd3..5c862e5f 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java @@ -2,33 +2,32 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - +import gov.nasa.pds.api.registry.ConnectionContext; +import gov.nasa.pds.api.registry.controllers.ProductsController; import gov.nasa.pds.api.registry.lexer.SearchBaseListener; import gov.nasa.pds.api.registry.lexer.SearchParser; - +import gov.nasa.pds.model.PropertiesListInner; +import jakarta.validation.constraints.NotNull; +import java.io.IOException; import java.util.ArrayDeque; import java.util.ArrayList; -import java.util.Arrays; import java.util.Deque; +import java.util.HashSet; import java.util.List; -import java.util.stream.Collectors; +import java.util.Set; +import java.util.regex.Pattern; import org.antlr.v4.runtime.misc.ParseCancellationException; +import org.opensearch.OpenSearchException; import org.opensearch.client.json.JsonData; import org.opensearch.client.opensearch._types.FieldValue; import org.opensearch.client.opensearch._types.query_dsl.BoolQuery; +import org.opensearch.client.opensearch._types.query_dsl.ExistsQuery; import org.opensearch.client.opensearch._types.query_dsl.MatchQuery; import org.opensearch.client.opensearch._types.query_dsl.Query; import org.opensearch.client.opensearch._types.query_dsl.RangeQuery; import org.opensearch.client.opensearch._types.query_dsl.SimpleQueryStringQuery; -import org.opensearch.client.opensearch._types.query_dsl.TermsQuery; -import org.opensearch.client.opensearch._types.query_dsl.TermsQueryField; -import org.opensearch.client.opensearch._types.query_dsl.TermsSetQuery; -import org.opensearch.client.opensearch._types.query_dsl.WildcardQuery; -import org.opensearch.client.opensearch._types.query_dsl.Operator; -import org.opensearch.index.query.RangeQueryBuilder; -import org.opensearch.index.query.SimpleQueryStringBuilder; -import org.opensearch.index.query.TermQueryBuilder; -import org.opensearch.index.query.QueryBuilder; +import org.opensearch.client.opensearch._types.query_dsl.TermQuery; + public class Antlr4SearchListener extends SearchBaseListener { enum conjunctions { @@ -41,27 +40,84 @@ enum operation { private static final Logger log = LoggerFactory.getLogger(Antlr4SearchListener.class); + private boolean isAnyWildcard = true; private BoolQuery.Builder queryBuilder = new BoolQuery.Builder(); private conjunctions conjunction = conjunctions.AND; // DEFAULT - final private Deque stackQueryBuilders = new ArrayDeque(); - final private Deque stack_conjunction = new ArrayDeque(); + private final ArrayList fieldNames = new ArrayList(); + private final ConnectionContext connectionContext; + private final Deque stackQueryBuilders = new ArrayDeque(); + private final Deque stackConjunction = new ArrayDeque(); + private final Set knownFieldNames = new HashSet(); private operation operator = null; - public Antlr4SearchListener() { + public Antlr4SearchListener(@NotNull ConnectionContext connectionContext) { super(); + this.connectionContext = connectionContext; + } + + // for testing purposes only + public Antlr4SearchListener(List knownFieldNames) { + super(); + this.connectionContext = null; + this.knownFieldNames.addAll(knownFieldNames); } @Override - public void exitQuery(SearchParser.QueryContext ctx) {} - + public void enterFields(SearchParser.FieldsContext ctx) { + this.fieldNames.clear(); + this.isAnyWildcard = true; + } + + @Override + public void exitFields(SearchParser.FieldsContext ctx) { + String fieldname = ""; + if (ctx.FIELDNAME() != null) { + fieldname = ctx.FIELDNAME().getText(); + } + if (ctx.ALL() != null ) { + fieldname = ctx.ALL().getText(); + } + if (ctx.ANY() != null) { + fieldname = ctx.ANY().getText(); + } + if (this.knownFieldNames.isEmpty()) { + try { + for (PropertiesListInner property : ProductsController.productPropertiesList(this.connectionContext).getBody()) { + this.knownFieldNames.add(property.getProperty()); + } + } catch (OpenSearchException | IOException e) { + throw new IllegalStateException("Could not load the mapping(s) from opensearch; meaning 'q=' with field names will not work", e); + } + } + if (fieldname.contains("*")) { + String theKey = fieldname.replace(".", "\\.").replace("*", ".*"); + Pattern regex = Pattern.compile(theKey); + for (String fn : this.knownFieldNames.stream() + .filter(s -> regex.matcher(s).matches()) + .toList()) { + this.fieldNames.add(SearchUtil.jsonPropertyToOpenProperty(fn)); + } + if (this.fieldNames.isEmpty()) { + throw new ParseCancellationException("Wildcarding request '" + fieldname + "' cannot match any field names in the LDD using regular expression " + theKey); + } + } else { + if (this.knownFieldNames.contains(fieldname)) { + this.fieldNames.add(SearchUtil.jsonPropertyToOpenProperty(fieldname)); + } else { + throw new ParseCancellationException("The request '" + fieldname + "' does not match any field name in the LDD."); + } + } + this.isAnyWildcard = ctx.ALL() == null && this.fieldNames.size() > 1; + } + @Override public void enterGroup(SearchParser.GroupContext ctx) { log.debug("Enter Group"); - this.stack_conjunction.push(this.conjunction); + this.stackConjunction.push(this.conjunction); this.conjunction = conjunctions.AND; // DEFAULT this.stackQueryBuilders.push(this.queryBuilder); @@ -75,7 +131,7 @@ public void exitGroup(SearchParser.GroupContext ctx) { log.debug("Exit Group"); BoolQuery.Builder upperBoolQueryBuilder = this.stackQueryBuilders.pop(); - this.conjunction = this.stack_conjunction.pop(); + this.conjunction = this.stackConjunction.pop(); Query innerQuery = this.queryBuilder.build().toQuery(); if (ctx.NOT() != null) { @@ -111,18 +167,14 @@ public void exitOrStatement(SearchParser.OrStatementContext ctx) { } - @Override - public void enterComparison(SearchParser.ComparisonContext ctx) {} - @Override public void exitComparison(SearchParser.ComparisonContext ctx) { log.debug("Exit comparison"); - final String left = SearchUtil.jsonPropertyToOpenProperty(ctx.FIELD().getSymbol().getText()); + BoolQuery.Builder wild = new BoolQuery.Builder(); String right; Query comparatorQuery = null; - if (ctx.NUMBER() != null) { right = ctx.NUMBER().getSymbol().getText(); } else if (ctx.STRINGVAL() != null) { @@ -132,74 +184,98 @@ public void exitComparison(SearchParser.ComparisonContext ctx) { throw new ParseCancellationException( "A right component (literal) of a comparison is neither a number or a string. Number and String are the only types supported for literals."); } - - if (this.operator == operation.eq || this.operator == operation.ne) { - - - FieldValue fieldValue = new FieldValue.Builder().stringValue(right).build(); - - MatchQuery matchQueryBuilder = new MatchQuery.Builder().field(left).query(fieldValue).build(); - - comparatorQuery = matchQueryBuilder.toQuery(); - - if (this.operator == operation.ne) { - comparatorQuery = new BoolQuery.Builder().mustNot(comparatorQuery).build().toQuery(); + if (this.isAnyWildcard) { + wild.minimumShouldMatch("1"); + } + for (String left : this.fieldNames) { + if (this.operator == operation.eq || this.operator == operation.ne) { + BoolQuery.Builder boolQueryBuilder = new BoolQuery.Builder(); + FieldValue fieldValue = new FieldValue.Builder().stringValue(right).build(); + MatchQuery matchQueryBuilder = new MatchQuery.Builder().field(left).query(fieldValue).build(); + TermQuery termQueryBulidler = new TermQuery.Builder().field(left).value(fieldValue).build(); + boolQueryBuilder.should(matchQueryBuilder.toQuery(), termQueryBulidler.toQuery()); + comparatorQuery = boolQueryBuilder.build().toQuery(); + + if (this.operator == operation.ne) { + comparatorQuery = new BoolQuery.Builder().mustNot(comparatorQuery).build().toQuery(); + } + } else { + RangeQuery.Builder rangeQueryBuilder = new RangeQuery.Builder(); + rangeQueryBuilder = rangeQueryBuilder.field(left); + + if (this.operator == operation.ge) + rangeQueryBuilder.gte(JsonData.of(right)); + else if (this.operator == operation.gt) + rangeQueryBuilder.gt(JsonData.of(right)); + else if (this.operator == operation.le) + rangeQueryBuilder.lte(JsonData.of(right)); + else if (this.operator == operation.lt) + rangeQueryBuilder.lt(JsonData.of(right)); + else { + throw new ParseCancellationException("Operator " + this.operator.name() + + " is not supported. Supported comparison operators are eq, ne, gt, gte, lt, lte."); + } + comparatorQuery = rangeQueryBuilder.build().toQuery(); } - - - - } else { - RangeQuery.Builder rangeQueryBuilder = new RangeQuery.Builder(); - - rangeQueryBuilder = rangeQueryBuilder.field(left); - - if (this.operator == operation.ge) - rangeQueryBuilder.gte(JsonData.of(right)); - else if (this.operator == operation.gt) - rangeQueryBuilder.gt(JsonData.of(right)); - else if (this.operator == operation.le) - rangeQueryBuilder.lte(JsonData.of(right)); - else if (this.operator == operation.lt) - rangeQueryBuilder.lt(JsonData.of(right)); - else { - throw new ParseCancellationException("Operator " + this.operator.name() - + " is not supported. Supported comparison operators are eq, ne, gt, gte, lt, lte."); + if (this.isAnyWildcard) { + wild.should(comparatorQuery); + } else { + wild.must(comparatorQuery); } - - comparatorQuery = rangeQueryBuilder.build().toQuery(); - } - if (this.conjunction == conjunctions.AND) { - this.queryBuilder.must(comparatorQuery); + this.queryBuilder.must(wild.build().toQuery()); } else { - this.queryBuilder.should(comparatorQuery); + this.queryBuilder.should(wild.build().toQuery()); } - } @Override - public void enterLikeComparison(SearchParser.LikeComparisonContext ctx) {} + public void exitExistence(SearchParser.ExistenceContext ctx) { + BoolQuery.Builder wild = new BoolQuery.Builder(); + if (this.isAnyWildcard) { + wild.minimumShouldMatch("1"); + } + for (String fieldName : this.fieldNames) { + if (this.isAnyWildcard) { + wild.should(new ExistsQuery.Builder().field(fieldName).build().toQuery()); + } else { + wild.must(new ExistsQuery.Builder().field(fieldName).build().toQuery()); + } + } + if (this.conjunction == conjunctions.AND) { + this.queryBuilder.must(wild.build().toQuery()); + } else { + this.queryBuilder.should(wild.build().toQuery()); + } + } @Override public void exitLikeComparison(SearchParser.LikeComparisonContext ctx) { log.debug("Exit likeComparison"); - final String left = SearchUtil.jsonPropertyToOpenProperty(ctx.FIELD().getSymbol().getText()); + BoolQuery.Builder wild = new BoolQuery.Builder(); String right = ctx.STRINGVAL().getText(); - // remove quotes - right = right.replaceAll("^\"|\"$", ""); - - SimpleQueryStringQuery simpleQueryString = new SimpleQueryStringQuery.Builder().fields(left) - .query(right).fuzzyMaxExpansions(0).build(); - - Query query = simpleQueryString.toQuery(); - log.debug("Exit Like comparison: left member is " + left + " right member is " + right); + right = right.replaceAll("^\"|\"$", ""); // remove the quotes + if (this.isAnyWildcard) { + wild.minimumShouldMatch("1"); + } + for (String left : this.fieldNames) { + SimpleQueryStringQuery simpleQueryString = new SimpleQueryStringQuery.Builder().fields(left) + .query(right).fuzzyMaxExpansions(0).build(); + if (this.isAnyWildcard) { + wild.should(simpleQueryString.toQuery()); + } else { + wild.must(simpleQueryString.toQuery()); + } + log.debug("Exit Like comparison: left member is {} right member is {}", left, right); + } + if (this.conjunction == conjunctions.AND) { - this.queryBuilder.must(query); + this.queryBuilder.must(wild.build().toQuery()); } else { - this.queryBuilder.should(query); + this.queryBuilder.should(wild.build().toQuery()); } } diff --git a/service/src/main/java/gov/nasa/pds/api/registry/model/RawMultipleProductResponse.java b/service/src/main/java/gov/nasa/pds/api/registry/model/RawMultipleProductResponse.java index a0e8c7bb..1447e1d6 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/model/RawMultipleProductResponse.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/model/RawMultipleProductResponse.java @@ -1,7 +1,6 @@ package gov.nasa.pds.api.registry.model; import java.util.*; -import java.util.stream.Collectors; import gov.nasa.pds.model.SummaryFacet; import org.opensearch.client.opensearch.core.SearchResponse; @@ -31,7 +30,7 @@ private List extractFacetsFromSearchResponse( }); } else if (aggregate.isLterms()) { aggregate.lterms().buckets().array().forEach(bucket -> { - facet.putCountsItem(bucket.key(), Math.toIntExact(bucket.docCount())); + facet.putCountsItem(bucket.key().toString(), Math.toIntExact(bucket.docCount())); }); } diff --git a/service/src/main/java/gov/nasa/pds/api/registry/model/SearchUtil.java b/service/src/main/java/gov/nasa/pds/api/registry/model/SearchUtil.java index d7b617c4..351c0f29 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/model/SearchUtil.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/model/SearchUtil.java @@ -8,20 +8,30 @@ import org.apache.http.client.utils.URIBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; import gov.nasa.pds.api.registry.exceptions.UnsupportedSearchProperty; import gov.nasa.pds.model.Metadata; import gov.nasa.pds.model.PdsProduct; import gov.nasa.pds.model.Reference; +@Component public class SearchUtil { private static final Logger log = LoggerFactory.getLogger(SearchUtil.class); - static public String jsonPropertyToOpenProperty(String jsonProperty) { - return jsonProperty.replace(".", "/"); + private static String fnArch; + @Value("${registry.field.name.architecture}") + public static void setFnArch(String fnArch) { + SearchUtil.fnArch = fnArch; } - static public String[] jsonPropertyToOpenProperty(String[] jsonProperties) { + public static String jsonPropertyToOpenProperty(String jsonProperty) { + if (SearchUtil.fnArch == null || SearchUtil.fnArch.equalsIgnoreCase("flat")) return jsonProperty.replace(".", "/"); + return jsonProperty; + } + + public static String[] jsonPropertyToOpenProperty(String[] jsonProperties) { if (jsonProperties != null && jsonProperties.length > 0) { for (int i = 0; i < jsonProperties.length; i++) { jsonProperties[i] = jsonPropertyToOpenProperty(jsonProperties[i]); @@ -30,7 +40,7 @@ static public String[] jsonPropertyToOpenProperty(String[] jsonProperties) { return jsonProperties; } - static public List jsonPropertyToOpenProperty(List jsonProperties) { + public static List jsonPropertyToOpenProperty(List jsonProperties) { if (jsonProperties != null && jsonProperties.size() > 0) { for (int i = 0; i < jsonProperties.size(); i++) { jsonProperties.set(i, jsonPropertyToOpenProperty(jsonProperties.get(i))); @@ -39,13 +49,13 @@ static public List jsonPropertyToOpenProperty(List jsonPropertie return jsonProperties; } - static public String openPropertyToJsonProperty(String openProperty) + public static String openPropertyToJsonProperty(String openProperty) throws UnsupportedSearchProperty { - - return openProperty.replace('/', '.'); + if (SearchUtil.fnArch == null || SearchUtil.fnArch.equalsIgnoreCase("flat")) return openProperty.replace('/', '.'); + return openProperty; } - static private void addReference(ArrayList to, String ID, URL baseURL) { + private static void addReference(ArrayList to, String ID, URL baseURL) { Reference reference = new Reference(); reference.setId(ID); @@ -76,7 +86,7 @@ static private void addReference(ArrayList to, String ID, URL baseURL to.add(reference); } - static private PdsProduct addPropertiesFromESEntity(PdsProduct product, EntityProduct ep, + private static PdsProduct addPropertiesFromESEntity(PdsProduct product, EntityProduct ep, URL baseURL) { product.setId(ep.getLidVid()); product.setType(ep.getProductClass()); @@ -145,7 +155,7 @@ static private PdsProduct addPropertiesFromESEntity(PdsProduct product, EntityPr return product; } - static public PdsProduct entityProductToAPIProduct(EntityProduct ep, URL baseURL) { + public static PdsProduct entityProductToAPIProduct(EntityProduct ep, URL baseURL) { log.debug("convert EntityProduct (ep) to API object without XML label"); PdsProduct product = new PdsProduct(); diff --git a/service/src/main/java/gov/nasa/pds/api/registry/model/exceptions/DeprecatedEndPointException.java b/service/src/main/java/gov/nasa/pds/api/registry/model/exceptions/DeprecatedEndPointException.java new file mode 100644 index 00000000..92925be5 --- /dev/null +++ b/service/src/main/java/gov/nasa/pds/api/registry/model/exceptions/DeprecatedEndPointException.java @@ -0,0 +1,17 @@ +package gov.nasa.pds.api.registry.model.exceptions; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + + +public class DeprecatedEndPointException extends RegistryApiException { + + private static final long serialVersionUID = -6704894264788325051L; + private static final Logger log = LoggerFactory.getLogger(DeprecatedEndPointException.class); + + public DeprecatedEndPointException(String msg) { + super(msg); + } + +} diff --git a/service/src/main/java/gov/nasa/pds/api/registry/search/HitIterator.java b/service/src/main/java/gov/nasa/pds/api/registry/search/HitIterator.java index 87ee07b8..a06b2305 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/search/HitIterator.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/search/HitIterator.java @@ -62,7 +62,7 @@ public String getCurrentId() { @Override public boolean hasNext() { return this.currentBatch == null ? false - : (this.at + this.page * this.size) < this.currentBatch.getTotalHits().value; + : (this.at + this.page * this.size) < this.currentBatch.getTotalHits().value(); } @Override diff --git a/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java b/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java index 07d8ede0..a5b267f3 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java @@ -5,7 +5,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.stream.Collectors; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; @@ -21,7 +20,6 @@ import org.antlr.v4.runtime.tree.ParseTreeWalker; import org.antlr.v4.runtime.RecognitionException; import org.antlr.v4.runtime.misc.ParseCancellationException; -import org.apache.commons.lang3.StringUtils; import org.opensearch.client.json.jackson.JacksonJsonpGenerator; import org.opensearch.client.opensearch._types.FieldSort; import org.opensearch.client.opensearch._types.FieldValue; @@ -63,7 +61,7 @@ public RegistrySearchRequestBuilder(ConnectionContext connectionContext) { this.connectionContext = connectionContext; this.registryIndices = this.connectionContext.getRegistryIndices(); - log.info("Use indices: " + String.join(",", registryIndices) + "End indices"); + log.info("Use indices: {}", String.join(",", registryIndices) + "End indices"); this.index(registryIndices); @@ -82,7 +80,7 @@ public RegistrySearchRequestBuilder(ConnectionContext connectionContext) { private static Query getMandatoryBaselineQuery(ConnectionContext connectionContext) { List archiveStatus = connectionContext.getArchiveStatus(); List archiveStatusFieldValues = archiveStatus.stream().map(FieldValue::of).toList(); - log.info("Only publishes archiveStatus: " + String.join(",", archiveStatus)); + log.info("Only publishes archiveStatus: {}", String.join(",", archiveStatus)); TermsQueryField archiveStatusTerms = new TermsQueryField.Builder().value(archiveStatusFieldValues).build(); @@ -136,6 +134,7 @@ public RegistrySearchRequestBuilder applyMultipleProductsDefaults( return this; } + @Override public SearchRequest build() { BoolQuery bQuery = this.queryBuilder.build(); this.query(bQuery.toQuery()); @@ -145,9 +144,9 @@ public SearchRequest build() { try { String requestJson = serializeSearchRequest(searchRequest); - log.debug("Generated OpenSearch SearchRequest with query:\n" + requestJson); + log.debug("Generated OpenSearch SearchRequest with query:\n{}", requestJson); } catch (Exception e) { - log.error("Failed to generate json serialization of SearchRequest: " + e); + log.error("Failed to generate json serialization of SearchRequest: ", e); } return searchRequest; @@ -235,12 +234,8 @@ public RegistrySearchRequestBuilder matchProductClass(PdsProductClasses productC return this.matchField(PdsProductClasses.getPropertyName(), productClass.getValue()); } - public RegistrySearchRequestBuilder matchMembersOfBundle(PdsLidVid identifier) { - return this.matchField("ops:Provenance/ops:parent_bundle_identifier", identifier); - } - - public RegistrySearchRequestBuilder matchMembersOfCollection(PdsLidVid identifier) { - return this.matchField("ops:Provenance/ops:parent_collection_identifier", identifier); + public RegistrySearchRequestBuilder matchMembers(PdsLidVid identifier) { + return this.matchField("ops:Provenance/ops:ancestor_refs", identifier); } public RegistrySearchRequestBuilder paginate(Integer pageSize, List sortFieldNames, @@ -305,7 +300,9 @@ public RegistrySearchRequestBuilder searchAfterFromStrings(List searchAf * need to be handled specfically. Method stringValue() implies yes * FieldValue.Builder().stringValue(fieldValue).build()); } */ - this.searchAfter(searchAfterValues); + this.searchAfter(searchAfterValues.stream() + .map(FieldValue::of) + .toList()); return this; @@ -356,7 +353,7 @@ public RegistrySearchRequestBuilder fieldsFromPdsProperties(List pd - private static BoolQuery parseQueryString(String queryString) { + private BoolQuery parseQueryString(String queryString) { CodePointCharStream input = CharStreams.fromString(queryString); SearchLexer lex = new SearchLexer(input); CommonTokenStream tokens = new CommonTokenStream(lex); @@ -369,7 +366,7 @@ private static BoolQuery parseQueryString(String queryString) { // Walk it and attach our listener ParseTreeWalker walker = new ParseTreeWalker(); - Antlr4SearchListener listener = new Antlr4SearchListener(); + Antlr4SearchListener listener = new Antlr4SearchListener(this.connectionContext); walker.walk(listener, tree); return listener.getBoolQuery(); @@ -386,12 +383,12 @@ public RegistrySearchRequestBuilder constrainByQueryString(String q) try { if ((q != null) && (q.length() > 0)) { - BoolQuery qBoolQuery = RegistrySearchRequestBuilder.parseQueryString(q); + BoolQuery qBoolQuery = this.parseQueryString(q); this.queryBuilder.must(qBoolQuery.toQuery()); } return this; } catch (RecognitionException | ParseCancellationException e) { - log.info("Unable to parse q " + LoggingAspect.sanitizeForLog(q) + "error message is " + e); + log.info("Unable to parse q {} error message is {}", LoggingAspect.sanitizeForLog(q), e); throw new UnparsableQParamException("Invalid q string value syntax " + e.getMessage()); } diff --git a/service/src/main/resources/application.properties b/service/src/main/resources/application.properties index b54a8032..9ffe8fef 100644 --- a/service/src/main/resources/application.properties +++ b/service/src/main/resources/application.properties @@ -51,3 +51,7 @@ filter.archiveStatus=archived,certified # source version from maven # need to be updated with actual value when runs outside of maven registry.service.version=@project.version@ + +# signal if the database being used is "flat" or "structured" +# used an enum of strings, "flat" or "structured" for future new types +registry.field.name.architecture=structured \ No newline at end of file diff --git a/service/src/test/java/gov/nasa/pds/api/registry/model/transformers/ResponseTransformerRegistryTest.java b/service/src/test/java/gov/nasa/pds/api/registry/model/transformers/ResponseTransformerRegistryTest.java index 3a43fe7b..a57d4d71 100644 --- a/service/src/test/java/gov/nasa/pds/api/registry/model/transformers/ResponseTransformerRegistryTest.java +++ b/service/src/test/java/gov/nasa/pds/api/registry/model/transformers/ResponseTransformerRegistryTest.java @@ -20,7 +20,7 @@ void selectFormatterClassFromSingleFormatSuccessfulTest() { String format = "text/html"; String expectedFormatterClassName = - "gov.nasa.pds.api.registry.model.api_responses.PdsProductBusinessObject"; + "gov.nasa.pds.api.registry.model.transformers.PdsProductTransformer"; String foundFormatterClassName; try { @@ -58,7 +58,7 @@ void selectFormatterClassFromMultipleFormatSuccessfulTest() { String format = "text/ms+word,text/html"; String expectedFormatterClassName = - "gov.nasa.pds.api.registry.model.api_responses.PdsProductBusinessObject"; + "gov.nasa.pds.api.registry.model.transformers.PdsProductTransformer"; String foundFormatterClassName; try { @@ -80,7 +80,7 @@ void selectFormatterClassFromMultipleFormatExtraSpacesSuccessfulTest() { String format = "text/ms+word,text/html ,anything/something"; String expectedFormatterClassName = - "gov.nasa.pds.api.registry.model.api_responses.PdsProductBusinessObject"; + "gov.nasa.pds.api.registry.model.transformers.PdsProductTransformer"; String foundFormatterClassName; try { diff --git a/service/src/test/java/gov/nasa/pds/api/registry/opensearch/Antlr4SearchListenerTest.java b/service/src/test/java/gov/nasa/pds/api/registry/opensearch/Antlr4SearchListenerTest.java index 16af90a6..98ac4698 100644 --- a/service/src/test/java/gov/nasa/pds/api/registry/opensearch/Antlr4SearchListenerTest.java +++ b/service/src/test/java/gov/nasa/pds/api/registry/opensearch/Antlr4SearchListenerTest.java @@ -18,13 +18,12 @@ import org.junit.jupiter.api.BeforeEach; import org.mockito.Mockito; import static org.junit.jupiter.api.Assertions.*; - - +import java.util.Arrays; import gov.nasa.pds.api.registry.lexer.SearchLexer; import gov.nasa.pds.api.registry.lexer.SearchParser; import gov.nasa.pds.api.registry.model.Antlr4SearchListener; -public class Antlr4SearchListenerTest { +class Antlr4SearchListenerTest { private class NegativeTester implements Executable { final private Antlr4SearchListenerTest parent; final private String qs; @@ -44,7 +43,14 @@ public void execute() { @BeforeEach void setUp() { - listener = new Antlr4SearchListener(); + listener = new Antlr4SearchListener(Arrays.asList( + "lid", + "pds:Time_Coordinates.pds:stop_date_time", + "ref_lid_target", + "timestamp", + "timestamp_A", + "timestamp_B" + )); } @@ -57,8 +63,7 @@ private BoolQuery run(String query) { ParseTree tree = par.query(); // Walk it and attach our listener ParseTreeWalker walker = new ParseTreeWalker(); - Antlr4SearchListener listener = new Antlr4SearchListener(); - walker.walk(listener, tree); + walker.walk(this.listener, tree); // System.out.println ("query string: " + query); // System.out.println("query tree: " + tree.toStringTree(par)); @@ -68,21 +73,33 @@ private BoolQuery run(String query) { @Test - public void testSimpleCompEq() { + void testSimpleCompEq() { String qs = "pds:Time_Coordinates.pds:stop_date_time eq \"2021-05-21T15:47:08Z\""; BoolQuery query = this.run(qs); // TODO: add asserts - Assertions.assertEquals(query.must().size(), 1); + Assertions.assertEquals(1, query.must().size()); Query matchQuery = (Query) query.must().get(0); - Assertions.assertEquals(matchQuery._kind(), Query.Kind.Match); - // Assertions.assertEquals((matchQuery).field(), "pds:Time_Coordinates/pds:stop_date_time"); - - + Assertions.assertEquals(Query.Kind.Bool, matchQuery._kind()); + query = matchQuery.bool(); + Assertions.assertEquals(1, query.must().size()); + matchQuery = (Query) query.must().get(0); + Assertions.assertEquals(Query.Kind.Bool, matchQuery._kind()); + query = matchQuery.bool(); + boolean match = false; + boolean term = false; + Assertions.assertEquals(2, query.should().size()); + for (int i = 0 ; i < 2 ; i++) { + matchQuery = (Query) query.should().get(i); + match = match || Query.Kind.Match == matchQuery._kind(); + term = term || Query.Kind.Term == matchQuery._kind(); + } + Assertions.assertTrue(match); + Assertions.assertTrue(term); } @Test - public void testLikeWildcard() { + void testLikeWildcard() { String qs = "lid like \"*pdart14_meap\""; BoolQuery query = this.run(qs); // TODO: add asserts @@ -91,7 +108,7 @@ public void testLikeWildcard() { } @Test - public void testEscape() { + void testEscape() { String qs = "lid eq \"*pdart14_meap?\""; BoolQuery query = this.run(qs); @@ -99,7 +116,7 @@ public void testEscape() { } @Test - public void testGroupedStatementAndExclusiveInequality() { + void testGroupedStatementAndExclusiveInequality() { String qs = "( timestamp gt 12 and timestamp lt 27 )"; BoolQuery query = this.run(qs); @@ -107,7 +124,7 @@ public void testGroupedStatementAndExclusiveInequality() { } @Test - public void testGroupedStatementAndInclusiveInequality() { + void testGroupedStatementAndInclusiveInequality() { String qs = "( timestamp_A ge 12 and timestamp_B le 27 )"; BoolQuery query = this.run(qs); @@ -115,7 +132,7 @@ public void testGroupedStatementAndInclusiveInequality() { } @Test - public void testNot() { + void testNot() { String qs = "not ( timestamp ge 12 and timestamp le 27 )"; BoolQuery query = this.run(qs); @@ -124,7 +141,7 @@ public void testNot() { @Test - public void testNestedGrouping() { + void testNestedGrouping() { String qs = "( ( timestamp ge 12 and timestamp le 27 ) or ( timestamp gt 13 and timestamp lt 37 ) )"; @@ -137,7 +154,7 @@ public void testNestedGrouping() { @Test - public void testNoWildcardQuoted() { + void testNoWildcardQuoted() { String qs = "ref_lid_target eq \"urn:nasa:pds:context:target:planet.mercury\""; BoolQuery query = this.run(qs); @@ -145,7 +162,7 @@ public void testNoWildcardQuoted() { } @Test - public void testExceptionsInParsing() { + void testExceptionsInParsing() { NegativeTester actor; String fails[] = {"( a eq b", "a eq b )", "not( a eq b )", "a eq b and c eq d and", "( a eq b and c eq d and )", "( a eq b and c eq d or e eq f )"}; @@ -189,7 +206,7 @@ void testEnterOrStatement() { void testExitOrStatement() { listener.enterOrStatement(Mockito.mock(SearchParser.OrStatementContext.class)); listener.exitOrStatement(Mockito.mock(SearchParser.OrStatementContext.class)); - assertTrue(listener.getBoolQuery().minimumShouldMatch() == "1", + assertEquals("1", listener.getBoolQuery().minimumShouldMatch(), "Minimum should match should be set"); } diff --git a/service/ut/base_ut.py b/service/ut/base_ut.py deleted file mode 100644 index 999a9d81..00000000 --- a/service/ut/base_ut.py +++ /dev/null @@ -1,360 +0,0 @@ - -import helpers -import unittest - -def test_bad_group(): - ep = '/classes/notreal' - status,data = helpers.fetch_kvp_json (helpers.make_url (ep)) - assert 406 == status - assert 'message' in data - assert 'request' in data - assert data['message'].startswith ("Unknown group 'notreal'. All known groups:") - assert data['request'] == ep - return - -def test_bad_lidvid(): - ep = '/products/notreal' - status,data = helpers.fetch_kvp_json (helpers.make_url (ep)) - assert 404 == status - assert 'message' in data - assert 'request' in data - assert 'The lidvid notreal was not found' == data['message'] - assert data['request'] == ep - return - -class TestAny(unittest.TestCase): - def test_products(self): - status,resp = helpers.fetch_kvp_json (helpers.make_url ('/classes/any')) - self.assertEqual (200, status) - self.assertIn ('summary', resp) - self.assertIn ('hits', resp['summary']) - self.assertEqual (17, resp['summary']['hits']) - self.assertIn ('data', resp) - self.assertEqual (resp['summary']['hits'], len(resp['data'])) - self.assertIn ('lidvid', resp['data'][0]) - return resp['data'][-1]['lidvid'] - - def test_lidvid(self): - lidvid = self.test_products() - status,resp = helpers.fetch_kvp_json (helpers.make_url - (f'/products/{lidvid}')) - self.assertEqual (200, status) - self.assertIn ('lidvid', resp) - self.assertEqual (lidvid, resp['lidvid']) - return - - def test_lidvid_latest(self): - lidvid = self.test_products() - status,resp = helpers.fetch_kvp_json(helpers.make_url - (f'/products/{lidvid}/latest')) - self.assertEqual (200, status) - self.assertIn ('lidvid', resp) - self.assertEqual (lidvid, resp['lidvid']) - return - - def test_lidvid_all(self): - lidvid = self.test_products() - status,resp = helpers.fetch_kvp_json (helpers.make_url - (f'/products/{lidvid}/all')) - self.assertEqual (200, status) - self.assertIn ('summary', resp) - self.assertIn ('hits', resp['summary']) - self.assertEqual (1, resp['summary']['hits']) - self.assertIn ('data', resp) - self.assertEqual (resp['summary']['hits'], len(resp['data'])) - self.assertIn ('lidvid', resp['data'][0]) - self.assertEqual (lidvid, resp['data'][0]['lidvid']) - return - - def test_collections(self): - lidvid = self.test_products() - status,resp = helpers.fetch_kvp_json (helpers.make_url - (f'/products/{lidvid}/member-of')) - self.assertEqual (200, status) - self.assertIn ('summary', resp) - self.assertIn ('hits', resp['summary']) - self.assertEqual (1, resp['summary']['hits']) - self.assertIn ('data', resp) - self.assertEqual (resp['summary']['hits'], len(resp['data'])) - self.assertIn ('lidvid', resp['data'][0]) - return - - def test_bundles(self): - lidvid = self.test_products() - status,resp = helpers.fetch_kvp_json (helpers.make_url - (f'/products/{lidvid}/member-of/member-of')) - self.assertEqual (200, status) - self.assertIn ('summary', resp) - self.assertIn ('hits', resp['summary']) - self.assertEqual (1, resp['summary']['hits']) - self.assertIn ('data', resp) - self.assertEqual (resp['summary']['hits'], len(resp['data'])) - self.assertIn ('lidvid', resp['data'][0]) - return - pass - -class TestBundles(unittest.TestCase): - def test_bundles(self): - status,resp = helpers.fetch_kvp_json (helpers.make_url ('/classes/bundles')) - self.assertEqual (200, status) - self.assertIn ('summary', resp) - self.assertIn ('hits', resp['summary']) - self.assertEqual (1, resp['summary']['hits']) - self.assertIn ('data', resp) - self.assertEqual (resp['summary']['hits'], len(resp['data'])) - self.assertIn ('lidvid', resp['data'][0]) - return resp['data'][0]['lidvid'] - - def test_lidvid(self): - lidvid = self.test_bundles() - status,resp = helpers.fetch_kvp_json (helpers.make_url - (f'/products/{lidvid}')) - self.assertEqual (200, status) - self.assertIn ('lidvid', resp) - self.assertEqual (lidvid, resp['lidvid']) - return - - def test_lidvid_latest(self): - lidvid = self.test_bundles() - status,resp = helpers.fetch_kvp_json(helpers.make_url - (f'/products/{lidvid}/latest')) - self.assertEqual (200, status) - self.assertIn ('lidvid', resp) - self.assertEqual (lidvid, resp['lidvid']) - return - - def test_lidvid_all(self): - lidvid = self.test_bundles() - status,resp = helpers.fetch_kvp_json (helpers.make_url - (f'/products/{lidvid}/all')) - self.assertEqual (200, status) - self.assertIn ('summary', resp) - self.assertIn ('hits', resp['summary']) - self.assertEqual (1, resp['summary']['hits']) - self.assertIn ('data', resp) - self.assertEqual (resp['summary']['hits'], len(resp['data'])) - self.assertIn ('lidvid', resp['data'][0]) - self.assertEqual (lidvid, resp['data'][0]['lidvid']) - return - - def test_collections(self): - lidvid = self.test_bundles() - status,resp = helpers.fetch_kvp_json \ - (helpers.make_url (f'/classes/bundles/{lidvid}/members')) - self.assertEqual (200, status) - self.assertIn ('summary', resp) - self.assertIn ('hits', resp['summary']) - self.assertEqual (2, resp['summary']['hits']) - self.assertIn ('data', resp) - self.assertEqual (resp['summary']['hits'], len(resp['data'])) - self.assertIn ('lidvid', resp['data'][0]) - return - - def test_collections_latest(self): - lidvid = self.test_bundles() - status,resp = helpers.fetch_kvp_json \ - (helpers.make_url (f'/classes/bundles/{lidvid}/members/latest')) - self.assertEqual (200, status) - self.assertIn ('summary', resp) - self.assertIn ('hits', resp['summary']) - self.assertEqual (2, resp['summary']['hits']) - self.assertIn ('data', resp) - self.assertEqual (resp['summary']['hits'], len(resp['data'])) - self.assertIn ('lidvid', resp['data'][0]) - return - - def test_collections_all(self): - lidvid = self.test_bundles() - status,resp = helpers.fetch_kvp_json \ - (helpers.make_url (f'/classes/bundles/{lidvid}/members/all')) - self.assertEqual (200, status) - self.assertIn ('summary', resp) - self.assertIn ('hits', resp['summary']) - self.assertEqual (2, resp['summary']['hits']) - self.assertIn ('data', resp) - self.assertEqual (resp['summary']['hits'], len(resp['data'])) - self.assertIn ('lidvid', resp['data'][0]) - return - - def test_products(self): - lidvid = self.test_bundles() - status,resp = helpers.fetch_kvp_json\ - (helpers.make_url (f'/classes/bundles/{lidvid}/members/members')) - self.assertEqual (200, status) - self.assertIn ('summary', resp) - self.assertIn ('hits', resp['summary']) - self.assertEqual (14, resp['summary']['hits']) - self.assertIn ('data', resp) - self.assertEqual (resp['summary']['hits'], len(resp['data'])) - self.assertIn ('lidvid', resp['data'][0]) - return - pass - -class TestCollections(unittest.TestCase): - def test_bundles(self): - lidvid = self.test_collections() - status,resp = helpers.fetch_kvp_json \ - (helpers.make_url (f'/classes/collections/{lidvid}/member-of')) - self.assertEqual (200, status) - self.assertIn ('summary', resp) - self.assertIn ('hits', resp['summary']) - self.assertEqual (1, resp['summary']['hits']) - self.assertIn ('data', resp) - self.assertEqual (resp['summary']['hits'], len(resp['data'])) - self.assertIn ('lidvid', resp['data'][0]) - return - - def test_collections(self): - status,resp = helpers.fetch_kvp_json \ - (helpers.make_url ('/classes/collections')) - self.assertEqual (200, status) - self.assertIn ('summary', resp) - self.assertIn ('hits', resp['summary']) - self.assertEqual (2, resp['summary']['hits']) - self.assertIn ('data', resp) - self.assertEqual (resp['summary']['hits'], len(resp['data'])) - self.assertIn ('lidvid', resp['data'][0]) - return resp['data'][0]['lidvid'] - - def test_lidvid(self): - lidvid = self.test_collections() - status,resp = helpers.fetch_kvp_json (helpers.make_url - (f'/products/{lidvid}')) - self.assertEqual (200, status) - self.assertIn ('lidvid', resp) - self.assertEqual (lidvid, resp['lidvid']) - return - - def test_lidvid_latest(self): - lidvid = self.test_collections() - status,resp = helpers.fetch_kvp_json(helpers.make_url - (f'/products/{lidvid}/latest')) - self.assertEqual (200, status) - self.assertIn ('lidvid', resp) - self.assertEqual (lidvid, resp['lidvid']) - return - - def test_lidvid_all(self): - lidvid = self.test_collections() - status,resp = helpers.fetch_kvp_json (helpers.make_url - (f'/products/{lidvid}/all')) - self.assertEqual (200, status) - self.assertIn ('summary', resp) - self.assertIn ('hits', resp['summary']) - self.assertEqual (1, resp['summary']['hits']) - self.assertIn ('data', resp) - self.assertEqual (resp['summary']['hits'], len(resp['data'])) - self.assertIn ('lidvid', resp['data'][0]) - self.assertEqual (lidvid, resp['data'][0]['lidvid']) - return - - def test_products(self): - lidvid = self.test_collections() - status,resp = helpers.fetch_kvp_json (helpers.make_url - (f'/classes/collections/{lidvid}/members')) - self.assertEqual (200, status) - self.assertIn ('summary', resp) - self.assertIn ('hits', resp['summary']) - self.assertEqual (7, resp['summary']['hits']) - self.assertIn ('data', resp) - self.assertEqual (resp['summary']['hits'], len(resp['data'])) - self.assertIn ('lidvid', resp['data'][0]) - return - - def test_products_latest(self): - lidvid = self.test_collections() - status,resp = helpers.fetch_kvp_json (helpers.make_url - (f'/classes/collections/{lidvid}/members/latest')) - self.assertEqual (200, status) - self.assertIn ('summary', resp) - self.assertIn ('hits', resp['summary']) - self.assertEqual (7, resp['summary']['hits']) - self.assertIn ('data', resp) - self.assertEqual (resp['summary']['hits'], len(resp['data'])) - self.assertIn ('lidvid', resp['data'][0]) - return - - def test_products_all(self): - lidvid = self.test_collections() - status,resp = helpers.fetch_kvp_json (helpers.make_url - (f'/classes/collections/{lidvid}/members/all')) - self.assertEqual (200, status) - self.assertIn ('summary', resp) - self.assertIn ('hits', resp['summary']) - self.assertEqual (7, resp['summary']['hits']) - self.assertIn ('data', resp) - self.assertEqual (resp['summary']['hits'], len(resp['data'])) - self.assertIn ('lidvid', resp['data'][0]) - return - pass - -class TestProducts(unittest.TestCase): - def test_products(self): - status,resp = helpers.fetch_kvp_json (helpers.make_url ('/classes/products')) - self.assertEqual (200, status) - self.assertIn ('summary', resp) - self.assertIn ('hits', resp['summary']) - self.assertEqual (14, resp['summary']['hits']) - self.assertIn ('data', resp) - self.assertEqual (resp['summary']['hits'], len(resp['data'])) - self.assertIn ('lidvid', resp['data'][0]) - return resp['data'][-1]['lidvid'] - - def test_lidvid(self): - lidvid = self.test_products() - status,resp = helpers.fetch_kvp_json (helpers.make_url - (f'/products/{lidvid}')) - self.assertEqual (200, status) - self.assertIn ('lidvid', resp) - self.assertEqual (lidvid, resp['lidvid']) - return - - def test_lidvid_latest(self): - lidvid = self.test_products() - status,resp = helpers.fetch_kvp_json(helpers.make_url - (f'/products/{lidvid}/latest')) - self.assertEqual (200, status) - self.assertIn ('lidvid', resp) - self.assertEqual (lidvid, resp['lidvid']) - return - - def test_lidvid_all(self): - lidvid = self.test_products() - status,resp = helpers.fetch_kvp_json (helpers.make_url - (f'/products/{lidvid}/all')) - self.assertEqual (200, status) - self.assertIn ('summary', resp) - self.assertIn ('hits', resp['summary']) - self.assertEqual (1, resp['summary']['hits']) - self.assertIn ('data', resp) - self.assertEqual (resp['summary']['hits'], len(resp['data'])) - self.assertIn ('lidvid', resp['data'][0]) - self.assertEqual (lidvid, resp['data'][0]['lidvid']) - return - - def test_collections(self): - lidvid = self.test_products() - status,resp = helpers.fetch_kvp_json (helpers.make_url - (f'/products/{lidvid}/member-of')) - self.assertEqual (200, status) - self.assertIn ('summary', resp) - self.assertIn ('hits', resp['summary']) - self.assertEqual (1, resp['summary']['hits']) - self.assertIn ('data', resp) - self.assertEqual (resp['summary']['hits'], len(resp['data'])) - self.assertIn ('lidvid', resp['data'][0]) - return - - def test_bundles(self): - lidvid = self.test_products() - status,resp = helpers.fetch_kvp_json (helpers.make_url - (f'/products/{lidvid}/member-of/member-of')) - self.assertEqual (200, status) - self.assertIn ('summary', resp) - self.assertIn ('hits', resp['summary']) - self.assertEqual (1, resp['summary']['hits']) - self.assertIn ('data', resp) - self.assertEqual (resp['summary']['hits'], len(resp['data'])) - self.assertIn ('lidvid', resp['data'][0]) - return - pass diff --git a/service/ut/helpers/__init__.py b/service/ut/helpers/__init__.py deleted file mode 100644 index aa0d04cc..00000000 --- a/service/ut/helpers/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ - -import os -import requests - -def fetch_kvp_json (url:str): - url += '?fields=lidvid' - print ('url:', url) - result = requests.get(url, headers={'Accept':'application/kvp+json'}) - return result.status_code,result.json() - -def make_url (endpoint:str)->str: - return os.environ.get ('REG_APO_UT_TYPE', 'http') + '://' + \ - os.environ.get ('REG_API_UT_HOSTNAME', 'localhost') + ':' + \ - os.environ.get ('REG_API_UT_PORT', '8080') + endpoint diff --git a/service/ut/regress.sh b/service/ut/regress.sh deleted file mode 100755 index 80c8aee0..00000000 --- a/service/ut/regress.sh +++ /dev/null @@ -1,9 +0,0 @@ -#! /bin/bash - -# make sure an instance of registry-api/service is runningn with the -# test data set - -base=$(realpath $0) -base=$(dirname $base) -PYTHONPATH=${base}:${PYTHONPATH} -pytest -v $(find $base -name \*_ut.py) diff --git a/terraform/README.md b/terraform/README.md index 13c0d4b7..6d84e119 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -24,18 +24,22 @@ These interfaces are going to be used a arguments of the terraform scripts. ## Deploy +Initialize the parameters, starting from the terraform.tfvars.example file provided. + +Copy it: + + cp terraform.tfvars.example terraform.tfvars + +And update the values. + Run the terraform scripts: + + + ``` - terraform apply \ - -var 'ecs_task_role=your-task-role-arn' \ - -var 'ecs_task_execution_role=your-task-execution-role-arn' \ - -var 'venue=your-venue' \ - -var 'aws_fg_vpc=your-vpc-arn' \ - -var 'aws_fg_security_groups=["your security group, e.g. sg-1223455..."]' \ - -var 'aws_fg_subnets=["your subnet e.g. subnet-1234..."]' \ - -var 'aws_fg_image=your-docker-image-available-on-ECR' \ - -var 'aws_acm_certificate_arn=ssl certificate for the load balancer listener' \ - -var 'spring_boot_args=--openSearch.host=your-opensearch-url-without-http --openSearch.CCSEnabled=true --openSearch.username=our-username-empty-for-opensearch-serverless --openSearch.disciplineNodes=the-prefixes-of-the-registry-indices-in-opensearch --registry.service.version=the-version-of-the-api-to-be-displayed-in-the-application' + terraform init -backend-config=backend-config.tfvars + terraform plan + terraform apply ``` diff --git a/terraform/backend-config.tfvars.example b/terraform/backend-config.tfvars.example new file mode 100644 index 00000000..c17c2064 --- /dev/null +++ b/terraform/backend-config.tfvars.example @@ -0,0 +1,12 @@ +# Example backend configuration for S3 +# Copy this file to backend-config.tfvars and customize the values +# Initialize with: terraform init -backend-config=backend-config.tfvars + +bucket = "my-terraform-state-bucket" +key = "registry/opensearch/terraform.tfstate" +region = "us-east-1" +dynamodb_table = "terraform-state-lock" +encrypt = true + +# Optional: Use a specific profile +# profile = "my-aws-profile" diff --git a/terraform/backend.tf b/terraform/backend.tf new file mode 100644 index 00000000..7644a18f --- /dev/null +++ b/terraform/backend.tf @@ -0,0 +1,25 @@ +# Backend configuration for S3 state storage +# Variables are not supported in backend blocks. Instead, provide configuration via: +# +# 1. backend-config.tfvars file: +# terraform init -backend-config=backend-config.tfvars +# +# 2. Command line arguments: +# terraform init -backend-config="bucket=${TFSTATE_BUCKET}" -backend-config="key=${TFSTATE_KEY}" +# +# 3. Environment variables or interactive prompts +# +# See https://stackoverflow.com/questions/63048738/how-to-declare-variables-for-s3-backend-in-terraform + +terraform { + backend "s3" { + # Backend configuration values provided via backend-config.tfvars + # Example backend-config.tfvars content: + # bucket = "pds-infra" + # key = "registry/opensearch/terraform.tfstate" + # region = "us-east-1" + # dynamodb_table = "terraform-state-lock" + # encrypt = true + # profile = "your-aws-profile" + } +} \ No newline at end of file diff --git a/terraform/ecs.tf b/terraform/main.tf similarity index 51% rename from terraform/ecs.tf rename to terraform/main.tf index 050a7846..135f1271 100644 --- a/terraform/ecs.tf +++ b/terraform/main.tf @@ -1,34 +1,31 @@ +locals { + + # Concatenate the load balancer domain to spring boot args + spring_boot_args_with_host = "${var.spring_boot_args} --server.authorizedForwardedHost=${aws_lb.registry-api-lb.dns_name},${var.cloudfront_dns}" +} + resource "aws_lb" "registry-api-lb" { - name = "registry-api-lb-new" + name = "registry-api-lb" internal = false load_balancer_type = "application" - security_groups = var.aws_fg_security_groups + security_groups = var.aws_lb_security_groups subnets = var.aws_lb_subnets enable_deletion_protection = false access_logs { bucket = var.aws_s3_bucket_logs_id - prefix = "registry-api-lb" + prefix = "registry/registry-api-lb" enabled = true } - tags = { - Alfa = var.node_name_abbr - Bravo = var.venue - Charlie = "registry" - } + tags = var.common_tags } -resource "aws_ssm_parameter" "load_balancer_domain" { - name = "/pds/registry/load-balancer-domain" - type = "String" - overwrite = true - value = aws_lb.registry-api-lb.dns_name -} + resource "aws_lb_target_group" "pds-registry-api-target-group" { - name = "pds-${var.venue}-registry-tgt" + name = "pds-registry-tg" port = 80 protocol = "HTTP" target_type = "ip" @@ -39,11 +36,13 @@ resource "aws_lb_target_group" "pds-registry-api-target-group" { } health_check { - enabled = true - path = "/health" - matcher = "200" + enabled = true + path = "/health" + matcher = "200" interval = 300 } + + tags = var.common_tags } resource "aws_lb_listener" "registry-api-ld-listener" { @@ -55,6 +54,7 @@ resource "aws_lb_listener" "registry-api-ld-listener" { type = "forward" target_group_arn = aws_lb_target_group.pds-registry-api-target-group.arn } + tags = var.common_tags } resource "aws_lb_listener_rule" "pds-registry-forward-rule" { @@ -62,7 +62,7 @@ resource "aws_lb_listener_rule" "pds-registry-forward-rule" { action { type = "forward" - target_group_arn = aws_lb_target_group.pds-registry-api-target-group.arn + target_group_arn = aws_lb_target_group.pds-registry-api-target-group.arn } # no condition for now @@ -70,49 +70,70 @@ resource "aws_lb_listener_rule" "pds-registry-forward-rule" { # used for multiple back-end service condition { path_pattern { - values = ["/*"] + values = ["/*"] } } } -# Define the cluster -resource "aws_ecs_cluster" "pds-registry-api-ecs" { - name = "pds-${var.venue}-registry-api-ecs" - tags = { - Alfa = var.node_name_abbr - Bravo = var.venue - Charlie = "registry" - } +# Credentials for ECR pull through cache from GHCR +resource "aws_secretsmanager_secret" "github_ecr_credentials" { + count = var.create_github_secret_credentials + + name = "ecr-pullthroughcache/github-credentials" + tags = var.common_tags +} + +resource "aws_secretsmanager_secret_version" "github_ecr_credentials" { + count = var.create_github_secret_credentials + + secret_id = aws_secretsmanager_secret.github_ecr_credentials[count.index].id + secret_string = jsonencode({ + username = var.github_username + accessToken = var.github_token + }) } -# Do we need individual dev/test/prod repositories? -# I don't think we do, but then we need to use prod account instead of the dev account, would that work ? -data "aws_ecr_repository" "pds-registry-api-service" { - name = "pds-registry-api-service" +# Look up the secret when it is not created by this script +data "aws_secretsmanager_secret" "github_ecr_credentials" { + count = 1 - var.create_github_secret_credentials + name = "ecr-pullthroughcache/github-credentials" +} + +locals { + github_ecr_credentials_arn = var.create_github_secret_credentials == 1 ? aws_secretsmanager_secret.github_ecr_credentials[0].arn : data.aws_secretsmanager_secret.github_ecr_credentials[0].arn +} + +# Add a Pull Through Cache rule for GHCR +resource "aws_ecr_pull_through_cache_rule" "ghcr" { + ecr_repository_prefix = "ghcr" + upstream_registry_url = "ghcr.io" + credential_arn = local.github_ecr_credentials_arn +} + +resource "aws_ecr_repository" "ghcr_registry_api" { + name = "ghcr/nasa-pds/registry-api" + tags = var.common_tags } # Log groups hold logs from our app. resource "aws_cloudwatch_log_group" "pds-registry-log-group" { - name = "/ecs/pds-${var.venue}-registry-api-svc-task" + name = "/ecs/pds-registry-api-task" - tags = { - Alfa = var.node_name_abbr - Bravo = var.venue - Charlie = "registry" - } + tags = var.common_tags } # The task definition for app. resource "aws_ecs_task_definition" "pds-registry-ecs-task" { - family = "pds-${var.venue}-registry-api-svc-task" + family = "pds-registry-api-task" + skip_destroy = true container_definitions = <"] +aws_fg_subnets = ["subnet-","subnet-"] +aws_lb_subnets = ["subnet-", "subnet-"] +aws_acm_certificate_arn = "arn:aws:acm:::certificate/" +ecs_task_role = "arn:aws:iam:::role/" +ecs_task_execution_role = "arn:aws:iam:::role/" + +# GitHub credentials for ECR pull through cache +github_username = "" +github_token = "" + +cloudfront_dns = "www.example.com" + +# Common tags applied to all resources +common_tags = { + tenant = "" + venue = "" + component = "registry" + cicd = "iac" + managedby = "" +} diff --git a/terraform/variables.tf b/terraform/variables.tf index bb82829d..b2ca0c11 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -1,16 +1,11 @@ variable "node_name_abbr" { description = "Node name abbreviation" - default="en" -} - -variable "venue" { - description = "Deployment venue (prod, test, dev)" - default = "delta" + default = "en" } variable "aws_region" { description = "AWS Region" - default = "us-west-2" + default = "us-west-2" } variable "spring_boot_args" { @@ -19,7 +14,7 @@ variable "spring_boot_args" { variable "aws_profile" { description = "AWS profile" - default = "default" + default = "" } variable "aws_fg_vpc" { @@ -28,17 +23,22 @@ variable "aws_fg_vpc" { variable "aws_fg_security_groups" { description = "AWS Security groups for Fargate" - type = list(string) + type = list(string) +} + +variable "aws_lb_security_groups" { + description = "AWS Security groups for Fargate" + type = list(string) } variable "aws_fg_subnets" { description = "AWS Subnets for Fargate" - type = list(string) + type = list(string) } variable "aws_lb_subnets" { description = "AWS Subnets for the load balancer" - type = list(string) + type = list(string) } variable "ecs_task_role" { @@ -49,7 +49,7 @@ variable "ecs_task_execution_role" { description = "ECS task execution role" } -variable "aws_fg_image" { +variable "registry_api_docker_image" { description = "AWS image name for Fargate" } @@ -59,14 +59,54 @@ variable "aws_s3_bucket_logs_id" { variable "aws_fg_cpu_units" { description = "CPU Units for fargate" - default = 256 + default = 256 } variable "aws_fg_ram_units" { description = "RAM Units for Fargate" - default = 512 + default = 512 } variable "aws_acm_certificate_arn" { description = "ACM SSL Certificate for the load balancer" } + +variable "component_name" { + description = "Component this subcomponents belongs to" + type = string + default = "registry" +} + +variable "common_tags" { + description = "Common tags to apply to all resources" + type = map(string) + default = { + Project = "registry" + ManagedBy = "terraform" + } +} + +variable "create_github_secret_credentials" { + description = "Whether to create GitHub secret credentials (1) or not (0)" + type = number + default = 0 +} + +# TODO remove as the ECR cache it is used for does not work, +# besides we would like to configure it in a infra module instead of this specific registry-api module +variable "github_username" { + description = "GitHub username for ECR pull through cache" + default = "" +} + +# TODO remove as the ECR cache it is used for does not work, +# besides we would like to configure it in a infra module instead of this specific registry-api module +variable "github_token" { + description = "GitHub personal access token for ECR pull through cache" + sensitive = true + default = "" +} + +variable "cloudfront_dns" { + description = "DNS of the cloudfront distribution giving access to the API" +} \ No newline at end of file