From e99e0e8f1322fa302747c7e2542d85d43630948f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 28 Mar 2026 13:47:33 +0000 Subject: [PATCH 1/3] Initial plan From 82bcf725aeb479fc46300b04cc7cc821e95ddbe9 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 28 Mar 2026 13:56:00 +0000 Subject: [PATCH 2/3] Fix set-archive-status: resolve bare LIDs, warn on empty bundle cascade, improve help text Agent-Logs-Url: https://github.com/NASA-PDS/registry-loader/sessions/ce52854d-a13e-4593-a0c6-86051ec310eb Co-authored-by: jordanpadams <33492486+jordanpadams@users.noreply.github.com> --- .../common/es/service/ProductService.java | 46 +++- .../test/java/service/TestProductService.java | 242 ++++++++++++++++++ .../mgr/cmd/data/SetArchiveStatusCmd.java | 6 +- 3 files changed, 283 insertions(+), 11 deletions(-) create mode 100644 common/src/test/java/service/TestProductService.java diff --git a/common/src/main/java/gov/nasa/pds/registry/common/es/service/ProductService.java b/common/src/main/java/gov/nasa/pds/registry/common/es/service/ProductService.java index 3776aab1..2721bcb4 100644 --- a/common/src/main/java/gov/nasa/pds/registry/common/es/service/ProductService.java +++ b/common/src/main/java/gov/nasa/pds/registry/common/es/service/ProductService.java @@ -2,6 +2,7 @@ import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Set; import java.util.TreeSet; @@ -36,8 +37,10 @@ public void updateArchiveStatus(List lidvids, String status) throws Exce } /** * Set archive status - * @param lidvid ID of a product to update. If it is a collection, - * update primary references from collection inventory. + * @param lidvid LID or LIDVID of a product to update. If a bare LID is provided it is + * resolved to the latest LIDVID. If the product is a collection, primary references from + * collection inventory are also updated. If it is a bundle, all referenced collections and + * their products are updated. * @param status new status * @throws Exception an exception */ @@ -46,26 +49,44 @@ public void updateArchiveStatus(String lidvid, String status) throws Exception log.info("Setting product status and its references if bundle or collection. LIDVID = " + lidvid + ", status = " + status); int total = 1; + String resolvedLidvid = lidvid; String pClass = dao.getProductClass(lidvid); - if(pClass == null) + if(pClass == null) { - log.warn("Unknown LIDVID: " + lidvid); - return; + // If the input has no version component it may be a bare LID; try to resolve it. + if(!lidvid.contains("::")) + { + List resolved = dao.getLatestLidVids(Collections.singletonList(lidvid)); + if(resolved != null && !resolved.isEmpty()) + { + resolvedLidvid = resolved.get(0); + log.info("Resolved bare LID " + lidvid + " to LIDVID " + resolvedLidvid); + pClass = dao.getProductClass(resolvedLidvid); + } + } + + if(pClass == null) + { + throw new Exception("Unknown LID/LIDVID: " + lidvid + + ". Verify that the identifier exists in the registry and that a full" + + " LIDVID (e.g. urn:nasa:pds:bundle::1.0) is provided when multiple" + + " versions are present."); + } } - + // Update the product - dao.updateArchiveStatus(Arrays.asList(lidvid), status); + dao.updateArchiveStatus(Arrays.asList(resolvedLidvid), status); // Update collection inventory if("Product_Collection".equals(pClass)) { log.info("Setting status of primary references from collection inventory"); - total += updateCollectionInventory(lidvid, status); + total += updateCollectionInventory(resolvedLidvid, status); } else if("Product_Bundle".equals(pClass)) { // Get collection IDs. There could be both LIDs and LIDVIDs at the same time. - LidvidSet collectionIds = dao.getCollectionIds(lidvid); + LidvidSet collectionIds = dao.getCollectionIds(resolvedLidvid); if(collectionIds == null) return; Set lidvids = new TreeSet(); @@ -74,6 +95,13 @@ else if("Product_Bundle".equals(pClass)) List tmp = dao.getLatestLidVids(collectionIds.lids); if(tmp != null) lidvids.addAll(tmp); + if(lidvids.isEmpty()) + { + log.warn("No collection references found for bundle " + resolvedLidvid + + ". Verify that the bundle document contains 'ref_lid_collection' or" + + " 'ref_lidvid_collection' fields."); + } + total += updateCollections(lidvids, status); } log.info("Updated a total of " + total + " products."); diff --git a/common/src/test/java/service/TestProductService.java b/common/src/test/java/service/TestProductService.java new file mode 100644 index 00000000..0bd4b1cd --- /dev/null +++ b/common/src/test/java/service/TestProductService.java @@ -0,0 +1,242 @@ +package service; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.junit.jupiter.api.Test; + +import gov.nasa.pds.registry.common.es.dao.LidvidSet; +import gov.nasa.pds.registry.common.es.dao.ProductDao; +import gov.nasa.pds.registry.common.es.service.ProductService; + +/** + * Unit tests for {@link ProductService#updateArchiveStatus(String, String)}. + * Uses a stub subclass of {@link ProductDao} to avoid requiring a live Elasticsearch instance. + */ +public class TestProductService { + + // --------------------------------------------------------------------------- + // Stub ProductDao + // --------------------------------------------------------------------------- + + /** + * Minimal stub of ProductDao that allows test cases to control responses. + */ + private static class StubProductDao extends ProductDao { + + /** Map: lidvid -> product class */ + private final java.util.Map productClasses = new java.util.LinkedHashMap<>(); + /** Map: lid -> latest lidvid */ + private final java.util.Map latestLidvids = new java.util.LinkedHashMap<>(); + /** Map: bundleLidvid -> LidvidSet of collection references */ + private final java.util.Map collectionIds = new java.util.LinkedHashMap<>(); + /** Map: collectionLidvid -> page count */ + private final java.util.Map refDocCounts = new java.util.LinkedHashMap<>(); + + /** Track which lidvids were updated and with what status */ + final List updatedLidvids = new ArrayList<>(); + String lastStatus; + + StubProductDao() { + // Pass nulls – the stub overrides all methods, so the real fields are unused. + super(null, null); + } + + void addProduct(String lidvid, String pClass) { + productClasses.put(lidvid, pClass); + } + + void addLatestLidvid(String lid, String lidvid) { + latestLidvids.put(lid, lidvid); + } + + void addCollectionIds(String bundleLidvid, Set lids, Set lidvids) { + collectionIds.put(bundleLidvid, new LidvidSet(lids, lidvids)); + } + + void setRefDocCount(String collectionLidvid, int pages) { + refDocCounts.put(collectionLidvid, pages); + } + + @Override + public String getProductClass(String lidvid) { + return productClasses.get(lidvid); + } + + @Override + public List getLatestLidVids(Collection lids) { + if (lids == null || lids.isEmpty()) return null; + List result = new ArrayList<>(); + for (String lid : lids) { + String resolved = latestLidvids.get(lid); + if (resolved != null) result.add(resolved); + } + return result.isEmpty() ? null : result; + } + + @Override + public LidvidSet getCollectionIds(String bundleLidvid) { + return collectionIds.get(bundleLidvid); + } + + @Override + public int getRefDocCount(String collectionLidVid, char type) { + return refDocCounts.getOrDefault(collectionLidVid, 0); + } + + @Override + public List getRefs(String collectionLidVid, char type, int page) { + // Return empty list; inventory test focus is on the count / status update tracking + return Collections.emptyList(); + } + + @Override + public void updateArchiveStatus(Collection lidvids, String status) { + if (lidvids == null) return; + updatedLidvids.addAll(lidvids); + lastStatus = status; + } + } + + // --------------------------------------------------------------------------- + // Tests for bare LID resolution + // --------------------------------------------------------------------------- + + /** Providing a bare LID that resolves to a known LIDVID should succeed. */ + @Test + void testBareLidResolvesToLatestLidvid() throws Exception { + StubProductDao dao = new StubProductDao(); + String lid = "urn:nasa:pds:my_bundle"; + String lidvid = "urn:nasa:pds:my_bundle::1.0"; + dao.addLatestLidvid(lid, lidvid); + dao.addProduct(lidvid, "Product_Bundle"); + // No collections configured – cascade will warn but not fail + dao.addCollectionIds(lidvid, Collections.emptySet(), Collections.emptySet()); + + ProductService svc = new ProductService(dao); + assertDoesNotThrow(() -> svc.updateArchiveStatus(lid, "archived")); + + // The bundle LIDVID should have been updated, not the bare LID + assertEquals(List.of(lidvid), dao.updatedLidvids); + assertEquals("archived", dao.lastStatus); + } + + /** Providing a bare LID that cannot be resolved should throw a descriptive exception. */ + @Test + void testBareLidNotFoundThrowsException() { + StubProductDao dao = new StubProductDao(); + String lid = "urn:nasa:pds:nonexistent"; + + ProductService svc = new ProductService(dao); + Exception ex = assertThrows(Exception.class, () -> svc.updateArchiveStatus(lid, "archived")); + // Message must mention the original identifier + assertTrue(ex.getMessage().contains(lid), "Expected message to contain '" + lid + "': " + ex.getMessage()); + } + + /** Providing a LIDVID that does not exist should throw a descriptive exception. */ + @Test + void testUnknownLidvidThrowsException() { + StubProductDao dao = new StubProductDao(); + String lidvid = "urn:nasa:pds:my_bundle::99.0"; + + ProductService svc = new ProductService(dao); + Exception ex = assertThrows(Exception.class, () -> svc.updateArchiveStatus(lidvid, "archived")); + assertTrue(ex.getMessage().contains(lidvid), "Expected message to contain '" + lidvid + "': " + ex.getMessage()); + } + + // --------------------------------------------------------------------------- + // Tests for bundle cascade + // --------------------------------------------------------------------------- + + /** Bundle cascade with empty collection references should not throw, but warn. */ + @Test + void testBundleCascadeWithNoCollectionsDoesNotThrow() throws Exception { + StubProductDao dao = new StubProductDao(); + String bundleLidvid = "urn:nasa:pds:my_bundle::1.0"; + dao.addProduct(bundleLidvid, "Product_Bundle"); + dao.addCollectionIds(bundleLidvid, Collections.emptySet(), Collections.emptySet()); + + ProductService svc = new ProductService(dao); + assertDoesNotThrow(() -> svc.updateArchiveStatus(bundleLidvid, "archived")); + + // Only the bundle itself should be updated + assertEquals(List.of(bundleLidvid), dao.updatedLidvids); + } + + /** Bundle cascade should update all collections referenced by LID. */ + @Test + void testBundleCascadeWithCollectionLidUpdatesCollections() throws Exception { + StubProductDao dao = new StubProductDao(); + String bundleLidvid = "urn:nasa:pds:my_bundle::1.0"; + String collectionLid = "urn:nasa:pds:my_collection"; + String collectionLidvid = "urn:nasa:pds:my_collection::1.0"; + + dao.addProduct(bundleLidvid, "Product_Bundle"); + dao.addCollectionIds(bundleLidvid, new HashSet<>(Arrays.asList(collectionLid)), Collections.emptySet()); + dao.addLatestLidvid(collectionLid, collectionLidvid); + dao.addProduct(collectionLidvid, "Product_Collection"); + // No primary products in inventory (count = 0, so just the collection itself) + + ProductService svc = new ProductService(dao); + assertDoesNotThrow(() -> svc.updateArchiveStatus(bundleLidvid, "archived")); + + // Bundle and the resolved collection LIDVID must be updated + assertTrue(dao.updatedLidvids.contains(bundleLidvid), "Bundle not updated"); + assertTrue(dao.updatedLidvids.contains(collectionLidvid), "Collection not updated"); + } + + /** Bundle cascade should update collections referenced by full LIDVID. */ + @Test + void testBundleCascadeWithCollectionLidvidUpdatesCollection() throws Exception { + StubProductDao dao = new StubProductDao(); + String bundleLidvid = "urn:nasa:pds:my_bundle::1.0"; + String collectionLidvid = "urn:nasa:pds:my_collection::2.0"; + + dao.addProduct(bundleLidvid, "Product_Bundle"); + dao.addCollectionIds(bundleLidvid, Collections.emptySet(), + new HashSet<>(Arrays.asList(collectionLidvid))); + dao.addProduct(collectionLidvid, "Product_Collection"); + + ProductService svc = new ProductService(dao); + assertDoesNotThrow(() -> svc.updateArchiveStatus(bundleLidvid, "archived")); + + assertTrue(dao.updatedLidvids.contains(bundleLidvid), "Bundle not updated"); + assertTrue(dao.updatedLidvids.contains(collectionLidvid), "Collection not updated"); + } + + // --------------------------------------------------------------------------- + // Tests for collection cascade + // --------------------------------------------------------------------------- + + /** Collection update should cascade to primary products when inventory exists. */ + @Test + void testCollectionCascadeWithInventory() throws Exception { + StubProductDao dao = new StubProductDao() { + @Override + public List getRefs(String collectionLidVid, char type, int page) { + return Arrays.asList("urn:nasa:pds:my_collection:data:product1::1.0", + "urn:nasa:pds:my_collection:data:product2::1.0"); + } + }; + String collectionLidvid = "urn:nasa:pds:my_collection::1.0"; + dao.addProduct(collectionLidvid, "Product_Collection"); + dao.setRefDocCount(collectionLidvid, 1); + + ProductService svc = new ProductService(dao); + assertDoesNotThrow(() -> svc.updateArchiveStatus(collectionLidvid, "archived")); + + assertTrue(dao.updatedLidvids.contains(collectionLidvid), "Collection not updated"); + assertTrue(dao.updatedLidvids.contains("urn:nasa:pds:my_collection:data:product1::1.0"), "Product1 not updated"); + assertTrue(dao.updatedLidvids.contains("urn:nasa:pds:my_collection:data:product2::1.0"), "Product2 not updated"); + } +} diff --git a/manager/src/main/java/gov/nasa/pds/registry/mgr/cmd/data/SetArchiveStatusCmd.java b/manager/src/main/java/gov/nasa/pds/registry/mgr/cmd/data/SetArchiveStatusCmd.java index 2e4ad1eb..f48f653b 100644 --- a/manager/src/main/java/gov/nasa/pds/registry/mgr/cmd/data/SetArchiveStatusCmd.java +++ b/manager/src/main/java/gov/nasa/pds/registry/mgr/cmd/data/SetArchiveStatusCmd.java @@ -118,8 +118,10 @@ public void printHelp() { System.out.println(" " + name); } - System.out.println(" -lidvid Update archive status of a document with given LIDVID."); - System.out.println(" For a collection also update primary references from collection inventory."); + System.out.println(" -lidvid Update archive status of a product with the given LID or LIDVID."); + System.out.println(" A bare LID is automatically resolved to the latest LIDVID."); + System.out.println(" For a bundle, all referenced collections and their products are also updated."); + System.out.println(" For a collection, all primary products in the collection inventory are also updated."); System.out.println(); } From ee2c8d1d46942a7d9b13a684dead6c05eb62c6b9 Mon Sep 17 00:00:00 2001 From: Jordan Padams Date: Thu, 2 Apr 2026 18:01:43 -0700 Subject: [PATCH 3/3] Refactor: improve test organization and bundle cascade handling - Move TestProductService to proper package (gov.nasa.pds.registry.common.es.service) - Fix early return in bundle cascade that skipped final log message - Handle null collectionIds gracefully to ensure consistent logging These changes improve code organization and ensure the "Updated a total of X products" message is always logged regardless of the bundle cascade outcome. Co-Authored-By: Claude Opus 4.6 --- CLAUDE.md | 111 ++++++++++++++++++ .../common/es/service/ProductService.java | 16 +-- .../es}/service/TestProductService.java | 2 +- 3 files changed, 121 insertions(+), 8 deletions(-) create mode 100644 CLAUDE.md rename common/src/test/java/{ => gov/nasa/pds/registry/common/es}/service/TestProductService.java (99%) diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..6d95dcad --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,111 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +Registry Loader is a NASA Planetary Data System (PDS) toolset for loading data into the PDS Registry (Elasticsearch/OpenSearch-based). It combines three Maven modules into a single multi-module project: + +- **common** (`registry-common`) - Shared library for Elasticsearch/OpenSearch connectivity, metadata extraction, and data dictionary operations +- **harvest** - CLI tool that crawls file systems to discover PDS4 products and indexes metadata into the Registry +- **manager** (`registry-manager`) - CLI tool for managing the Registry: creating indices, loading data dictionaries, setting archive status, and data operations + +## Build Commands + +```bash +# Build all modules +mvn package + +# Build specific module +mvn -pl common package +mvn -pl harvest package +mvn -pl manager package + +# Run tests +mvn test + +# Run single test class +mvn -pl common test -Dtest=TestBulkResponseParser + +# Skip tests +mvn package -DskipTests + +# Clean build +mvn clean package + +# Install to local repository +mvn install + +# Deploy release (requires GPG setup) +mvn -P release clean deploy +``` + +## Running the Tools + +After building, executable JARs are in `{module}/target/`: + +```bash +# Harvest - requires config file +java -jar harvest/target/harvest-*.jar -c + +# Registry Manager - various subcommands +java -jar manager/target/registry-manager-*.jar + +# Registry Manager commands: +# create-registry, delete-registry +# list-dd, load-dd, delete-dd, export-dd, upgrade-dd +# delete-data, export-file, set-archive-status, update-alt-ids +``` + +## Architecture + +### Module Dependencies +``` +harvest ──────┐ + ├──> common ──> Elasticsearch/OpenSearch +manager ──────┘ +``` + +### Key Packages + +**common** (`gov.nasa.pds.registry.common`): +- `connection/` - Registry connection handling (AWS OpenSearch Serverless, direct ES/OS) +- `connection/aws/` - AWS-specific implementations using OpenSearch SDK +- `connection/es/` - Standard Elasticsearch REST client implementations +- `es/dao/` - Data Access Objects for registry operations +- `es/service/` - High-level services (schema updates, data loading) +- `meta/` - Metadata extractors for PDS4 labels +- `dd/` - Data dictionary parsing and loading + +**harvest** (`gov.nasa.pds.harvest`): +- `HarvestCli` - CLI entry point +- `cmd/` - Command implementations +- `cfg/` - Configuration parsing + +**manager** (`gov.nasa.pds.registry.mgr`): +- `RegistryManagerCli` - CLI entry point with command dispatcher +- `cmd/reg/` - Registry management commands +- `cmd/dd/` - Data dictionary commands +- `cmd/data/` - Data manipulation commands + +### Connection Configuration + +Registry connections use XML configuration files. The `common` module provides: +- Direct connections to Elasticsearch/OpenSearch +- AWS OpenSearch Serverless with Cognito authentication +- Connection factory pattern via `EstablishConnectionFactory` + +## Docker + +Build Docker image containing both tools: +```bash +docker image build -t nasapds/registry-loader -f docker/Dockerfile \ + --build-arg harvest_package_path=harvest/target/harvest-*-bin.tar.gz \ + --build-arg reg_manager_package_path=manager/target/registry-manager-*-bin.tar.gz . +``` + +## CI/CD + +- Releases triggered by pushing `release/*` tags +- Uses NASA-PDS Roundup Action for automated releases +- Docker images published to Docker Hub on stable releases diff --git a/common/src/main/java/gov/nasa/pds/registry/common/es/service/ProductService.java b/common/src/main/java/gov/nasa/pds/registry/common/es/service/ProductService.java index 2721bcb4..5c66bfc4 100644 --- a/common/src/main/java/gov/nasa/pds/registry/common/es/service/ProductService.java +++ b/common/src/main/java/gov/nasa/pds/registry/common/es/service/ProductService.java @@ -87,13 +87,15 @@ else if("Product_Bundle".equals(pClass)) { // Get collection IDs. There could be both LIDs and LIDVIDs at the same time. LidvidSet collectionIds = dao.getCollectionIds(resolvedLidvid); - if(collectionIds == null) return; - - Set lidvids = new TreeSet(); - if(collectionIds.lidvids != null) lidvids.addAll(collectionIds.lidvids); - - List tmp = dao.getLatestLidVids(collectionIds.lids); - if(tmp != null) lidvids.addAll(tmp); + + Set lidvids = new TreeSet(); + if(collectionIds != null) + { + if(collectionIds.lidvids != null) lidvids.addAll(collectionIds.lidvids); + + List tmp = dao.getLatestLidVids(collectionIds.lids); + if(tmp != null) lidvids.addAll(tmp); + } if(lidvids.isEmpty()) { diff --git a/common/src/test/java/service/TestProductService.java b/common/src/test/java/gov/nasa/pds/registry/common/es/service/TestProductService.java similarity index 99% rename from common/src/test/java/service/TestProductService.java rename to common/src/test/java/gov/nasa/pds/registry/common/es/service/TestProductService.java index 0bd4b1cd..086d1a05 100644 --- a/common/src/test/java/service/TestProductService.java +++ b/common/src/test/java/gov/nasa/pds/registry/common/es/service/TestProductService.java @@ -1,4 +1,4 @@ -package service; +package gov.nasa.pds.registry.common.es.service; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals;