From 86361f264cee91e04a720bf640451046bee1846d Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Thu, 23 Oct 2025 09:52:53 -0700 Subject: [PATCH 001/137] existance Updated the language to allow for finding documents that have a specific field name. If the FIELD is given, then it assumes that an exact match is being requested. If a STRINGVAL is given, then it treats it as a regular expression. Processing regular expressions collects all of the known field names from the mapping and pulls out all matching field names. It then creates an existance check for each of the matching field names. --- .../gov/nasa/pds/api/registry/lexer/Search.g4 | 4 +- .../api_search_query_lexer/TestParsing.java | 13 ++++++ .../registry/model/Antlr4SearchListener.java | 44 ++++++++++++++----- 3 files changed, 49 insertions(+), 12 deletions(-) diff --git a/lexer/src/main/antlr4/gov/nasa/pds/api/registry/lexer/Search.g4 b/lexer/src/main/antlr4/gov/nasa/pds/api/registry/lexer/Search.g4 index dbccf1f7..1af8dfdb 100644 --- a/lexer/src/main/antlr4/gov/nasa/pds/api/registry/lexer/Search.g4 +++ b/lexer/src/main/antlr4/gov/nasa/pds/api/registry/lexer/Search.g4 @@ -1,8 +1,9 @@ grammar Search; query : queryTerm EOF ; -queryTerm : comparison | likeComparison | group ; +queryTerm : comparison | likeComparison | existence | group ; group : NOT? LPAREN expression RPAREN ; +existence : ( FIELD | STRINGVAL ) EXISTS ; expression : andStatement | orStatement | queryTerm ; andStatement : queryTerm (AND queryTerm)+ ; orStatement : queryTerm (OR queryTerm)+ ; @@ -19,6 +20,7 @@ GE : G E ; LT : L T ; LE : L E ; +EXISTS: E X I S T S; LIKE: L I K E; LPAREN : '(' ; diff --git a/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/TestParsing.java b/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/TestParsing.java index 74775f2d..181f10f9 100644 --- a/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/TestParsing.java +++ b/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/TestParsing.java @@ -18,6 +18,7 @@ import gov.nasa.pds.api.registry.lexer.SearchParser; import gov.nasa.pds.api.registry.lexer.SearchParser.AndStatementContext; import gov.nasa.pds.api.registry.lexer.SearchParser.ComparisonContext; +import gov.nasa.pds.api.registry.lexer.SearchParser.ExistenceContext; import gov.nasa.pds.api.registry.lexer.SearchParser.ExpressionContext; import gov.nasa.pds.api.registry.lexer.SearchParser.GroupContext; import gov.nasa.pds.api.registry.lexer.SearchParser.LikeComparisonContext; @@ -244,5 +245,17 @@ public void exitLikeComparison(LikeComparisonContext ctx) { } + @Override + public void enterExistence(ExistenceContext ctx) { + // TODO Auto-generated method stub + + } + + @Override + public void exitExistence(ExistenceContext ctx) { + // TODO Auto-generated method stub + + } + } diff --git a/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java b/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java index f5fb4fd3..e3a46beb 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java @@ -2,33 +2,25 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import gov.nasa.pds.api.registry.lexer.SearchBaseListener; import gov.nasa.pds.api.registry.lexer.SearchParser; import java.util.ArrayDeque; import java.util.ArrayList; -import java.util.Arrays; import java.util.Deque; import java.util.List; +import java.util.regex.Pattern; import java.util.stream.Collectors; import org.antlr.v4.runtime.misc.ParseCancellationException; import org.opensearch.client.json.JsonData; import org.opensearch.client.opensearch._types.FieldValue; import org.opensearch.client.opensearch._types.query_dsl.BoolQuery; +import org.opensearch.client.opensearch._types.query_dsl.ExistsQuery; import org.opensearch.client.opensearch._types.query_dsl.MatchQuery; import org.opensearch.client.opensearch._types.query_dsl.Query; import org.opensearch.client.opensearch._types.query_dsl.RangeQuery; import org.opensearch.client.opensearch._types.query_dsl.SimpleQueryStringQuery; -import org.opensearch.client.opensearch._types.query_dsl.TermsQuery; -import org.opensearch.client.opensearch._types.query_dsl.TermsQueryField; -import org.opensearch.client.opensearch._types.query_dsl.TermsSetQuery; -import org.opensearch.client.opensearch._types.query_dsl.WildcardQuery; -import org.opensearch.client.opensearch._types.query_dsl.Operator; -import org.opensearch.index.query.RangeQueryBuilder; -import org.opensearch.index.query.SimpleQueryStringBuilder; -import org.opensearch.index.query.TermQueryBuilder; -import org.opensearch.index.query.QueryBuilder; + public class Antlr4SearchListener extends SearchBaseListener { enum conjunctions { @@ -178,6 +170,36 @@ else if (this.operator == operation.lt) } + @Override + public void exitExistence(SearchParser.ExistenceContext ctx) { + ArrayList checks = new ArrayList(); + final String fieldName = SearchUtil.jsonPropertyToOpenProperty(ctx.FIELD().getSymbol().getText()); + final String regexp = ctx.STRINGVAL().getText(); + String theKey = "''"; + + if (fieldName != null && !fieldName.isBlank()) { + theKey = fieldName; + checks.add(new ExistsQuery.Builder().field(fieldName).build().toQuery()); + } else if (regexp != null && !regexp.isBlank()) { + theKey = regexp; + List fieldNames = new ArrayList(); // FIXME: need to get the mapping for field names here + Pattern regex = Pattern.compile(regexp); + for (String fn : fieldNames.stream() + .flatMap(s -> regex.matcher(s).results()) + .map(matchResults -> matchResults.group()) + .collect(Collectors.toList())) { + checks.add(new ExistsQuery.Builder().field(fn).build().toQuery()); + } + if (checks.isEmpty()) + throw new ParseCancellationException("For existence testing, cannot match any field names to " + theKey); + } + if (this.conjunction == conjunctions.AND) { + this.queryBuilder.must(checks); + } else { + this.queryBuilder.should(checks); + } + } + @Override public void enterLikeComparison(SearchParser.LikeComparisonContext ctx) {} From 9a34e2d8e1eec17c13ccd8094f696ce9117e8802 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Wed, 29 Oct 2025 08:56:26 -0700 Subject: [PATCH 002/137] get mappings from opensearch Use the functionality in ProductsController to collect the property mappings. Use the names in mappings to then find the set of matches for the exists. --- .../controllers/ProductsController.java | 10 ++++--- .../registry/model/Antlr4SearchListener.java | 26 +++++++++++++++---- .../search/RegistrySearchRequestBuilder.java | 6 ++--- 3 files changed, 30 insertions(+), 12 deletions(-) diff --git a/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java b/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java index 562fb1ea..fbeac719 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java @@ -3,7 +3,6 @@ import java.lang.reflect.InvocationTargetException; import java.io.IOException; import java.util.*; -import java.util.stream.Collectors; import gov.nasa.pds.api.base.ClassesApi; import gov.nasa.pds.api.base.PropertiesApi; import gov.nasa.pds.api.registry.model.exceptions.*; @@ -598,7 +597,7 @@ public ResponseEntity> classes() throws Exception { /** * Resolve the appropriate enumerated user type hint from an OpenSearch Property */ - protected PropertiesListInner.TypeEnum _resolvePropertyToEnumType(Property property) { + protected static PropertiesListInner.TypeEnum _resolvePropertyToEnumType(Property property) { if (property.isBoolean()) { return PropertiesListInner.TypeEnum.BOOLEAN; } else if (property.isKeyword() || property.isText()) { @@ -616,12 +615,15 @@ protected PropertiesListInner.TypeEnum _resolvePropertyToEnumType(Property prope @Override public ResponseEntity> productPropertiesList() throws Exception { + return ProductsController.productPropertiesList(this.connectionContext); + } + public static ResponseEntity> productPropertiesList(ConnectionContext connectionContext) throws OpenSearchException, IOException { - List indexNames = this.connectionContext.getRegistryIndices(); + List indexNames = connectionContext.getRegistryIndices(); GetMappingRequest getMappingRequest = new GetMappingRequest.Builder().index(indexNames).build(); - OpenSearchIndicesClient indicesClient = this.openSearchClient.indices(); + OpenSearchIndicesClient indicesClient = connectionContext.getOpenSearchClient().indices(); GetMappingResponse getMappingResponse = indicesClient.getMapping(getMappingRequest); diff --git a/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java b/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java index e3a46beb..9acfd61f 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java @@ -2,16 +2,21 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import gov.nasa.pds.api.registry.ConnectionContext; +import gov.nasa.pds.api.registry.controllers.ProductsController; import gov.nasa.pds.api.registry.lexer.SearchBaseListener; import gov.nasa.pds.api.registry.lexer.SearchParser; - +import gov.nasa.pds.model.PropertiesListInner; +import java.io.IOException; import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Deque; -import java.util.List; +import java.util.HashSet; +import java.util.Set; import java.util.regex.Pattern; import java.util.stream.Collectors; import org.antlr.v4.runtime.misc.ParseCancellationException; +import org.opensearch.OpenSearchException; import org.opensearch.client.json.JsonData; import org.opensearch.client.opensearch._types.FieldValue; import org.opensearch.client.opensearch._types.query_dsl.BoolQuery; @@ -36,13 +41,16 @@ enum operation { private BoolQuery.Builder queryBuilder = new BoolQuery.Builder(); private conjunctions conjunction = conjunctions.AND; // DEFAULT + final private ConnectionContext connectionContext; final private Deque stackQueryBuilders = new ArrayDeque(); final private Deque stack_conjunction = new ArrayDeque(); + final private Set knownFieldNames = new HashSet(); private operation operator = null; - public Antlr4SearchListener() { + public Antlr4SearchListener(ConnectionContext connectionContext) { super(); + this.connectionContext = connectionContext; } @@ -182,9 +190,17 @@ public void exitExistence(SearchParser.ExistenceContext ctx) { checks.add(new ExistsQuery.Builder().field(fieldName).build().toQuery()); } else if (regexp != null && !regexp.isBlank()) { theKey = regexp; - List fieldNames = new ArrayList(); // FIXME: need to get the mapping for field names here + if (knownFieldNames.isEmpty()) { + try { + for (PropertiesListInner property : ProductsController.productPropertiesList(this.connectionContext).getBody()) { + knownFieldNames.add(property.getProperty()); + } + } catch (OpenSearchException | IOException e) { + log.error("Could not load the mapping(s) from opensearch; meaning 'exists' will not work"); + } + } Pattern regex = Pattern.compile(regexp); - for (String fn : fieldNames.stream() + for (String fn : knownFieldNames.stream() .flatMap(s -> regex.matcher(s).results()) .map(matchResults -> matchResults.group()) .collect(Collectors.toList())) { diff --git a/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java b/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java index 07d8ede0..b12b5d34 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java @@ -356,7 +356,7 @@ public RegistrySearchRequestBuilder fieldsFromPdsProperties(List pd - private static BoolQuery parseQueryString(String queryString) { + private BoolQuery parseQueryString(String queryString) { CodePointCharStream input = CharStreams.fromString(queryString); SearchLexer lex = new SearchLexer(input); CommonTokenStream tokens = new CommonTokenStream(lex); @@ -369,7 +369,7 @@ private static BoolQuery parseQueryString(String queryString) { // Walk it and attach our listener ParseTreeWalker walker = new ParseTreeWalker(); - Antlr4SearchListener listener = new Antlr4SearchListener(); + Antlr4SearchListener listener = new Antlr4SearchListener(this.connectionContext); walker.walk(listener, tree); return listener.getBoolQuery(); @@ -386,7 +386,7 @@ public RegistrySearchRequestBuilder constrainByQueryString(String q) try { if ((q != null) && (q.length() > 0)) { - BoolQuery qBoolQuery = RegistrySearchRequestBuilder.parseQueryString(q); + BoolQuery qBoolQuery = this.parseQueryString(q); this.queryBuilder.must(qBoolQuery.toQuery()); } return this; From ae05410d20f7f2bbf22e78034cf96555142e8eb6 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Wed, 29 Oct 2025 09:22:20 -0700 Subject: [PATCH 003/137] sonar cleanup --- .../gov/api_search_query_lexer/TestParsing.java | 12 ++++++------ .../registry/controllers/ProductsController.java | 13 ++++++------- .../api/registry/model/Antlr4SearchListener.java | 12 ++++++------ .../search/RegistrySearchRequestBuilder.java | 13 ++++++------- 4 files changed, 24 insertions(+), 26 deletions(-) diff --git a/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/TestParsing.java b/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/TestParsing.java index 181f10f9..5fa7aa40 100644 --- a/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/TestParsing.java +++ b/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/TestParsing.java @@ -46,10 +46,10 @@ public void testNumber() { walker.walk(this, tree); Assertions.assertNotNull(this.field); - Assertions.assertEquals(this.field.getSymbol().getText(), "lid"); + Assertions.assertEquals("lid", this.field.getSymbol().getText()); Assertions.assertNotEquals(this.number, null); - Assertions.assertEquals(this.number.getSymbol().getText(), "1234"); + Assertions.assertEquals("1234", this.number.getSymbol().getText()); } @@ -65,10 +65,10 @@ public void testStringVal() { walker.walk(this, tree); Assertions.assertNotNull(this.field); - Assertions.assertEquals(this.field.getSymbol().getText(), "lid"); + Assertions.assertEquals("lid", this.field.getSymbol().getText()); Assertions.assertNotNull(this.strval); - Assertions.assertEquals(this.strval.getSymbol().getText(), "\"*text*\""); + Assertions.assertEquals("\"*text*\"", this.strval.getSymbol().getText()); } @@ -84,10 +84,10 @@ public void testLike() { walker.walk(this, tree); Assertions.assertNotNull(this.field); - Assertions.assertEquals(this.field.getText(), "lid"); + Assertions.assertEquals("lid", this.field.getText()); Assertions.assertNotNull(this.strval); - Assertions.assertEquals(this.strval.getText(), "\"*text*\""); + Assertions.assertEquals("\"*text*\"", this.strval.getText()); } diff --git a/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java b/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java index fbeac719..87f4a92c 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java @@ -274,7 +274,7 @@ private HashMap getLidVid(PdsProductIdentifier identifier, throw new NotFoundException("No product found with identifier " + identifier.toString()); } HashMap product = searchResponse.hits().hits().get(0).source(); - ProductsController.log.debug("Found product with lid=" + product.get("lid")); + ProductsController.log.debug("Found product with lid={}", product.get("lid")); return product; } @@ -300,8 +300,8 @@ private HashMap getLatestLidVid(PdsProductIdentifier identifier, } HashMap product = searchResponse.hits().hits().get(0).source(); - ProductsController.log.debug("Found product with lid=" + product.get("lid")); - return (HashMap) searchResponse.hits().hits().get(0).source(); + ProductsController.log.debug("Found product with lid={}", product.get("lid")); + return searchResponse.hits().hits().get(0).source(); } @@ -377,8 +377,7 @@ private PdsLidVid resolveLatestLidvid(PdsProductIdentifier identifier) * @throws AcceptFormatNotSupportedException */ private PdsLidVid resolveIdentifierToLidvid(PdsProductIdentifier identifier) - throws NotFoundException, IOException, AcceptFormatNotSupportedException, UnhandledException, - OpenSearchException { + throws NotFoundException, IOException, OpenSearchException { return identifier.isLidvid() ? (PdsLidVid) identifier : resolveLatestLidvid(identifier); } @@ -597,7 +596,7 @@ public ResponseEntity> classes() throws Exception { /** * Resolve the appropriate enumerated user type hint from an OpenSearch Property */ - protected static PropertiesListInner.TypeEnum _resolvePropertyToEnumType(Property property) { + protected static PropertiesListInner.TypeEnum resolvePropertyToEnumType(Property property) { if (property.isBoolean()) { return PropertiesListInner.TypeEnum.BOOLEAN; } else if (property.isKeyword() || property.isText()) { @@ -636,7 +635,7 @@ public static ResponseEntity> productPropertiesList(Co String jsonPropertyName = PdsProperty.toJsonPropertyString(property.getKey()); Property openPropertyName = property.getValue(); PropertiesListInner.TypeEnum propertyEnumType = - _resolvePropertyToEnumType(openPropertyName); + resolvePropertyToEnumType(openPropertyName); // No consistency-checking between duplicates, for now. TODO: add error log for mismatching // duplicates diff --git a/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java b/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java index 9acfd61f..c90fc248 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java @@ -41,10 +41,10 @@ enum operation { private BoolQuery.Builder queryBuilder = new BoolQuery.Builder(); private conjunctions conjunction = conjunctions.AND; // DEFAULT - final private ConnectionContext connectionContext; - final private Deque stackQueryBuilders = new ArrayDeque(); - final private Deque stack_conjunction = new ArrayDeque(); - final private Set knownFieldNames = new HashSet(); + private final ConnectionContext connectionContext; + private final Deque stackQueryBuilders = new ArrayDeque(); + private final Deque stack_conjunction = new ArrayDeque(); + private final Set knownFieldNames = new HashSet(); private operation operator = null; @@ -196,7 +196,7 @@ public void exitExistence(SearchParser.ExistenceContext ctx) { knownFieldNames.add(property.getProperty()); } } catch (OpenSearchException | IOException e) { - log.error("Could not load the mapping(s) from opensearch; meaning 'exists' will not work"); + log.error("Could not load the mapping(s) from opensearch; meaning 'exists' will not work", e); } } Pattern regex = Pattern.compile(regexp); @@ -232,7 +232,7 @@ public void exitLikeComparison(SearchParser.LikeComparisonContext ctx) { .query(right).fuzzyMaxExpansions(0).build(); Query query = simpleQueryString.toQuery(); - log.debug("Exit Like comparison: left member is " + left + " right member is " + right); + log.debug("Exit Like comparison: left member is {} right member is {}", left, right); if (this.conjunction == conjunctions.AND) { this.queryBuilder.must(query); diff --git a/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java b/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java index b12b5d34..7da9334d 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java @@ -5,7 +5,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.stream.Collectors; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; @@ -21,7 +20,6 @@ import org.antlr.v4.runtime.tree.ParseTreeWalker; import org.antlr.v4.runtime.RecognitionException; import org.antlr.v4.runtime.misc.ParseCancellationException; -import org.apache.commons.lang3.StringUtils; import org.opensearch.client.json.jackson.JacksonJsonpGenerator; import org.opensearch.client.opensearch._types.FieldSort; import org.opensearch.client.opensearch._types.FieldValue; @@ -63,7 +61,7 @@ public RegistrySearchRequestBuilder(ConnectionContext connectionContext) { this.connectionContext = connectionContext; this.registryIndices = this.connectionContext.getRegistryIndices(); - log.info("Use indices: " + String.join(",", registryIndices) + "End indices"); + log.info("Use indices: {}", String.join(",", registryIndices) + "End indices"); this.index(registryIndices); @@ -82,7 +80,7 @@ public RegistrySearchRequestBuilder(ConnectionContext connectionContext) { private static Query getMandatoryBaselineQuery(ConnectionContext connectionContext) { List archiveStatus = connectionContext.getArchiveStatus(); List archiveStatusFieldValues = archiveStatus.stream().map(FieldValue::of).toList(); - log.info("Only publishes archiveStatus: " + String.join(",", archiveStatus)); + log.info("Only publishes archiveStatus: {}", String.join(",", archiveStatus)); TermsQueryField archiveStatusTerms = new TermsQueryField.Builder().value(archiveStatusFieldValues).build(); @@ -136,6 +134,7 @@ public RegistrySearchRequestBuilder applyMultipleProductsDefaults( return this; } + @Override public SearchRequest build() { BoolQuery bQuery = this.queryBuilder.build(); this.query(bQuery.toQuery()); @@ -145,9 +144,9 @@ public SearchRequest build() { try { String requestJson = serializeSearchRequest(searchRequest); - log.debug("Generated OpenSearch SearchRequest with query:\n" + requestJson); + log.debug("Generated OpenSearch SearchRequest with query:\n{}", requestJson); } catch (Exception e) { - log.error("Failed to generate json serialization of SearchRequest: " + e); + log.error("Failed to generate json serialization of SearchRequest: {}", e); } return searchRequest; @@ -391,7 +390,7 @@ public RegistrySearchRequestBuilder constrainByQueryString(String q) } return this; } catch (RecognitionException | ParseCancellationException e) { - log.info("Unable to parse q " + LoggingAspect.sanitizeForLog(q) + "error message is " + e); + log.info("Unable to parse q {} error message is {}", LoggingAspect.sanitizeForLog(q), e); throw new UnparsableQParamException("Invalid q string value syntax " + e.getMessage()); } From bcc4d5f2e319bc686314435dace2eb29bacf78d6 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Wed, 29 Oct 2025 09:34:16 -0700 Subject: [PATCH 004/137] sonar cleanup --- .../pds/nasa/gov/api_search_query_lexer/TestParsing.java | 2 +- .../pds/api/registry/model/Antlr4SearchListener.java | 9 ++++----- .../registry/search/RegistrySearchRequestBuilder.java | 2 +- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/TestParsing.java b/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/TestParsing.java index 5fa7aa40..e4b7d493 100644 --- a/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/TestParsing.java +++ b/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/TestParsing.java @@ -48,7 +48,7 @@ public void testNumber() { Assertions.assertNotNull(this.field); Assertions.assertEquals("lid", this.field.getSymbol().getText()); - Assertions.assertNotEquals(this.number, null); + Assertions.assertNotNull(this.number); Assertions.assertEquals("1234", this.number.getSymbol().getText()); } diff --git a/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java b/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java index c90fc248..a6361838 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java @@ -14,7 +14,6 @@ import java.util.HashSet; import java.util.Set; import java.util.regex.Pattern; -import java.util.stream.Collectors; import org.antlr.v4.runtime.misc.ParseCancellationException; import org.opensearch.OpenSearchException; import org.opensearch.client.json.JsonData; @@ -43,7 +42,7 @@ enum operation { private final ConnectionContext connectionContext; private final Deque stackQueryBuilders = new ArrayDeque(); - private final Deque stack_conjunction = new ArrayDeque(); + private final Deque stackConjunction = new ArrayDeque(); private final Set knownFieldNames = new HashSet(); private operation operator = null; @@ -61,7 +60,7 @@ public void exitQuery(SearchParser.QueryContext ctx) {} public void enterGroup(SearchParser.GroupContext ctx) { log.debug("Enter Group"); - this.stack_conjunction.push(this.conjunction); + this.stackConjunction.push(this.conjunction); this.conjunction = conjunctions.AND; // DEFAULT this.stackQueryBuilders.push(this.queryBuilder); @@ -75,7 +74,7 @@ public void exitGroup(SearchParser.GroupContext ctx) { log.debug("Exit Group"); BoolQuery.Builder upperBoolQueryBuilder = this.stackQueryBuilders.pop(); - this.conjunction = this.stack_conjunction.pop(); + this.conjunction = this.stackConjunction.pop(); Query innerQuery = this.queryBuilder.build().toQuery(); if (ctx.NOT() != null) { @@ -203,7 +202,7 @@ public void exitExistence(SearchParser.ExistenceContext ctx) { for (String fn : knownFieldNames.stream() .flatMap(s -> regex.matcher(s).results()) .map(matchResults -> matchResults.group()) - .collect(Collectors.toList())) { + .toList()) { checks.add(new ExistsQuery.Builder().field(fn).build().toQuery()); } if (checks.isEmpty()) diff --git a/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java b/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java index 7da9334d..e84a81e5 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java @@ -146,7 +146,7 @@ public SearchRequest build() { String requestJson = serializeSearchRequest(searchRequest); log.debug("Generated OpenSearch SearchRequest with query:\n{}", requestJson); } catch (Exception e) { - log.error("Failed to generate json serialization of SearchRequest: {}", e); + log.error("Failed to generate json serialization of SearchRequest: ", e); } return searchRequest; From 718b3b6b1a17917663634f34d2c2bf2438d1930e Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Wed, 29 Oct 2025 09:48:50 -0700 Subject: [PATCH 005/137] missed test dependencies --- .../pds/api/registry/opensearch/Antlr4SearchListenerTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/service/src/test/java/gov/nasa/pds/api/registry/opensearch/Antlr4SearchListenerTest.java b/service/src/test/java/gov/nasa/pds/api/registry/opensearch/Antlr4SearchListenerTest.java index 16af90a6..b2b2e15d 100644 --- a/service/src/test/java/gov/nasa/pds/api/registry/opensearch/Antlr4SearchListenerTest.java +++ b/service/src/test/java/gov/nasa/pds/api/registry/opensearch/Antlr4SearchListenerTest.java @@ -44,7 +44,7 @@ public void execute() { @BeforeEach void setUp() { - listener = new Antlr4SearchListener(); + listener = new Antlr4SearchListener(null); } @@ -57,7 +57,7 @@ private BoolQuery run(String query) { ParseTree tree = par.query(); // Walk it and attach our listener ParseTreeWalker walker = new ParseTreeWalker(); - Antlr4SearchListener listener = new Antlr4SearchListener(); + Antlr4SearchListener listener = new Antlr4SearchListener(null); walker.walk(listener, tree); // System.out.println ("query string: " + query); From bc4b2f94cf7a4c93488f35310d31c7bcdb42a52c Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Wed, 29 Oct 2025 09:54:50 -0700 Subject: [PATCH 006/137] sonar cleanup --- .../opensearch/Antlr4SearchListenerTest.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/service/src/test/java/gov/nasa/pds/api/registry/opensearch/Antlr4SearchListenerTest.java b/service/src/test/java/gov/nasa/pds/api/registry/opensearch/Antlr4SearchListenerTest.java index b2b2e15d..bd67783e 100644 --- a/service/src/test/java/gov/nasa/pds/api/registry/opensearch/Antlr4SearchListenerTest.java +++ b/service/src/test/java/gov/nasa/pds/api/registry/opensearch/Antlr4SearchListenerTest.java @@ -24,7 +24,7 @@ import gov.nasa.pds.api.registry.lexer.SearchParser; import gov.nasa.pds.api.registry.model.Antlr4SearchListener; -public class Antlr4SearchListenerTest { +class Antlr4SearchListenerTest { private class NegativeTester implements Executable { final private Antlr4SearchListenerTest parent; final private String qs; @@ -68,13 +68,13 @@ private BoolQuery run(String query) { @Test - public void testSimpleCompEq() { + void testSimpleCompEq() { String qs = "pds:Time_Coordinates.pds:stop_date_time eq \"2021-05-21T15:47:08Z\""; BoolQuery query = this.run(qs); // TODO: add asserts - Assertions.assertEquals(query.must().size(), 1); + Assertions.assertEquals(1, query.must().size()); Query matchQuery = (Query) query.must().get(0); - Assertions.assertEquals(matchQuery._kind(), Query.Kind.Match); + Assertions.assertEquals(Query.Kind.Match, matchQuery._kind()); // Assertions.assertEquals((matchQuery).field(), "pds:Time_Coordinates/pds:stop_date_time"); @@ -82,7 +82,7 @@ public void testSimpleCompEq() { @Test - public void testLikeWildcard() { + void testLikeWildcard() { String qs = "lid like \"*pdart14_meap\""; BoolQuery query = this.run(qs); // TODO: add asserts @@ -91,7 +91,7 @@ public void testLikeWildcard() { } @Test - public void testEscape() { + void testEscape() { String qs = "lid eq \"*pdart14_meap?\""; BoolQuery query = this.run(qs); @@ -99,7 +99,7 @@ public void testEscape() { } @Test - public void testGroupedStatementAndExclusiveInequality() { + void testGroupedStatementAndExclusiveInequality() { String qs = "( timestamp gt 12 and timestamp lt 27 )"; BoolQuery query = this.run(qs); @@ -107,7 +107,7 @@ public void testGroupedStatementAndExclusiveInequality() { } @Test - public void testGroupedStatementAndInclusiveInequality() { + void testGroupedStatementAndInclusiveInequality() { String qs = "( timestamp_A ge 12 and timestamp_B le 27 )"; BoolQuery query = this.run(qs); @@ -115,7 +115,7 @@ public void testGroupedStatementAndInclusiveInequality() { } @Test - public void testNot() { + void testNot() { String qs = "not ( timestamp ge 12 and timestamp le 27 )"; BoolQuery query = this.run(qs); @@ -124,7 +124,7 @@ public void testNot() { @Test - public void testNestedGrouping() { + void testNestedGrouping() { String qs = "( ( timestamp ge 12 and timestamp le 27 ) or ( timestamp gt 13 and timestamp lt 37 ) )"; @@ -137,7 +137,7 @@ public void testNestedGrouping() { @Test - public void testNoWildcardQuoted() { + void testNoWildcardQuoted() { String qs = "ref_lid_target eq \"urn:nasa:pds:context:target:planet.mercury\""; BoolQuery query = this.run(qs); @@ -145,7 +145,7 @@ public void testNoWildcardQuoted() { } @Test - public void testExceptionsInParsing() { + void testExceptionsInParsing() { NegativeTester actor; String fails[] = {"( a eq b", "a eq b )", "not( a eq b )", "a eq b and c eq d and", "( a eq b and c eq d and )", "( a eq b and c eq d or e eq f )"}; From c9396a6043f0dae2864ed0d6c88fc3d24c5c7858 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Thu, 30 Oct 2025 07:59:19 -0700 Subject: [PATCH 007/137] sonar cleanup --- .../pds/api/registry/opensearch/Antlr4SearchListenerTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/src/test/java/gov/nasa/pds/api/registry/opensearch/Antlr4SearchListenerTest.java b/service/src/test/java/gov/nasa/pds/api/registry/opensearch/Antlr4SearchListenerTest.java index bd67783e..0baa2748 100644 --- a/service/src/test/java/gov/nasa/pds/api/registry/opensearch/Antlr4SearchListenerTest.java +++ b/service/src/test/java/gov/nasa/pds/api/registry/opensearch/Antlr4SearchListenerTest.java @@ -189,7 +189,7 @@ void testEnterOrStatement() { void testExitOrStatement() { listener.enterOrStatement(Mockito.mock(SearchParser.OrStatementContext.class)); listener.exitOrStatement(Mockito.mock(SearchParser.OrStatementContext.class)); - assertTrue(listener.getBoolQuery().minimumShouldMatch() == "1", + assertSame("1", listener.getBoolQuery().minimumShouldMatch(), "Minimum should match should be set"); } From 908e6c14549f8df1565e92ca911aa65d45729b9b Mon Sep 17 00:00:00 2001 From: al-niessner <1130658+al-niessner@users.noreply.github.com> Date: Mon, 3 Nov 2025 08:48:14 -0800 Subject: [PATCH 008/137] Update service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../gov/nasa/pds/api/registry/model/Antlr4SearchListener.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java b/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java index a6361838..d9ff09ac 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java @@ -200,8 +200,7 @@ public void exitExistence(SearchParser.ExistenceContext ctx) { } Pattern regex = Pattern.compile(regexp); for (String fn : knownFieldNames.stream() - .flatMap(s -> regex.matcher(s).results()) - .map(matchResults -> matchResults.group()) + .filter(s -> regex.matcher(s).matches()) .toList()) { checks.add(new ExistsQuery.Builder().field(fn).build().toQuery()); } From 85146321c8bbe00e5dbcd46abdc3eb1862ffffc0 Mon Sep 17 00:00:00 2001 From: al-niessner <1130658+al-niessner@users.noreply.github.com> Date: Mon, 3 Nov 2025 08:52:17 -0800 Subject: [PATCH 009/137] Update CHANGELOG.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cd1403fd..544c5246 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## [1.6.0](https://github.com/NASA-PDS/registry-api/tree/1.6.0) (2025-10-14) -[Full Changelog](https://github.com/NASA-PDS/registry-api/compare/v1.6.2...1.6.0) +[Full Changelog](https://github.com/NASA-PDS/registry-api/compare/1.6.0...v1.6.2) **Defects:** From 6a4d6005589046a22bd2d7fc25d2cc9677852604 Mon Sep 17 00:00:00 2001 From: al-niessner <1130658+al-niessner@users.noreply.github.com> Date: Mon, 3 Nov 2025 08:53:09 -0800 Subject: [PATCH 010/137] Update service/src/test/java/gov/nasa/pds/api/registry/opensearch/Antlr4SearchListenerTest.java Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../pds/api/registry/opensearch/Antlr4SearchListenerTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/src/test/java/gov/nasa/pds/api/registry/opensearch/Antlr4SearchListenerTest.java b/service/src/test/java/gov/nasa/pds/api/registry/opensearch/Antlr4SearchListenerTest.java index 0baa2748..48070952 100644 --- a/service/src/test/java/gov/nasa/pds/api/registry/opensearch/Antlr4SearchListenerTest.java +++ b/service/src/test/java/gov/nasa/pds/api/registry/opensearch/Antlr4SearchListenerTest.java @@ -189,7 +189,7 @@ void testEnterOrStatement() { void testExitOrStatement() { listener.enterOrStatement(Mockito.mock(SearchParser.OrStatementContext.class)); listener.exitOrStatement(Mockito.mock(SearchParser.OrStatementContext.class)); - assertSame("1", listener.getBoolQuery().minimumShouldMatch(), + assertEquals("1", listener.getBoolQuery().minimumShouldMatch(), "Minimum should match should be set"); } From 40ac16686cc3a812db29719ba29fb1541d04c2f6 Mon Sep 17 00:00:00 2001 From: Jordan Padams Date: Mon, 3 Nov 2025 11:41:23 -0800 Subject: [PATCH 011/137] Update developer docs to point to registry docs for integration testing --- CONTRIBUTING.md | 18 +++++++++++++++++- README.md | 22 +++++++++------------- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 296d8288..61c4984a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -63,8 +63,24 @@ There are a few steps required to prepare for merging code back into the main br 1. Create a pull request if have not done this already. 1. Address all automated messages. -1. Run all regression checks to make sure changes have re-introduced already fixed bugs. +1. **Add required integration tests** (see [Integration Testing Requirements](#integration-testing-requirements) below). +1. Run all regression checks to make sure changes have not re-introduced already fixed bugs. 1. Move from draft to ready for review if in draft mode. 1. Request review. +## Integration Testing Requirements + +**IMPORTANT**: Each new feature, requirement, or bug fix must include at least one integration test added to the Postman collection. + +Integration tests are maintained in the [`registry` repository](https://github.com/NASA-PDS/registry) and must be updated as part of your contribution. For detailed instructions on creating and submitting integration tests, see: + +**[Integration Testing Guide](https://nasa-pds.github.io/registry/developer/integration-testing.html)** + +The guide covers: +- When tests are required +- Step-by-step process for adding tests to Postman +- TestRail integration (for internal developers) +- Running and validating tests locally +- Submitting test updates via pull request + diff --git a/README.md b/README.md index 745a43a3..d064d301 100644 --- a/README.md +++ b/README.md @@ -113,23 +113,19 @@ The integration tests will be automatically applied. Check the results, update/c ## Tests -**Important note:** As a developer you are asked to complete the postman test suite according to the new feature you are developing. Do a pull request in the `registry` project to submit the updates. +### Testing Requirements -Integration test are maintained in postman. +**IMPORTANT:** As a developer, you are **required** to add integration tests to the Postman test suite for: +- Each new feature or requirement +- Each bug fix +- Any changes to existing API behavior -### Edit/Run of the integration tests in postman GUI +### Integration Testing Guide -Install the postman desktop, from https://www.postman.com/downloads/ +Integration tests are maintained in the `registry` repository as Postman collections. For complete instructions on creating, running, and submitting integration tests, see: -Download and open the test suite found in https://github.com/NASA-PDS/registry/tree/main/docker/postman +**[Integration Testing Guide](https://nasa-pds.github.io/registry/developer/integration-testing.html)** -### Run the integration tests in command line - -In the `registry` project. - -Launch the test in command line: - - npm install newman - newman run docker/postman/postman_collection.json --env-var baseUrl=http://localhost:8080 +All test updates must be submitted as pull requests to the [`registry` repository](https://github.com/NASA-PDS/registry). From 4e2c487b11b3d0d0b01764b2c1975790377347d1 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Thu, 13 Nov 2025 07:45:12 -0800 Subject: [PATCH 012/137] clean up the docs --- docker/Dockerfile.local | 37 ---- docker/README.md | 9 - service/ut/base_ut.py | 360 --------------------------------- service/ut/helpers/__init__.py | 14 -- service/ut/regress.sh | 9 - 5 files changed, 429 deletions(-) delete mode 100644 docker/Dockerfile.local delete mode 100644 service/ut/base_ut.py delete mode 100644 service/ut/helpers/__init__.py delete mode 100755 service/ut/regress.sh diff --git a/docker/Dockerfile.local b/docker/Dockerfile.local deleted file mode 100644 index be5a76f5..00000000 --- a/docker/Dockerfile.local +++ /dev/null @@ -1,37 +0,0 @@ -FROM ubuntu:24.04 - - # Get arguments from the build command line - ARG version - ENV VERSION=$version - - # Build up the OS - RUN export DEBIAN_FRONTEND=noninteractive && \ - apt-get update && \ - apt-get install -y curl \ - libtcnative-1 \ - maven \ - openjdk-17-jdk-headless \ - tar - - # Make room for the app - RUN mkdir -p /usr/local/registry-${VERSION} - - # Copy the data into the building container - COPY LICENSE.md /usr/local/registry-${VERSION}/ - COPY pom.xml /usr/local/registry-${VERSION}/ - COPY SECURITY.md /usr/local/registry-${VERSION}/ - COPY lexer /usr/local/registry-${VERSION}/lexer - COPY model /usr/local/registry-${VERSION}/model - COPY service /usr/local/registry-${VERSION}/service - - # Resources shared with the rest of the world - EXPOSE 8080 - - # Build the application and deploy it inside the container - RUN set -x && \ - cd /usr/local/registry-${VERSION} && \ - mvn clean install - - # Run the sevice by default - WORKDIR /usr/local/registry-${VERSION}/service - CMD ["mvn", "spring-boot:run"] diff --git a/docker/README.md b/docker/README.md index 54163268..b95a11f9 100644 --- a/docker/README.md +++ b/docker/README.md @@ -46,12 +46,3 @@ For example on AWS, with OpenSearch serverless as a back-end: SPRING_BOOT_APP_ARGS=--openSearch.host= --openSearch.CCSEnabled=true --openSearch.username="" --openSearch.disciplineNodes=atm-delta,en-delta --registry.service.version=1.5.0-SNAPSHOT SERVER_PORT=80 - - - - - - -## 📍 Dockerfile.local - -You can ignore `Dockerfile.local` unless you're @al-niessner. diff --git a/service/ut/base_ut.py b/service/ut/base_ut.py deleted file mode 100644 index 999a9d81..00000000 --- a/service/ut/base_ut.py +++ /dev/null @@ -1,360 +0,0 @@ - -import helpers -import unittest - -def test_bad_group(): - ep = '/classes/notreal' - status,data = helpers.fetch_kvp_json (helpers.make_url (ep)) - assert 406 == status - assert 'message' in data - assert 'request' in data - assert data['message'].startswith ("Unknown group 'notreal'. All known groups:") - assert data['request'] == ep - return - -def test_bad_lidvid(): - ep = '/products/notreal' - status,data = helpers.fetch_kvp_json (helpers.make_url (ep)) - assert 404 == status - assert 'message' in data - assert 'request' in data - assert 'The lidvid notreal was not found' == data['message'] - assert data['request'] == ep - return - -class TestAny(unittest.TestCase): - def test_products(self): - status,resp = helpers.fetch_kvp_json (helpers.make_url ('/classes/any')) - self.assertEqual (200, status) - self.assertIn ('summary', resp) - self.assertIn ('hits', resp['summary']) - self.assertEqual (17, resp['summary']['hits']) - self.assertIn ('data', resp) - self.assertEqual (resp['summary']['hits'], len(resp['data'])) - self.assertIn ('lidvid', resp['data'][0]) - return resp['data'][-1]['lidvid'] - - def test_lidvid(self): - lidvid = self.test_products() - status,resp = helpers.fetch_kvp_json (helpers.make_url - (f'/products/{lidvid}')) - self.assertEqual (200, status) - self.assertIn ('lidvid', resp) - self.assertEqual (lidvid, resp['lidvid']) - return - - def test_lidvid_latest(self): - lidvid = self.test_products() - status,resp = helpers.fetch_kvp_json(helpers.make_url - (f'/products/{lidvid}/latest')) - self.assertEqual (200, status) - self.assertIn ('lidvid', resp) - self.assertEqual (lidvid, resp['lidvid']) - return - - def test_lidvid_all(self): - lidvid = self.test_products() - status,resp = helpers.fetch_kvp_json (helpers.make_url - (f'/products/{lidvid}/all')) - self.assertEqual (200, status) - self.assertIn ('summary', resp) - self.assertIn ('hits', resp['summary']) - self.assertEqual (1, resp['summary']['hits']) - self.assertIn ('data', resp) - self.assertEqual (resp['summary']['hits'], len(resp['data'])) - self.assertIn ('lidvid', resp['data'][0]) - self.assertEqual (lidvid, resp['data'][0]['lidvid']) - return - - def test_collections(self): - lidvid = self.test_products() - status,resp = helpers.fetch_kvp_json (helpers.make_url - (f'/products/{lidvid}/member-of')) - self.assertEqual (200, status) - self.assertIn ('summary', resp) - self.assertIn ('hits', resp['summary']) - self.assertEqual (1, resp['summary']['hits']) - self.assertIn ('data', resp) - self.assertEqual (resp['summary']['hits'], len(resp['data'])) - self.assertIn ('lidvid', resp['data'][0]) - return - - def test_bundles(self): - lidvid = self.test_products() - status,resp = helpers.fetch_kvp_json (helpers.make_url - (f'/products/{lidvid}/member-of/member-of')) - self.assertEqual (200, status) - self.assertIn ('summary', resp) - self.assertIn ('hits', resp['summary']) - self.assertEqual (1, resp['summary']['hits']) - self.assertIn ('data', resp) - self.assertEqual (resp['summary']['hits'], len(resp['data'])) - self.assertIn ('lidvid', resp['data'][0]) - return - pass - -class TestBundles(unittest.TestCase): - def test_bundles(self): - status,resp = helpers.fetch_kvp_json (helpers.make_url ('/classes/bundles')) - self.assertEqual (200, status) - self.assertIn ('summary', resp) - self.assertIn ('hits', resp['summary']) - self.assertEqual (1, resp['summary']['hits']) - self.assertIn ('data', resp) - self.assertEqual (resp['summary']['hits'], len(resp['data'])) - self.assertIn ('lidvid', resp['data'][0]) - return resp['data'][0]['lidvid'] - - def test_lidvid(self): - lidvid = self.test_bundles() - status,resp = helpers.fetch_kvp_json (helpers.make_url - (f'/products/{lidvid}')) - self.assertEqual (200, status) - self.assertIn ('lidvid', resp) - self.assertEqual (lidvid, resp['lidvid']) - return - - def test_lidvid_latest(self): - lidvid = self.test_bundles() - status,resp = helpers.fetch_kvp_json(helpers.make_url - (f'/products/{lidvid}/latest')) - self.assertEqual (200, status) - self.assertIn ('lidvid', resp) - self.assertEqual (lidvid, resp['lidvid']) - return - - def test_lidvid_all(self): - lidvid = self.test_bundles() - status,resp = helpers.fetch_kvp_json (helpers.make_url - (f'/products/{lidvid}/all')) - self.assertEqual (200, status) - self.assertIn ('summary', resp) - self.assertIn ('hits', resp['summary']) - self.assertEqual (1, resp['summary']['hits']) - self.assertIn ('data', resp) - self.assertEqual (resp['summary']['hits'], len(resp['data'])) - self.assertIn ('lidvid', resp['data'][0]) - self.assertEqual (lidvid, resp['data'][0]['lidvid']) - return - - def test_collections(self): - lidvid = self.test_bundles() - status,resp = helpers.fetch_kvp_json \ - (helpers.make_url (f'/classes/bundles/{lidvid}/members')) - self.assertEqual (200, status) - self.assertIn ('summary', resp) - self.assertIn ('hits', resp['summary']) - self.assertEqual (2, resp['summary']['hits']) - self.assertIn ('data', resp) - self.assertEqual (resp['summary']['hits'], len(resp['data'])) - self.assertIn ('lidvid', resp['data'][0]) - return - - def test_collections_latest(self): - lidvid = self.test_bundles() - status,resp = helpers.fetch_kvp_json \ - (helpers.make_url (f'/classes/bundles/{lidvid}/members/latest')) - self.assertEqual (200, status) - self.assertIn ('summary', resp) - self.assertIn ('hits', resp['summary']) - self.assertEqual (2, resp['summary']['hits']) - self.assertIn ('data', resp) - self.assertEqual (resp['summary']['hits'], len(resp['data'])) - self.assertIn ('lidvid', resp['data'][0]) - return - - def test_collections_all(self): - lidvid = self.test_bundles() - status,resp = helpers.fetch_kvp_json \ - (helpers.make_url (f'/classes/bundles/{lidvid}/members/all')) - self.assertEqual (200, status) - self.assertIn ('summary', resp) - self.assertIn ('hits', resp['summary']) - self.assertEqual (2, resp['summary']['hits']) - self.assertIn ('data', resp) - self.assertEqual (resp['summary']['hits'], len(resp['data'])) - self.assertIn ('lidvid', resp['data'][0]) - return - - def test_products(self): - lidvid = self.test_bundles() - status,resp = helpers.fetch_kvp_json\ - (helpers.make_url (f'/classes/bundles/{lidvid}/members/members')) - self.assertEqual (200, status) - self.assertIn ('summary', resp) - self.assertIn ('hits', resp['summary']) - self.assertEqual (14, resp['summary']['hits']) - self.assertIn ('data', resp) - self.assertEqual (resp['summary']['hits'], len(resp['data'])) - self.assertIn ('lidvid', resp['data'][0]) - return - pass - -class TestCollections(unittest.TestCase): - def test_bundles(self): - lidvid = self.test_collections() - status,resp = helpers.fetch_kvp_json \ - (helpers.make_url (f'/classes/collections/{lidvid}/member-of')) - self.assertEqual (200, status) - self.assertIn ('summary', resp) - self.assertIn ('hits', resp['summary']) - self.assertEqual (1, resp['summary']['hits']) - self.assertIn ('data', resp) - self.assertEqual (resp['summary']['hits'], len(resp['data'])) - self.assertIn ('lidvid', resp['data'][0]) - return - - def test_collections(self): - status,resp = helpers.fetch_kvp_json \ - (helpers.make_url ('/classes/collections')) - self.assertEqual (200, status) - self.assertIn ('summary', resp) - self.assertIn ('hits', resp['summary']) - self.assertEqual (2, resp['summary']['hits']) - self.assertIn ('data', resp) - self.assertEqual (resp['summary']['hits'], len(resp['data'])) - self.assertIn ('lidvid', resp['data'][0]) - return resp['data'][0]['lidvid'] - - def test_lidvid(self): - lidvid = self.test_collections() - status,resp = helpers.fetch_kvp_json (helpers.make_url - (f'/products/{lidvid}')) - self.assertEqual (200, status) - self.assertIn ('lidvid', resp) - self.assertEqual (lidvid, resp['lidvid']) - return - - def test_lidvid_latest(self): - lidvid = self.test_collections() - status,resp = helpers.fetch_kvp_json(helpers.make_url - (f'/products/{lidvid}/latest')) - self.assertEqual (200, status) - self.assertIn ('lidvid', resp) - self.assertEqual (lidvid, resp['lidvid']) - return - - def test_lidvid_all(self): - lidvid = self.test_collections() - status,resp = helpers.fetch_kvp_json (helpers.make_url - (f'/products/{lidvid}/all')) - self.assertEqual (200, status) - self.assertIn ('summary', resp) - self.assertIn ('hits', resp['summary']) - self.assertEqual (1, resp['summary']['hits']) - self.assertIn ('data', resp) - self.assertEqual (resp['summary']['hits'], len(resp['data'])) - self.assertIn ('lidvid', resp['data'][0]) - self.assertEqual (lidvid, resp['data'][0]['lidvid']) - return - - def test_products(self): - lidvid = self.test_collections() - status,resp = helpers.fetch_kvp_json (helpers.make_url - (f'/classes/collections/{lidvid}/members')) - self.assertEqual (200, status) - self.assertIn ('summary', resp) - self.assertIn ('hits', resp['summary']) - self.assertEqual (7, resp['summary']['hits']) - self.assertIn ('data', resp) - self.assertEqual (resp['summary']['hits'], len(resp['data'])) - self.assertIn ('lidvid', resp['data'][0]) - return - - def test_products_latest(self): - lidvid = self.test_collections() - status,resp = helpers.fetch_kvp_json (helpers.make_url - (f'/classes/collections/{lidvid}/members/latest')) - self.assertEqual (200, status) - self.assertIn ('summary', resp) - self.assertIn ('hits', resp['summary']) - self.assertEqual (7, resp['summary']['hits']) - self.assertIn ('data', resp) - self.assertEqual (resp['summary']['hits'], len(resp['data'])) - self.assertIn ('lidvid', resp['data'][0]) - return - - def test_products_all(self): - lidvid = self.test_collections() - status,resp = helpers.fetch_kvp_json (helpers.make_url - (f'/classes/collections/{lidvid}/members/all')) - self.assertEqual (200, status) - self.assertIn ('summary', resp) - self.assertIn ('hits', resp['summary']) - self.assertEqual (7, resp['summary']['hits']) - self.assertIn ('data', resp) - self.assertEqual (resp['summary']['hits'], len(resp['data'])) - self.assertIn ('lidvid', resp['data'][0]) - return - pass - -class TestProducts(unittest.TestCase): - def test_products(self): - status,resp = helpers.fetch_kvp_json (helpers.make_url ('/classes/products')) - self.assertEqual (200, status) - self.assertIn ('summary', resp) - self.assertIn ('hits', resp['summary']) - self.assertEqual (14, resp['summary']['hits']) - self.assertIn ('data', resp) - self.assertEqual (resp['summary']['hits'], len(resp['data'])) - self.assertIn ('lidvid', resp['data'][0]) - return resp['data'][-1]['lidvid'] - - def test_lidvid(self): - lidvid = self.test_products() - status,resp = helpers.fetch_kvp_json (helpers.make_url - (f'/products/{lidvid}')) - self.assertEqual (200, status) - self.assertIn ('lidvid', resp) - self.assertEqual (lidvid, resp['lidvid']) - return - - def test_lidvid_latest(self): - lidvid = self.test_products() - status,resp = helpers.fetch_kvp_json(helpers.make_url - (f'/products/{lidvid}/latest')) - self.assertEqual (200, status) - self.assertIn ('lidvid', resp) - self.assertEqual (lidvid, resp['lidvid']) - return - - def test_lidvid_all(self): - lidvid = self.test_products() - status,resp = helpers.fetch_kvp_json (helpers.make_url - (f'/products/{lidvid}/all')) - self.assertEqual (200, status) - self.assertIn ('summary', resp) - self.assertIn ('hits', resp['summary']) - self.assertEqual (1, resp['summary']['hits']) - self.assertIn ('data', resp) - self.assertEqual (resp['summary']['hits'], len(resp['data'])) - self.assertIn ('lidvid', resp['data'][0]) - self.assertEqual (lidvid, resp['data'][0]['lidvid']) - return - - def test_collections(self): - lidvid = self.test_products() - status,resp = helpers.fetch_kvp_json (helpers.make_url - (f'/products/{lidvid}/member-of')) - self.assertEqual (200, status) - self.assertIn ('summary', resp) - self.assertIn ('hits', resp['summary']) - self.assertEqual (1, resp['summary']['hits']) - self.assertIn ('data', resp) - self.assertEqual (resp['summary']['hits'], len(resp['data'])) - self.assertIn ('lidvid', resp['data'][0]) - return - - def test_bundles(self): - lidvid = self.test_products() - status,resp = helpers.fetch_kvp_json (helpers.make_url - (f'/products/{lidvid}/member-of/member-of')) - self.assertEqual (200, status) - self.assertIn ('summary', resp) - self.assertIn ('hits', resp['summary']) - self.assertEqual (1, resp['summary']['hits']) - self.assertIn ('data', resp) - self.assertEqual (resp['summary']['hits'], len(resp['data'])) - self.assertIn ('lidvid', resp['data'][0]) - return - pass diff --git a/service/ut/helpers/__init__.py b/service/ut/helpers/__init__.py deleted file mode 100644 index aa0d04cc..00000000 --- a/service/ut/helpers/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ - -import os -import requests - -def fetch_kvp_json (url:str): - url += '?fields=lidvid' - print ('url:', url) - result = requests.get(url, headers={'Accept':'application/kvp+json'}) - return result.status_code,result.json() - -def make_url (endpoint:str)->str: - return os.environ.get ('REG_APO_UT_TYPE', 'http') + '://' + \ - os.environ.get ('REG_API_UT_HOSTNAME', 'localhost') + ':' + \ - os.environ.get ('REG_API_UT_PORT', '8080') + endpoint diff --git a/service/ut/regress.sh b/service/ut/regress.sh deleted file mode 100755 index 80c8aee0..00000000 --- a/service/ut/regress.sh +++ /dev/null @@ -1,9 +0,0 @@ -#! /bin/bash - -# make sure an instance of registry-api/service is runningn with the -# test data set - -base=$(realpath $0) -base=$(dirname $base) -PYTHONPATH=${base}:${PYTHONPATH} -pytest -v $(find $base -name \*_ut.py) From 385316f85c4277c0d32931d1709ce7e41835642d Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Thu, 13 Nov 2025 08:07:26 -0800 Subject: [PATCH 013/137] add unit testing to show exists is parsing as expected --- .../api_search_query_lexer/TestParsing.java | 70 +++++++++++++++++-- 1 file changed, 63 insertions(+), 7 deletions(-) diff --git a/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/TestParsing.java b/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/TestParsing.java index e4b7d493..e54ec8e5 100644 --- a/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/TestParsing.java +++ b/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/TestParsing.java @@ -99,7 +99,6 @@ void testTemporalRange() { CommonTokenStream tokens = new CommonTokenStream(lex); SearchParser par = new SearchParser(tokens); ParseTree tree = par.query(); - ParseTreeWalker walker = new ParseTreeWalker(); walker.walk(this, tree); @@ -107,7 +106,66 @@ void testTemporalRange() { } - + @Test + void testFieldExistence() { + String queryString = "apple exists"; + CodePointCharStream input = CharStreams.fromString(queryString); + SearchLexer lex = new SearchLexer(input); + CommonTokenStream tokens = new CommonTokenStream(lex); + SearchParser par = new SearchParser(tokens); + ParseTree tree = par.query(); + ParseTreeWalker walker = new ParseTreeWalker(); + walker.walk(this, tree); + + Assertions.assertNotNull(this.field); + Assertions.assertNull(this.strval); + Assertions.assertEquals("apple", this.field.getSymbol().getText()); + } + @Test + void testParenFieldExistence() { + String queryString = "(apple exists)"; + CodePointCharStream input = CharStreams.fromString(queryString); + SearchLexer lex = new SearchLexer(input); + CommonTokenStream tokens = new CommonTokenStream(lex); + SearchParser par = new SearchParser(tokens); + ParseTree tree = par.query(); + ParseTreeWalker walker = new ParseTreeWalker(); + walker.walk(this, tree); + + Assertions.assertNotNull(this.field); + Assertions.assertNull(this.strval); + Assertions.assertEquals("apple", this.field.getSymbol().getText()); + } + @Test + void testStrvalExistence() { + String queryString = "\".*apple\" exists"; + CodePointCharStream input = CharStreams.fromString(queryString); + SearchLexer lex = new SearchLexer(input); + CommonTokenStream tokens = new CommonTokenStream(lex); + SearchParser par = new SearchParser(tokens); + ParseTree tree = par.query(); + ParseTreeWalker walker = new ParseTreeWalker(); + walker.walk(this, tree); + + Assertions.assertNull(this.field); + Assertions.assertNotNull(this.strval); + Assertions.assertEquals("\".*apple\"", this.strval.getSymbol().getText()); + } + @Test + void testParenStrvalExistence() { + String queryString = "(\".*apple\" exists)"; + CodePointCharStream input = CharStreams.fromString(queryString); + SearchLexer lex = new SearchLexer(input); + CommonTokenStream tokens = new CommonTokenStream(lex); + SearchParser par = new SearchParser(tokens); + ParseTree tree = par.query(); + ParseTreeWalker walker = new ParseTreeWalker(); + walker.walk(this, tree); + + Assertions.assertNull(this.field); + Assertions.assertNotNull(this.strval); + Assertions.assertEquals("\".*apple\"", this.strval.getSymbol().getText()); + } @Override public void enterQuery(QueryContext ctx) { // TODO Auto-generated method stub @@ -236,7 +294,7 @@ public void enterLikeComparison(LikeComparisonContext ctx) { String op = ctx.getChild(1).getText(); if ("not".equals(op)) - isNot = true; + this.isNot = true; } @Override @@ -253,9 +311,7 @@ public void enterExistence(ExistenceContext ctx) { @Override public void exitExistence(ExistenceContext ctx) { - // TODO Auto-generated method stub - + this.field = ctx.FIELD(); + this.strval = ctx.STRINGVAL(); } - - } From f2e76d2b3e744b6f35f2bb4ee595c06efbf04673 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Wed, 19 Nov 2025 10:06:51 -0800 Subject: [PATCH 014/137] postman testing Added various tests for specific names and wildcards. Both exists and not exists. Flushed out some bugs and fixed them. --- .../registry/model/Antlr4SearchListener.java | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java b/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java index d9ff09ac..1cd50bf6 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java @@ -180,31 +180,32 @@ else if (this.operator == operation.lt) @Override public void exitExistence(SearchParser.ExistenceContext ctx) { ArrayList checks = new ArrayList(); - final String fieldName = SearchUtil.jsonPropertyToOpenProperty(ctx.FIELD().getSymbol().getText()); - final String regexp = ctx.STRINGVAL().getText(); + final String fieldName = ctx.FIELD() == null ? "" : SearchUtil.jsonPropertyToOpenProperty(ctx.FIELD().getSymbol().getText()); + final String regexp = ctx.STRINGVAL() == null ? "" : ctx.STRINGVAL().getText(); String theKey = "''"; - if (fieldName != null && !fieldName.isBlank()) { + if (!fieldName.isBlank()) { theKey = fieldName; checks.add(new ExistsQuery.Builder().field(fieldName).build().toQuery()); - } else if (regexp != null && !regexp.isBlank()) { - theKey = regexp; - if (knownFieldNames.isEmpty()) { + } else if (!regexp.isBlank()) { + theKey = regexp.substring(1, regexp.length()-1); + if (this.knownFieldNames.isEmpty()) { try { for (PropertiesListInner property : ProductsController.productPropertiesList(this.connectionContext).getBody()) { - knownFieldNames.add(property.getProperty()); + this.knownFieldNames.add(property.getProperty()); } } catch (OpenSearchException | IOException e) { log.error("Could not load the mapping(s) from opensearch; meaning 'exists' will not work", e); } } - Pattern regex = Pattern.compile(regexp); - for (String fn : knownFieldNames.stream() + Pattern regex = Pattern.compile(theKey); + for (String fn : this.knownFieldNames.stream() .filter(s -> regex.matcher(s).matches()) .toList()) { - checks.add(new ExistsQuery.Builder().field(fn).build().toQuery()); + checks.add(new ExistsQuery.Builder().field(SearchUtil.jsonPropertyToOpenProperty(fn)).build().toQuery()); } - if (checks.isEmpty()) + } + if (checks.isEmpty()) { throw new ParseCancellationException("For existence testing, cannot match any field names to " + theKey); } if (this.conjunction == conjunctions.AND) { From e8d677161f31303da47c7007a8b6a10b830215d7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Nov 2025 14:57:08 +0000 Subject: [PATCH 015/137] Bump actions/checkout from 5 to 6 Bumps [actions/checkout](https://github.com/actions/checkout) from 5 to 6. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/branch-cicd.yaml | 2 +- .github/workflows/codeql-analysis.yml | 4 ++-- .github/workflows/secrets-detection.yaml | 2 +- .github/workflows/stable-cicd.yaml | 2 +- .github/workflows/unstable-cicd.yaml | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/branch-cicd.yaml b/.github/workflows/branch-cicd.yaml index ed099e77..62268b6c 100644 --- a/.github/workflows/branch-cicd.yaml +++ b/.github/workflows/branch-cicd.yaml @@ -40,7 +40,7 @@ jobs: steps: - name: 💳 Checkout - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: lfs: true fetch-depth: 0 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 7e2dccec..adbc1854 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -23,7 +23,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: # We must fetch at least the immediate parents so that if this is # a pull request then we can checkout the head. @@ -93,7 +93,7 @@ jobs: steps: - name: 💳 Checkout - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: lfs: true fetch-depth: 0 diff --git a/.github/workflows/secrets-detection.yaml b/.github/workflows/secrets-detection.yaml index 65c29546..92f4d894 100644 --- a/.github/workflows/secrets-detection.yaml +++ b/.github/workflows/secrets-detection.yaml @@ -13,7 +13,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Install necessary packages run: | diff --git a/.github/workflows/stable-cicd.yaml b/.github/workflows/stable-cicd.yaml index fc5bca5b..1be47589 100644 --- a/.github/workflows/stable-cicd.yaml +++ b/.github/workflows/stable-cicd.yaml @@ -50,7 +50,7 @@ jobs: steps: - name: 💳 Checkout - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: lfs: true token: ${{secrets.ADMIN_GITHUB_TOKEN}} diff --git a/.github/workflows/unstable-cicd.yaml b/.github/workflows/unstable-cicd.yaml index 8753bc7b..d1e7480f 100644 --- a/.github/workflows/unstable-cicd.yaml +++ b/.github/workflows/unstable-cicd.yaml @@ -50,7 +50,7 @@ jobs: steps: - name: 💳 Checkout - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: lfs: true fetch-depth: 0 From f8bb99a891428b9957770efdfd893fd15bdbe145 Mon Sep 17 00:00:00 2001 From: al-niessner <1130658+al-niessner@users.noreply.github.com> Date: Wed, 10 Dec 2025 10:55:21 -0800 Subject: [PATCH 016/137] Update TestParsing.java --- .../api/pds/nasa/gov/api_search_query_lexer/TestParsing.java | 1 - 1 file changed, 1 deletion(-) diff --git a/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/TestParsing.java b/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/TestParsing.java index b08e6d1f..58fe9c2d 100644 --- a/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/TestParsing.java +++ b/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/TestParsing.java @@ -130,7 +130,6 @@ void testTemporalRange() { // TODO: Parse } -} @Test void testFieldExistence() { From 9072d4da2676be39640e2816223153ee62f02cfe Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Thu, 11 Dec 2025 15:46:24 -0800 Subject: [PATCH 017/137] fix lexer unit tests --- .../MockedListener.java | 14 ++ .../api_search_query_lexer/TestParsing.java | 230 ++++-------------- 2 files changed, 63 insertions(+), 181 deletions(-) diff --git a/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/MockedListener.java b/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/MockedListener.java index 8d10f4fc..64973663 100644 --- a/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/MockedListener.java +++ b/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/MockedListener.java @@ -10,6 +10,7 @@ import gov.nasa.pds.api.registry.lexer.SearchParser.ExpressionContext; import gov.nasa.pds.api.registry.lexer.SearchParser.GroupContext; import gov.nasa.pds.api.registry.lexer.SearchParser.LikeComparisonContext; +import gov.nasa.pds.api.registry.lexer.SearchParser.ExistenceContext; import gov.nasa.pds.api.registry.lexer.SearchParser.OperatorContext; import gov.nasa.pds.api.registry.lexer.SearchParser.OrStatementContext; import gov.nasa.pds.api.registry.lexer.SearchParser.QueryContext; @@ -158,4 +159,17 @@ public void exitEveryRule(ParserRuleContext ctx) { } + + @Override + public void enterExistence(ExistenceContext ctx) { + // TODO Auto-generated method stub + + } + + @Override + public void exitExistence(ExistenceContext ctx) { + this.field = ctx.FIELD(); + this.strval = ctx.STRINGVAL(); + } + } diff --git a/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/TestParsing.java b/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/TestParsing.java index 58fe9c2d..838ee6ee 100644 --- a/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/TestParsing.java +++ b/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/TestParsing.java @@ -16,7 +16,6 @@ import gov.nasa.pds.api.registry.lexer.SearchParser; import gov.nasa.pds.api.registry.lexer.SearchParser.AndStatementContext; import gov.nasa.pds.api.registry.lexer.SearchParser.ComparisonContext; -import gov.nasa.pds.api.registry.lexer.SearchParser.ExistenceContext; import gov.nasa.pds.api.registry.lexer.SearchParser.ExpressionContext; import gov.nasa.pds.api.registry.lexer.SearchParser.GroupContext; import gov.nasa.pds.api.registry.lexer.SearchParser.LikeComparisonContext; @@ -67,11 +66,11 @@ public void testNumber() { MockedListener listener = new MockedListener(); walker.walk(listener, tree); - Assertions.assertNotNull(this.field); - Assertions.assertEquals("lid", this.field.getSymbol().getText()); + Assertions.assertNotNull(listener.field); + Assertions.assertEquals(listener.field.getSymbol().getText(), "lid"); - Assertions.assertNotNull(this.number); - Assertions.assertEquals("1234", this.number.getSymbol().getText()); + Assertions.assertNotEquals(listener.number, null); + Assertions.assertEquals(listener.number.getSymbol().getText(), "1234"); } @@ -87,11 +86,11 @@ public void testStringVal() { MockedListener listener = new MockedListener(); walker.walk(listener, tree); - Assertions.assertNotNull(this.field); - Assertions.assertEquals("lid", this.field.getSymbol().getText()); + Assertions.assertNotNull(listener.field); + Assertions.assertEquals(listener.field.getSymbol().getText(), "lid"); - Assertions.assertNotNull(this.strval); - Assertions.assertEquals("\"*text*\"", this.strval.getSymbol().getText()); + Assertions.assertNotNull(listener.strval); + Assertions.assertEquals(listener.strval.getSymbol().getText(), "\"*text*\""); } @@ -107,11 +106,11 @@ public void testLike() { MockedListener listener = new MockedListener(); walker.walk(listener, tree); - Assertions.assertNotNull(this.field); - Assertions.assertEquals("lid", this.field.getText()); + Assertions.assertNotNull(listener.field); + Assertions.assertEquals(listener.field.getText(), "lid"); - Assertions.assertNotNull(this.strval); - Assertions.assertEquals("\"*text*\"", this.strval.getText()); + Assertions.assertNotNull(listener.strval); + Assertions.assertEquals(listener.strval.getText(), "\"*text*\""); } @@ -123,6 +122,7 @@ void testTemporalRange() { CommonTokenStream tokens = new CommonTokenStream(lex); SearchParser par = new SearchParser(tokens); ParseTree tree = par.query(); + ParseTreeWalker walker = new ParseTreeWalker(); MockedListener listener = new MockedListener(); walker.walk(listener, tree); @@ -133,210 +133,78 @@ void testTemporalRange() { @Test void testFieldExistence() { - String queryString = "apple exists"; + String queryString = "apple exists"; CodePointCharStream input = CharStreams.fromString(queryString); SearchLexer lex = new SearchLexer(input); CommonTokenStream tokens = new CommonTokenStream(lex); SearchParser par = new SearchParser(tokens); ParseTree tree = par.query(); + ParseTreeWalker walker = new ParseTreeWalker(); - walker.walk(this, tree); - - Assertions.assertNotNull(this.field); - Assertions.assertNull(this.strval); - Assertions.assertEquals("apple", this.field.getSymbol().getText()); + MockedListener listener = new MockedListener(); + walker.walk(listener, tree); + + Assertions.assertNotNull(listener.field); + Assertions.assertNull(listener.strval); + Assertions.assertEquals("apple", listener.field.getSymbol().getText()); } + + @Test void testParenFieldExistence() { - String queryString = "(apple exists)"; + String queryString = "(apple exists)"; CodePointCharStream input = CharStreams.fromString(queryString); SearchLexer lex = new SearchLexer(input); CommonTokenStream tokens = new CommonTokenStream(lex); SearchParser par = new SearchParser(tokens); ParseTree tree = par.query(); + ParseTreeWalker walker = new ParseTreeWalker(); - walker.walk(this, tree); - - Assertions.assertNotNull(this.field); - Assertions.assertNull(this.strval); - Assertions.assertEquals("apple", this.field.getSymbol().getText()); + MockedListener listener = new MockedListener(); + walker.walk(listener, tree); + + Assertions.assertNotNull(listener.field); + Assertions.assertNull(listener.strval); + Assertions.assertEquals("apple", listener.field.getSymbol().getText()); } + @Test void testStrvalExistence() { - String queryString = "\".*apple\" exists"; + String queryString = "\".*apple\" exists"; CodePointCharStream input = CharStreams.fromString(queryString); SearchLexer lex = new SearchLexer(input); CommonTokenStream tokens = new CommonTokenStream(lex); SearchParser par = new SearchParser(tokens); ParseTree tree = par.query(); + ParseTreeWalker walker = new ParseTreeWalker(); - walker.walk(this, tree); - - Assertions.assertNull(this.field); - Assertions.assertNotNull(this.strval); - Assertions.assertEquals("\".*apple\"", this.strval.getSymbol().getText()); + MockedListener listener = new MockedListener(); + walker.walk(listener, tree); + + Assertions.assertNull(listener.field); + Assertions.assertNotNull(listener.strval); + Assertions.assertEquals("\".*apple\"", listener.strval.getSymbol().getText()); } + + @Test void testParenStrvalExistence() { - String queryString = "(\".*apple\" exists)"; + String queryString = "(\".*apple\" exists)"; CodePointCharStream input = CharStreams.fromString(queryString); SearchLexer lex = new SearchLexer(input); CommonTokenStream tokens = new CommonTokenStream(lex); SearchParser par = new SearchParser(tokens); ParseTree tree = par.query(); - ParseTreeWalker walker = new ParseTreeWalker(); - walker.walk(this, tree); - - Assertions.assertNull(this.field); - Assertions.assertNotNull(this.strval); - Assertions.assertEquals("\".*apple\"", this.strval.getSymbol().getText()); - } - @Override - public void enterQuery(QueryContext ctx) { - // TODO Auto-generated method stub - - } - - @Override - public void exitQuery(QueryContext ctx) { - // TODO Auto-generated method stub - - } - - @Override - public void enterQueryTerm(QueryTermContext ctx) { - // TODO Auto-generated method stub - - } - - @Override - public void exitQueryTerm(QueryTermContext ctx) { - // TODO Auto-generated method stub - - } - - @Override - public void enterGroup(GroupContext ctx) { - // TODO Auto-generated method stub - - } - - @Override - public void exitGroup(GroupContext ctx) { - // TODO Auto-generated method stub - - } - - @Override - public void enterExpression(ExpressionContext ctx) { - // TODO Auto-generated method stub - - } - - @Override - public void exitExpression(ExpressionContext ctx) { - // TODO Auto-generated method stub - - } - @Override - public void enterAndStatement(AndStatementContext ctx) { - // TODO Auto-generated method stub - - } - - @Override - public void exitAndStatement(AndStatementContext ctx) { - // TODO Auto-generated method stub - - } - - @Override - public void enterOrStatement(OrStatementContext ctx) { - // TODO Auto-generated method stub - - } - - @Override - public void exitOrStatement(OrStatementContext ctx) { - // TODO Auto-generated method stub - - } - - @Override - public void enterComparison(ComparisonContext ctx) { - this.field = ctx.FIELD(); - this.number = ctx.NUMBER(); - this.strval = ctx.STRINGVAL(); - } - - @Override - public void exitComparison(ComparisonContext ctx) { - // TODO Auto-generated method stub - - } - - @Override - public void enterOperator(OperatorContext ctx) { - // TODO Auto-generated method stub - - } - - @Override - public void exitOperator(OperatorContext ctx) { - // TODO Auto-generated method stub - - } - - @Override - public void enterEveryRule(ParserRuleContext arg0) { - // TODO Auto-generated method stub - - } - - @Override - public void exitEveryRule(ParserRuleContext arg0) { - // TODO Auto-generated method stub - - } - - @Override - public void visitErrorNode(ErrorNode arg0) { - // TODO Auto-generated method stub - - } - - @Override - public void visitTerminal(TerminalNode arg0) { - // TODO Auto-generated method stub - - } - - @Override - public void enterLikeComparison(LikeComparisonContext ctx) { - this.field = ctx.FIELD(); - this.strval = ctx.STRINGVAL(); + ParseTreeWalker walker = new ParseTreeWalker(); + MockedListener listener = new MockedListener(); + walker.walk(listener, tree); - String op = ctx.getChild(1).getText(); - if ("not".equals(op)) - this.isNot = true; + Assertions.assertNull(listener.field); + Assertions.assertNotNull(listener.strval); + Assertions.assertEquals("\".*apple\"", listener.strval.getSymbol().getText()); } - @Override - public void exitLikeComparison(LikeComparisonContext ctx) { - // TODO Auto-generated method stub - } - @Override - public void enterExistence(ExistenceContext ctx) { - // TODO Auto-generated method stub - - } - - @Override - public void exitExistence(ExistenceContext ctx) { - this.field = ctx.FIELD(); - this.strval = ctx.STRINGVAL(); - } } From cd136b0efe853cb94b74dfeead5bb28bb18f5bc5 Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Thu, 11 Dec 2025 17:20:29 -0800 Subject: [PATCH 018/137] remove TODOs, add critical command for Thomas L --- .../MockedListener.java | 42 +++++++++---------- service/README.md | 11 ++++- 2 files changed, 31 insertions(+), 22 deletions(-) diff --git a/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/MockedListener.java b/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/MockedListener.java index 64973663..2fe5393e 100644 --- a/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/MockedListener.java +++ b/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/MockedListener.java @@ -24,73 +24,73 @@ public class MockedListener implements ParseTreeListener, SearchListener { @Override public void enterQuery(QueryContext ctx) { - // TODO Auto-generated method stub + // Nothing useful to do in this mocked version } @Override public void exitQuery(QueryContext ctx) { - // TODO Auto-generated method stub + // Nothing useful to do in this mocked version } @Override public void enterQueryTerm(QueryTermContext ctx) { - // TODO Auto-generated method stub + // Nothing useful to do in this mocked version } @Override public void exitQueryTerm(QueryTermContext ctx) { - // TODO Auto-generated method stub + // Nothing useful to do in this mocked version } @Override public void enterGroup(GroupContext ctx) { - // TODO Auto-generated method stub + // Nothing useful to do in this mocked version } @Override public void exitGroup(GroupContext ctx) { - // TODO Auto-generated method stub + // Nothing useful to do in this mocked version } @Override public void enterExpression(ExpressionContext ctx) { - // TODO Auto-generated method stub + // Nothing useful to do in this mocked version } @Override public void exitExpression(ExpressionContext ctx) { - // TODO Auto-generated method stub + // Nothing useful to do in this mocked version } @Override public void enterAndStatement(AndStatementContext ctx) { - // TODO Auto-generated method stub + // Nothing useful to do in this mocked version } @Override public void exitAndStatement(AndStatementContext ctx) { - // TODO Auto-generated method stub + // Nothing useful to do in this mocked version } @Override public void enterOrStatement(OrStatementContext ctx) { - // TODO Auto-generated method stub + // Nothing useful to do in this mocked version } @Override public void exitOrStatement(OrStatementContext ctx) { - // TODO Auto-generated method stub + // Nothing useful to do in this mocked version } @@ -103,7 +103,7 @@ public void enterComparison(ComparisonContext ctx) { @Override public void exitComparison(ComparisonContext ctx) { - // TODO Auto-generated method stub + // Nothing useful to do in this mocked version } @@ -119,50 +119,50 @@ public void enterLikeComparison(LikeComparisonContext ctx) { @Override public void exitLikeComparison(LikeComparisonContext ctx) { - // TODO Auto-generated method stub + // Nothing useful to do in this mocked version } @Override public void enterOperator(OperatorContext ctx) { - // TODO Auto-generated method stub + // Nothing useful to do in this mocked version } @Override public void exitOperator(OperatorContext ctx) { - // TODO Auto-generated method stub + // Nothing useful to do in this mocked version } @Override public void visitTerminal(TerminalNode node) { - // TODO Auto-generated method stub + // Nothing useful to do in this mocked version } @Override public void visitErrorNode(ErrorNode node) { - // TODO Auto-generated method stub + // Nothing useful to do in this mocked version } @Override public void enterEveryRule(ParserRuleContext ctx) { - // TODO Auto-generated method stub + // Nothing useful to do in this mocked version } @Override public void exitEveryRule(ParserRuleContext ctx) { - // TODO Auto-generated method stub + // Nothing useful to do in this mocked version } @Override public void enterExistence(ExistenceContext ctx) { - // TODO Auto-generated method stub + // Nothing useful to do in this mocked version } diff --git a/service/README.md b/service/README.md index 1bcf5d35..ca69f082 100644 --- a/service/README.md +++ b/service/README.md @@ -9,7 +9,7 @@ For more information, please visit https://nasa-pds.github.io/registry-api-servi ## Prerequisites -This software requires open jdk 17. +This software requires open jdk 25. ## Administrator @@ -32,7 +32,16 @@ Note, the registry index in elasticSearch is hard-coded. It need to be `registry mvn clean mvn install + cd service mvn spring-boot:run + + The API will now be accessible on (by default) https://localhost:8080 + + With a specific configuration profile you can run the application with a specific configuration. Define a dedicated application.properties, for example application-dev.properties that does not need to be committed on git. Launch it as follows: + + mvn -Dspring-boot.run.profiles=dev spring-boot:run + + 👉 **Note:** in order to run in this way, you will need to modify the `spring-boot-starter-thymeleaf` dependency by pinning it to version `1.5.1.RELEASE` and excluding the `logback-classic` artifact in the `pom.xml` file as follows: From 11b7f80433dbfcf7a3027ff91a747d7ae054a5d9 Mon Sep 17 00:00:00 2001 From: PDSEN CI Bot Date: Fri, 12 Dec 2025 13:52:53 +0000 Subject: [PATCH 019/137] Update changelog --- CHANGELOG.md | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 544c5246..3d3601d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,8 @@ # Changelog -## [1.6.0](https://github.com/NASA-PDS/registry-api/tree/1.6.0) (2025-10-14) +## [«unknown»](https://github.com/NASA-PDS/registry-api/tree/«unknown») (2025-12-12) -[Full Changelog](https://github.com/NASA-PDS/registry-api/compare/1.6.0...v1.6.2) +[Full Changelog](https://github.com/NASA-PDS/registry-api/compare/v1.6.2...«unknown») **Defects:** @@ -31,11 +31,7 @@ ## [v1.6.0](https://github.com/NASA-PDS/registry-api/tree/v1.6.0) (2025-06-26) -[Full Changelog](https://github.com/NASA-PDS/registry-api/compare/release/1.6.0...v1.6.0) - -## [release/1.6.0](https://github.com/NASA-PDS/registry-api/tree/release/1.6.0) (2025-06-25) - -[Full Changelog](https://github.com/NASA-PDS/registry-api/compare/v1.5.0...release/1.6.0) +[Full Changelog](https://github.com/NASA-PDS/registry-api/compare/v1.5.0...v1.6.0) **Requirements:** @@ -65,11 +61,7 @@ ## [v1.5.0](https://github.com/NASA-PDS/registry-api/tree/v1.5.0) (2024-09-03) -[Full Changelog](https://github.com/NASA-PDS/registry-api/compare/release/1.5.0...v1.5.0) - -## [release/1.5.0](https://github.com/NASA-PDS/registry-api/tree/release/1.5.0) (2024-09-03) - -[Full Changelog](https://github.com/NASA-PDS/registry-api/compare/v1.4.1...release/1.5.0) +[Full Changelog](https://github.com/NASA-PDS/registry-api/compare/v1.4.1...v1.5.0) **Requirements:** @@ -117,11 +109,7 @@ ## [v1.4.1](https://github.com/NASA-PDS/registry-api/tree/v1.4.1) (2024-02-29) -[Full Changelog](https://github.com/NASA-PDS/registry-api/compare/release/1.4.1...v1.4.1) - -## [release/1.4.1](https://github.com/NASA-PDS/registry-api/tree/release/1.4.1) (2024-02-29) - -[Full Changelog](https://github.com/NASA-PDS/registry-api/compare/v1.4.0...release/1.4.1) +[Full Changelog](https://github.com/NASA-PDS/registry-api/compare/v1.4.0...v1.4.1) **Defects:** From e5e40e5dfdb420e66a9e557e3db5ca678aa36744 Mon Sep 17 00:00:00 2001 From: PDSEN CI Bot Date: Fri, 12 Dec 2025 17:39:10 +0000 Subject: [PATCH 020/137] Update changelog From 27da157758e8c0bc6750f543dedf7453f5fe2781 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Dec 2025 14:14:56 +0000 Subject: [PATCH 021/137] Bump actions/cache from 4 to 5 Bumps [actions/cache](https://github.com/actions/cache) from 4 to 5. - [Release notes](https://github.com/actions/cache/releases) - [Changelog](https://github.com/actions/cache/blob/main/RELEASES.md) - [Commits](https://github.com/actions/cache/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/cache dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/branch-cicd.yaml | 2 +- .github/workflows/stable-cicd.yaml | 2 +- .github/workflows/unstable-cicd.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/branch-cicd.yaml b/.github/workflows/branch-cicd.yaml index 62268b6c..00130ce4 100644 --- a/.github/workflows/branch-cicd.yaml +++ b/.github/workflows/branch-cicd.yaml @@ -47,7 +47,7 @@ jobs: token: ${{secrets.ADMIN_GITHUB_TOKEN || github.token}} - name: 💵 Maven Cache - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: ~/.m2/repository # The "key" used to indicate a set of cached files is the operating system runner diff --git a/.github/workflows/stable-cicd.yaml b/.github/workflows/stable-cicd.yaml index 1be47589..74d7054a 100644 --- a/.github/workflows/stable-cicd.yaml +++ b/.github/workflows/stable-cicd.yaml @@ -57,7 +57,7 @@ jobs: fetch-depth: 0 - name: 💵 Maven Cache - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: ~/.m2/repository # The "key" used to indicate a set of cached files is the operating system runner diff --git a/.github/workflows/unstable-cicd.yaml b/.github/workflows/unstable-cicd.yaml index d1e7480f..ada5052d 100644 --- a/.github/workflows/unstable-cicd.yaml +++ b/.github/workflows/unstable-cicd.yaml @@ -57,7 +57,7 @@ jobs: token: ${{secrets.ADMIN_GITHUB_TOKEN}} - name: 💵 Maven Cache - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: ~/.m2/repository # The "key" used to indicate a set of cached files is the operating system runner From 32df333441bdc4c55d42aa3dde0e47e28e112546 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Dec 2025 14:15:00 +0000 Subject: [PATCH 022/137] Bump actions/upload-artifact from 5 to 6 Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 5 to 6. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index adbc1854..1cfcc398 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -80,7 +80,7 @@ jobs: - name: Upload CodeQL Artifacts - uses: actions/upload-artifact@v5 + uses: actions/upload-artifact@v6 with: name: codeql-artifacts path: ${{ env.RESULTS_DIR }} @@ -108,7 +108,7 @@ jobs: - name: Upload SLOC - uses: actions/upload-artifact@v5 + uses: actions/upload-artifact@v6 with: name: sloc-count path: ${{ github.workspace }}/cloc.md From e58ff6bdf08ae11a251ac3f3b8185ed0781010b4 Mon Sep 17 00:00:00 2001 From: PDSEN CI Bot Date: Tue, 6 Jan 2026 18:30:00 +0000 Subject: [PATCH 023/137] Update changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d3601d6..491d3d4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,12 @@ # Changelog -## [«unknown»](https://github.com/NASA-PDS/registry-api/tree/«unknown») (2025-12-12) +## [«unknown»](https://github.com/NASA-PDS/registry-api/tree/«unknown») (2026-01-06) [Full Changelog](https://github.com/NASA-PDS/registry-api/compare/v1.6.2...«unknown») **Defects:** +- Inconsistent support for `application/vnd.nasa.pds.pds4+json` response format [\#705](https://github.com/NASA-PDS/registry-api/issues/705) [[s.high](https://github.com/NASA-PDS/registry-api/labels/s.high)] - API search results using "search-after" returns empty \[data\] block even though I can find the product by lidvid [\#677](https://github.com/NASA-PDS/registry-api/issues/677) [[s.high](https://github.com/NASA-PDS/registry-api/labels/s.high)] **Other closed issues:** From f05ffb4e7f5130d69cd80bd737858b7a03f38802 Mon Sep 17 00:00:00 2001 From: edunn Date: Tue, 6 Jan 2026 10:43:12 -0800 Subject: [PATCH 024/137] document behaviour fixes https://github.com/NASA-PDS/registry/issues/456 --- model/swagger.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/model/swagger.yml b/model/swagger.yml index 2b9fe25e..030a17df 100644 --- a/model/swagger.yml +++ b/model/swagger.yml @@ -2,6 +2,8 @@ openapi: 3.0.0 info: description: | Registry API enabling advanced search on PDS data and metadata. The API provides end-points to search for bundles, collections and any PDS products with advanced search queries. It also enables to browse the archive hierarchically downward (e.g. collection/s products) or upward (e.g. bundles containing a product). + + Property values are cast to string in responses due to limitations of JSON typing, and should be interpreted by the client/user according to the data dictionary. version: 1.3.0 title: PDS Registry Search API termsOfService: 'http://pds.nasa.gov' From f9fe1ea6e21b28303db93d49ca1fcd9b55e848b9 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Mon, 12 Jan 2026 14:47:17 -0800 Subject: [PATCH 025/137] updated antlr language Add ALL(FIELDNAME) and ANY(FIELDNAME) where FIELDNAME is same as ANY(FIELDNAME). Tests say that it works not as expected but good enough. Also, may the wildcard character * such that *apple will be treated as all field names ending in apple. --- .../gov/nasa/pds/api/registry/lexer/Search.g4 | 13 +++-- .../MockedListener.java | 34 +++++++++-- .../api_search_query_lexer/TestParsing.java | 56 +++++++------------ 3 files changed, 57 insertions(+), 46 deletions(-) diff --git a/lexer/src/main/antlr4/gov/nasa/pds/api/registry/lexer/Search.g4 b/lexer/src/main/antlr4/gov/nasa/pds/api/registry/lexer/Search.g4 index 1af8dfdb..aa5221aa 100644 --- a/lexer/src/main/antlr4/gov/nasa/pds/api/registry/lexer/Search.g4 +++ b/lexer/src/main/antlr4/gov/nasa/pds/api/registry/lexer/Search.g4 @@ -1,14 +1,15 @@ grammar Search; -query : queryTerm EOF ; +query : queryTerm EOF ; queryTerm : comparison | likeComparison | existence | group ; +fields : FIELDNAME | ALL LPAREN FIELDNAME RPAREN | ANY LPAREN FIELDNAME RPAREN ; group : NOT? LPAREN expression RPAREN ; -existence : ( FIELD | STRINGVAL ) EXISTS ; +existence : fields EXISTS ; expression : andStatement | orStatement | queryTerm ; andStatement : queryTerm (AND queryTerm)+ ; orStatement : queryTerm (OR queryTerm)+ ; -comparison : FIELD operator ( NUMBER | STRINGVAL ) ; -likeComparison : FIELD LIKE STRINGVAL ; +comparison : fields operator ( NUMBER | STRINGVAL ) ; +likeComparison : fields LIKE STRINGVAL ; operator : EQ | NE | GT | GE | LT | LE ; NOT : 'NOT' | 'not' ; @@ -26,10 +27,12 @@ LIKE: L I K E; LPAREN : '(' ; RPAREN : ')' ; +ALL : A L L ; AND : A N D ; +ANY : A N Y ; OR : O R ; -FIELD : [A-Za-z_] [A-Za-z0-9_.:/]* ; +FIELDNAME : [A-Za-z_*] [A-Za-z0-9_.:/*]* ; STRINGVAL : '"' ~["\r\n]* '"' ; NUMBER : ('-')? [0-9]+ ('.' [0-9]*)? ; diff --git a/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/MockedListener.java b/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/MockedListener.java index 2fe5393e..df003be4 100644 --- a/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/MockedListener.java +++ b/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/MockedListener.java @@ -1,5 +1,6 @@ package api.pds.nasa.gov.api_search_query_lexer; +import java.util.ArrayList; import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.tree.ErrorNode; import org.antlr.v4.runtime.tree.ParseTreeListener; @@ -8,6 +9,7 @@ import gov.nasa.pds.api.registry.lexer.SearchParser.AndStatementContext; import gov.nasa.pds.api.registry.lexer.SearchParser.ComparisonContext; import gov.nasa.pds.api.registry.lexer.SearchParser.ExpressionContext; +import gov.nasa.pds.api.registry.lexer.SearchParser.FieldsContext; import gov.nasa.pds.api.registry.lexer.SearchParser.GroupContext; import gov.nasa.pds.api.registry.lexer.SearchParser.LikeComparisonContext; import gov.nasa.pds.api.registry.lexer.SearchParser.ExistenceContext; @@ -18,8 +20,8 @@ public class MockedListener implements ParseTreeListener, SearchListener { - - TerminalNode field = null, number = null, strval = null; + ArrayList fields = new ArrayList(); + TerminalNode number = null, strval = null; boolean isNot = false; @Override @@ -96,7 +98,6 @@ public void exitOrStatement(OrStatementContext ctx) { @Override public void enterComparison(ComparisonContext ctx) { - this.field = ctx.FIELD(); this.number = ctx.NUMBER(); this.strval = ctx.STRINGVAL(); } @@ -109,7 +110,6 @@ public void exitComparison(ComparisonContext ctx) { @Override public void enterLikeComparison(LikeComparisonContext ctx) { - this.field = ctx.FIELD(); this.strval = ctx.STRINGVAL(); String op = ctx.getChild(1).getText(); @@ -168,8 +168,30 @@ public void enterExistence(ExistenceContext ctx) { @Override public void exitExistence(ExistenceContext ctx) { - this.field = ctx.FIELD(); - this.strval = ctx.STRINGVAL(); + } + + @Override + public void enterFields(FieldsContext ctx) { + } + + @Override + public void exitFields(FieldsContext ctx) { + boolean any = ctx.ALL() == null; + String fieldname = ""; + if (ctx.FIELDNAME() != null) { + fieldname = ctx.FIELDNAME().getText(); + } + if (ctx.ALL() != null ) { + fieldname = ctx.ALL().getText(); + } + if (ctx.ANY() != null) { + fieldname = ctx.ANY().getText(); + } + if (fieldname.contains("*")) { + fields.add(fieldname.replace(".", "\\.").replace("*", ".*")); + } else { + fields.add(fieldname); + } } } diff --git a/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/TestParsing.java b/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/TestParsing.java index 838ee6ee..8241de25 100644 --- a/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/TestParsing.java +++ b/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/TestParsing.java @@ -5,24 +5,12 @@ import org.antlr.v4.runtime.CharStreams; import org.antlr.v4.runtime.CodePointCharStream; import org.antlr.v4.runtime.CommonTokenStream; -import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.misc.ParseCancellationException; -import org.antlr.v4.runtime.tree.ErrorNode; import org.antlr.v4.runtime.tree.ParseTree; -import org.antlr.v4.runtime.tree.ParseTreeListener; import org.antlr.v4.runtime.tree.ParseTreeWalker; import org.junit.jupiter.api.Test; import gov.nasa.pds.api.registry.lexer.SearchLexer; import gov.nasa.pds.api.registry.lexer.SearchParser; -import gov.nasa.pds.api.registry.lexer.SearchParser.AndStatementContext; -import gov.nasa.pds.api.registry.lexer.SearchParser.ComparisonContext; -import gov.nasa.pds.api.registry.lexer.SearchParser.ExpressionContext; -import gov.nasa.pds.api.registry.lexer.SearchParser.GroupContext; -import gov.nasa.pds.api.registry.lexer.SearchParser.LikeComparisonContext; -import gov.nasa.pds.api.registry.lexer.SearchParser.OperatorContext; -import gov.nasa.pds.api.registry.lexer.SearchParser.OrStatementContext; -import gov.nasa.pds.api.registry.lexer.SearchParser.QueryContext; -import gov.nasa.pds.api.registry.lexer.SearchParser.QueryTermContext; import org.junit.jupiter.api.Assertions; @@ -48,8 +36,6 @@ public void testMaliciousQuery() { par.setErrorHandler(new BailErrorStrategy()); ParseTree tree = par.query(); }, "Expected code to throw, but it didn't"); - - } @@ -66,8 +52,8 @@ public void testNumber() { MockedListener listener = new MockedListener(); walker.walk(listener, tree); - Assertions.assertNotNull(listener.field); - Assertions.assertEquals(listener.field.getSymbol().getText(), "lid"); + Assertions.assertEquals(listener.fields.size(), 1); + Assertions.assertEquals(listener.fields.get(0), "lid"); Assertions.assertNotEquals(listener.number, null); Assertions.assertEquals(listener.number.getSymbol().getText(), "1234"); @@ -86,8 +72,8 @@ public void testStringVal() { MockedListener listener = new MockedListener(); walker.walk(listener, tree); - Assertions.assertNotNull(listener.field); - Assertions.assertEquals(listener.field.getSymbol().getText(), "lid"); + Assertions.assertEquals(listener.fields.size(), 1); + Assertions.assertEquals(listener.fields.get(0), "lid"); Assertions.assertNotNull(listener.strval); Assertions.assertEquals(listener.strval.getSymbol().getText(), "\"*text*\""); @@ -106,8 +92,8 @@ public void testLike() { MockedListener listener = new MockedListener(); walker.walk(listener, tree); - Assertions.assertNotNull(listener.field); - Assertions.assertEquals(listener.field.getText(), "lid"); + Assertions.assertNotNull(listener.fields); + Assertions.assertEquals(listener.fields.get(0), "lid"); Assertions.assertNotNull(listener.strval); Assertions.assertEquals(listener.strval.getText(), "\"*text*\""); @@ -144,9 +130,9 @@ void testFieldExistence() { MockedListener listener = new MockedListener(); walker.walk(listener, tree); - Assertions.assertNotNull(listener.field); + Assertions.assertEquals(1, listener.fields.size()); Assertions.assertNull(listener.strval); - Assertions.assertEquals("apple", listener.field.getSymbol().getText()); + Assertions.assertEquals("apple", listener.fields.get(0)); } @@ -163,14 +149,14 @@ void testParenFieldExistence() { MockedListener listener = new MockedListener(); walker.walk(listener, tree); - Assertions.assertNotNull(listener.field); - Assertions.assertNull(listener.strval); - Assertions.assertEquals("apple", listener.field.getSymbol().getText()); + Assertions.assertEquals(1, listener.fields.size()); + Assertions.assertNull(listener.strval, "strval should be null not: " + listener.strval); + Assertions.assertEquals("apple", listener.fields.get(0)); } @Test - void testStrvalExistence() { - String queryString = "\".*apple\" exists"; + void testWildExistence() { + String queryString = "*.apple exists"; CodePointCharStream input = CharStreams.fromString(queryString); SearchLexer lex = new SearchLexer(input); CommonTokenStream tokens = new CommonTokenStream(lex); @@ -181,15 +167,15 @@ void testStrvalExistence() { MockedListener listener = new MockedListener(); walker.walk(listener, tree); - Assertions.assertNull(listener.field); - Assertions.assertNotNull(listener.strval); - Assertions.assertEquals("\".*apple\"", listener.strval.getSymbol().getText()); + Assertions.assertEquals(1, listener.fields.size()); + Assertions.assertNull(listener.strval, "strval should be null not: " + listener.strval); + Assertions.assertEquals(".*\\.apple", listener.fields.get(0)); } @Test - void testParenStrvalExistence() { - String queryString = "(\".*apple\" exists)"; + void testParenWildExistence() { + String queryString = "(*apple exists)"; CodePointCharStream input = CharStreams.fromString(queryString); SearchLexer lex = new SearchLexer(input); CommonTokenStream tokens = new CommonTokenStream(lex); @@ -200,9 +186,9 @@ void testParenStrvalExistence() { MockedListener listener = new MockedListener(); walker.walk(listener, tree); - Assertions.assertNull(listener.field); - Assertions.assertNotNull(listener.strval); - Assertions.assertEquals("\".*apple\"", listener.strval.getSymbol().getText()); + Assertions.assertEquals(listener.fields.size(), 1); + Assertions.assertNull(listener.strval); + Assertions.assertEquals(".*apple", listener.fields.get(0)); } From 79a1fe5cf83452e7dd8dc074bf89cf0aaf01da6a Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Tue, 13 Jan 2026 08:13:34 -0800 Subject: [PATCH 026/137] update processor to match new wildcarding Moved where the LDD field names are loaded to a common area. All field names are now processed equally. If no field names are found using the the wildcarding, processing is aborted via an exception explaining so. Do we move the downloading of the LDD fieldnames out of the loop and check non-wildcard values as well and return the same/similar error message? All of the uses of field names have been updated to reflect the new array of field names and concatentate them in a giant bool query using must (ALL or 1 item) and should (ANY with many items). --- .../registry/model/Antlr4SearchListener.java | 184 ++++++++++-------- 1 file changed, 100 insertions(+), 84 deletions(-) diff --git a/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java b/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java index 1cd50bf6..27a2623f 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java @@ -37,9 +37,11 @@ enum operation { private static final Logger log = LoggerFactory.getLogger(Antlr4SearchListener.class); + private boolean isAnyWildcard = true; private BoolQuery.Builder queryBuilder = new BoolQuery.Builder(); private conjunctions conjunction = conjunctions.AND; // DEFAULT + private final ArrayList fieldnames = new ArrayList(); private final ConnectionContext connectionContext; private final Deque stackQueryBuilders = new ArrayDeque(); private final Deque stackConjunction = new ArrayDeque(); @@ -54,8 +56,49 @@ public Antlr4SearchListener(ConnectionContext connectionContext) { @Override - public void exitQuery(SearchParser.QueryContext ctx) {} - + public void enterFields(SearchParser.FieldsContext ctx) { + this.fieldnames.clear(); + this.isAnyWildcard = true; + } + + @Override + public void exitFields(SearchParser.FieldsContext ctx) { + String fieldname = ""; + if (ctx.FIELDNAME() != null) { + fieldname = ctx.FIELDNAME().getText(); + } + if (ctx.ALL() != null ) { + fieldname = ctx.ALL().getText(); + } + if (ctx.ANY() != null) { + fieldname = ctx.ANY().getText(); + } + if (fieldname.contains("*")) { + if (this.knownFieldNames.isEmpty()) { + try { + for (PropertiesListInner property : ProductsController.productPropertiesList(this.connectionContext).getBody()) { + this.knownFieldNames.add(property.getProperty()); + } + } catch (OpenSearchException | IOException e) { + log.error("Could not load the mapping(s) from opensearch; meaning 'wildcarding' will not work", e); + } + } + String theKey = fieldname.replace(".", "\\.").replace("*", ".*"); + Pattern regex = Pattern.compile(theKey); + for (String fn : this.knownFieldNames.stream() + .filter(s -> regex.matcher(s).matches()) + .toList()) { + this.fieldnames.add(SearchUtil.jsonPropertyToOpenProperty(fn)); + } + if (this.fieldnames.isEmpty()) { + throw new ParseCancellationException("Wildcarding request '" + fieldname + "' cannot match any field names in the LDD using regular expression " + theKey); + } + } else { + this.fieldnames.add(fieldname); + } + this.isAnyWildcard = ctx.ALL() == null && this.fieldnames.size() > 1; + } + @Override public void enterGroup(SearchParser.GroupContext ctx) { log.debug("Enter Group"); @@ -110,14 +153,11 @@ public void exitOrStatement(SearchParser.OrStatementContext ctx) { } - @Override - public void enterComparison(SearchParser.ComparisonContext ctx) {} - @Override public void exitComparison(SearchParser.ComparisonContext ctx) { log.debug("Exit comparison"); - final String left = SearchUtil.jsonPropertyToOpenProperty(ctx.FIELD().getSymbol().getText()); + BoolQuery.Builder wild = new BoolQuery.Builder(); String right; Query comparatorQuery = null; @@ -132,111 +172,87 @@ public void exitComparison(SearchParser.ComparisonContext ctx) { "A right component (literal) of a comparison is neither a number or a string. Number and String are the only types supported for literals."); } - if (this.operator == operation.eq || this.operator == operation.ne) { - - - FieldValue fieldValue = new FieldValue.Builder().stringValue(right).build(); - - MatchQuery matchQueryBuilder = new MatchQuery.Builder().field(left).query(fieldValue).build(); + for (String left : this.fieldnames) { + if (this.operator == operation.eq || this.operator == operation.ne) { + FieldValue fieldValue = new FieldValue.Builder().stringValue(right).build(); + MatchQuery matchQueryBuilder = new MatchQuery.Builder().field(left).query(fieldValue).build(); + comparatorQuery = matchQueryBuilder.toQuery(); - comparatorQuery = matchQueryBuilder.toQuery(); - - if (this.operator == operation.ne) { - comparatorQuery = new BoolQuery.Builder().mustNot(comparatorQuery).build().toQuery(); + if (this.operator == operation.ne) { + comparatorQuery = new BoolQuery.Builder().mustNot(comparatorQuery).build().toQuery(); + } + } else { + RangeQuery.Builder rangeQueryBuilder = new RangeQuery.Builder(); + rangeQueryBuilder = rangeQueryBuilder.field(left); + + if (this.operator == operation.ge) + rangeQueryBuilder.gte(JsonData.of(right)); + else if (this.operator == operation.gt) + rangeQueryBuilder.gt(JsonData.of(right)); + else if (this.operator == operation.le) + rangeQueryBuilder.lte(JsonData.of(right)); + else if (this.operator == operation.lt) + rangeQueryBuilder.lt(JsonData.of(right)); + else { + throw new ParseCancellationException("Operator " + this.operator.name() + + " is not supported. Supported comparison operators are eq, ne, gt, gte, lt, lte."); + } + comparatorQuery = rangeQueryBuilder.build().toQuery(); } - - - - } else { - RangeQuery.Builder rangeQueryBuilder = new RangeQuery.Builder(); - - rangeQueryBuilder = rangeQueryBuilder.field(left); - - if (this.operator == operation.ge) - rangeQueryBuilder.gte(JsonData.of(right)); - else if (this.operator == operation.gt) - rangeQueryBuilder.gt(JsonData.of(right)); - else if (this.operator == operation.le) - rangeQueryBuilder.lte(JsonData.of(right)); - else if (this.operator == operation.lt) - rangeQueryBuilder.lt(JsonData.of(right)); - else { - throw new ParseCancellationException("Operator " + this.operator.name() - + " is not supported. Supported comparison operators are eq, ne, gt, gte, lt, lte."); + if (this.isAnyWildcard) { + wild.should(comparatorQuery); + } else { + wild.must(comparatorQuery); } - - comparatorQuery = rangeQueryBuilder.build().toQuery(); - } - if (this.conjunction == conjunctions.AND) { - this.queryBuilder.must(comparatorQuery); + this.queryBuilder.must(wild.build().toQuery()); } else { - this.queryBuilder.should(comparatorQuery); + this.queryBuilder.should(wild.build().toQuery()); } } @Override public void exitExistence(SearchParser.ExistenceContext ctx) { - ArrayList checks = new ArrayList(); - final String fieldName = ctx.FIELD() == null ? "" : SearchUtil.jsonPropertyToOpenProperty(ctx.FIELD().getSymbol().getText()); - final String regexp = ctx.STRINGVAL() == null ? "" : ctx.STRINGVAL().getText(); - String theKey = "''"; - - if (!fieldName.isBlank()) { - theKey = fieldName; - checks.add(new ExistsQuery.Builder().field(fieldName).build().toQuery()); - } else if (!regexp.isBlank()) { - theKey = regexp.substring(1, regexp.length()-1); - if (this.knownFieldNames.isEmpty()) { - try { - for (PropertiesListInner property : ProductsController.productPropertiesList(this.connectionContext).getBody()) { - this.knownFieldNames.add(property.getProperty()); - } - } catch (OpenSearchException | IOException e) { - log.error("Could not load the mapping(s) from opensearch; meaning 'exists' will not work", e); - } - } - Pattern regex = Pattern.compile(theKey); - for (String fn : this.knownFieldNames.stream() - .filter(s -> regex.matcher(s).matches()) - .toList()) { - checks.add(new ExistsQuery.Builder().field(SearchUtil.jsonPropertyToOpenProperty(fn)).build().toQuery()); + BoolQuery.Builder wild = new BoolQuery.Builder(); + for (String fieldName : this.fieldnames) { + if (this.isAnyWildcard) { + wild.should(new ExistsQuery.Builder().field(fieldName).build().toQuery()); + } else { + wild.must(new ExistsQuery.Builder().field(fieldName).build().toQuery()); } } - if (checks.isEmpty()) { - throw new ParseCancellationException("For existence testing, cannot match any field names to " + theKey); - } if (this.conjunction == conjunctions.AND) { - this.queryBuilder.must(checks); + this.queryBuilder.must(wild.build().toQuery()); } else { - this.queryBuilder.should(checks); + this.queryBuilder.should(wild.build().toQuery()); } } - @Override - public void enterLikeComparison(SearchParser.LikeComparisonContext ctx) {} - @Override public void exitLikeComparison(SearchParser.LikeComparisonContext ctx) { log.debug("Exit likeComparison"); - final String left = SearchUtil.jsonPropertyToOpenProperty(ctx.FIELD().getSymbol().getText()); + BoolQuery.Builder wild = new BoolQuery.Builder(); String right = ctx.STRINGVAL().getText(); - // remove quotes - right = right.replaceAll("^\"|\"$", ""); - - SimpleQueryStringQuery simpleQueryString = new SimpleQueryStringQuery.Builder().fields(left) - .query(right).fuzzyMaxExpansions(0).build(); - - Query query = simpleQueryString.toQuery(); - log.debug("Exit Like comparison: left member is {} right member is {}", left, right); + right = right.replaceAll("^\"|\"$", ""); // remove the quotes + for (String left : this.fieldnames) { + SimpleQueryStringQuery simpleQueryString = new SimpleQueryStringQuery.Builder().fields(left) + .query(right).fuzzyMaxExpansions(0).build(); + if (this.isAnyWildcard) { + wild.should(simpleQueryString.toQuery()); + } else { + wild.must(simpleQueryString.toQuery()); + } + log.debug("Exit Like comparison: left member is {} right member is {}", left, right); + } + if (this.conjunction == conjunctions.AND) { - this.queryBuilder.must(query); + this.queryBuilder.must(wild.build().toQuery()); } else { - this.queryBuilder.should(query); + this.queryBuilder.should(wild.build().toQuery()); } } From dc1a2ade6f5b95d28f4121df6ab33695f4c5a098 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Tue, 13 Jan 2026 08:19:41 -0800 Subject: [PATCH 027/137] make sure at least on should matches --- .../pds/api/registry/model/Antlr4SearchListener.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java b/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java index 27a2623f..43367b91 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java @@ -161,7 +161,6 @@ public void exitComparison(SearchParser.ComparisonContext ctx) { String right; Query comparatorQuery = null; - if (ctx.NUMBER() != null) { right = ctx.NUMBER().getSymbol().getText(); } else if (ctx.STRINGVAL() != null) { @@ -171,7 +170,9 @@ public void exitComparison(SearchParser.ComparisonContext ctx) { throw new ParseCancellationException( "A right component (literal) of a comparison is neither a number or a string. Number and String are the only types supported for literals."); } - + if (this.isAnyWildcard) { + wild.minimumShouldMatch("1"); + } for (String left : this.fieldnames) { if (this.operator == operation.eq || this.operator == operation.ne) { FieldValue fieldValue = new FieldValue.Builder().stringValue(right).build(); @@ -216,6 +217,9 @@ else if (this.operator == operation.lt) @Override public void exitExistence(SearchParser.ExistenceContext ctx) { BoolQuery.Builder wild = new BoolQuery.Builder(); + if (this.isAnyWildcard) { + wild.minimumShouldMatch("1"); + } for (String fieldName : this.fieldnames) { if (this.isAnyWildcard) { wild.should(new ExistsQuery.Builder().field(fieldName).build().toQuery()); @@ -238,6 +242,9 @@ public void exitLikeComparison(SearchParser.LikeComparisonContext ctx) { String right = ctx.STRINGVAL().getText(); right = right.replaceAll("^\"|\"$", ""); // remove the quotes + if (this.isAnyWildcard) { + wild.minimumShouldMatch("1"); + } for (String left : this.fieldnames) { SimpleQueryStringQuery simpleQueryString = new SimpleQueryStringQuery.Builder().fields(left) .query(right).fuzzyMaxExpansions(0).build(); From 9c66d46a7189a88e066704ed35756a268c411147 Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Thu, 22 Jan 2026 08:13:35 -0800 Subject: [PATCH 028/137] upgrade jdk dependency in cicd --- .github/workflows/unstable-cicd.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unstable-cicd.yaml b/.github/workflows/unstable-cicd.yaml index ada5052d..4068ab97 100644 --- a/.github/workflows/unstable-cicd.yaml +++ b/.github/workflows/unstable-cicd.yaml @@ -73,7 +73,7 @@ jobs: uses: NASA-PDS/roundup-action@stable with: assembly: unstable - packages: openjdk17-jdk + packages: openjdk25-jdk maven-doc-phases: package env: central_portal_username: ${{secrets.CENTRAL_REPOSITORY_USERNAME}} From a156e2573546f45363f4337c85588d8fb4c35680 Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Thu, 22 Jan 2026 09:03:34 -0800 Subject: [PATCH 029/137] upgrade jdk in docker image --- docker/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 220a79a3..59c85c45 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -41,8 +41,8 @@ # Normally we'd prefer Alpine Linux, but JDK 11 isn't available with it, so # we go with a slim Debian. Debian's good too. -#FROM tomcat:9.0.58-jdk17-openjdk-slim -FROM tomcat:10.1.0-jdk17-openjdk-slim +FROM tomcat:jre25-temurin-noble + # API JAR file # ------------ @@ -92,4 +92,4 @@ CMD java $JAVA_OPTS -jar /usr/local/registry-api-service/registry-api-service.ja LABEL "org.label-schema.name" "PDS Registry API" LABEL "org.label-schema.description" "Planetary Data System's Application Programmer's Interface for the Registry" -LABEL "org.label-schema.url" "https://github.com/NASA-PDS/registry-api" \ No newline at end of file +LABEL "org.label-schema.url" "https://github.com/NASA-PDS/registry-api" From 06f493fd7abdfb7313b9b02d95b7d045ed3e9af4 Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Thu, 22 Jan 2026 09:14:25 -0800 Subject: [PATCH 030/137] try a different jdk package name --- .github/workflows/unstable-cicd.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unstable-cicd.yaml b/.github/workflows/unstable-cicd.yaml index 4068ab97..624e2d95 100644 --- a/.github/workflows/unstable-cicd.yaml +++ b/.github/workflows/unstable-cicd.yaml @@ -73,7 +73,7 @@ jobs: uses: NASA-PDS/roundup-action@stable with: assembly: unstable - packages: openjdk25-jdk + packages: openjdk25 maven-doc-phases: package env: central_portal_username: ${{secrets.CENTRAL_REPOSITORY_USERNAME}} From 0ba5f43755bac82bb8556afa1265bf89eb4490e1 Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Thu, 22 Jan 2026 13:24:31 -0800 Subject: [PATCH 031/137] downgrade to jdk21 since 25 is not available alpine 3.22 --- .github/workflows/unstable-cicd.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unstable-cicd.yaml b/.github/workflows/unstable-cicd.yaml index 624e2d95..60e11a37 100644 --- a/.github/workflows/unstable-cicd.yaml +++ b/.github/workflows/unstable-cicd.yaml @@ -73,7 +73,7 @@ jobs: uses: NASA-PDS/roundup-action@stable with: assembly: unstable - packages: openjdk25 + packages: openjdk21 maven-doc-phases: package env: central_portal_username: ${{secrets.CENTRAL_REPOSITORY_USERNAME}} From 5048ea1cf62154432125479e2e370e49c16c2265 Mon Sep 17 00:00:00 2001 From: PDSEN CI Bot Date: Thu, 22 Jan 2026 21:31:28 +0000 Subject: [PATCH 032/137] Update changelog --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 491d3d4a..48b9d447 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,16 +1,18 @@ # Changelog -## [«unknown»](https://github.com/NASA-PDS/registry-api/tree/«unknown») (2026-01-06) +## [«unknown»](https://github.com/NASA-PDS/registry-api/tree/«unknown») (2026-01-22) [Full Changelog](https://github.com/NASA-PDS/registry-api/compare/v1.6.2...«unknown») **Defects:** +- API in production is unstable and returns 500 errors [\#716](https://github.com/NASA-PDS/registry-api/issues/716) [[s.critical](https://github.com/NASA-PDS/registry-api/labels/s.critical)] - Inconsistent support for `application/vnd.nasa.pds.pds4+json` response format [\#705](https://github.com/NASA-PDS/registry-api/issues/705) [[s.high](https://github.com/NASA-PDS/registry-api/labels/s.high)] - API search results using "search-after" returns empty \[data\] block even though I can find the product by lidvid [\#677](https://github.com/NASA-PDS/registry-api/issues/677) [[s.high](https://github.com/NASA-PDS/registry-api/labels/s.high)] **Other closed issues:** +- B13.1 Registry + API [\#724](https://github.com/NASA-PDS/registry-api/issues/724) - Registry API Test Suite is failing [\#680](https://github.com/NASA-PDS/registry-api/issues/680) [[s.critical](https://github.com/NASA-PDS/registry-api/labels/s.critical)] - Manage errors as recommended in the spring mvc framework [\#286](https://github.com/NASA-PDS/registry-api/issues/286) From 12d25ed708153205400109f8e5d63159127f2432 Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Mon, 26 Jan 2026 10:23:56 -0800 Subject: [PATCH 033/137] add detail on string cast in the responses --- model/swagger.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/model/swagger.yml b/model/swagger.yml index 030a17df..79f26eef 100644 --- a/model/swagger.yml +++ b/model/swagger.yml @@ -4,6 +4,7 @@ info: Registry API enabling advanced search on PDS data and metadata. The API provides end-points to search for bundles, collections and any PDS products with advanced search queries. It also enables to browse the archive hierarchically downward (e.g. collection/s products) or upward (e.g. bundles containing a product). Property values are cast to string in responses due to limitations of JSON typing, and should be interpreted by the client/user according to the data dictionary. + As a result of the string cast, some missing values might be found as "null". version: 1.3.0 title: PDS Registry Search API termsOfService: 'http://pds.nasa.gov' From 10b74aaf016bdd75dc7c119eb2f100b4152fbddc Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Mon, 26 Jan 2026 11:53:57 -0800 Subject: [PATCH 034/137] move description in code as that is what is displayed. Did not have time to investigate how to get the description from the openapi.yml file. --- model/swagger.yml | 5 ++--- .../api/registry/configuration/OpenApiConfiguration.java | 7 ++++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/model/swagger.yml b/model/swagger.yml index 79f26eef..a41fbceb 100644 --- a/model/swagger.yml +++ b/model/swagger.yml @@ -2,13 +2,12 @@ openapi: 3.0.0 info: description: | Registry API enabling advanced search on PDS data and metadata. The API provides end-points to search for bundles, collections and any PDS products with advanced search queries. It also enables to browse the archive hierarchically downward (e.g. collection/s products) or upward (e.g. bundles containing a product). - Property values are cast to string in responses due to limitations of JSON typing, and should be interpreted by the client/user according to the data dictionary. As a result of the string cast, some missing values might be found as "null". - version: 1.3.0 title: PDS Registry Search API termsOfService: 'http://pds.nasa.gov' contact: + name: "Contact PDS Engineering Node Support" email: pds-operator@jpl.nasa.gov license: name: Apache 2.0 @@ -667,7 +666,7 @@ components: schema: $ref: '#/components/schemas/errorMessage' Plural: - description: Successful request + description: Successful request. content: "*": schema: diff --git a/service/src/main/java/gov/nasa/pds/api/registry/configuration/OpenApiConfiguration.java b/service/src/main/java/gov/nasa/pds/api/registry/configuration/OpenApiConfiguration.java index 1fa153d3..8003f7f8 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/configuration/OpenApiConfiguration.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/configuration/OpenApiConfiguration.java @@ -22,9 +22,10 @@ public class OpenApiConfiguration { @Bean public OpenAPI customOpenAPI() { OpenAPI customOpenAPI = new OpenAPI() - .info(new Info().title("PDS Registry Search API") - .description( - "RestFul web API provided to search all classes of products in the PDS registries.") + .info(new Info().title("PDS Registry Search API").description( + "Registry API enabling advanced search on PDS data and metadata. The API provides end-points to search for bundles, collections and any PDS products with advanced search queries. It also enables to browse the archive hierarchically downward (e.g. collection/s products) or upward (e.g. bundles containing a product).\n" + + " Property values are cast to string in responses due to limitations of JSON typing, and should be interpreted by the client/user according to the data dictionary.\n" + + " As a result of the string cast, some missing values might be found as \"null\".") .version(this.version) .contact(new Contact().name("Contact PDS Engineering Node Support") .email("pds_operator@jpl.nasa.gov")) From 5d07f74b5221ec75c624fc3da0d7254124e0cfba Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Mon, 26 Jan 2026 13:29:35 -0800 Subject: [PATCH 035/137] trigger tests From 4839e754143baa4516eecafe06b4ca696c08b539 Mon Sep 17 00:00:00 2001 From: PDSEN CI Bot Date: Mon, 26 Jan 2026 22:05:48 +0000 Subject: [PATCH 036/137] Update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 48b9d447..f1189328 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## [«unknown»](https://github.com/NASA-PDS/registry-api/tree/«unknown») (2026-01-22) +## [«unknown»](https://github.com/NASA-PDS/registry-api/tree/«unknown») (2026-01-26) [Full Changelog](https://github.com/NASA-PDS/registry-api/compare/v1.6.2...«unknown») From 95778f75cf709ef14fa51f025995bfb4a0a514a7 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Tue, 27 Jan 2026 13:00:17 -0800 Subject: [PATCH 037/137] forget to change single names --- .../registry/model/Antlr4SearchListener.java | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java b/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java index 43367b91..12122afc 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java @@ -41,7 +41,7 @@ enum operation { private BoolQuery.Builder queryBuilder = new BoolQuery.Builder(); private conjunctions conjunction = conjunctions.AND; // DEFAULT - private final ArrayList fieldnames = new ArrayList(); + private final ArrayList fieldNames = new ArrayList(); private final ConnectionContext connectionContext; private final Deque stackQueryBuilders = new ArrayDeque(); private final Deque stackConjunction = new ArrayDeque(); @@ -57,7 +57,7 @@ public Antlr4SearchListener(ConnectionContext connectionContext) { @Override public void enterFields(SearchParser.FieldsContext ctx) { - this.fieldnames.clear(); + this.fieldNames.clear(); this.isAnyWildcard = true; } @@ -88,15 +88,15 @@ public void exitFields(SearchParser.FieldsContext ctx) { for (String fn : this.knownFieldNames.stream() .filter(s -> regex.matcher(s).matches()) .toList()) { - this.fieldnames.add(SearchUtil.jsonPropertyToOpenProperty(fn)); + this.fieldNames.add(SearchUtil.jsonPropertyToOpenProperty(fn)); } - if (this.fieldnames.isEmpty()) { + if (this.fieldNames.isEmpty()) { throw new ParseCancellationException("Wildcarding request '" + fieldname + "' cannot match any field names in the LDD using regular expression " + theKey); } } else { - this.fieldnames.add(fieldname); + this.fieldNames.add(SearchUtil.jsonPropertyToOpenProperty(fieldname)); } - this.isAnyWildcard = ctx.ALL() == null && this.fieldnames.size() > 1; + this.isAnyWildcard = ctx.ALL() == null && this.fieldNames.size() > 1; } @Override @@ -173,7 +173,7 @@ public void exitComparison(SearchParser.ComparisonContext ctx) { if (this.isAnyWildcard) { wild.minimumShouldMatch("1"); } - for (String left : this.fieldnames) { + for (String left : this.fieldNames) { if (this.operator == operation.eq || this.operator == operation.ne) { FieldValue fieldValue = new FieldValue.Builder().stringValue(right).build(); MatchQuery matchQueryBuilder = new MatchQuery.Builder().field(left).query(fieldValue).build(); @@ -220,7 +220,8 @@ public void exitExistence(SearchParser.ExistenceContext ctx) { if (this.isAnyWildcard) { wild.minimumShouldMatch("1"); } - for (String fieldName : this.fieldnames) { + for (String fieldName : this.fieldNames) { + log.error("************************* field name: " + fieldName); if (this.isAnyWildcard) { wild.should(new ExistsQuery.Builder().field(fieldName).build().toQuery()); } else { @@ -245,7 +246,7 @@ public void exitLikeComparison(SearchParser.LikeComparisonContext ctx) { if (this.isAnyWildcard) { wild.minimumShouldMatch("1"); } - for (String left : this.fieldnames) { + for (String left : this.fieldNames) { SimpleQueryStringQuery simpleQueryString = new SimpleQueryStringQuery.Builder().fields(left) .query(right).fuzzyMaxExpansions(0).build(); if (this.isAnyWildcard) { From 2ff80fb3c629c1af14f9f70d2596861377c2449a Mon Sep 17 00:00:00 2001 From: PDSEN CI Bot Date: Wed, 28 Jan 2026 01:21:55 +0000 Subject: [PATCH 038/137] Update changelog --- CHANGELOG.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f1189328..4651ccc7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,15 @@ # Changelog -## [«unknown»](https://github.com/NASA-PDS/registry-api/tree/«unknown») (2026-01-26) +## [«unknown»](https://github.com/NASA-PDS/registry-api/tree/«unknown») (2026-01-28) [Full Changelog](https://github.com/NASA-PDS/registry-api/compare/v1.6.2...«unknown») +**Requirements:** + +- As a user, I want the exists operator to match OpenSearch's native behavior for all fields [\#712](https://github.com/NASA-PDS/registry-api/issues/712) +- As a user, I want to search by a full/unique hierarchical path for a specific attribute [\#611](https://github.com/NASA-PDS/registry-api/issues/611) +- As a user, I want to query for documents where a specific search field exists in the document [\#406](https://github.com/NASA-PDS/registry-api/issues/406) + **Defects:** - API in production is unstable and returns 500 errors [\#716](https://github.com/NASA-PDS/registry-api/issues/716) [[s.critical](https://github.com/NASA-PDS/registry-api/labels/s.critical)] From efe000cba1aa81c9a5dbaca81e5f155240a16ead Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Mon, 2 Feb 2026 09:23:56 -0800 Subject: [PATCH 039/137] last step In order for the registry-api to use the structured field names, need to quit doing the property name conversions. Made it selectable via the properties file just because the transition from one to the other may take a while. This gives us the freedom to use the same API code for older or newer registry-apis. --- .../nasa/pds/api/registry/model/SearchUtil.java | 16 +++++++++++++--- .../src/main/resources/application.properties | 4 ++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/service/src/main/java/gov/nasa/pds/api/registry/model/SearchUtil.java b/service/src/main/java/gov/nasa/pds/api/registry/model/SearchUtil.java index d7b617c4..826f0a90 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/model/SearchUtil.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/model/SearchUtil.java @@ -8,17 +8,27 @@ import org.apache.http.client.utils.URIBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; import gov.nasa.pds.api.registry.exceptions.UnsupportedSearchProperty; import gov.nasa.pds.model.Metadata; import gov.nasa.pds.model.PdsProduct; import gov.nasa.pds.model.Reference; +@Component public class SearchUtil { private static final Logger log = LoggerFactory.getLogger(SearchUtil.class); + private static String fnArch; + @Value("${registry.field.name.architecture}") + public void setFnArch(String fnArch) { + SearchUtil.fnArch = fnArch; + } + static public String jsonPropertyToOpenProperty(String jsonProperty) { - return jsonProperty.replace(".", "/"); + if ("flat".equalsIgnoreCase(SearchUtil.fnArch)) return jsonProperty.replace(".", "/"); + return jsonProperty; } static public String[] jsonPropertyToOpenProperty(String[] jsonProperties) { @@ -41,8 +51,8 @@ static public List jsonPropertyToOpenProperty(List jsonPropertie static public String openPropertyToJsonProperty(String openProperty) throws UnsupportedSearchProperty { - - return openProperty.replace('/', '.'); + if ("flat".equalsIgnoreCase(SearchUtil.fnArch)) return openProperty.replace('/', '.'); + return openProperty; } static private void addReference(ArrayList to, String ID, URL baseURL) { diff --git a/service/src/main/resources/application.properties b/service/src/main/resources/application.properties index b54a8032..9ffe8fef 100644 --- a/service/src/main/resources/application.properties +++ b/service/src/main/resources/application.properties @@ -51,3 +51,7 @@ filter.archiveStatus=archived,certified # source version from maven # need to be updated with actual value when runs outside of maven registry.service.version=@project.version@ + +# signal if the database being used is "flat" or "structured" +# used an enum of strings, "flat" or "structured" for future new types +registry.field.name.architecture=structured \ No newline at end of file From ba9b4e79f6a4f7e16c5e1d5eef3431cb4c027e33 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Mon, 2 Feb 2026 09:27:12 -0800 Subject: [PATCH 040/137] makes it backward compatible --- .../main/java/gov/nasa/pds/api/registry/model/SearchUtil.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/service/src/main/java/gov/nasa/pds/api/registry/model/SearchUtil.java b/service/src/main/java/gov/nasa/pds/api/registry/model/SearchUtil.java index 826f0a90..e099c27a 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/model/SearchUtil.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/model/SearchUtil.java @@ -27,7 +27,7 @@ public void setFnArch(String fnArch) { } static public String jsonPropertyToOpenProperty(String jsonProperty) { - if ("flat".equalsIgnoreCase(SearchUtil.fnArch)) return jsonProperty.replace(".", "/"); + if (SearchUtil.fnArch == null || SearchUtil.fnArch.equalsIgnoreCase("flat")) return jsonProperty.replace(".", "/"); return jsonProperty; } @@ -51,7 +51,7 @@ static public List jsonPropertyToOpenProperty(List jsonPropertie static public String openPropertyToJsonProperty(String openProperty) throws UnsupportedSearchProperty { - if ("flat".equalsIgnoreCase(SearchUtil.fnArch)) return openProperty.replace('/', '.'); + if (SearchUtil.fnArch == null || SearchUtil.fnArch.equalsIgnoreCase("flat")) return openProperty.replace('/', '.'); return openProperty; } From 7537998eb621a4e0535cd6e91aa446e1087a85c7 Mon Sep 17 00:00:00 2001 From: edunn Date: Mon, 12 Jan 2026 14:52:31 -0800 Subject: [PATCH 041/137] merge matchMembersOfCollection() and matchMembersOfBundle() into a unified matchMembers() which leverages the new post-overhaul ancestry metadata --- .../registry/controllers/ProductsController.java | 14 +++++++++----- .../search/RegistrySearchRequestBuilder.java | 8 ++------ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java b/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java index 87f4a92c..2e5bd169 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java @@ -397,10 +397,10 @@ public ResponseEntity productMembers(String identifier, List use new RegistrySearchRequestBuilder(this.connectionContext); if (productClass.isBundle()) { - searchRequestBuilder.matchMembersOfBundle(lidvid); + searchRequestBuilder.matchMembers(lidvid); searchRequestBuilder.onlyCollections(); } else if (productClass.isCollection()) { - searchRequestBuilder.matchMembersOfCollection(lidvid); + searchRequestBuilder.matchMembers(lidvid); searchRequestBuilder.onlyBasicProducts(); } else { throw new BadRequestException( @@ -424,6 +424,8 @@ public ResponseEntity productMembersMembers(String identifier, throws NotFoundException, UnhandledException, SortSearchAfterMismatchException, BadRequestException, AcceptFormatNotSupportedException, UnparsableQParamException { +// TODO: This functionality is currently deprecated and requires reimplementation or removal + try { PdsProductIdentifier pdsIdentifier = PdsProductIdentifier.fromString(identifier); PdsProductClasses productClass = resolveProductClass(pdsIdentifier); @@ -433,7 +435,7 @@ public ResponseEntity productMembersMembers(String identifier, new RegistrySearchRequestBuilder(this.connectionContext); if (productClass.isBundle()) { - searchRequestBuilder.matchMembersOfBundle(lidvid); + searchRequestBuilder.matchMembers(lidvid); searchRequestBuilder.onlyBasicProducts(); } else { throw new BadRequestException( @@ -507,10 +509,10 @@ public ResponseEntity productMemberOf(String identifier, List us List parentIds; if (productClass.isCollection()) { parentIds = - resolveLidVidsFromProductField(lidvid, "ops:Provenance/ops:parent_bundle_identifier"); + resolveLidVidsFromProductField(lidvid, "ops:Provenance/ops:ancestry"); } else if (productClass.isBasicProduct()) { parentIds = resolveLidVidsFromProductField(lidvid, - "ops:Provenance/ops:parent_collection_identifier"); + "ops:Provenance/ops:ancestry"); } else { throw new BadRequestException( "productMembersOf endpoint is not valid for products with Product_Class '" @@ -536,6 +538,8 @@ public ResponseEntity productMemberOfOf(String identifier, throws NotFoundException, UnhandledException, SortSearchAfterMismatchException, BadRequestException, AcceptFormatNotSupportedException, UnparsableQParamException { +// TODO: This functionality is currently deprecated and requires reimplementation or removal + try { PdsProductIdentifier pdsIdentifier = PdsProductIdentifier.fromString(identifier); PdsProductClasses productClass = resolveProductClass(pdsIdentifier); diff --git a/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java b/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java index e84a81e5..479e0bc5 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java @@ -234,12 +234,8 @@ public RegistrySearchRequestBuilder matchProductClass(PdsProductClasses productC return this.matchField(PdsProductClasses.getPropertyName(), productClass.getValue()); } - public RegistrySearchRequestBuilder matchMembersOfBundle(PdsLidVid identifier) { - return this.matchField("ops:Provenance/ops:parent_bundle_identifier", identifier); - } - - public RegistrySearchRequestBuilder matchMembersOfCollection(PdsLidVid identifier) { - return this.matchField("ops:Provenance/ops:parent_collection_identifier", identifier); + public RegistrySearchRequestBuilder matchMembers(PdsLidVid identifier) { + return this.matchField("ops:Provenance/ops:ancestry", identifier); } public RegistrySearchRequestBuilder paginate(Integer pageSize, List sortFieldNames, From a48755260a348c81df7400c18d72452c5d67c19b Mon Sep 17 00:00:00 2001 From: edunn Date: Mon, 12 Jan 2026 15:15:34 -0800 Subject: [PATCH 042/137] Correct ancestry metadata attribute name --- .../nasa/pds/api/registry/controllers/ProductsController.java | 4 ++-- .../pds/api/registry/search/RegistrySearchRequestBuilder.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java b/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java index 2e5bd169..caab4461 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java @@ -509,10 +509,10 @@ public ResponseEntity productMemberOf(String identifier, List us List parentIds; if (productClass.isCollection()) { parentIds = - resolveLidVidsFromProductField(lidvid, "ops:Provenance/ops:ancestry"); + resolveLidVidsFromProductField(lidvid, "ops:Provenance/ops:ancestor_refs"); } else if (productClass.isBasicProduct()) { parentIds = resolveLidVidsFromProductField(lidvid, - "ops:Provenance/ops:ancestry"); + "ops:Provenance/ops:ancestor_refs"); } else { throw new BadRequestException( "productMembersOf endpoint is not valid for products with Product_Class '" diff --git a/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java b/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java index 479e0bc5..41607a8b 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java @@ -235,7 +235,7 @@ public RegistrySearchRequestBuilder matchProductClass(PdsProductClasses productC } public RegistrySearchRequestBuilder matchMembers(PdsLidVid identifier) { - return this.matchField("ops:Provenance/ops:ancestry", identifier); + return this.matchField("ops:Provenance/ops:ancestor_refs", identifier); } public RegistrySearchRequestBuilder paginate(Integer pageSize, List sortFieldNames, From 84215fa21859a7c8ee8d4aff888bfa7f80c850e9 Mon Sep 17 00:00:00 2001 From: edunn Date: Tue, 27 Jan 2026 15:50:01 -0800 Subject: [PATCH 043/137] implement recursive step to resolve LID references to the set of all LIDVIDs --- .../controllers/ProductsController.java | 58 +++++++++++++++---- 1 file changed, 48 insertions(+), 10 deletions(-) diff --git a/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java b/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java index caab4461..b81a0ac7 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java @@ -3,6 +3,8 @@ import java.lang.reflect.InvocationTargetException; import java.io.IOException; import java.util.*; +import java.util.stream.Stream; + import gov.nasa.pds.api.base.ClassesApi; import gov.nasa.pds.api.base.PropertiesApi; import gov.nasa.pds.api.registry.model.exceptions.*; @@ -458,16 +460,30 @@ public ResponseEntity productMembersMembers(String identifier, * * @param identifier the LID/LIDVID for which to retrieve documents * @param fieldName the name of the document _source property/field from which to extract results - * @return a deduplicated list of the aggregated property/field contents, converted to - * PdsProductLidvids + * @return a list of the aggregated property/field contents, converted to PdsProductLidvids * @throws AcceptFormatNotSupportedException */ private List resolveLidVidsFromProductField(PdsProductIdentifier identifier, - String fieldName) - throws OpenSearchException, IOException, NotFoundException, UnhandledException { + String fieldName) + throws OpenSearchException, IOException, NotFoundException, UnhandledException { + return resolveLidVidsFromProductField(identifier, fieldName, 0); + } + + /** + * Internal implementation with recursion depth protection against the unlikely event that a LID value ever turns up + * erroneously in the lidvid property. + */ + private List resolveLidVidsFromProductField(PdsProductIdentifier identifier, + String fieldName, int recursionDepth) + throws OpenSearchException, IOException, NotFoundException, UnhandledException { + + if (recursionDepth > 1) { + throw new UnhandledException( + "Recursion depth exceeded in resolveLidVidsFromProductField. Maximum depth is 1."); + } RegistrySearchRequestBuilder searchRequestBuilder = - new RegistrySearchRequestBuilder(this.connectionContext); + new RegistrySearchRequestBuilder(this.connectionContext); if (identifier.isLid()) { searchRequestBuilder.matchLid(identifier); @@ -475,22 +491,44 @@ private List resolveLidVidsFromProductField(PdsProductIdentifier iden searchRequestBuilder.matchLidvid(identifier); } else { throw new UnhandledException( - "PdsProductIdentifier identifier is neither LID nor LIDVID. This should never occur"); + "PdsProductIdentifier identifier is neither LID nor LIDVID. This should never occur"); } SearchRequest searchRequest = - searchRequestBuilder.matchLid(identifier).fieldsFromStrings(List.of(fieldName)).build(); + searchRequestBuilder.matchLid(identifier).fieldsFromStrings(List.of(fieldName)).build(); SearchResponse searchResponse = - this.openSearchClient.search(searchRequest, HashMap.class); + this.openSearchClient.search(searchRequest, HashMap.class); if (searchResponse.hits().total().value() == 0) { throw new NotFoundException("No product found with identifier " + identifier); } return searchResponse.hits().hits().stream() - .map(hit -> (List) hit.source().get(fieldName)).filter(Objects::nonNull) - .flatMap(Collection::stream).map(PdsLidVid::fromString).toList(); + .map(hit -> (List) hit.source().get(fieldName)) + .filter(Objects::nonNull) + .flatMap(Collection::stream) + .flatMap(idString -> { + try { + PdsProductIdentifier parsedId = PdsProductIdentifier.fromString(idString); + + if (parsedId != null && parsedId.isLidvid()) { + return Stream.of((PdsLidVid) parsedId); + } else if (parsedId != null && parsedId.isLid()) { + // Recurse to resolve LID to LIDVIDs + return resolveLidVidsFromProductField(parsedId, "lidvid", recursionDepth + 1).stream(); + } else { + throw new UnhandledException( + "Parsed identifier is neither LID nor LIDVID: " + idString); + } + } catch (NotFoundException e) { + log.warn("Product not found for identifier {}: {}", idString, e.getMessage()); + return Stream.empty(); + } catch (IOException | UnhandledException | OpenSearchException e) { + throw new RuntimeException(e); + } + }) + .toList(); } From 4b77de15dcbfef50d0bd3972ef937fbf07356fe6 Mon Sep 17 00:00:00 2001 From: edunn Date: Tue, 27 Jan 2026 15:50:28 -0800 Subject: [PATCH 044/137] implement deduplication which was missing --- .../nasa/pds/api/registry/controllers/ProductsController.java | 1 + 1 file changed, 1 insertion(+) diff --git a/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java b/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java index b81a0ac7..8b420fc1 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java @@ -528,6 +528,7 @@ private List resolveLidVidsFromProductField(PdsProductIdentifier iden throw new RuntimeException(e); } }) + .distinct() .toList(); } From 14eca0c2b2a00f1b86fc33a918b9cb0a80cd705a Mon Sep 17 00:00:00 2001 From: edunn Date: Mon, 2 Feb 2026 16:00:22 -0800 Subject: [PATCH 045/137] fix interpretation of non-array-like fields such as lidvid et al. --- .../pds/api/registry/controllers/ProductsController.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java b/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java index 8b420fc1..0c584105 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java @@ -505,8 +505,11 @@ private List resolveLidVidsFromProductField(PdsProductIdentifier iden } return searchResponse.hits().hits().stream() - .map(hit -> (List) hit.source().get(fieldName)) + .map(hit -> hit.source().get(fieldName)) .filter(Objects::nonNull) + // the following map() is necessary to support non-array fields like 'lidvid' by normalising them to multi-element collections + .map(el -> el instanceof Collection ? el : List.of(el)) + .map(x -> (List) x) .flatMap(Collection::stream) .flatMap(idString -> { try { From 8c612186d901ba8a4d97d647ba8575ef38e1bc43 Mon Sep 17 00:00:00 2001 From: PDSEN CI Bot Date: Thu, 5 Feb 2026 17:17:34 +0000 Subject: [PATCH 046/137] Update changelog --- CHANGELOG.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4651ccc7..b2792bd8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,15 +1,18 @@ # Changelog -## [«unknown»](https://github.com/NASA-PDS/registry-api/tree/«unknown») (2026-01-28) +## [«unknown»](https://github.com/NASA-PDS/registry-api/tree/«unknown») (2026-02-05) [Full Changelog](https://github.com/NASA-PDS/registry-api/compare/v1.6.2...«unknown») **Requirements:** -- As a user, I want the exists operator to match OpenSearch's native behavior for all fields [\#712](https://github.com/NASA-PDS/registry-api/issues/712) - As a user, I want to search by a full/unique hierarchical path for a specific attribute [\#611](https://github.com/NASA-PDS/registry-api/issues/611) - As a user, I want to query for documents where a specific search field exists in the document [\#406](https://github.com/NASA-PDS/registry-api/issues/406) +**Improvements:** + +- Update registry API `/members/members` algorithm per deprecation of `parent_bundle_identifier` metadata non-aggregate products [\#699](https://github.com/NASA-PDS/registry-api/issues/699) + **Defects:** - API in production is unstable and returns 500 errors [\#716](https://github.com/NASA-PDS/registry-api/issues/716) [[s.critical](https://github.com/NASA-PDS/registry-api/labels/s.critical)] From b69ce34c814d739b723a1f6708ecbbd50eb05b7b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Mar 2026 16:09:47 +0000 Subject: [PATCH 047/137] Bump actions/upload-artifact from 6 to 7 Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 6 to 7. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v6...v7) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 1cfcc398..d996aefe 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -80,7 +80,7 @@ jobs: - name: Upload CodeQL Artifacts - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v7 with: name: codeql-artifacts path: ${{ env.RESULTS_DIR }} @@ -108,7 +108,7 @@ jobs: - name: Upload SLOC - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v7 with: name: sloc-count path: ${{ github.workspace }}/cloc.md From f4fcfe2922427ddca6b2c7c9b4ded4f0235e8814 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Mar 2026 16:23:01 +0000 Subject: [PATCH 048/137] Bump docker/login-action from 3 to 4 Bumps [docker/login-action](https://github.com/docker/login-action) from 3 to 4. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/v3...v4) --- updated-dependencies: - dependency-name: docker/login-action dependency-version: '4' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/stable-cicd.yaml | 2 +- .github/workflows/unstable-cicd.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/stable-cicd.yaml b/.github/workflows/stable-cicd.yaml index 74d7054a..7a8fd412 100644 --- a/.github/workflows/stable-cicd.yaml +++ b/.github/workflows/stable-cicd.yaml @@ -89,7 +89,7 @@ jobs: echo "image_tag=$(echo ${{github.ref}} | awk -F/ '{print $NF}')" >> $GITHUB_OUTPUT - name: 💳 Docker Hub Identification - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: username: ${{secrets.DOCKERHUB_USERNAME}} password: ${{secrets.DOCKERHUB_TOKEN}} diff --git a/.github/workflows/unstable-cicd.yaml b/.github/workflows/unstable-cicd.yaml index 60e11a37..6091749e 100644 --- a/.github/workflows/unstable-cicd.yaml +++ b/.github/workflows/unstable-cicd.yaml @@ -87,7 +87,7 @@ jobs: run: echo "jar_file=$(find ./service/target/ -maxdepth 1 -regextype posix-extended -regex '.*/registry-api-service-[0-9]+\.[0-9]+\.[0-9]+(-SNAPSHOT)?\.jar')" >> $GITHUB_OUTPUT - name: 💳 Docker Hub Identification - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: username: ${{secrets.DOCKERHUB_USERNAME}} password: ${{secrets.DOCKERHUB_TOKEN}} From 0cbe156ff8a53595ecc85836c78d85b157658aed Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Mar 2026 16:23:07 +0000 Subject: [PATCH 049/137] Bump docker/setup-buildx-action from 3 to 4 Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 3 to 4. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/v3...v4) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-version: '4' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/branch-cicd.yaml | 2 +- .github/workflows/stable-cicd.yaml | 2 +- .github/workflows/unstable-cicd.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/branch-cicd.yaml b/.github/workflows/branch-cicd.yaml index 00130ce4..0056d800 100644 --- a/.github/workflows/branch-cicd.yaml +++ b/.github/workflows/branch-cicd.yaml @@ -78,7 +78,7 @@ jobs: uses: docker/setup-qemu-action@v3 - name: 🚢 Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: 🧱 Image Construction and Publication uses: docker/build-push-action@v6 diff --git a/.github/workflows/stable-cicd.yaml b/.github/workflows/stable-cicd.yaml index 74d7054a..f7730817 100644 --- a/.github/workflows/stable-cicd.yaml +++ b/.github/workflows/stable-cicd.yaml @@ -98,7 +98,7 @@ jobs: uses: docker/setup-qemu-action@v3 - name: 🚢 Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: 🧱 Image Construction and Publication uses: docker/build-push-action@v6 diff --git a/.github/workflows/unstable-cicd.yaml b/.github/workflows/unstable-cicd.yaml index 60e11a37..5b7956ad 100644 --- a/.github/workflows/unstable-cicd.yaml +++ b/.github/workflows/unstable-cicd.yaml @@ -96,7 +96,7 @@ jobs: uses: docker/setup-qemu-action@v3 - name: 🚢 Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: 🧱 Image Construction and Publication uses: docker/build-push-action@v6 From ed66b3fd1512b933eef76cc24fb589327c9fcc7b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Mar 2026 16:23:13 +0000 Subject: [PATCH 050/137] Bump docker/build-push-action from 6 to 7 Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 6 to 7. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/v6...v7) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/branch-cicd.yaml | 2 +- .github/workflows/stable-cicd.yaml | 2 +- .github/workflows/unstable-cicd.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/branch-cicd.yaml b/.github/workflows/branch-cicd.yaml index 00130ce4..2ba31abc 100644 --- a/.github/workflows/branch-cicd.yaml +++ b/.github/workflows/branch-cicd.yaml @@ -81,7 +81,7 @@ jobs: uses: docker/setup-buildx-action@v3 - name: 🧱 Image Construction and Publication - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 with: context: ./ file: ./docker/Dockerfile diff --git a/.github/workflows/stable-cicd.yaml b/.github/workflows/stable-cicd.yaml index 74d7054a..6de81367 100644 --- a/.github/workflows/stable-cicd.yaml +++ b/.github/workflows/stable-cicd.yaml @@ -101,7 +101,7 @@ jobs: uses: docker/setup-buildx-action@v3 - name: 🧱 Image Construction and Publication - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 with: context: ./ file: ./docker/Dockerfile diff --git a/.github/workflows/unstable-cicd.yaml b/.github/workflows/unstable-cicd.yaml index 60e11a37..ecb07f34 100644 --- a/.github/workflows/unstable-cicd.yaml +++ b/.github/workflows/unstable-cicd.yaml @@ -99,7 +99,7 @@ jobs: uses: docker/setup-buildx-action@v3 - name: 🧱 Image Construction and Publication - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 with: context: ./ file: ./docker/Dockerfile From f4e0c2062e2ebb446a0df0bfe12f4a2684be7740 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Mar 2026 22:50:29 +0000 Subject: [PATCH 051/137] Bump docker/setup-qemu-action from 3 to 4 Bumps [docker/setup-qemu-action](https://github.com/docker/setup-qemu-action) from 3 to 4. - [Release notes](https://github.com/docker/setup-qemu-action/releases) - [Commits](https://github.com/docker/setup-qemu-action/compare/v3...v4) --- updated-dependencies: - dependency-name: docker/setup-qemu-action dependency-version: '4' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/branch-cicd.yaml | 2 +- .github/workflows/stable-cicd.yaml | 2 +- .github/workflows/unstable-cicd.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/branch-cicd.yaml b/.github/workflows/branch-cicd.yaml index dc86150f..c9702090 100644 --- a/.github/workflows/branch-cicd.yaml +++ b/.github/workflows/branch-cicd.yaml @@ -75,7 +75,7 @@ jobs: run: echo "jar_file=$(find ./service/target/ -maxdepth 1 -regextype posix-extended -regex '.*/registry-api-service-[0-9]+\.[0-9]+\.[0-9]+(-SNAPSHOT)?\.jar')" >> $GITHUB_OUTPUT - name: 🎰 QEMU Multiple Machine Emulation - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@v4 - name: 🚢 Docker Buildx uses: docker/setup-buildx-action@v4 diff --git a/.github/workflows/stable-cicd.yaml b/.github/workflows/stable-cicd.yaml index 54473e2d..28fd6c1d 100644 --- a/.github/workflows/stable-cicd.yaml +++ b/.github/workflows/stable-cicd.yaml @@ -95,7 +95,7 @@ jobs: password: ${{secrets.DOCKERHUB_TOKEN}} - name: 🎰 QEMU Multiple Machine Emulation - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@v4 - name: 🚢 Docker Buildx uses: docker/setup-buildx-action@v4 diff --git a/.github/workflows/unstable-cicd.yaml b/.github/workflows/unstable-cicd.yaml index 45571417..17853172 100644 --- a/.github/workflows/unstable-cicd.yaml +++ b/.github/workflows/unstable-cicd.yaml @@ -93,7 +93,7 @@ jobs: password: ${{secrets.DOCKERHUB_TOKEN}} - name: 🎰 QEMU Multiple Machine Emulation - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@v4 - name: 🚢 Docker Buildx uses: docker/setup-buildx-action@v4 From ab8e65a80ffc38bfa24306ff343cee74b998ee89 Mon Sep 17 00:00:00 2001 From: PDSEN CI Bot Date: Mon, 9 Mar 2026 22:55:50 +0000 Subject: [PATCH 052/137] Update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b2792bd8..805fa08b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## [«unknown»](https://github.com/NASA-PDS/registry-api/tree/«unknown») (2026-02-05) +## [«unknown»](https://github.com/NASA-PDS/registry-api/tree/«unknown») (2026-03-09) [Full Changelog](https://github.com/NASA-PDS/registry-api/compare/v1.6.2...«unknown») From 1f88b60d865837233ab6f8812d0d4b27d66090c1 Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Thu, 19 Mar 2026 10:00:26 -0700 Subject: [PATCH 053/137] add the missing comments sonarcube was complaining about --- .../nasa/gov/api_search_query_lexer/MockedListener.java | 2 ++ terraform/ecs.tf | 8 ++------ terraform/variables.tf | 4 ---- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/MockedListener.java b/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/MockedListener.java index df003be4..bea1625a 100644 --- a/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/MockedListener.java +++ b/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/MockedListener.java @@ -168,10 +168,12 @@ public void enterExistence(ExistenceContext ctx) { @Override public void exitExistence(ExistenceContext ctx) { + // Nothing useful to do in this mocked version } @Override public void enterFields(FieldsContext ctx) { + // Nothing useful to do in this mocked version } @Override diff --git a/terraform/ecs.tf b/terraform/ecs.tf index 050a7846..f2696a32 100644 --- a/terraform/ecs.tf +++ b/terraform/ecs.tf @@ -86,11 +86,7 @@ resource "aws_ecs_cluster" "pds-registry-api-ecs" { } } -# Do we need individual dev/test/prod repositories? -# I don't think we do, but then we need to use prod account instead of the dev account, would that work ? -data "aws_ecr_repository" "pds-registry-api-service" { - name = "pds-registry-api-service" -} +# ECR repository is now defined in ecr.tf # Log groups hold logs from our app. resource "aws_cloudwatch_log_group" "pds-registry-log-group" { @@ -112,7 +108,7 @@ resource "aws_ecs_task_definition" "pds-registry-ecs-task" { [ { "name": "pds-${var.venue}-reg-container", - "image": "${var.aws_fg_image}", + "image": "${aws_ecr_repository.pds-registry-api-service.repository_url}:latest", "portMappings": [ { "containerPort": 80 diff --git a/terraform/variables.tf b/terraform/variables.tf index bb82829d..ac5c92fb 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -49,10 +49,6 @@ variable "ecs_task_execution_role" { description = "ECS task execution role" } -variable "aws_fg_image" { - description = "AWS image name for Fargate" -} - variable "aws_s3_bucket_logs_id" { description = "AWS S3 bucket with the logs" } From 61cc4c31baac13c73ba40a222bb42bcddebb3a4d Mon Sep 17 00:00:00 2001 From: PDSEN CI Bot Date: Thu, 19 Mar 2026 17:41:52 +0000 Subject: [PATCH 054/137] Update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 805fa08b..e029106f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## [«unknown»](https://github.com/NASA-PDS/registry-api/tree/«unknown») (2026-03-09) +## [«unknown»](https://github.com/NASA-PDS/registry-api/tree/«unknown») (2026-03-19) [Full Changelog](https://github.com/NASA-PDS/registry-api/compare/v1.6.2...«unknown») From 09f87037766013a8bfc67daa6fdd9fd7bbe8a591 Mon Sep 17 00:00:00 2001 From: al-niessner <1130658+al-niessner@users.noreply.github.com> Date: Mon, 23 Mar 2026 11:25:32 -0700 Subject: [PATCH 055/137] Update Search.g4 --- .../src/main/antlr4/gov/nasa/pds/api/registry/lexer/Search.g4 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lexer/src/main/antlr4/gov/nasa/pds/api/registry/lexer/Search.g4 b/lexer/src/main/antlr4/gov/nasa/pds/api/registry/lexer/Search.g4 index aa5221aa..3502488e 100644 --- a/lexer/src/main/antlr4/gov/nasa/pds/api/registry/lexer/Search.g4 +++ b/lexer/src/main/antlr4/gov/nasa/pds/api/registry/lexer/Search.g4 @@ -4,7 +4,7 @@ query : queryTerm EOF ; queryTerm : comparison | likeComparison | existence | group ; fields : FIELDNAME | ALL LPAREN FIELDNAME RPAREN | ANY LPAREN FIELDNAME RPAREN ; group : NOT? LPAREN expression RPAREN ; -existence : fields EXISTS ; +existence : EXISTS fields; expression : andStatement | orStatement | queryTerm ; andStatement : queryTerm (AND queryTerm)+ ; orStatement : queryTerm (OR queryTerm)+ ; @@ -65,4 +65,4 @@ fragment V : [vV]; fragment W : [wW]; fragment X : [xX]; fragment Y : [yY]; -fragment Z : [zZ]; \ No newline at end of file +fragment Z : [zZ]; From 48196ae9e26e2261811efd302a46badc60f29698 Mon Sep 17 00:00:00 2001 From: al-niessner <1130658+al-niessner@users.noreply.github.com> Date: Tue, 24 Mar 2026 07:33:15 -0700 Subject: [PATCH 056/137] Update TestParsing.java --- .../pds/nasa/gov/api_search_query_lexer/TestParsing.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/TestParsing.java b/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/TestParsing.java index 8241de25..36651e39 100644 --- a/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/TestParsing.java +++ b/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/TestParsing.java @@ -119,7 +119,7 @@ void testTemporalRange() { @Test void testFieldExistence() { - String queryString = "apple exists"; + String queryString = "exists apple"; CodePointCharStream input = CharStreams.fromString(queryString); SearchLexer lex = new SearchLexer(input); CommonTokenStream tokens = new CommonTokenStream(lex); @@ -138,7 +138,7 @@ void testFieldExistence() { @Test void testParenFieldExistence() { - String queryString = "(apple exists)"; + String queryString = "(exists apple)"; CodePointCharStream input = CharStreams.fromString(queryString); SearchLexer lex = new SearchLexer(input); CommonTokenStream tokens = new CommonTokenStream(lex); @@ -156,7 +156,7 @@ void testParenFieldExistence() { @Test void testWildExistence() { - String queryString = "*.apple exists"; + String queryString = "exists *.apple"; CodePointCharStream input = CharStreams.fromString(queryString); SearchLexer lex = new SearchLexer(input); CommonTokenStream tokens = new CommonTokenStream(lex); @@ -175,7 +175,7 @@ void testWildExistence() { @Test void testParenWildExistence() { - String queryString = "(*apple exists)"; + String queryString = "(exists *apple)"; CodePointCharStream input = CharStreams.fromString(queryString); SearchLexer lex = new SearchLexer(input); CommonTokenStream tokens = new CommonTokenStream(lex); From 7c19d51f9044820af64a277f3412a2e388582719 Mon Sep 17 00:00:00 2001 From: PDSEN CI Bot Date: Thu, 2 Apr 2026 16:13:08 +0000 Subject: [PATCH 057/137] Update changelog --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e029106f..2f6afcfd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## [«unknown»](https://github.com/NASA-PDS/registry-api/tree/«unknown») (2026-03-19) +## [«unknown»](https://github.com/NASA-PDS/registry-api/tree/«unknown») (2026-04-02) [Full Changelog](https://github.com/NASA-PDS/registry-api/compare/v1.6.2...«unknown») @@ -11,10 +11,12 @@ **Improvements:** +- As a user, I want the exists operator to be prepended to the query [\#727](https://github.com/NASA-PDS/registry-api/issues/727) - Update registry API `/members/members` algorithm per deprecation of `parent_bundle_identifier` metadata non-aggregate products [\#699](https://github.com/NASA-PDS/registry-api/issues/699) **Defects:** +- A query to pds.nasa.gov does not respond the same as a query to pds.mcp.nasa.gov [\#742](https://github.com/NASA-PDS/registry-api/issues/742) [[s.medium](https://github.com/NASA-PDS/registry-api/labels/s.medium)] - API in production is unstable and returns 500 errors [\#716](https://github.com/NASA-PDS/registry-api/issues/716) [[s.critical](https://github.com/NASA-PDS/registry-api/labels/s.critical)] - Inconsistent support for `application/vnd.nasa.pds.pds4+json` response format [\#705](https://github.com/NASA-PDS/registry-api/issues/705) [[s.high](https://github.com/NASA-PDS/registry-api/labels/s.high)] - API search results using "search-after" returns empty \[data\] block even though I can find the product by lidvid [\#677](https://github.com/NASA-PDS/registry-api/issues/677) [[s.high](https://github.com/NASA-PDS/registry-api/labels/s.high)] From e38bf460fca6ad58c4ccffeade1c9d3c35ce85d6 Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Mon, 6 Apr 2026 16:57:22 -0700 Subject: [PATCH 058/137] fix member-of/member-of with updated opensearch schema --- .../controllers/ProductsController.java | 119 ++++++++++-------- 1 file changed, 66 insertions(+), 53 deletions(-) diff --git a/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java b/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java index 0c584105..b07af125 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java @@ -3,6 +3,8 @@ import java.lang.reflect.InvocationTargetException; import java.io.IOException; import java.util.*; +import java.util.function.Function; +import java.util.stream.Collectors; import java.util.stream.Stream; import gov.nasa.pds.api.base.ClassesApi; @@ -426,7 +428,7 @@ public ResponseEntity productMembersMembers(String identifier, throws NotFoundException, UnhandledException, SortSearchAfterMismatchException, BadRequestException, AcceptFormatNotSupportedException, UnparsableQParamException { -// TODO: This functionality is currently deprecated and requires reimplementation or removal + // TODO: This functionality is currently deprecated and requires reimplementation or removal try { PdsProductIdentifier pdsIdentifier = PdsProductIdentifier.fromString(identifier); @@ -464,26 +466,26 @@ public ResponseEntity productMembersMembers(String identifier, * @throws AcceptFormatNotSupportedException */ private List resolveLidVidsFromProductField(PdsProductIdentifier identifier, - String fieldName) - throws OpenSearchException, IOException, NotFoundException, UnhandledException { + String fieldName) + throws OpenSearchException, IOException, NotFoundException, UnhandledException { return resolveLidVidsFromProductField(identifier, fieldName, 0); } /** - * Internal implementation with recursion depth protection against the unlikely event that a LID value ever turns up - * erroneously in the lidvid property. + * Internal implementation with recursion depth protection against the unlikely event that a LID + * value ever turns up erroneously in the lidvid property. */ private List resolveLidVidsFromProductField(PdsProductIdentifier identifier, - String fieldName, int recursionDepth) - throws OpenSearchException, IOException, NotFoundException, UnhandledException { + String fieldName, int recursionDepth) + throws OpenSearchException, IOException, NotFoundException, UnhandledException { if (recursionDepth > 1) { throw new UnhandledException( - "Recursion depth exceeded in resolveLidVidsFromProductField. Maximum depth is 1."); + "Recursion depth exceeded in resolveLidVidsFromProductField. Maximum depth is 1."); } RegistrySearchRequestBuilder searchRequestBuilder = - new RegistrySearchRequestBuilder(this.connectionContext); + new RegistrySearchRequestBuilder(this.connectionContext); if (identifier.isLid()) { searchRequestBuilder.matchLid(identifier); @@ -491,48 +493,45 @@ private List resolveLidVidsFromProductField(PdsProductIdentifier iden searchRequestBuilder.matchLidvid(identifier); } else { throw new UnhandledException( - "PdsProductIdentifier identifier is neither LID nor LIDVID. This should never occur"); + "PdsProductIdentifier identifier is neither LID nor LIDVID. This should never occur"); } SearchRequest searchRequest = - searchRequestBuilder.matchLid(identifier).fieldsFromStrings(List.of(fieldName)).build(); + searchRequestBuilder.matchLid(identifier).fieldsFromStrings(List.of(fieldName)).build(); SearchResponse searchResponse = - this.openSearchClient.search(searchRequest, HashMap.class); + this.openSearchClient.search(searchRequest, HashMap.class); if (searchResponse.hits().total().value() == 0) { throw new NotFoundException("No product found with identifier " + identifier); } - return searchResponse.hits().hits().stream() - .map(hit -> hit.source().get(fieldName)) - .filter(Objects::nonNull) - // the following map() is necessary to support non-array fields like 'lidvid' by normalising them to multi-element collections - .map(el -> el instanceof Collection ? el : List.of(el)) - .map(x -> (List) x) - .flatMap(Collection::stream) - .flatMap(idString -> { - try { - PdsProductIdentifier parsedId = PdsProductIdentifier.fromString(idString); - - if (parsedId != null && parsedId.isLidvid()) { - return Stream.of((PdsLidVid) parsedId); - } else if (parsedId != null && parsedId.isLid()) { - // Recurse to resolve LID to LIDVIDs - return resolveLidVidsFromProductField(parsedId, "lidvid", recursionDepth + 1).stream(); - } else { - throw new UnhandledException( - "Parsed identifier is neither LID nor LIDVID: " + idString); - } - } catch (NotFoundException e) { - log.warn("Product not found for identifier {}: {}", idString, e.getMessage()); - return Stream.empty(); - } catch (IOException | UnhandledException | OpenSearchException e) { - throw new RuntimeException(e); - } - }) - .distinct() - .toList(); + return searchResponse.hits().hits().stream().map(hit -> hit.source().get(fieldName)) + .filter(Objects::nonNull) + // the following map() is necessary to support non-array fields like 'lidvid' by normalising + // them to multi-element collections + .map(el -> el instanceof Collection ? el : List.of(el)).map(x -> (List) x) + .flatMap(Collection::stream).flatMap(idString -> { + try { + PdsProductIdentifier parsedId = PdsProductIdentifier.fromString(idString); + + if (parsedId != null && parsedId.isLidvid()) { + return Stream.of((PdsLidVid) parsedId); + } else if (parsedId != null && parsedId.isLid()) { + // Recurse to resolve LID to LIDVIDs + return resolveLidVidsFromProductField(parsedId, "lidvid", recursionDepth + 1) + .stream(); + } else { + throw new UnhandledException( + "Parsed identifier is neither LID nor LIDVID: " + idString); + } + } catch (NotFoundException e) { + log.warn("Product not found for identifier {}: {}", idString, e.getMessage()); + return Stream.empty(); + } catch (IOException | UnhandledException | OpenSearchException e) { + throw new RuntimeException(e); + } + }).distinct().toList(); } @@ -550,11 +549,9 @@ public ResponseEntity productMemberOf(String identifier, List us List parentIds; if (productClass.isCollection()) { - parentIds = - resolveLidVidsFromProductField(lidvid, "ops:Provenance/ops:ancestor_refs"); + parentIds = resolveLidVidsFromProductField(lidvid, "ops:Provenance/ops:ancestor_refs"); } else if (productClass.isBasicProduct()) { - parentIds = resolveLidVidsFromProductField(lidvid, - "ops:Provenance/ops:ancestor_refs"); + parentIds = resolveLidVidsFromProductField(lidvid, "ops:Provenance/ops:ancestor_refs"); } else { throw new BadRequestException( "productMembersOf endpoint is not valid for products with Product_Class '" @@ -573,6 +570,16 @@ public ResponseEntity productMemberOf(String identifier, List us } } + + private Stream safeResolveLidVidsFromAncestor(PdsLidVid obj) { + try { + return resolveLidVidsFromProductField(obj, "ops:Provenance/ops:ancestor_refs").stream(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override public ResponseEntity productMemberOfOf(String identifier, List userRequestedFields, Integer limit, String q, List sort, @@ -580,17 +587,22 @@ public ResponseEntity productMemberOfOf(String identifier, throws NotFoundException, UnhandledException, SortSearchAfterMismatchException, BadRequestException, AcceptFormatNotSupportedException, UnparsableQParamException { -// TODO: This functionality is currently deprecated and requires reimplementation or removal + // TODO: This functionality is currently deprecated and requires reimplementation or removal try { PdsProductIdentifier pdsIdentifier = PdsProductIdentifier.fromString(identifier); PdsProductClasses productClass = resolveProductClass(pdsIdentifier); PdsLidVid lidvid = resolveIdentifierToLidvid(pdsIdentifier); - List parentIds; + Stream parentIdStream; + List greatParentIds; if (productClass.isBasicProduct()) { - parentIds = - resolveLidVidsFromProductField(lidvid, "ops:Provenance/ops:parent_bundle_identifier"); + + greatParentIds = safeResolveLidVidsFromAncestor(lidvid) + .flatMap(this::safeResolveLidVidsFromAncestor).toList(); + + + } else { throw new BadRequestException( "productMembersOf endpoint is not valid for products with Product_Class '" @@ -600,7 +612,7 @@ public ResponseEntity productMemberOfOf(String identifier, RegistrySearchRequestBuilder searchRequestBuilder = new RegistrySearchRequestBuilder(this.connectionContext).matchFieldAnyOfIdentifiers("_id", - parentIds); + greatParentIds); return searchAndTransform(userRequestedFields, List.of(), limit, q, sort, searchAfter, facetFields, facetLimit, searchRequestBuilder); @@ -662,7 +674,9 @@ protected static PropertiesListInner.TypeEnum resolvePropertyToEnumType(Property public ResponseEntity> productPropertiesList() throws Exception { return ProductsController.productPropertiesList(this.connectionContext); } - public static ResponseEntity> productPropertiesList(ConnectionContext connectionContext) throws OpenSearchException, IOException { + + public static ResponseEntity> productPropertiesList( + ConnectionContext connectionContext) throws OpenSearchException, IOException { List indexNames = connectionContext.getRegistryIndices(); @@ -680,8 +694,7 @@ public static ResponseEntity> productPropertiesList(Co for (Map.Entry property : indexProperties) { String jsonPropertyName = PdsProperty.toJsonPropertyString(property.getKey()); Property openPropertyName = property.getValue(); - PropertiesListInner.TypeEnum propertyEnumType = - resolvePropertyToEnumType(openPropertyName); + PropertiesListInner.TypeEnum propertyEnumType = resolvePropertyToEnumType(openPropertyName); // No consistency-checking between duplicates, for now. TODO: add error log for mismatching // duplicates From 3eb2d50f946a44a5c66708410086bd21125cbf2e Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Mon, 6 Apr 2026 17:50:34 -0700 Subject: [PATCH 059/137] deprecate the members/members endpoint with documentation and error messages for the work around --- model/swagger.yml | 8 +++++ .../controllers/ProductsController.java | 32 ++++--------------- ...stryApiResponseEntityExceptionHandler.java | 5 +++ .../DeprecatedEndPointException.java | 17 ++++++++++ 4 files changed, 36 insertions(+), 26 deletions(-) create mode 100644 service/src/main/java/gov/nasa/pds/api/registry/model/exceptions/DeprecatedEndPointException.java diff --git a/model/swagger.yml b/model/swagger.yml index a41fbceb..a8fb6c89 100644 --- a/model/swagger.yml +++ b/model/swagger.yml @@ -321,6 +321,14 @@ paths: - 2. product references summary: | returns all of the members of the members of the given lid/lidvid + deprecated: true + description: | + ⚠️ This endpoint is deprecated and does not work anymore. It will be removed in a future release.\n\n + + Please call `/{id}/members` instead, as follows:\n + 1. Get the collection members of the bundle {id} with a first call.\n + 2. Use the collection ids found and get their products by calling the `/{coll_id}/members` for each.\n + operationId: product-members-members responses: '200': diff --git a/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java b/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java index b07af125..38313814 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java @@ -425,34 +425,16 @@ public ResponseEntity productMembers(String identifier, List use public ResponseEntity productMembersMembers(String identifier, List userRequestedFields, Integer limit, String q, List sort, List searchAfter, List facetFields, Integer facetLimit) - throws NotFoundException, UnhandledException, SortSearchAfterMismatchException, - BadRequestException, AcceptFormatNotSupportedException, UnparsableQParamException { - - // TODO: This functionality is currently deprecated and requires reimplementation or removal - - try { - PdsProductIdentifier pdsIdentifier = PdsProductIdentifier.fromString(identifier); - PdsProductClasses productClass = resolveProductClass(pdsIdentifier); - PdsLidVid lidvid = resolveIdentifierToLidvid(pdsIdentifier); + throws DeprecatedEndPointException { - RegistrySearchRequestBuilder searchRequestBuilder = - new RegistrySearchRequestBuilder(this.connectionContext); - if (productClass.isBundle()) { - searchRequestBuilder.matchMembers(lidvid); - searchRequestBuilder.onlyBasicProducts(); - } else { - throw new BadRequestException( - "productMembers endpoint is only valid for products with Product_Class '" - + PdsProductClasses.Product_Bundle + "' (got '" + productClass + "')"); - } + throw new DeprecatedEndPointException( + "This endpoint is deprecated and does not work anymore. It will be removed in a future release.\n" + + "\n" + "Please call `/{id}/members` instead, as follows:\n" + + " 1. Get the collection members of the bundle {id} with a first call.\n" + + " 2. Use the collection ids found and get their products by calling the `/{coll_id}/members` for each."); - return searchAndTransform(userRequestedFields, List.of(), limit, q, sort, searchAfter, - facetFields, facetLimit, searchRequestBuilder); - } catch (IOException | OpenSearchException e) { - throw new UnhandledException(e); - } } /** @@ -587,8 +569,6 @@ public ResponseEntity productMemberOfOf(String identifier, throws NotFoundException, UnhandledException, SortSearchAfterMismatchException, BadRequestException, AcceptFormatNotSupportedException, UnparsableQParamException { - // TODO: This functionality is currently deprecated and requires reimplementation or removal - try { PdsProductIdentifier pdsIdentifier = PdsProductIdentifier.fromString(identifier); PdsProductClasses productClass = resolveProductClass(pdsIdentifier); diff --git a/service/src/main/java/gov/nasa/pds/api/registry/controllers/RegistryApiResponseEntityExceptionHandler.java b/service/src/main/java/gov/nasa/pds/api/registry/controllers/RegistryApiResponseEntityExceptionHandler.java index c9bd1768..757e3896 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/controllers/RegistryApiResponseEntityExceptionHandler.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/controllers/RegistryApiResponseEntityExceptionHandler.java @@ -92,5 +92,10 @@ public ResponseEntity unknownQueryParameter(UnauthorizedForwardedHostExc return genericExceptionHandler(ex, request, "", HttpStatus.BAD_REQUEST); } + @ExceptionHandler(value = {DeprecatedEndPointException.class}) + public ResponseEntity deprecatedEndPoint(DeprecatedEndPointException ex, + WebRequest request) { + return genericExceptionHandler(ex, request, "", HttpStatus.GONE); + } } diff --git a/service/src/main/java/gov/nasa/pds/api/registry/model/exceptions/DeprecatedEndPointException.java b/service/src/main/java/gov/nasa/pds/api/registry/model/exceptions/DeprecatedEndPointException.java new file mode 100644 index 00000000..92925be5 --- /dev/null +++ b/service/src/main/java/gov/nasa/pds/api/registry/model/exceptions/DeprecatedEndPointException.java @@ -0,0 +1,17 @@ +package gov.nasa.pds.api.registry.model.exceptions; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + + +public class DeprecatedEndPointException extends RegistryApiException { + + private static final long serialVersionUID = -6704894264788325051L; + private static final Logger log = LoggerFactory.getLogger(DeprecatedEndPointException.class); + + public DeprecatedEndPointException(String msg) { + super(msg); + } + +} From 4f79981f7447b810d314ed08d0b0305cca55b4bd Mon Sep 17 00:00:00 2001 From: PDSEN CI Bot Date: Tue, 7 Apr 2026 07:06:43 +0000 Subject: [PATCH 060/137] Update changelog --- CHANGELOG.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f6afcfd..5f8eb751 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## [«unknown»](https://github.com/NASA-PDS/registry-api/tree/«unknown») (2026-04-02) +## [«unknown»](https://github.com/NASA-PDS/registry-api/tree/«unknown») (2026-04-07) [Full Changelog](https://github.com/NASA-PDS/registry-api/compare/v1.6.2...«unknown») @@ -12,7 +12,6 @@ **Improvements:** - As a user, I want the exists operator to be prepended to the query [\#727](https://github.com/NASA-PDS/registry-api/issues/727) -- Update registry API `/members/members` algorithm per deprecation of `parent_bundle_identifier` metadata non-aggregate products [\#699](https://github.com/NASA-PDS/registry-api/issues/699) **Defects:** @@ -88,11 +87,8 @@ - As a user, I want to apply an additional query filter \(`q=`\) to the `/classes/{class}` result set [\#493](https://github.com/NASA-PDS/registry-api/issues/493) - As a user, I want to apply an additional query filter \(`q=`\) to the `/products/{identifier}/member-of/member-of` result set [\#492](https://github.com/NASA-PDS/registry-api/issues/492) - As a user, I want to apply an additional query filter \(`q=`\) to the `/products/{identifier}/member-of` result set [\#491](https://github.com/NASA-PDS/registry-api/issues/491) -- As a user, I want to apply an additional query filter \(`q=`\) to members of the members of an aggregate product \(`/products/{identifier}/members/members`\) [\#490](https://github.com/NASA-PDS/registry-api/issues/490) - As a user, by default, I want to search for the latest versions of all products on the `/classes/{class}` endpoint unless explicitly requested [\#488](https://github.com/NASA-PDS/registry-api/issues/488) -- As a user, by default, I want to search only for the latest versions of all products on the `/products/{identifier}/member-of/member-of` endpoint [\#487](https://github.com/NASA-PDS/registry-api/issues/487) - As a user, by default, I want to search for only the latest versions of all products on the `/products/{identifier}/member-of` endpoint [\#486](https://github.com/NASA-PDS/registry-api/issues/486) -- As a user, by default, I want to search for only the latest versions of all products on the `/products/{identifier}/members/members` endpoint [\#485](https://github.com/NASA-PDS/registry-api/issues/485) - As a user, by default, I want to search for only the latest versions of all products on the `/products/{identifier}/members` endpoint [\#484](https://github.com/NASA-PDS/registry-api/issues/484) - As a user, I want to filter the products by any available PDS4 property using a combination of comparison, logical, and precedence grouping operators [\#469](https://github.com/NASA-PDS/registry-api/issues/469) - As a user, I want to get all product versions associated to one lid [\#436](https://github.com/NASA-PDS/registry-api/issues/436) From 99655932a18dca9c68fd15c7466d22bdc982ff08 Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Tue, 7 Apr 2026 10:48:22 -0700 Subject: [PATCH 061/137] retrigger branch test From fbcb4a956b7d57ad80868ebc8cadf748824fdce5 Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Tue, 7 Apr 2026 12:34:37 -0700 Subject: [PATCH 062/137] retrigger branch test From bf0607be782218b4076e83daa54ad5a52d0f302c Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Tue, 7 Apr 2026 15:07:58 -0700 Subject: [PATCH 063/137] retrigger branch test From a84db8eda6d66639d5b18f6d3ec4a0e2bedd8d52 Mon Sep 17 00:00:00 2001 From: thomas loubrieu <60993872+tloubrieu-jpl@users.noreply.github.com> Date: Tue, 7 Apr 2026 15:19:35 -0700 Subject: [PATCH 064/137] Add docker compose logs --- .github/workflows/branch-cicd.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/branch-cicd.yaml b/.github/workflows/branch-cicd.yaml index c9702090..bed24677 100644 --- a/.github/workflows/branch-cicd.yaml +++ b/.github/workflows/branch-cicd.yaml @@ -99,10 +99,9 @@ jobs: docker image inspect nasapds/registry-api-service:latest >/dev/null docker compose \ --ansi never --profile int-registry-batch-loader --project-name registry \ - up --quiet-pull --detach - # --abort-on-container-exit - #echo "===== Docker logs =====" - #docker compose logs --no-color + up --quiet-pull --detach --abort-on-container-exit + echo "===== Docker logs =====" + docker compose logs --no-color docker compose \ --ansi never --profile int-registry-batch-loader --project-name registry \ run --rm --no-TTY reg-api-integration-test-with-wait From f4765a773e91f394046d8b223794dece9cb101e9 Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Tue, 7 Apr 2026 15:19:47 -0700 Subject: [PATCH 065/137] retrigger branch test From fd668325a817882e6d70fbf27ac59a4ae94d8f96 Mon Sep 17 00:00:00 2001 From: thomas loubrieu <60993872+tloubrieu-jpl@users.noreply.github.com> Date: Tue, 7 Apr 2026 15:24:56 -0700 Subject: [PATCH 066/137] Tryng to debug the github action --- .github/workflows/branch-cicd.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/branch-cicd.yaml b/.github/workflows/branch-cicd.yaml index bed24677..3a7ad852 100644 --- a/.github/workflows/branch-cicd.yaml +++ b/.github/workflows/branch-cicd.yaml @@ -99,7 +99,8 @@ jobs: docker image inspect nasapds/registry-api-service:latest >/dev/null docker compose \ --ansi never --profile int-registry-batch-loader --project-name registry \ - up --quiet-pull --detach --abort-on-container-exit + up --quiet-pull --detach + # --abort-on-container-exit echo "===== Docker logs =====" docker compose logs --no-color docker compose \ From a8a727499dc243ce19ef412a3c5b0a2b9b5e2778 Mon Sep 17 00:00:00 2001 From: thomas loubrieu <60993872+tloubrieu-jpl@users.noreply.github.com> Date: Tue, 7 Apr 2026 15:31:20 -0700 Subject: [PATCH 067/137] Add log on top to investigate docker compose issue --- .github/workflows/branch-cicd.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/branch-cicd.yaml b/.github/workflows/branch-cicd.yaml index 3a7ad852..bb63b8ac 100644 --- a/.github/workflows/branch-cicd.yaml +++ b/.github/workflows/branch-cicd.yaml @@ -97,12 +97,12 @@ jobs: ./generate-certs.sh cd .. docker image inspect nasapds/registry-api-service:latest >/dev/null + echo "===== Docker logs =====" + docker compose logs --no-color docker compose \ --ansi never --profile int-registry-batch-loader --project-name registry \ - up --quiet-pull --detach + up --quiet-pull --abort-on-container-exit --detach # --abort-on-container-exit - echo "===== Docker logs =====" - docker compose logs --no-color docker compose \ --ansi never --profile int-registry-batch-loader --project-name registry \ run --rm --no-TTY reg-api-integration-test-with-wait From 96d5484e9ce50c076a3675424ef67ec60a65c19f Mon Sep 17 00:00:00 2001 From: thomas loubrieu <60993872+tloubrieu-jpl@users.noreply.github.com> Date: Tue, 7 Apr 2026 15:34:31 -0700 Subject: [PATCH 068/137] Update branch-cicd.yaml --- .github/workflows/branch-cicd.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/branch-cicd.yaml b/.github/workflows/branch-cicd.yaml index bb63b8ac..66f4ad4f 100644 --- a/.github/workflows/branch-cicd.yaml +++ b/.github/workflows/branch-cicd.yaml @@ -101,7 +101,7 @@ jobs: docker compose logs --no-color docker compose \ --ansi never --profile int-registry-batch-loader --project-name registry \ - up --quiet-pull --abort-on-container-exit --detach + up --quiet-pull --detach # --abort-on-container-exit docker compose \ --ansi never --profile int-registry-batch-loader --project-name registry \ From f89f543ba0f8e77f11d0f0343b334a39f3b7a333 Mon Sep 17 00:00:00 2001 From: thomas loubrieu <60993872+tloubrieu-jpl@users.noreply.github.com> Date: Tue, 7 Apr 2026 15:40:22 -0700 Subject: [PATCH 069/137] Update branch-cicd.yaml --- .github/workflows/branch-cicd.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/branch-cicd.yaml b/.github/workflows/branch-cicd.yaml index 66f4ad4f..5876a118 100644 --- a/.github/workflows/branch-cicd.yaml +++ b/.github/workflows/branch-cicd.yaml @@ -101,7 +101,8 @@ jobs: docker compose logs --no-color docker compose \ --ansi never --profile int-registry-batch-loader --project-name registry \ - up --quiet-pull --detach + up --quiet-pull --abort-on-container-exit + #--detach # --abort-on-container-exit docker compose \ --ansi never --profile int-registry-batch-loader --project-name registry \ From 339cf11ee0eb6cd4312eea775b4cece60e8a5a2e Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Tue, 7 Apr 2026 16:15:53 -0700 Subject: [PATCH 070/137] retrigger branch test From 143ce054204c5ac13eeac4b8a4b4c684f8050f68 Mon Sep 17 00:00:00 2001 From: thomas loubrieu <60993872+tloubrieu-jpl@users.noreply.github.com> Date: Tue, 7 Apr 2026 17:18:37 -0700 Subject: [PATCH 071/137] Update branch-cicd.yaml --- .github/workflows/branch-cicd.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/branch-cicd.yaml b/.github/workflows/branch-cicd.yaml index 5876a118..eabd7577 100644 --- a/.github/workflows/branch-cicd.yaml +++ b/.github/workflows/branch-cicd.yaml @@ -99,13 +99,14 @@ jobs: docker image inspect nasapds/registry-api-service:latest >/dev/null echo "===== Docker logs =====" docker compose logs --no-color + export COMPOSE_PROJECT_NAME=registry docker compose \ - --ansi never --profile int-registry-batch-loader --project-name registry \ + --ansi never --profile int-registry-batch-loader --project-name ${COMPOSE_PROJECT_NAME} \ up --quiet-pull --abort-on-container-exit #--detach # --abort-on-container-exit docker compose \ - --ansi never --profile int-registry-batch-loader --project-name registry \ + --ansi never --profile int-registry-batch-loader --project-name ${COMPOSE_PROJECT_NAME} \ run --rm --no-TTY reg-api-integration-test-with-wait - From 4d82915eefd1c7aec46a1a757679a1116ccbaf24 Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Tue, 7 Apr 2026 17:25:22 -0700 Subject: [PATCH 072/137] retrigger branch test From 23dcf9567e60effe956385e07ace2a903591bc7a Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Wed, 8 Apr 2026 07:45:43 -0700 Subject: [PATCH 073/137] trigger github action tests From 5c972d35ae6b36b09dda09029c531c3c201e3729 Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Wed, 8 Apr 2026 12:18:44 -0700 Subject: [PATCH 074/137] add content to health check to avoid verbose warning in API log on helathchecks --- README.md | 2 +- .../nasa/pds/api/registry/controllers/HealthController.java | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d064d301..bdb35cf0 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ Follow instructions in README.txt in the decompressed folder To build and run the application you need: -- jdk 17 +- jdk 25 - maven Additionally, harvested data will only be picked up correctly by the API if all of the following are true: diff --git a/service/src/main/java/gov/nasa/pds/api/registry/controllers/HealthController.java b/service/src/main/java/gov/nasa/pds/api/registry/controllers/HealthController.java index 9ddd127b..d9406bc6 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/controllers/HealthController.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/controllers/HealthController.java @@ -11,9 +11,8 @@ public class HealthController implements HealthApi { @Override public ResponseEntity> health() { - // To Be Completed - return new ResponseEntity<>(HttpStatus.OK); - + Map response = Map.of("status", "ok"); + return new ResponseEntity<>(response, HttpStatus.OK); } } From 2cee08706b91a7e642543484de779816e5c9a9a6 Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Wed, 8 Apr 2026 12:22:41 -0700 Subject: [PATCH 075/137] upgrade deendency for security prupose --- service/pom.xml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/service/pom.xml b/service/pom.xml index 4ccbb754..1e461102 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -322,8 +322,7 @@ org.apache.httpcomponents.client5 httpclient5 - - 5.4.1 + 5.4.3 From 0ab7cd669c6eb9e8fd9b603e5656c7bfe16b65d8 Mon Sep 17 00:00:00 2001 From: thomas loubrieu <60993872+tloubrieu-jpl@users.noreply.github.com> Date: Thu, 9 Apr 2026 08:44:56 -0700 Subject: [PATCH 076/137] Run without stop on exit because it interrupts the service for no good reason --- .github/workflows/branch-cicd.yaml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/branch-cicd.yaml b/.github/workflows/branch-cicd.yaml index eabd7577..dd6507ab 100644 --- a/.github/workflows/branch-cicd.yaml +++ b/.github/workflows/branch-cicd.yaml @@ -97,14 +97,13 @@ jobs: ./generate-certs.sh cd .. docker image inspect nasapds/registry-api-service:latest >/dev/null - echo "===== Docker logs =====" - docker compose logs --no-color + export COMPOSE_PROJECT_NAME=registry docker compose \ --ansi never --profile int-registry-batch-loader --project-name ${COMPOSE_PROJECT_NAME} \ - up --quiet-pull --abort-on-container-exit - #--detach - # --abort-on-container-exit + up --quiet-pull --detach + echo "===== Docker logs =====" + docker compose logs --no-color docker compose \ --ansi never --profile int-registry-batch-loader --project-name ${COMPOSE_PROJECT_NAME} \ run --rm --no-TTY reg-api-integration-test-with-wait From fb614f5a2485e0e2b8af322105aa2e9f56de859f Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Thu, 9 Apr 2026 10:48:41 -0700 Subject: [PATCH 077/137] adjust dependency to avoid vulnerability --- service/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/service/pom.xml b/service/pom.xml index 1e461102..51bc2e15 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -330,7 +330,7 @@ org.apache.httpcomponents.core5 httpcore5 - 5.3.3 + 5.4.2 @@ -338,7 +338,7 @@ org.apache.httpcomponents.core5 httpcore5-h2 - 5.3.3 + 5.4.2 From f98ffe2e8f7f86c2519d3ee15ffc7e40c82d6986 Mon Sep 17 00:00:00 2001 From: thomas loubrieu <60993872+tloubrieu-jpl@users.noreply.github.com> Date: Thu, 9 Apr 2026 15:24:48 -0700 Subject: [PATCH 078/137] Update branch-cicd.yaml --- .github/workflows/branch-cicd.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/branch-cicd.yaml b/.github/workflows/branch-cicd.yaml index dd6507ab..c6bb3324 100644 --- a/.github/workflows/branch-cicd.yaml +++ b/.github/workflows/branch-cicd.yaml @@ -88,7 +88,7 @@ jobs: build-args: api_jar=${{steps.jarrer.outputs.jar_file}} push: false load: true - tags: nasapds/registry-api-service:latest + tags: nasapds/registry-api-service:develop - name: ∫ Integration tests … hold onto your hats, pardners run: | @@ -96,9 +96,10 @@ jobs: cd registry/docker/certs ./generate-certs.sh cd .. - docker image inspect nasapds/registry-api-service:latest >/dev/null + docker image inspect nasapds/registry-api-service:develop >/dev/null export COMPOSE_PROJECT_NAME=registry + export REG_API_IMAGE=nasapds/registry-api-service:develop docker compose \ --ansi never --profile int-registry-batch-loader --project-name ${COMPOSE_PROJECT_NAME} \ up --quiet-pull --detach From e332a87e040f71c528cf9d4d8f59e1a40739d413 Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Thu, 9 Apr 2026 15:47:41 -0700 Subject: [PATCH 079/137] use constant for string used in 3 places. --- .../pds/api/registry/controllers/ProductsController.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java b/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java index 38313814..4792e890 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java @@ -53,6 +53,8 @@ // corresponding controllers public class ProductsController implements ProductsApi, ClassesApi, PropertiesApi { + private static final String OPS_PROVENANCE_OPS_ANCESTOR_REFS = "ops:Provenance/ops:ancestor_refs"; + @Override // TODO: Remove this when the common controller code is refactored out - it is only necessary // because additional @@ -531,9 +533,9 @@ public ResponseEntity productMemberOf(String identifier, List us List parentIds; if (productClass.isCollection()) { - parentIds = resolveLidVidsFromProductField(lidvid, "ops:Provenance/ops:ancestor_refs"); + parentIds = resolveLidVidsFromProductField(lidvid, OPS_PROVENANCE_OPS_ANCESTOR_REFS); } else if (productClass.isBasicProduct()) { - parentIds = resolveLidVidsFromProductField(lidvid, "ops:Provenance/ops:ancestor_refs"); + parentIds = resolveLidVidsFromProductField(lidvid, OPS_PROVENANCE_OPS_ANCESTOR_REFS); } else { throw new BadRequestException( "productMembersOf endpoint is not valid for products with Product_Class '" @@ -555,7 +557,7 @@ public ResponseEntity productMemberOf(String identifier, List us private Stream safeResolveLidVidsFromAncestor(PdsLidVid obj) { try { - return resolveLidVidsFromProductField(obj, "ops:Provenance/ops:ancestor_refs").stream(); + return resolveLidVidsFromProductField(obj, OPS_PROVENANCE_OPS_ANCESTOR_REFS).stream(); } catch (Exception e) { throw new RuntimeException(e); } From db3f8aa47c62e75750bc3dd2f3aaa88fd587ead6 Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Thu, 9 Apr 2026 15:58:24 -0700 Subject: [PATCH 080/137] fix snonacloud medium gravity review --- .../nasa/pds/api/registry/controllers/ProductsController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java b/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java index 4792e890..6229caf7 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java @@ -558,7 +558,7 @@ public ResponseEntity productMemberOf(String identifier, List us private Stream safeResolveLidVidsFromAncestor(PdsLidVid obj) { try { return resolveLidVidsFromProductField(obj, OPS_PROVENANCE_OPS_ANCESTOR_REFS).stream(); - } catch (Exception e) { + } catch (OpenSearchException | IOException | NotFoundException | UnhandledException e) { throw new RuntimeException(e); } } From 969084280b28544cc3797191b8b9f7f4c52139fd Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Thu, 9 Apr 2026 16:21:56 -0700 Subject: [PATCH 081/137] fix minor sonnaQube issue --- .../registry/controllers/ProductsController.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java b/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java index 6229caf7..fe98a529 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java @@ -429,12 +429,16 @@ public ResponseEntity productMembersMembers(String identifier, List searchAfter, List facetFields, Integer facetLimit) throws DeprecatedEndPointException { + String message = + """ + This endpoint is deprecated and does not work anymore. It will be removed in a future release. - throw new DeprecatedEndPointException( - "This endpoint is deprecated and does not work anymore. It will be removed in a future release.\n" - + "\n" + "Please call `/{id}/members` instead, as follows:\n" - + " 1. Get the collection members of the bundle {id} with a first call.\n" - + " 2. Use the collection ids found and get their products by calling the `/{coll_id}/members` for each."); + Please call `/{id}/members` instead, as follows: + 1. Get the collection members of the bundle {id} with a first call. + 2. Use the collection ids found and get their products by calling the `/{coll_id}/members` for each. + """; + + throw new DeprecatedEndPointException(message); } @@ -576,7 +580,6 @@ public ResponseEntity productMemberOfOf(String identifier, PdsProductClasses productClass = resolveProductClass(pdsIdentifier); PdsLidVid lidvid = resolveIdentifierToLidvid(pdsIdentifier); - Stream parentIdStream; List greatParentIds; if (productClass.isBasicProduct()) { From 428029d667c5abfec04b3c1e324f849e0c3d5311 Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Thu, 9 Apr 2026 17:23:57 -0700 Subject: [PATCH 082/137] add push docker images to GHCR for development versions. --- .github/workflows/unstable-cicd.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/unstable-cicd.yaml b/.github/workflows/unstable-cicd.yaml index 17853172..1a9d9cd1 100644 --- a/.github/workflows/unstable-cicd.yaml +++ b/.github/workflows/unstable-cicd.yaml @@ -107,6 +107,21 @@ jobs: platforms: linux/amd64,linux/arm64 push: true tags: ${{secrets.DOCKERHUB_USERNAME}}/registry-api-service:develop + # also push the image to GHCR to synchronize with AWS ECR + # we don't pull from docker hub as the AWS Pull Through Cache requires authentication + # and don't have an organization account to manage read-only logins for the cache + - + name: Log in to GHCR + run: | + echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + - + name: Build Docker image + run: | + docker build -t ghcr.io/${{ github.repository }}:latest . + - + name: Push Docker image + run: | + docker push ghcr.io/${{ github.repository }}:latest - name: ∫ Integration tests … hold onto your hats, pardners run: | From a3898a0c9d549378b27e95dd7cfe2ecde81dcaf9 Mon Sep 17 00:00:00 2001 From: PDSEN CI Bot Date: Fri, 10 Apr 2026 00:30:26 +0000 Subject: [PATCH 083/137] Update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f8eb751..29ec361a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## [«unknown»](https://github.com/NASA-PDS/registry-api/tree/«unknown») (2026-04-07) +## [«unknown»](https://github.com/NASA-PDS/registry-api/tree/«unknown») (2026-04-10) [Full Changelog](https://github.com/NASA-PDS/registry-api/compare/v1.6.2...«unknown») From 43e1abdb002aa5ddcf3fb86cbb507f5c3a478f7e Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Fri, 10 Apr 2026 08:10:15 -0700 Subject: [PATCH 084/137] adjust the naming of the ghcr docker image to comply with the repository rules --- .github/workflows/unstable-cicd.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/unstable-cicd.yaml b/.github/workflows/unstable-cicd.yaml index 1a9d9cd1..e918340d 100644 --- a/.github/workflows/unstable-cicd.yaml +++ b/.github/workflows/unstable-cicd.yaml @@ -115,13 +115,13 @@ jobs: run: | echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - - name: Build Docker image + name: Tag Docker image for GHCR run: | - docker build -t ghcr.io/${{ github.repository }}:latest . + docker tag ${{secrets.DOCKERHUB_USERNAME}}/registry-api-service:develop ghcr.io/${{ lower(github.repository) }}:develop - name: Push Docker image run: | - docker push ghcr.io/${{ github.repository }}:latest + docker push ghcr.io/${{ lower(github.repository) }}:develop - name: ∫ Integration tests … hold onto your hats, pardners run: | From 2a783e23e3135e5500b371284d232bdfd6850934 Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Fri, 10 Apr 2026 08:14:24 -0700 Subject: [PATCH 085/137] fix lower function which was not supported in github actions --- .github/workflows/unstable-cicd.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unstable-cicd.yaml b/.github/workflows/unstable-cicd.yaml index e918340d..d755323b 100644 --- a/.github/workflows/unstable-cicd.yaml +++ b/.github/workflows/unstable-cicd.yaml @@ -117,11 +117,13 @@ jobs: - name: Tag Docker image for GHCR run: | - docker tag ${{secrets.DOCKERHUB_USERNAME}}/registry-api-service:develop ghcr.io/${{ lower(github.repository) }}:develop + REPO_LOWER=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]') + docker tag ${{secrets.DOCKERHUB_USERNAME}}/registry-api-service:develop ghcr.io/${REPO_LOWER}:develop - name: Push Docker image run: | - docker push ghcr.io/${{ lower(github.repository) }}:develop + REPO_LOWER=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]') + docker push ghcr.io/${REPO_LOWER}:develop - name: ∫ Integration tests … hold onto your hats, pardners run: | From a217c453b8a84f14d9f7a069549f85db9a5a97b3 Mon Sep 17 00:00:00 2001 From: PDSEN CI Bot Date: Fri, 10 Apr 2026 15:20:48 +0000 Subject: [PATCH 086/137] Update changelog From 183e3019c5634716726728667991a612cfe91270 Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Fri, 10 Apr 2026 09:12:40 -0700 Subject: [PATCH 087/137] load=true so that the build image can be re-tagged in later step. --- .github/workflows/unstable-cicd.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/unstable-cicd.yaml b/.github/workflows/unstable-cicd.yaml index d755323b..bec837d2 100644 --- a/.github/workflows/unstable-cicd.yaml +++ b/.github/workflows/unstable-cicd.yaml @@ -106,6 +106,7 @@ jobs: build-args: api_jar=${{steps.jarrer.outputs.jar_file}} platforms: linux/amd64,linux/arm64 push: true + load: true tags: ${{secrets.DOCKERHUB_USERNAME}}/registry-api-service:develop # also push the image to GHCR to synchronize with AWS ECR # we don't pull from docker hub as the AWS Pull Through Cache requires authentication From 71f31208625e9350f2cdee490ccb6e6fc396251e Mon Sep 17 00:00:00 2001 From: PDSEN CI Bot Date: Fri, 10 Apr 2026 16:20:03 +0000 Subject: [PATCH 088/137] Update changelog From dbedc30dcff71bd3655499b8da0f91161dff8ac4 Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Fri, 10 Apr 2026 10:40:26 -0700 Subject: [PATCH 089/137] push docker image to ghcr first and then sync it with dockerhub --- .github/workflows/unstable-cicd.yaml | 42 +++++++++++++--------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/.github/workflows/unstable-cicd.yaml b/.github/workflows/unstable-cicd.yaml index bec837d2..b3bac820 100644 --- a/.github/workflows/unstable-cicd.yaml +++ b/.github/workflows/unstable-cicd.yaml @@ -47,6 +47,8 @@ jobs: name: 🧩 Unstable Assembly runs-on: ubuntu-latest if: github.actor != 'pdsen-ci' + env: + LOWER_GHCR_REPO: $(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]') steps: - name: 💳 Checkout @@ -85,18 +87,19 @@ jobs: name: 🫙 Jar File Determination id: jarrer run: echo "jar_file=$(find ./service/target/ -maxdepth 1 -regextype posix-extended -regex '.*/registry-api-service-[0-9]+\.[0-9]+\.[0-9]+(-SNAPSHOT)?\.jar')" >> $GITHUB_OUTPUT - - - name: 💳 Docker Hub Identification - uses: docker/login-action@v4 - with: - username: ${{secrets.DOCKERHUB_USERNAME}} - password: ${{secrets.DOCKERHUB_TOKEN}} - name: 🎰 QEMU Multiple Machine Emulation uses: docker/setup-qemu-action@v4 - name: 🚢 Docker Buildx uses: docker/setup-buildx-action@v4 + # we want to publish the docker image ""locally" to GHCR as the AWS Pull Through Cache requires authentication + # and we don't have an organization account to manage read-only logins for the cache + - + name: Log in to GHCR + run: | + echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + - name: 🧱 Image Construction and Publication uses: docker/build-push-action@v7 @@ -106,25 +109,20 @@ jobs: build-args: api_jar=${{steps.jarrer.outputs.jar_file}} platforms: linux/amd64,linux/arm64 push: true - load: true - tags: ${{secrets.DOCKERHUB_USERNAME}}/registry-api-service:develop - # also push the image to GHCR to synchronize with AWS ECR - # we don't pull from docker hub as the AWS Pull Through Cache requires authentication - # and don't have an organization account to manage read-only logins for the cache + tags: ghcr.io/${LOWER_GHCR_REPO}:develop + # also push the image to DockerHub because we keep providing that distribution channel. - - name: Log in to GHCR - run: | - echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - - - name: Tag Docker image for GHCR - run: | - REPO_LOWER=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]') - docker tag ${{secrets.DOCKERHUB_USERNAME}}/registry-api-service:develop ghcr.io/${REPO_LOWER}:develop + name: 💳 Docker Hub Identification + uses: docker/login-action@v4 + with: + username: ${{secrets.DOCKERHUB_USERNAME}} + password: ${{secrets.DOCKERHUB_TOKEN}}- - - name: Push Docker image + name: Tag Docker image for Docker hub run: | - REPO_LOWER=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]') - docker push ghcr.io/${REPO_LOWER}:develop + docker buildx imagetools create \ + -t ghcr.io/${LOWER_GHCR_REPO}:develop \ + ${{secrets.DOCKERHUB_USERNAME}}/registry-api-service:develop - name: ∫ Integration tests … hold onto your hats, pardners run: | From 3dd95378605b73b9cd7aba307421493c3b18652b Mon Sep 17 00:00:00 2001 From: PDSEN CI Bot Date: Fri, 10 Apr 2026 17:47:25 +0000 Subject: [PATCH 090/137] Update changelog From 01ce2568985eedc46db0e5b01bdbca5c69c0fd81 Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Fri, 10 Apr 2026 11:04:50 -0700 Subject: [PATCH 091/137] change GHCR authentication, for the better --- .github/workflows/unstable-cicd.yaml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/unstable-cicd.yaml b/.github/workflows/unstable-cicd.yaml index b3bac820..914631c3 100644 --- a/.github/workflows/unstable-cicd.yaml +++ b/.github/workflows/unstable-cicd.yaml @@ -96,10 +96,11 @@ jobs: # we want to publish the docker image ""locally" to GHCR as the AWS Pull Through Cache requires authentication # and we don't have an organization account to manage read-only logins for the cache - - name: Log in to GHCR - run: | - echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - + name: 💳 GHCR Identification + uses: docker/login-action@v4 + with: + username: ${{github.actor}} + password: ${{secrets.GITHUB_TOKEN}} - name: 🧱 Image Construction and Publication uses: docker/build-push-action@v7 @@ -116,7 +117,7 @@ jobs: uses: docker/login-action@v4 with: username: ${{secrets.DOCKERHUB_USERNAME}} - password: ${{secrets.DOCKERHUB_TOKEN}}- + password: ${{secrets.DOCKERHUB_TOKEN}} - name: Tag Docker image for Docker hub run: | From 13dc0a42dd17d29e77667bfec755fcb2ad194c26 Mon Sep 17 00:00:00 2001 From: PDSEN CI Bot Date: Fri, 10 Apr 2026 18:10:53 +0000 Subject: [PATCH 092/137] Update changelog From ba62357309e0f6316f9f9fe56ff63641e00e1dbb Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Fri, 10 Apr 2026 11:19:55 -0700 Subject: [PATCH 093/137] add missing registry --- .github/workflows/unstable-cicd.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/unstable-cicd.yaml b/.github/workflows/unstable-cicd.yaml index 914631c3..3b33267c 100644 --- a/.github/workflows/unstable-cicd.yaml +++ b/.github/workflows/unstable-cicd.yaml @@ -99,6 +99,7 @@ jobs: name: 💳 GHCR Identification uses: docker/login-action@v4 with: + registry: ghcr.io username: ${{github.actor}} password: ${{secrets.GITHUB_TOKEN}} - From 5d3a1ce034ddc6753bd2600a04c1330396f7338d Mon Sep 17 00:00:00 2001 From: PDSEN CI Bot Date: Fri, 10 Apr 2026 18:26:09 +0000 Subject: [PATCH 094/137] Update changelog From cbd3ddf967fed6eeb4c598b31e11a767eb77b166 Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Fri, 10 Apr 2026 11:37:57 -0700 Subject: [PATCH 095/137] condense the docker push code --- .github/workflows/unstable-cicd.yaml | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/.github/workflows/unstable-cicd.yaml b/.github/workflows/unstable-cicd.yaml index 3b33267c..52e40326 100644 --- a/.github/workflows/unstable-cicd.yaml +++ b/.github/workflows/unstable-cicd.yaml @@ -102,6 +102,13 @@ jobs: registry: ghcr.io username: ${{github.actor}} password: ${{secrets.GITHUB_TOKEN}} + # also push the image to DockerHub because we keep providing that distribution channel. + - + name: 💳 Docker Hub Identification + uses: docker/login-action@v4 + with: + username: ${{secrets.DOCKERHUB_USERNAME}} + password: ${{secrets.DOCKERHUB_TOKEN}} - name: 🧱 Image Construction and Publication uses: docker/build-push-action@v7 @@ -111,20 +118,9 @@ jobs: build-args: api_jar=${{steps.jarrer.outputs.jar_file}} platforms: linux/amd64,linux/arm64 push: true - tags: ghcr.io/${LOWER_GHCR_REPO}:develop - # also push the image to DockerHub because we keep providing that distribution channel. - - - name: 💳 Docker Hub Identification - uses: docker/login-action@v4 - with: - username: ${{secrets.DOCKERHUB_USERNAME}} - password: ${{secrets.DOCKERHUB_TOKEN}} - - - name: Tag Docker image for Docker hub - run: | - docker buildx imagetools create \ - -t ghcr.io/${LOWER_GHCR_REPO}:develop \ - ${{secrets.DOCKERHUB_USERNAME}}/registry-api-service:develop + tags: | + ghcr.io/${{LOWER_GHCR_REPO}}:develop + ${{secrets.DOCKERHUB_USERNAME}}/registry-api-service:develop - name: ∫ Integration tests … hold onto your hats, pardners run: | From 6e7c65082d6c12db09d8efa55e2cddd9f3cfaf1f Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Fri, 10 Apr 2026 11:39:57 -0700 Subject: [PATCH 096/137] fix env variable call syntax --- .github/workflows/unstable-cicd.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unstable-cicd.yaml b/.github/workflows/unstable-cicd.yaml index 52e40326..61920aaf 100644 --- a/.github/workflows/unstable-cicd.yaml +++ b/.github/workflows/unstable-cicd.yaml @@ -119,7 +119,7 @@ jobs: platforms: linux/amd64,linux/arm64 push: true tags: | - ghcr.io/${{LOWER_GHCR_REPO}}:develop + ghcr.io/${{env.LOWER_GHCR_REPO}}:develop ${{secrets.DOCKERHUB_USERNAME}}/registry-api-service:develop - name: ∫ Integration tests … hold onto your hats, pardners From 1de091fb30b82869a0e8c3e5ca459372a61858e9 Mon Sep 17 00:00:00 2001 From: PDSEN CI Bot Date: Fri, 10 Apr 2026 18:46:27 +0000 Subject: [PATCH 097/137] Update changelog From 502f88aed95cc186623ac17b9c8298a46fae4d3b Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Fri, 10 Apr 2026 12:21:11 -0700 Subject: [PATCH 098/137] fix initialization of github action environment variable --- .github/workflows/unstable-cicd.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unstable-cicd.yaml b/.github/workflows/unstable-cicd.yaml index 61920aaf..b0fa48ea 100644 --- a/.github/workflows/unstable-cicd.yaml +++ b/.github/workflows/unstable-cicd.yaml @@ -47,8 +47,6 @@ jobs: name: 🧩 Unstable Assembly runs-on: ubuntu-latest if: github.actor != 'pdsen-ci' - env: - LOWER_GHCR_REPO: $(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]') steps: - name: 💳 Checkout @@ -109,6 +107,9 @@ jobs: with: username: ${{secrets.DOCKERHUB_USERNAME}} password: ${{secrets.DOCKERHUB_TOKEN}} + - + name: Set lowercase repo name + run: echo "LOWER_GHCR_REPO=$(echo '${{ github.repository }}' | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV - name: 🧱 Image Construction and Publication uses: docker/build-push-action@v7 From 2d505a76cdc3236d8b54cb083a4a4fd26e029ec7 Mon Sep 17 00:00:00 2001 From: PDSEN CI Bot Date: Fri, 10 Apr 2026 19:27:26 +0000 Subject: [PATCH 099/137] Update changelog From 5396c8f947f18597fbd2f412984cdf73305a8d9a Mon Sep 17 00:00:00 2001 From: PDSEN CI Bot Date: Tue, 14 Apr 2026 17:26:15 +0000 Subject: [PATCH 100/137] Update changelog --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 29ec361a..087d3e46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## [«unknown»](https://github.com/NASA-PDS/registry-api/tree/«unknown») (2026-04-10) +## [«unknown»](https://github.com/NASA-PDS/registry-api/tree/«unknown») (2026-04-14) [Full Changelog](https://github.com/NASA-PDS/registry-api/compare/v1.6.2...«unknown») @@ -12,9 +12,11 @@ **Improvements:** - As a user, I want the exists operator to be prepended to the query [\#727](https://github.com/NASA-PDS/registry-api/issues/727) +- Update registry API `/members/members` algorithm per deprecation of `parent_bundle_identifier` metadata non-aggregate products [\#699](https://github.com/NASA-PDS/registry-api/issues/699) **Defects:** +- Investigate and fix skipped `product/{id}/member*` integration tests [\#748](https://github.com/NASA-PDS/registry-api/issues/748) [[s.medium](https://github.com/NASA-PDS/registry-api/labels/s.medium)] - A query to pds.nasa.gov does not respond the same as a query to pds.mcp.nasa.gov [\#742](https://github.com/NASA-PDS/registry-api/issues/742) [[s.medium](https://github.com/NASA-PDS/registry-api/labels/s.medium)] - API in production is unstable and returns 500 errors [\#716](https://github.com/NASA-PDS/registry-api/issues/716) [[s.critical](https://github.com/NASA-PDS/registry-api/labels/s.critical)] - Inconsistent support for `application/vnd.nasa.pds.pds4+json` response format [\#705](https://github.com/NASA-PDS/registry-api/issues/705) [[s.high](https://github.com/NASA-PDS/registry-api/labels/s.high)] From 4ba436e9e741664d865aec41dd3ce22a1480c5e0 Mon Sep 17 00:00:00 2001 From: al-niessner <1130658+al-niessner@users.noreply.github.com> Date: Wed, 15 Apr 2026 13:21:27 -0700 Subject: [PATCH 101/137] #744 Group integration tests in an external script (#747) * Pulled the integration tests from github actions to local script 1. github action machines are tiny (2GB mem) and tests desire a bit more with opensearch running in a container. 2. the deep archive checks require the opensearch be up and running with the registry-api. * fixed starting point 1. remove all of the existing docker images that are used for testing 2 reload the docker images to make sure they are the latest that have been published 3. if the branch name in registry-api is something like issue-1234 then the script will use the branch api-1234 in the registry repository. It will use develop if the more specific branch does not exist. --- .github/workflows/branch-cicd.yaml | 32 +-- .github/workflows/integration_tests.sh | 205 +++++++++++++++++++ .github/workflows/last_integration_test.json | 5 + .gitignore | 3 + 4 files changed, 217 insertions(+), 28 deletions(-) create mode 100755 .github/workflows/integration_tests.sh create mode 100644 .github/workflows/last_integration_test.json diff --git a/.github/workflows/branch-cicd.yaml b/.github/workflows/branch-cicd.yaml index c6bb3324..f8f8f65a 100644 --- a/.github/workflows/branch-cicd.yaml +++ b/.github/workflows/branch-cicd.yaml @@ -89,39 +89,15 @@ jobs: push: false load: true tags: nasapds/registry-api-service:develop - - - name: ∫ Integration tests … hold onto your hats, pardners - run: | - git clone --quiet https://github.com/NASA-PDS/registry.git - cd registry/docker/certs - ./generate-certs.sh - cd .. - docker image inspect nasapds/registry-api-service:develop >/dev/null - - export COMPOSE_PROJECT_NAME=registry - export REG_API_IMAGE=nasapds/registry-api-service:develop - docker compose \ - --ansi never --profile int-registry-batch-loader --project-name ${COMPOSE_PROJECT_NAME} \ - up --quiet-pull --detach - echo "===== Docker logs =====" - docker compose logs --no-color - docker compose \ - --ansi never --profile int-registry-batch-loader --project-name ${COMPOSE_PROJECT_NAME} \ - run --rm --no-TTY reg-api-integration-test-with-wait - - name: Set up Python 3 - uses: actions/setup-python@v6 - with: - python-version: '3.13' + name: Install jq + uses: dcarbone/install-jq-action@v3 - - name: ∫ Test PDS Deep Archive compatibility + name: ∫ Integration and deep archive tests … hold onto your hats, pardners run: | - git clone --quiet https://github.com/NASA-PDS/deep-archive.git - cd deep-archive - pip install . - pds-deep-registry-archive -u http://localhost:8080 -s PDS_ENG urn:nasa:pds:insight_rad::2.1 --debug + .github/workflows/integration_tests.sh --verify ... diff --git a/.github/workflows/integration_tests.sh b/.github/workflows/integration_tests.sh new file mode 100755 index 00000000..e0143fb7 --- /dev/null +++ b/.github/workflows/integration_tests.sh @@ -0,0 +1,205 @@ +#! /usr/bin/env bash +# +# requires: docker, git, mvn, and shellcheck +# +# docker - builds the registry api image, uses compose to run a host of services +# git clones registry repo +# jq - bash JSON tool +# mvn - to build the jar file for the current source registry api source code +# "shellcheck" - linter to keep this script clean +# + +build() { + mvn --quiet clean package + jar_file="$(find ./service/target/ -maxdepth 1 -regextype posix-extended -regex '.*/registry-api-service-[0-9]+\.[0-9]+\.[0-9]+(-SNAPSHOT)?\.jar')" + docker build --build-arg api_jar="$jar_file" -t nasapds/registry-api-service:latest -f docker/Dockerfile . +} + +clean() { + # shellcheck disable=SC2086 # for correct docker interpretation + docker compose \ + --ansi never \ + --profile int-registry-batch-loader \ + --project-name registry \ + down ${IT_CLEANSE:---rmi all} +} + +deep_archive() { + cd "$tdir" || return 1 + python3 -m venv "$tdir"/da + # shellcheck disable=SC1091 # cannot find dynamically created script + source "$tdir"/da/bin/activate + git clone --quiet https://github.com/NASA-PDS/deep-archive.git + cd deep-archive || return 1 + pip install . + pds-deep-registry-archive -u http://localhost:8080 -s PDS_ENG urn:nasa:pds:insight_rad::2.1 --debug +} + +double_check_logfile() { + echo "everything looked ok, so double check postman logs" + [ -s "$1" ] || { echo "$1 is an empty file"; return 1; } + grep -Eq "[[:space:]]*#[[:space:]]+failure[[:space:]]+detail" "$1" \ + && { echo "postman log file reported failures" ; return 2; } + return 0 +} + +record() { + cat > last_integration_test.json </dev/null + echo "launch services" + docker compose \ + --ansi never \ + --profile int-registry-batch-loader \ + --project-name registry \ + up --detach --quiet-pull || return 5 + echo "launch tests" + if docker compose \ + --ansi never \ + --profile int-registry-batch-loader \ + --project-name registry \ + run --rm --no-TTY reg-api-integration-test-with-wait + then + deep_archive + status=$? + else + status=1 + fi + echo "run status: ${status}" + clean + # shellcheck disable=SC2086 # because we need to return an int + return $status +} + +if [ $# -gt 1 ] +then + echo "Usage: $0 [--verify]" + exit 1 +fi + +if [ $# -eq 1 ] && [ "$1" != "--verify" ] +then + echo "Error: Invalid argument '$1'" + echo "Usage: $0 [--verify]" + exit 1 +fi + +bdir=$(dirname "$(realpath "$0")") +rdir=$(realpath "$bdir/../..") +cd "$rdir" || exit 1 +api_gitrev=$(git describe --always --abbrev=40 --dirty='+' --exclude '*') +branchname=$(git branch --show-current) +branchname=${branchname/issue/api} +branchname=${branchname/_/-} +tdir=$(mktemp -d) +# The EXIT pseudo-signal covers normal exits, errors, and interruptions (Ctrl+C) +trap 'rm -rf "$tdir"' EXIT +export tdir +cd "$tdir" || exit 1 +git clone --quiet https://github.com/NASA-PDS/registry.git +cd registry || exit 1 +if git show-ref --verify --quiet refs/remotes/origin/"$branchname" +then + git switch "$branchname" +fi +echo "registry being used" +git status +reg_gitrev=$(git describe --always --abbrev=40 --dirty='+' --exclude '*') +if [ "$1" == "--verify" ]; then + echo "Running in VERIFY mode..." + status=failure + cd "$tdir" || exit 1 + record "$api_gitrev" "$reg_gitrev" "$status" + cd "$rdir" || exit 1 + test_key=$(jq -r '.api_gitrev' "$bdir"/last_integration_test.json | sed 's/+$//') + files=$(git diff --name-only -r "$test_key") + # shellcheck disable=SC2046 # because comparing integers + if [ $(echo "$files" | wc -l) -eq 1 ] + then + if [ "$files" == ".github/workflows/last_integration_test.json" ] + then + if [ -s "$files" ] + then + # do a one line diff from last test run + # look at additions or subtractions + # ignore --- and +++ because those are the filenames + # ignore the api_gitrev because that must be different + # count all other changes + # if there are none, then status is meaningful + # shellcheck disable=SC2126 # because simpler to understand + if [ $(git diff -U0 -r "$test_key" | \ + grep "^[+-]" | \ + grep -v "^---" | \ + grep -v "^+++" | \ + grep -v "api_gitrev" | \ + wc -l) == 0 ] + then + status=$(jq -r '.status' "$bdir"/last_integration_test.json) + echo "Found the I&T test to be: ${status}" + else + git diff -r "$test_key" + fi + else + echo "Reporting file is empty" + fi + else + echo "the file changed was not for I&T: $files" + fi + else + echo "commit contains edits beyond those of last_integration_test.json" + echo "files changed: $files" + fi + if [ "$status" == "failure" ] + then + echo + echo "If you are reading this in the github actions log, then it seems" + echo "this test cannot verify that this registry-api repository branch" + echo "has been successfully tested. The first step at resolving this" + echo "message is to run the script .github/workflows/integration_tests.sh" + echo "locally. If it is successful, then commit all changes and push." + echo "Otherwise, fix any problems demonstrated from running the tests," + echo "then commit and push all changes when the script is successful." + echo "Once commited, run this script again to generate the single file" + echo "last_integration_test.json, commit it, and push it." + echo + echo "Note: there are timing tests that can cause temporary failures." + echo " If those failures occur, just run the script again until" + echo " a success is achived." + echo + echo "Note: to determine if the latest commit will pass, run the script" + echo " with 'integration_tests.sh --verify'" + else + echo "Verified tests completed and successful" + fi +else + cd "$rdir" || exit 1 + clean || exit 2 + build || exit 3 + cd "$tdir"/registry || exit 1 + ( set -o pipefail ; run 2>&1 | tee "$rdir"/integration_tests.rpt.txt ) \ + && status=success || status=failure + if [ "$status" == "success" ] + then + double_check_logfile "$rdir"/integration_tests.rpt.txt \ + || status=failure + else + echo "docker run or deep archive did not return success" + fi + cd "$bdir" || exit 1 + record "$api_gitrev" "$reg_gitrev" "$status" + [ "$status" == "success" ] && rm "$rdir"/integration_tests.rpt.txt +fi + +echo "Status: $status" +[ "$status" == "success" ] && exit 0 || exit 1 diff --git a/.github/workflows/last_integration_test.json b/.github/workflows/last_integration_test.json new file mode 100644 index 00000000..a11b725c --- /dev/null +++ b/.github/workflows/last_integration_test.json @@ -0,0 +1,5 @@ +{ + "api_gitrev": "0563642e7c8e6ff153176adebc188f615db93a0e", + "reg_gitrev": "4faf4ff0ccafac6b9d4b77acdc395dfe074ef332", + "status": "success" +} diff --git a/.gitignore b/.gitignore index c9fffc88..b8e47750 100644 --- a/.gitignore +++ b/.gitignore @@ -87,3 +87,6 @@ application-*.properties # macOS specific stuff .DS_Store + +# reports to help separate testing from actions due to limited resources +*.rpt.txt From eaaa37bbedaa43fa942a8ecf60b9620ee41a3851 Mon Sep 17 00:00:00 2001 From: PDSEN CI Bot Date: Wed, 15 Apr 2026 20:28:18 +0000 Subject: [PATCH 102/137] Update changelog --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 087d3e46..d8e488bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## [«unknown»](https://github.com/NASA-PDS/registry-api/tree/«unknown») (2026-04-14) +## [«unknown»](https://github.com/NASA-PDS/registry-api/tree/«unknown») (2026-04-15) [Full Changelog](https://github.com/NASA-PDS/registry-api/compare/v1.6.2...«unknown») @@ -17,6 +17,8 @@ **Defects:** - Investigate and fix skipped `product/{id}/member*` integration tests [\#748](https://github.com/NASA-PDS/registry-api/issues/748) [[s.medium](https://github.com/NASA-PDS/registry-api/labels/s.medium)] +- Integration tests in unstable build suite do not pass when run locally [\#745](https://github.com/NASA-PDS/registry-api/issues/745) [[s.medium](https://github.com/NASA-PDS/registry-api/labels/s.medium)] +- Unstable build does not complete on develop branch due to GitHub Actions runner timeout [\#744](https://github.com/NASA-PDS/registry-api/issues/744) [[s.medium](https://github.com/NASA-PDS/registry-api/labels/s.medium)] - A query to pds.nasa.gov does not respond the same as a query to pds.mcp.nasa.gov [\#742](https://github.com/NASA-PDS/registry-api/issues/742) [[s.medium](https://github.com/NASA-PDS/registry-api/labels/s.medium)] - API in production is unstable and returns 500 errors [\#716](https://github.com/NASA-PDS/registry-api/issues/716) [[s.critical](https://github.com/NASA-PDS/registry-api/labels/s.critical)] - Inconsistent support for `application/vnd.nasa.pds.pds4+json` response format [\#705](https://github.com/NASA-PDS/registry-api/issues/705) [[s.high](https://github.com/NASA-PDS/registry-api/labels/s.high)] From a81918ec4505cf8092b761ff0b4b44c7cf97c92f Mon Sep 17 00:00:00 2001 From: al-niessner <1130658+al-niessner@users.noreply.github.com> Date: Wed, 22 Apr 2026 12:16:12 -0700 Subject: [PATCH 103/137] Consistent error messaging for field names in query language (#734) * WIP: fixed two antlr problems 1. version lexer was built with did not match runtime version. caused null pointer error deep in setting up antlr long before any string was being parsed. 2. using wrong field name to search ldd. converted to open search style too soon. finds it, but does not return the desired response. --- .github/workflows/last_integration_test.json | 4 +-- service/pom.xml | 2 +- .../registry/model/Antlr4SearchListener.java | 30 +++++++++++-------- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/.github/workflows/last_integration_test.json b/.github/workflows/last_integration_test.json index a11b725c..50196960 100644 --- a/.github/workflows/last_integration_test.json +++ b/.github/workflows/last_integration_test.json @@ -1,5 +1,5 @@ { - "api_gitrev": "0563642e7c8e6ff153176adebc188f615db93a0e", - "reg_gitrev": "4faf4ff0ccafac6b9d4b77acdc395dfe074ef332", + "api_gitrev": "e810b3a98a0caed79bf17e4cc5f000388a0d2945", + "reg_gitrev": "5a3a1789265928dc625efb0f6a1eff2f5685ab6e", "status": "success" } diff --git a/service/pom.xml b/service/pom.xml index 51bc2e15..43e34325 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -307,7 +307,7 @@ org.antlr antlr4-runtime - 4.11.1 + 4.13.2 diff --git a/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java b/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java index 12122afc..2e32b66e 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java @@ -24,6 +24,7 @@ import org.opensearch.client.opensearch._types.query_dsl.Query; import org.opensearch.client.opensearch._types.query_dsl.RangeQuery; import org.opensearch.client.opensearch._types.query_dsl.SimpleQueryStringQuery; +import org.opensearch.client.opensearch._types.query_dsl.TermQuery; public class Antlr4SearchListener extends SearchBaseListener { @@ -73,16 +74,16 @@ public void exitFields(SearchParser.FieldsContext ctx) { if (ctx.ANY() != null) { fieldname = ctx.ANY().getText(); } - if (fieldname.contains("*")) { - if (this.knownFieldNames.isEmpty()) { - try { - for (PropertiesListInner property : ProductsController.productPropertiesList(this.connectionContext).getBody()) { - this.knownFieldNames.add(property.getProperty()); - } - } catch (OpenSearchException | IOException e) { - log.error("Could not load the mapping(s) from opensearch; meaning 'wildcarding' will not work", e); + if (this.knownFieldNames.isEmpty()) { + try { + for (PropertiesListInner property : ProductsController.productPropertiesList(this.connectionContext).getBody()) { + this.knownFieldNames.add(property.getProperty()); } + } catch (OpenSearchException | IOException e) { + throw new IllegalStateException("Could not load the mapping(s) from opensearch; meaning 'q=' with field names will not work", e); } + } + if (fieldname.contains("*")) { String theKey = fieldname.replace(".", "\\.").replace("*", ".*"); Pattern regex = Pattern.compile(theKey); for (String fn : this.knownFieldNames.stream() @@ -94,7 +95,11 @@ public void exitFields(SearchParser.FieldsContext ctx) { throw new ParseCancellationException("Wildcarding request '" + fieldname + "' cannot match any field names in the LDD using regular expression " + theKey); } } else { - this.fieldNames.add(SearchUtil.jsonPropertyToOpenProperty(fieldname)); + if (this.knownFieldNames.contains(fieldname)) { + this.fieldNames.add(SearchUtil.jsonPropertyToOpenProperty(fieldname)); + } else { + throw new ParseCancellationException("The request '" + fieldname + "' does not match any field name in the LDD."); + } } this.isAnyWildcard = ctx.ALL() == null && this.fieldNames.size() > 1; } @@ -175,9 +180,12 @@ public void exitComparison(SearchParser.ComparisonContext ctx) { } for (String left : this.fieldNames) { if (this.operator == operation.eq || this.operator == operation.ne) { + BoolQuery.Builder boolQueryBuilder = new BoolQuery.Builder(); FieldValue fieldValue = new FieldValue.Builder().stringValue(right).build(); MatchQuery matchQueryBuilder = new MatchQuery.Builder().field(left).query(fieldValue).build(); - comparatorQuery = matchQueryBuilder.toQuery(); + TermQuery termQueryBulidler = new TermQuery.Builder().field(left).value(fieldValue).build(); + boolQueryBuilder.should(matchQueryBuilder.toQuery(), termQueryBulidler.toQuery()); + comparatorQuery = boolQueryBuilder.build().toQuery(); if (this.operator == operation.ne) { comparatorQuery = new BoolQuery.Builder().mustNot(comparatorQuery).build().toQuery(); @@ -211,7 +219,6 @@ else if (this.operator == operation.lt) } else { this.queryBuilder.should(wild.build().toQuery()); } - } @Override @@ -221,7 +228,6 @@ public void exitExistence(SearchParser.ExistenceContext ctx) { wild.minimumShouldMatch("1"); } for (String fieldName : this.fieldNames) { - log.error("************************* field name: " + fieldName); if (this.isAnyWildcard) { wild.should(new ExistsQuery.Builder().field(fieldName).build().toQuery()); } else { From c910a5c810eb8ab1ee1c4a188c497ce50b3d1998 Mon Sep 17 00:00:00 2001 From: PDSEN CI Bot Date: Wed, 22 Apr 2026 19:22:55 +0000 Subject: [PATCH 104/137] Update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8e488bd..1497b3c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## [«unknown»](https://github.com/NASA-PDS/registry-api/tree/«unknown») (2026-04-15) +## [«unknown»](https://github.com/NASA-PDS/registry-api/tree/«unknown») (2026-04-22) [Full Changelog](https://github.com/NASA-PDS/registry-api/compare/v1.6.2...«unknown») From b39d1f1694758b82d8ca899a521e58114f0dc63a Mon Sep 17 00:00:00 2001 From: al-niessner <1130658+al-niessner@users.noreply.github.com> Date: Tue, 28 Apr 2026 13:23:39 -0700 Subject: [PATCH 105/137] Upgrade dependencies per NASA-PDS/outlaw-tracker issues (#758) * Bump org.springframework:spring-context from 6.2.2 to 6.2.7 Bumps [org.springframework:spring-context](https://github.com/spring-projects/spring-framework) from 6.2.2 to 6.2.7. - [Release notes](https://github.com/spring-projects/spring-framework/releases) - [Commits](https://github.com/spring-projects/spring-framework/compare/v6.2.2...v6.2.7) --- updated-dependencies: - dependency-name: org.springframework:spring-context dependency-version: 6.2.7 dependency-type: direct:production ... Signed-off-by: dependabot[bot] * Bump org.springframework:spring-web from 6.2.2 to 6.2.8 Bumps [org.springframework:spring-web](https://github.com/spring-projects/spring-framework) from 6.2.2 to 6.2.8. - [Release notes](https://github.com/spring-projects/spring-framework/releases) - [Commits](https://github.com/spring-projects/spring-framework/compare/v6.2.2...v6.2.8) --- updated-dependencies: - dependency-name: org.springframework:spring-web dependency-version: 6.2.8 dependency-type: direct:production ... Signed-off-by: dependabot[bot] * Bump jackson-version from 2.18.2 to 2.19.1 Bumps `jackson-version` from 2.18.2 to 2.19.1. Updates `com.fasterxml.jackson.jaxrs:jackson-jaxrs-base` from 2.18.2 to 2.19.1 - [Commits](https://github.com/FasterXML/jackson-jaxrs-providers/compare/jackson-jaxrs-providers-2.18.2...jackson-jaxrs-providers-2.19.1) Updates `com.fasterxml.jackson.core:jackson-core` from 2.18.2 to 2.19.1 - [Commits](https://github.com/FasterXML/jackson-core/compare/jackson-core-2.18.2...jackson-core-2.19.1) Updates `com.fasterxml.jackson.core:jackson-databind` from 2.18.2 to 2.19.1 - [Commits](https://github.com/FasterXML/jackson/commits) Updates `com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider` from 2.18.2 to 2.19.1 Updates `com.fasterxml.jackson.dataformat:jackson-dataformat-xml` from 2.18.2 to 2.19.1 - [Commits](https://github.com/FasterXML/jackson-dataformat-xml/compare/jackson-dataformat-xml-2.18.2...jackson-dataformat-xml-2.19.1) --- updated-dependencies: - dependency-name: com.fasterxml.jackson.jaxrs:jackson-jaxrs-base dependency-version: 2.19.1 dependency-type: direct:production update-type: version-update:semver-minor - dependency-name: com.fasterxml.jackson.core:jackson-core dependency-version: 2.19.1 dependency-type: direct:production update-type: version-update:semver-minor - dependency-name: com.fasterxml.jackson.core:jackson-databind dependency-version: 2.19.1 dependency-type: direct:production update-type: version-update:semver-minor - dependency-name: com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider dependency-version: 2.19.1 dependency-type: direct:production update-type: version-update:semver-minor - dependency-name: com.fasterxml.jackson.dataformat:jackson-dataformat-xml dependency-version: 2.19.1 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Bump joda-time:joda-time from 2.13.0 to 2.14.0 Bumps [joda-time:joda-time](https://github.com/JodaOrg/joda-time) from 2.13.0 to 2.14.0. - [Release notes](https://github.com/JodaOrg/joda-time/releases) - [Changelog](https://github.com/JodaOrg/joda-time/blob/main/RELEASE-NOTES.txt) - [Commits](https://github.com/JodaOrg/joda-time/compare/v2.13.0...v2.14.0) --- updated-dependencies: - dependency-name: joda-time:joda-time dependency-version: 2.14.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Bump jakarta.servlet:jakarta.servlet-api from 6.0.0 to 6.1.0 Bumps [jakarta.servlet:jakarta.servlet-api](https://github.com/eclipse-ee4j/servlet-api) from 6.0.0 to 6.1.0. - [Commits](https://github.com/eclipse-ee4j/servlet-api/compare/6.0.0-RELEASE...6.1.0-RELEASE) --- updated-dependencies: - dependency-name: jakarta.servlet:jakarta.servlet-api dependency-version: 6.1.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Bump org.apache.maven.plugins:maven-gpg-plugin from 3.0.1 to 3.2.8 Bumps [org.apache.maven.plugins:maven-gpg-plugin](https://github.com/apache/maven-gpg-plugin) from 3.0.1 to 3.2.8. - [Release notes](https://github.com/apache/maven-gpg-plugin/releases) - [Commits](https://github.com/apache/maven-gpg-plugin/compare/maven-gpg-plugin-3.0.1...maven-gpg-plugin-3.2.8) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-gpg-plugin dependency-version: 3.2.8 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Bump org.opensearch.client:opensearch-rest-high-level-client Bumps [org.opensearch.client:opensearch-rest-high-level-client](https://github.com/opensearch-project/OpenSearch) from 1.2.4 to 3.2.0. - [Release notes](https://github.com/opensearch-project/OpenSearch/releases) - [Changelog](https://github.com/opensearch-project/OpenSearch/blob/main/CHANGELOG.md) - [Commits](https://github.com/opensearch-project/OpenSearch/compare/1.2.4...3.2.0) --- updated-dependencies: - dependency-name: org.opensearch.client:opensearch-rest-high-level-client dependency-version: 3.2.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * Update changelog * Bump com.fasterxml.jackson.core:jackson-core in /service Bumps [com.fasterxml.jackson.core:jackson-core](https://github.com/FasterXML/jackson-core) from 2.18.2 to 2.18.6. - [Commits](https://github.com/FasterXML/jackson-core/compare/jackson-core-2.18.2...jackson-core-2.18.6) --- updated-dependencies: - dependency-name: com.fasterxml.jackson.core:jackson-core dependency-version: 2.18.6 dependency-type: direct:production ... Signed-off-by: dependabot[bot] * Bump org.springframework:spring-webmvc from 6.2.2 to 6.2.17 Bumps [org.springframework:spring-webmvc](https://github.com/spring-projects/spring-framework) from 6.2.2 to 6.2.17. - [Release notes](https://github.com/spring-projects/spring-framework/releases) - [Commits](https://github.com/spring-projects/spring-framework/compare/v6.2.2...v6.2.17) --- updated-dependencies: - dependency-name: org.springframework:spring-webmvc dependency-version: 6.2.17 dependency-type: direct:production ... Signed-off-by: dependabot[bot] * Bump dependency versions to address 8 Dependabot security vulnerabilities - spring-framework: 6.2.2 -> 6.2.17 (fixes 5 CVEs: spring-webmvc path traversal, RFD, SSE corruption, script view templates; spring-context DataBinder; spring-web RFD) - jackson-core: 2.18.2 -> 2.18.6 (fixes DoS via async parser number length constraint bypass) - httpclient5: 5.4.1 -> 5.4.3 (HIGH: fixes domain check bypass) - commons-lang3: 3.4 -> 3.18.0 (fixes uncontrolled recursion on long inputs) Co-Authored-By: Claude Sonnet 4.6 * API changes require hand code changes * compilation works * clean up testing too * try and fix boot error * final pom clean up. * done * rollback to java 17 * Update to openJDK21 to match unstable/stable builds * lock onto Java 21 * ready --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: PDSEN CI Bot Co-authored-by: Jordan Padams Co-authored-by: Claude Sonnet 4.6 Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: jordanpadams <33492486+jordanpadams@users.noreply.github.com> Co-authored-by: Al Niessner --- .github/workflows/branch-cicd.yaml | 2 +- .github/workflows/last_integration_test.json | 4 +- lexer/pom.xml | 2 +- model/pom.xml | 6 +- pom.xml | 4 +- service/pom.xml | 518 ++++++------------ .../nasa/pds/api/registry/SpringBootMain.java | 27 +- .../registry/model/Antlr4SearchListener.java | 11 +- .../model/RawMultipleProductResponse.java | 3 +- .../pds/api/registry/search/HitIterator.java | 2 +- .../search/RegistrySearchRequestBuilder.java | 4 +- .../ResponseTransformerRegistryTest.java | 6 +- .../opensearch/Antlr4SearchListenerTest.java | 35 +- 13 files changed, 248 insertions(+), 376 deletions(-) diff --git a/.github/workflows/branch-cicd.yaml b/.github/workflows/branch-cicd.yaml index f8f8f65a..26ab2702 100644 --- a/.github/workflows/branch-cicd.yaml +++ b/.github/workflows/branch-cicd.yaml @@ -35,7 +35,7 @@ jobs: strategy: matrix: - java-version: [17] + java-version: [21] steps: - diff --git a/.github/workflows/last_integration_test.json b/.github/workflows/last_integration_test.json index 50196960..45df6e74 100644 --- a/.github/workflows/last_integration_test.json +++ b/.github/workflows/last_integration_test.json @@ -1,5 +1,5 @@ { - "api_gitrev": "e810b3a98a0caed79bf17e4cc5f000388a0d2945", - "reg_gitrev": "5a3a1789265928dc625efb0f6a1eff2f5685ab6e", + "api_gitrev": "426d29b016fc118e77855d2807456d1ccc592366", + "reg_gitrev": "d1be5e2574c073227cebac29367521be34f74b4d", "status": "success" } diff --git a/lexer/pom.xml b/lexer/pom.xml index 94a4b143..15149cee 100644 --- a/lexer/pom.xml +++ b/lexer/pom.xml @@ -57,7 +57,7 @@ POSSIBILITY OF SUCH DAMAGE. org.apache.commons commons-lang3 - 3.4 + 3.18.0 diff --git a/model/pom.xml b/model/pom.xml index de09232a..505fb635 100644 --- a/model/pom.xml +++ b/model/pom.xml @@ -138,7 +138,7 @@ jakarta.servlet jakarta.servlet-api - 6.0.0 + 6.1.0 provided @@ -180,7 +180,7 @@ com.fasterxml.jackson.dataformat jackson-dataformat-xml - 2.18.2 + 2.19.1 @@ -200,7 +200,7 @@ joda-time joda-time - 2.13.0 + 2.14.0 diff --git a/pom.xml b/pom.xml index 523043d7..3c62e849 100644 --- a/pom.xml +++ b/pom.xml @@ -50,7 +50,7 @@ Go through this file line-by-line and replace the template values with your own. Registry API UTF-8 17 - 6.2.2 + 6.2.17 gov.nasa.pds @@ -304,7 +304,7 @@ Go through this file line-by-line and replace the template values with your own. org.apache.maven.plugins maven-gpg-plugin - 3.0.1 + 3.2.8 sign-artifacts diff --git a/service/pom.xml b/service/pom.xml index 43e34325..1a8a49a5 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -30,6 +30,8 @@ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --> + 4.0.0 @@ -42,197 +44,114 @@ registry-api-service Registry API Service - Registry API Service contributing to the PDS Federated Search API - 17 + 21 UTF-8 - 3.4.1 - 2.18.2 + + 3.5.12 + + 6.2.14 + 2.19.1 2.31.54 - - - - src/main/resources - true - - - - - org.springframework.boot - spring-boot-maven-plugin - ${spring-boot-version} - - - - repackage - - - - - gov.nasa.pds.api.registry.SpringBootMain - nasapds/registry-api-service - JAR - - - 17 - -XX:MaxDirectMemorySize=1G - - - - - - - org.apache.maven.plugins - maven-assembly-plugin - 3.1.1 - - - bin-release - package - - single - - - true - - src/main/assembly/tar-assembly.xml - src/main/assembly/zip-assembly.xml - - - jar-with-dependencies - - - - - - posix - - - - org.apache.maven.plugins - maven-compiler-plugin - - - com.iluwatar.urm - urm-maven-plugin - 2.0.0 - - ${project.basedir}/target - - gov.nasa.pds.api.registry - - - - true - false - mermaid - - jar-with-dependencies - - - - - process-classes - - map - - - - - - - + + + + + + org.springframework + spring-framework-bom + ${spring-framework.version} + pom + import + + + + org.springframework.boot + spring-boot-dependencies + ${spring-boot-version} + pom + import + + + + com.fasterxml.jackson + jackson-bom + ${jackson-version} + pom + import + + + + software.amazon.awssdk + bom + 2.31.54 + pom + import + + + - + org.springframework.boot spring-boot-starter-web - - org.springframework.boot - spring-boot-starter-actuator + org.springframework.boot + spring-boot-starter-actuator - - org.springframework.data - spring-data-commons - - - - org.springdoc - springdoc-openapi-starter-webmvc-ui - 2.8.4 - - - - - org.springdoc - springdoc-openapi-starter-common - 2.8.4 - - - org.springframework.boot spring-boot-starter-thymeleaf - - - - io.swagger.core.v3 - swagger-core - 2.2.28 - - - - org.springframework.boot spring-boot-autoconfigure - - - + + + org.springframework.data + spring-data-commons + - - - jakarta.validation - jakarta.validation-api - 3.0.2 - - - - - jakarta.annotation - jakarta.annotation-api - 3.0.0 - + + + org.springdoc + springdoc-openapi-starter-webmvc-ui + 2.8.4 + + + org.springdoc + springdoc-openapi-starter-common + 2.8.4 + + + io.swagger.core.v3 + swagger-core + 2.2.28 + + + + jakarta.validation + jakarta.validation-api + + + jakarta.annotation + jakarta.annotation-api + + com.fasterxml.jackson.jaxrs jackson-jaxrs-base - ${jackson-version} com.fasterxml.jackson.core jackson-core - ${jackson-version} com.fasterxml.jackson.core @@ -241,34 +160,27 @@ com.fasterxml.jackson.core jackson-databind - ${jackson-version} com.fasterxml.jackson.jaxrs jackson-jaxrs-json-provider - ${jackson-version} - com.fasterxml.jackson.dataformat jackson-dataformat-xml - ${jackson-version} - + com.github.joschi.jackson jackson-datatype-threetenbp 2.12.5 - - joda-time joda-time - 2.13.0 + 2.14.0 - com.sun.xml.bind jaxb-core @@ -284,226 +196,140 @@ javassist 3.30.2-GA - - + - org.threeten - threetenbp - 1.4.4 + org.apache.httpcomponents.client5 + httpclient5 - + + org.springframework.boot + spring-boot-starter-aop + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.platform + junit-platform-launcher + test + + + org.opensearch.client + spring-data-opensearch-starter + 1.4.0 + + gov.nasa.pds.registry-api registry-api-model ${project.version} - gov.nasa.pds.registry-api registry-api-lexer ${project.version} - + - org.antlr - antlr4-runtime - 4.13.2 - - - - - org.opensearch.client - opensearch-java - - 2.24.0 - - - - - org.apache.httpcomponents.client5 - httpclient5 - 5.4.3 - - - - - - org.apache.httpcomponents.core5 - httpcore5 - 5.4.2 - - - - - - org.apache.httpcomponents.core5 - httpcore5-h2 - 5.4.2 - - - - - - - - - software.amazon.awssdk - apache-client - ${awssdk-version} - - - - - software.amazon.awssdk - checksums - ${awssdk-version} - - - - - software.amazon.awssdk - regions - ${awssdk-version} - - - - - - - - software.amazon.awssdk - auth - ${awssdk-version} - - - - - software.amazon.awssdk - secretsmanager - ${awssdk-version} + org.opensearch.client + opensearch-java + 3.8.0 - - - - software.amazon.awssdk - sdk-core - ${awssdk-version} - - - - - org.opensearch.client opensearch-rest-client 2.18.0 - - + org.opensearch.client opensearch-rest-high-level-client - 1.2.4 + 3.2.0 - - - - + - org.apache.httpcomponents - httpclient - 4.5.13 + software.amazon.awssdk + apache-client - - - org.apache.commons - commons-collections4 - 4.2 + software.amazon.awssdk + auth - - - - org.springframework.boot - spring-boot-starter-test - - - junit - junit - - + software.amazon.awssdk + aws-core - - - org.junit.jupiter - junit-jupiter-engine - 5.7.0 - test + software.amazon.awssdk + opensearch - - - org.mockito - mockito-core - 3.6.28 - test + software.amazon.awssdk + regions - - - org.springframework.boot - spring-boot-starter-validation + software.amazon.awssdk + sdk-core - - - - + + software.amazon.awssdk + secretsmanager + + com.google.guava guava 33.4.8-jre - - - - jakarta.servlet - jakarta.servlet-api - 6.0.0 - provided - - - - - org.springframework - spring-aspects - 6.2.5 - - + + + org.apache.commons + commons-collections4 + 4.5.0 + + + + org.antlr + antlr4-runtime + 4.13.2 + - - - - + + + + src/main/resources + true + + + + org.springframework.boot - spring-boot-dependencies + spring-boot-maven-plugin ${spring-boot-version} - pom - import - - - - + + + + repackage + + + + + gov.nasa.pds.api.registry.SpringBootMain + nasapds/registry-api-service + + + 21 + -XX:MaxDirectMemorySize=1G + + + + + + diff --git a/service/src/main/java/gov/nasa/pds/api/registry/SpringBootMain.java b/service/src/main/java/gov/nasa/pds/api/registry/SpringBootMain.java index a2bba5fa..1baad868 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/SpringBootMain.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/SpringBootMain.java @@ -1,12 +1,16 @@ package gov.nasa.pds.api.registry; import java.lang.IllegalArgumentException; +import org.opensearch.spring.boot.autoconfigure.OpenSearchRestHighLevelClientAutoConfiguration; +import org.opensearch.spring.boot.autoconfigure.data.OpenSearchDataAutoConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.CommandLineRunner; import org.springframework.boot.ExitCodeGenerator; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.data.elasticsearch.ElasticsearchDataAutoConfiguration; +import org.springframework.boot.autoconfigure.data.elasticsearch.ElasticsearchRepositoriesAutoConfiguration; import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.context.annotation.AnnotationConfigApplicationContext; import org.springframework.context.annotation.ComponentScan; @@ -16,12 +20,27 @@ // add archive status filter // add other resolver endpoints -@SpringBootApplication +@SpringBootApplication(exclude = { + // 1. Prevents the 'elasticsearchTemplate' creation error you are seeing + ElasticsearchDataAutoConfiguration.class, + // 2. Prevents conflict with OpenSearch repositories + ElasticsearchRepositoriesAutoConfiguration.class, + // 3. Resolves the 'opensearchClient' name collision + OpenSearchRestHighLevelClientAutoConfiguration.class, + // 4. Prevents the elasticsearchTemplate alias conflict in 2.x + OpenSearchDataAutoConfiguration.class + +}) @OpenAPIDefinition @EnableScheduling -@ComponentScan(basePackages = {"gov.nasa.pds.api.registry.configuration", - "gov.nasa.pds.api.registry.controllers", "gov.nasa.pds.api.registry.model", - "gov.nasa.pds.api.registry.search", "gov.nasa.pds.api.registry.util", "javax.servlet.http"}) +@ComponentScan(basePackages = { + "gov.nasa.pds.api.registry.configuration", + "gov.nasa.pds.api.registry.controllers", + "gov.nasa.pds.api.registry.model", + "gov.nasa.pds.api.registry.search", + "gov.nasa.pds.api.registry.util" + // jakarta.servlet.http is loaded and configured automatically with springboot 3 + }) public class SpringBootMain implements CommandLineRunner { private static final Logger log = LoggerFactory.getLogger(SpringBootMain.class); diff --git a/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java b/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java index 2e32b66e..5c862e5f 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/model/Antlr4SearchListener.java @@ -7,11 +7,13 @@ import gov.nasa.pds.api.registry.lexer.SearchBaseListener; import gov.nasa.pds.api.registry.lexer.SearchParser; import gov.nasa.pds.model.PropertiesListInner; +import jakarta.validation.constraints.NotNull; import java.io.IOException; import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Deque; import java.util.HashSet; +import java.util.List; import java.util.Set; import java.util.regex.Pattern; import org.antlr.v4.runtime.misc.ParseCancellationException; @@ -50,10 +52,17 @@ enum operation { private operation operator = null; - public Antlr4SearchListener(ConnectionContext connectionContext) { + public Antlr4SearchListener(@NotNull ConnectionContext connectionContext) { super(); this.connectionContext = connectionContext; } + + // for testing purposes only + public Antlr4SearchListener(List knownFieldNames) { + super(); + this.connectionContext = null; + this.knownFieldNames.addAll(knownFieldNames); + } @Override diff --git a/service/src/main/java/gov/nasa/pds/api/registry/model/RawMultipleProductResponse.java b/service/src/main/java/gov/nasa/pds/api/registry/model/RawMultipleProductResponse.java index a0e8c7bb..1447e1d6 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/model/RawMultipleProductResponse.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/model/RawMultipleProductResponse.java @@ -1,7 +1,6 @@ package gov.nasa.pds.api.registry.model; import java.util.*; -import java.util.stream.Collectors; import gov.nasa.pds.model.SummaryFacet; import org.opensearch.client.opensearch.core.SearchResponse; @@ -31,7 +30,7 @@ private List extractFacetsFromSearchResponse( }); } else if (aggregate.isLterms()) { aggregate.lterms().buckets().array().forEach(bucket -> { - facet.putCountsItem(bucket.key(), Math.toIntExact(bucket.docCount())); + facet.putCountsItem(bucket.key().toString(), Math.toIntExact(bucket.docCount())); }); } diff --git a/service/src/main/java/gov/nasa/pds/api/registry/search/HitIterator.java b/service/src/main/java/gov/nasa/pds/api/registry/search/HitIterator.java index 87ee07b8..a06b2305 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/search/HitIterator.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/search/HitIterator.java @@ -62,7 +62,7 @@ public String getCurrentId() { @Override public boolean hasNext() { return this.currentBatch == null ? false - : (this.at + this.page * this.size) < this.currentBatch.getTotalHits().value; + : (this.at + this.page * this.size) < this.currentBatch.getTotalHits().value(); } @Override diff --git a/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java b/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java index 41607a8b..a5b267f3 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java @@ -300,7 +300,9 @@ public RegistrySearchRequestBuilder searchAfterFromStrings(List searchAf * need to be handled specfically. Method stringValue() implies yes * FieldValue.Builder().stringValue(fieldValue).build()); } */ - this.searchAfter(searchAfterValues); + this.searchAfter(searchAfterValues.stream() + .map(FieldValue::of) + .toList()); return this; diff --git a/service/src/test/java/gov/nasa/pds/api/registry/model/transformers/ResponseTransformerRegistryTest.java b/service/src/test/java/gov/nasa/pds/api/registry/model/transformers/ResponseTransformerRegistryTest.java index 3a43fe7b..a57d4d71 100644 --- a/service/src/test/java/gov/nasa/pds/api/registry/model/transformers/ResponseTransformerRegistryTest.java +++ b/service/src/test/java/gov/nasa/pds/api/registry/model/transformers/ResponseTransformerRegistryTest.java @@ -20,7 +20,7 @@ void selectFormatterClassFromSingleFormatSuccessfulTest() { String format = "text/html"; String expectedFormatterClassName = - "gov.nasa.pds.api.registry.model.api_responses.PdsProductBusinessObject"; + "gov.nasa.pds.api.registry.model.transformers.PdsProductTransformer"; String foundFormatterClassName; try { @@ -58,7 +58,7 @@ void selectFormatterClassFromMultipleFormatSuccessfulTest() { String format = "text/ms+word,text/html"; String expectedFormatterClassName = - "gov.nasa.pds.api.registry.model.api_responses.PdsProductBusinessObject"; + "gov.nasa.pds.api.registry.model.transformers.PdsProductTransformer"; String foundFormatterClassName; try { @@ -80,7 +80,7 @@ void selectFormatterClassFromMultipleFormatExtraSpacesSuccessfulTest() { String format = "text/ms+word,text/html ,anything/something"; String expectedFormatterClassName = - "gov.nasa.pds.api.registry.model.api_responses.PdsProductBusinessObject"; + "gov.nasa.pds.api.registry.model.transformers.PdsProductTransformer"; String foundFormatterClassName; try { diff --git a/service/src/test/java/gov/nasa/pds/api/registry/opensearch/Antlr4SearchListenerTest.java b/service/src/test/java/gov/nasa/pds/api/registry/opensearch/Antlr4SearchListenerTest.java index 48070952..98ac4698 100644 --- a/service/src/test/java/gov/nasa/pds/api/registry/opensearch/Antlr4SearchListenerTest.java +++ b/service/src/test/java/gov/nasa/pds/api/registry/opensearch/Antlr4SearchListenerTest.java @@ -18,8 +18,7 @@ import org.junit.jupiter.api.BeforeEach; import org.mockito.Mockito; import static org.junit.jupiter.api.Assertions.*; - - +import java.util.Arrays; import gov.nasa.pds.api.registry.lexer.SearchLexer; import gov.nasa.pds.api.registry.lexer.SearchParser; import gov.nasa.pds.api.registry.model.Antlr4SearchListener; @@ -44,7 +43,14 @@ public void execute() { @BeforeEach void setUp() { - listener = new Antlr4SearchListener(null); + listener = new Antlr4SearchListener(Arrays.asList( + "lid", + "pds:Time_Coordinates.pds:stop_date_time", + "ref_lid_target", + "timestamp", + "timestamp_A", + "timestamp_B" + )); } @@ -57,8 +63,7 @@ private BoolQuery run(String query) { ParseTree tree = par.query(); // Walk it and attach our listener ParseTreeWalker walker = new ParseTreeWalker(); - Antlr4SearchListener listener = new Antlr4SearchListener(null); - walker.walk(listener, tree); + walker.walk(this.listener, tree); // System.out.println ("query string: " + query); // System.out.println("query tree: " + tree.toStringTree(par)); @@ -74,10 +79,22 @@ void testSimpleCompEq() { // TODO: add asserts Assertions.assertEquals(1, query.must().size()); Query matchQuery = (Query) query.must().get(0); - Assertions.assertEquals(Query.Kind.Match, matchQuery._kind()); - // Assertions.assertEquals((matchQuery).field(), "pds:Time_Coordinates/pds:stop_date_time"); - - + Assertions.assertEquals(Query.Kind.Bool, matchQuery._kind()); + query = matchQuery.bool(); + Assertions.assertEquals(1, query.must().size()); + matchQuery = (Query) query.must().get(0); + Assertions.assertEquals(Query.Kind.Bool, matchQuery._kind()); + query = matchQuery.bool(); + boolean match = false; + boolean term = false; + Assertions.assertEquals(2, query.should().size()); + for (int i = 0 ; i < 2 ; i++) { + matchQuery = (Query) query.should().get(i); + match = match || Query.Kind.Match == matchQuery._kind(); + term = term || Query.Kind.Term == matchQuery._kind(); + } + Assertions.assertTrue(match); + Assertions.assertTrue(term); } From c2325eb45889caab7ee3a42ec96342b4c5dd3c98 Mon Sep 17 00:00:00 2001 From: PDSEN CI Bot Date: Tue, 28 Apr 2026 20:29:33 +0000 Subject: [PATCH 106/137] Update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1497b3c5..c03b6272 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## [«unknown»](https://github.com/NASA-PDS/registry-api/tree/«unknown») (2026-04-22) +## [«unknown»](https://github.com/NASA-PDS/registry-api/tree/«unknown») (2026-04-28) [Full Changelog](https://github.com/NASA-PDS/registry-api/compare/v1.6.2...«unknown») From a766359e0300388564e24f4e30a6f6072d6dd7a7 Mon Sep 17 00:00:00 2001 From: thomas loubrieu <60993872+tloubrieu-jpl@users.noreply.github.com> Date: Tue, 26 May 2026 15:59:12 -0700 Subject: [PATCH 107/137] Modernized terraform script for deployment as a module of nasa-pds/registry (#765) * wip: renew terraform for registry integration * ready to deploy in dev, not tested yet though * modernize the terraform script so to integrate it as a module called in the registry main deployment. * switch error status from 400 ot 404, as expected in integration tests * remove unused github credentials --------- Co-authored-by: Thomas Loubrieu --- .gitignore | 3 + .../controllers/ProductsController.java | 2 +- terraform/README.md | 24 ++-- terraform/backend-config.tfvars.example | 12 ++ terraform/backend.tf | 25 ++++ terraform/{ecs.tf => main.tf} | 121 ++++++++++-------- terraform/output.tf | 19 +++ terraform/provider.tf | 10 +- terraform/terraform.tfvars.example | 30 +++++ terraform/variables.tf | 37 +++++- 10 files changed, 212 insertions(+), 71 deletions(-) create mode 100644 terraform/backend-config.tfvars.example create mode 100644 terraform/backend.tf rename terraform/{ecs.tf => main.tf} (57%) create mode 100644 terraform/output.tf create mode 100644 terraform/terraform.tfvars.example diff --git a/.gitignore b/.gitignore index b8e47750..8cfbf204 100644 --- a/.gitignore +++ b/.gitignore @@ -65,6 +65,9 @@ src/test/temp/ # terraform .terraform/ +terraform/*.tfvars +!terraform/*.tfvars.example +!.terraform.lock.hcl # other stuff *.xpr diff --git a/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java b/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java index fe98a529..bfa5357c 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java @@ -617,7 +617,7 @@ public ResponseEntity classList(String propertyClass, List userR try { pdsProductClass = PdsProductClasses.fromSwaggerName(propertyClass); } catch (IllegalArgumentException err) { - throw new BadRequestException(err.getMessage()); + throw new NotFoundException(err.getMessage()); } RegistrySearchRequestBuilder searchRequestBuilder = diff --git a/terraform/README.md b/terraform/README.md index 13c0d4b7..6d84e119 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -24,18 +24,22 @@ These interfaces are going to be used a arguments of the terraform scripts. ## Deploy +Initialize the parameters, starting from the terraform.tfvars.example file provided. + +Copy it: + + cp terraform.tfvars.example terraform.tfvars + +And update the values. + Run the terraform scripts: + + + ``` - terraform apply \ - -var 'ecs_task_role=your-task-role-arn' \ - -var 'ecs_task_execution_role=your-task-execution-role-arn' \ - -var 'venue=your-venue' \ - -var 'aws_fg_vpc=your-vpc-arn' \ - -var 'aws_fg_security_groups=["your security group, e.g. sg-1223455..."]' \ - -var 'aws_fg_subnets=["your subnet e.g. subnet-1234..."]' \ - -var 'aws_fg_image=your-docker-image-available-on-ECR' \ - -var 'aws_acm_certificate_arn=ssl certificate for the load balancer listener' \ - -var 'spring_boot_args=--openSearch.host=your-opensearch-url-without-http --openSearch.CCSEnabled=true --openSearch.username=our-username-empty-for-opensearch-serverless --openSearch.disciplineNodes=the-prefixes-of-the-registry-indices-in-opensearch --registry.service.version=the-version-of-the-api-to-be-displayed-in-the-application' + terraform init -backend-config=backend-config.tfvars + terraform plan + terraform apply ``` diff --git a/terraform/backend-config.tfvars.example b/terraform/backend-config.tfvars.example new file mode 100644 index 00000000..c17c2064 --- /dev/null +++ b/terraform/backend-config.tfvars.example @@ -0,0 +1,12 @@ +# Example backend configuration for S3 +# Copy this file to backend-config.tfvars and customize the values +# Initialize with: terraform init -backend-config=backend-config.tfvars + +bucket = "my-terraform-state-bucket" +key = "registry/opensearch/terraform.tfstate" +region = "us-east-1" +dynamodb_table = "terraform-state-lock" +encrypt = true + +# Optional: Use a specific profile +# profile = "my-aws-profile" diff --git a/terraform/backend.tf b/terraform/backend.tf new file mode 100644 index 00000000..7644a18f --- /dev/null +++ b/terraform/backend.tf @@ -0,0 +1,25 @@ +# Backend configuration for S3 state storage +# Variables are not supported in backend blocks. Instead, provide configuration via: +# +# 1. backend-config.tfvars file: +# terraform init -backend-config=backend-config.tfvars +# +# 2. Command line arguments: +# terraform init -backend-config="bucket=${TFSTATE_BUCKET}" -backend-config="key=${TFSTATE_KEY}" +# +# 3. Environment variables or interactive prompts +# +# See https://stackoverflow.com/questions/63048738/how-to-declare-variables-for-s3-backend-in-terraform + +terraform { + backend "s3" { + # Backend configuration values provided via backend-config.tfvars + # Example backend-config.tfvars content: + # bucket = "pds-infra" + # key = "registry/opensearch/terraform.tfstate" + # region = "us-east-1" + # dynamodb_table = "terraform-state-lock" + # encrypt = true + # profile = "your-aws-profile" + } +} \ No newline at end of file diff --git a/terraform/ecs.tf b/terraform/main.tf similarity index 57% rename from terraform/ecs.tf rename to terraform/main.tf index 050a7846..b07236f1 100644 --- a/terraform/ecs.tf +++ b/terraform/main.tf @@ -1,5 +1,11 @@ +locals { + + # Concatenate the load balancer domain to spring boot args + spring_boot_args_with_host = "${var.spring_boot_args} --server.authorizedForwardedHost=${aws_lb.registry-api-lb.dns_name},${var.cloudfront_dns}" +} + resource "aws_lb" "registry-api-lb" { - name = "registry-api-lb-new" + name = "registry-api-lb" internal = false load_balancer_type = "application" security_groups = var.aws_fg_security_groups @@ -9,26 +15,17 @@ resource "aws_lb" "registry-api-lb" { access_logs { bucket = var.aws_s3_bucket_logs_id - prefix = "registry-api-lb" + prefix = "registry/registry-api-lb" enabled = true } - tags = { - Alfa = var.node_name_abbr - Bravo = var.venue - Charlie = "registry" - } + tags = var.common_tags } -resource "aws_ssm_parameter" "load_balancer_domain" { - name = "/pds/registry/load-balancer-domain" - type = "String" - overwrite = true - value = aws_lb.registry-api-lb.dns_name -} + resource "aws_lb_target_group" "pds-registry-api-target-group" { - name = "pds-${var.venue}-registry-tgt" + name = "pds-registry-tg" port = 80 protocol = "HTTP" target_type = "ip" @@ -44,6 +41,8 @@ resource "aws_lb_target_group" "pds-registry-api-target-group" { matcher = "200" interval = 300 } + + tags = var.common_tags } resource "aws_lb_listener" "registry-api-ld-listener" { @@ -55,6 +54,7 @@ resource "aws_lb_listener" "registry-api-ld-listener" { type = "forward" target_group_arn = aws_lb_target_group.pds-registry-api-target-group.arn } + tags = var.common_tags } resource "aws_lb_listener_rule" "pds-registry-forward-rule" { @@ -75,44 +75,64 @@ resource "aws_lb_listener_rule" "pds-registry-forward-rule" { } } -# Define the cluster -resource "aws_ecs_cluster" "pds-registry-api-ecs" { - name = "pds-${var.venue}-registry-api-ecs" - tags = { - Alfa = var.node_name_abbr - Bravo = var.venue - Charlie = "registry" - } +# Credentials for ECR pull through cache from GHCR +resource "aws_secretsmanager_secret" "github_ecr_credentials" { + count = var.create_github_secret_credentials + + name = "ecr-pullthroughcache/github-credentials" + tags = var.common_tags } -# Do we need individual dev/test/prod repositories? -# I don't think we do, but then we need to use prod account instead of the dev account, would that work ? -data "aws_ecr_repository" "pds-registry-api-service" { - name = "pds-registry-api-service" +resource "aws_secretsmanager_secret_version" "github_ecr_credentials" { + count = var.create_github_secret_credentials + + secret_id = aws_secretsmanager_secret.github_ecr_credentials[count.index].id + secret_string = jsonencode({ + username = var.github_username + accessToken = var.github_token + }) +} + +# Look up the secret when it is not created by this script +data "aws_secretsmanager_secret" "github_ecr_credentials" { + count = 1 - var.create_github_secret_credentials + name = "ecr-pullthroughcache/github-credentials" +} + +locals { + github_ecr_credentials_arn = var.create_github_secret_credentials == 1 ? aws_secretsmanager_secret.github_ecr_credentials[0].arn : data.aws_secretsmanager_secret.github_ecr_credentials[0].arn +} + +# Add a Pull Through Cache rule for GHCR +resource "aws_ecr_pull_through_cache_rule" "ghcr" { + ecr_repository_prefix = "ghcr" + upstream_registry_url = "ghcr.io" + credential_arn = local.github_ecr_credentials_arn +} + +resource "aws_ecr_repository" "ghcr_registry_api" { + name = "ghcr/nasa-pds/registry-api" + tags = var.common_tags } # Log groups hold logs from our app. resource "aws_cloudwatch_log_group" "pds-registry-log-group" { - name = "/ecs/pds-${var.venue}-registry-api-svc-task" + name = "/ecs/pds-registry-api-task" - tags = { - Alfa = var.node_name_abbr - Bravo = var.venue - Charlie = "registry" - } + tags = var.common_tags } # The task definition for app. resource "aws_ecs_task_definition" "pds-registry-ecs-task" { - family = "pds-${var.venue}-registry-api-svc-task" + family = "pds-registry-api-task" container_definitions = <"] +aws_fg_subnets = ["subnet-","subnet-"] +aws_lb_subnets = ["subnet-", "subnet-"] +aws_acm_certificate_arn = "arn:aws:acm:::certificate/" +ecs_task_role = "arn:aws:iam:::role/" +ecs_task_execution_role = "arn:aws:iam:::role/" + +# GitHub credentials for ECR pull through cache +github_username = "" +github_token = "" + +cloudfront_dns = "www.example.com" + +# Common tags applied to all resources +common_tags = { + tenant = "" + venue = "" + component = "registry" + cicd = "iac" + managedby = "" +} diff --git a/terraform/variables.tf b/terraform/variables.tf index bb82829d..886746de 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -3,11 +3,6 @@ variable "node_name_abbr" { default="en" } -variable "venue" { - description = "Deployment venue (prod, test, dev)" - default = "delta" -} - variable "aws_region" { description = "AWS Region" default = "us-west-2" @@ -19,7 +14,7 @@ variable "spring_boot_args" { variable "aws_profile" { description = "AWS profile" - default = "default" + default = "" } variable "aws_fg_vpc" { @@ -49,7 +44,7 @@ variable "ecs_task_execution_role" { description = "ECS task execution role" } -variable "aws_fg_image" { +variable "registry_api_docker_image" { description = "AWS image name for Fargate" } @@ -70,3 +65,31 @@ variable "aws_fg_ram_units" { variable "aws_acm_certificate_arn" { description = "ACM SSL Certificate for the load balancer" } + +variable "component_name" { + description = "Component this subcomponents belongs to" + type = string + default = "registry" +} + +variable "common_tags" { + description = "Common tags to apply to all resources" + type = map(string) + default = { + Project = "registry" + ManagedBy = "terraform" + } +} + +variable "github_username" { + description = "GitHub username for ECR pull through cache" +} + +variable "github_token" { + description = "GitHub personal access token for ECR pull through cache" + sensitive = true +} + +variable "cloudfront_dns" { + description = "DNS of the cloudfront distribution giving access to the API" +} \ No newline at end of file From 1db04c96855ee3f174c1965c679b89ff06c03e94 Mon Sep 17 00:00:00 2001 From: PDSEN CI Bot Date: Tue, 26 May 2026 23:05:06 +0000 Subject: [PATCH 108/137] Update changelog --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c03b6272..8c2f8ca1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,12 @@ # Changelog -## [«unknown»](https://github.com/NASA-PDS/registry-api/tree/«unknown») (2026-04-28) +## [«unknown»](https://github.com/NASA-PDS/registry-api/tree/«unknown») (2026-05-26) [Full Changelog](https://github.com/NASA-PDS/registry-api/compare/v1.6.2...«unknown») **Requirements:** +- As a user, I want the exists operator to match OpenSearch's native behavior for all fields [\#712](https://github.com/NASA-PDS/registry-api/issues/712) - As a user, I want to search by a full/unique hierarchical path for a specific attribute [\#611](https://github.com/NASA-PDS/registry-api/issues/611) - As a user, I want to query for documents where a specific search field exists in the document [\#406](https://github.com/NASA-PDS/registry-api/issues/406) @@ -16,6 +17,7 @@ **Defects:** +- When a request /classes/{product class} does not match an existing product class, I want a 404 error. [\#767](https://github.com/NASA-PDS/registry-api/issues/767) [[s.medium](https://github.com/NASA-PDS/registry-api/labels/s.medium)] - Investigate and fix skipped `product/{id}/member*` integration tests [\#748](https://github.com/NASA-PDS/registry-api/issues/748) [[s.medium](https://github.com/NASA-PDS/registry-api/labels/s.medium)] - Integration tests in unstable build suite do not pass when run locally [\#745](https://github.com/NASA-PDS/registry-api/issues/745) [[s.medium](https://github.com/NASA-PDS/registry-api/labels/s.medium)] - Unstable build does not complete on develop branch due to GitHub Actions runner timeout [\#744](https://github.com/NASA-PDS/registry-api/issues/744) [[s.medium](https://github.com/NASA-PDS/registry-api/labels/s.medium)] From 95fda445bd34f646383a7708d2e4fee42201fe85 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Fri, 19 Jun 2026 10:15:25 -0700 Subject: [PATCH 109/137] fix security eror Use a security focused HTML string encoder. Made it very aggressive so that the largest string is encoded rather than the smallest. Did this to protect against deeper exception strings from containing malicious strings by accident. --- service/pom.xml | 6 ++++++ .../RegistryApiResponseEntityExceptionHandler.java | 5 ++--- .../api/registry/controllers/SecurityValidationFilter.java | 4 ++-- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/service/pom.xml b/service/pom.xml index 1a8a49a5..65d07f9e 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -298,6 +298,12 @@ antlr4-runtime 4.13.2 + + + org.owasp.encoder + encoder + 1.4.0 + diff --git a/service/src/main/java/gov/nasa/pds/api/registry/controllers/RegistryApiResponseEntityExceptionHandler.java b/service/src/main/java/gov/nasa/pds/api/registry/controllers/RegistryApiResponseEntityExceptionHandler.java index 757e3896..e4d537ad 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/controllers/RegistryApiResponseEntityExceptionHandler.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/controllers/RegistryApiResponseEntityExceptionHandler.java @@ -4,7 +4,7 @@ import java.util.Set; import gov.nasa.pds.api.registry.model.exceptions.*; import gov.nasa.pds.api.registry.model.transformers.ResponseTransformerRegistry; - +import org.owasp.encoder.Encode; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; @@ -17,7 +17,6 @@ @ControllerAdvice public class RegistryApiResponseEntityExceptionHandler extends ResponseEntityExceptionHandler { - private String errorDisclaimerHeader = "An error occured.\n"; private String errorDisclaimerFooter = "For assistance, forward this error message to pds-operator@jpl.nasa.gov"; @@ -34,7 +33,7 @@ private ResponseEntity genericExceptionHandler(RegistryApiException ex, String bodyOfResponse = status.toString() + "\n Request " + requestDescription + " failed with message:\n" + errorDescription + "(ref:" + errorIdentifier + ")\n" + errorDisclaimerFooter; - return handleExceptionInternal(ex, bodyOfResponse, new HttpHeaders(), status, request); + return handleExceptionInternal(ex, Encode.forHtml(bodyOfResponse), new HttpHeaders(), status, request); } diff --git a/service/src/main/java/gov/nasa/pds/api/registry/controllers/SecurityValidationFilter.java b/service/src/main/java/gov/nasa/pds/api/registry/controllers/SecurityValidationFilter.java index 925363d5..1b6396e5 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/controllers/SecurityValidationFilter.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/controllers/SecurityValidationFilter.java @@ -5,12 +5,12 @@ import jakarta.servlet.http.HttpServletResponse; import gov.nasa.pds.api.registry.model.exceptions.UnauthorizedForwardedHostException; import gov.nasa.pds.api.registry.model.exceptions.UnknownQueryParameterException; -import io.micrometer.core.instrument.util.StringEscapeUtils; import java.util.Arrays; import java.util.Enumeration; import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.owasp.encoder.Encode; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import org.springframework.web.servlet.HandlerInterceptor; @@ -44,7 +44,7 @@ public boolean preHandle(HttpServletRequest request, HttpServletResponse respons if (!ALLOWED_QUERY_PARAMETERS.contains(paramName)) { throw new UnknownQueryParameterException( "Query parameter not enumerated in SecurityValidationFilter.ALLOWED_QUERY_PARAMETERS: " - + paramName); + + Encode.forHtml(paramName)); } } From 9095ea5b47ff2edcf21763399a4a4b8c04b45f30 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Fri, 26 Jun 2026 11:11:01 -0700 Subject: [PATCH 110/137] it works again! --- .github/workflows/integration_tests.sh | 5 +++++ .github/workflows/last_integration_test.json | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration_tests.sh b/.github/workflows/integration_tests.sh index e0143fb7..cc39ed53 100755 --- a/.github/workflows/integration_tests.sh +++ b/.github/workflows/integration_tests.sh @@ -32,6 +32,11 @@ deep_archive() { git clone --quiet https://github.com/NASA-PDS/deep-archive.git cd deep-archive || return 1 pip install . + if [ "$(python3 -c "import sys; print(sys.version_info.minor)")" -gt 12 ] + then + echo "Python 3.$MINOR_VERSION detected. Upgrading zope.interface..." + pip install --upgrade "zope.interface>=8.0.0" + fi pds-deep-registry-archive -u http://localhost:8080 -s PDS_ENG urn:nasa:pds:insight_rad::2.1 --debug } diff --git a/.github/workflows/last_integration_test.json b/.github/workflows/last_integration_test.json index 45df6e74..0974041e 100644 --- a/.github/workflows/last_integration_test.json +++ b/.github/workflows/last_integration_test.json @@ -1,5 +1,5 @@ { - "api_gitrev": "426d29b016fc118e77855d2807456d1ccc592366", - "reg_gitrev": "d1be5e2574c073227cebac29367521be34f74b4d", + "api_gitrev": "95fda445bd34f646383a7708d2e4fee42201fe85+", + "reg_gitrev": "0fcb3cfb684511ac071739f10aafcf07e6007379", "status": "success" } From 5ffbe9af43e2e0236fb8919200fe6ad591fc5410 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Fri, 26 Jun 2026 11:30:26 -0700 Subject: [PATCH 111/137] once again for github --- .github/workflows/last_integration_test.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/last_integration_test.json b/.github/workflows/last_integration_test.json index 0974041e..4b899ba3 100644 --- a/.github/workflows/last_integration_test.json +++ b/.github/workflows/last_integration_test.json @@ -1,5 +1,5 @@ { - "api_gitrev": "95fda445bd34f646383a7708d2e4fee42201fe85+", + "api_gitrev": "9095ea5b47ff2edcf21763399a4a4b8c04b45f30", "reg_gitrev": "0fcb3cfb684511ac071739f10aafcf07e6007379", "status": "success" } From 4f26ea99867a31d8f35c7fd099ea4557ff1661ba Mon Sep 17 00:00:00 2001 From: Thomas Loubrieu Date: Mon, 29 Jun 2026 11:54:49 -0700 Subject: [PATCH 112/137] make integration test work on my laptop, force python3.12 for deep-archive --- .github/workflows/integration_tests.sh | 58 +++++++++++++------ .github/workflows/last_integration_test.json | 4 +- .../api_search_query_lexer/TestParsing.java | 2 - 3 files changed, 43 insertions(+), 21 deletions(-) diff --git a/.github/workflows/integration_tests.sh b/.github/workflows/integration_tests.sh index e0143fb7..c9d5a9e5 100755 --- a/.github/workflows/integration_tests.sh +++ b/.github/workflows/integration_tests.sh @@ -11,7 +11,9 @@ build() { mvn --quiet clean package - jar_file="$(find ./service/target/ -maxdepth 1 -regextype posix-extended -regex '.*/registry-api-service-[0-9]+\.[0-9]+\.[0-9]+(-SNAPSHOT)?\.jar')" + jar_file="$(find ./service/target/ -maxdepth 1 -name 'registry-api-service-*.jar')" + echo "jar file: $jar_file" + [ -s "$jar_file" ] || { echo "jar file not found or empty"; return 1; } docker build --build-arg api_jar="$jar_file" -t nasapds/registry-api-service:latest -f docker/Dockerfile . } @@ -26,7 +28,7 @@ clean() { deep_archive() { cd "$tdir" || return 1 - python3 -m venv "$tdir"/da + python3.12 -m venv "$tdir"/da # shellcheck disable=SC1091 # cannot find dynamically created script source "$tdir"/da/bin/activate git clone --quiet https://github.com/NASA-PDS/deep-archive.git @@ -55,6 +57,7 @@ EOF run() { cd docker || exit 1 + ddir=$(pwd) ( cd certs || exit 1 ; ./generate-certs.sh ) export REG_API_IMAGE=nasapds/registry-api-service:latest docker image inspect nasapds/registry-api-service:latest >/dev/null @@ -63,13 +66,22 @@ run() { --ansi never \ --profile int-registry-batch-loader \ --project-name registry \ - up --detach --quiet-pull || return 5 + up --detach --quiet-pull || { + echo "--- docker compose ps ---" + docker compose --ansi never --project-name registry ps -a + if $verbose; then + echo "--- docker compose logs ---" + docker compose --ansi never --project-name registry logs + fi + return 5 + } echo "launch tests" if docker compose \ --ansi never \ --profile int-registry-batch-loader \ --project-name registry \ - run --rm --no-TTY reg-api-integration-test-with-wait + run --rm --no-TTY reg-api-integration-test-with-wait \ + 2>&1 | tee "$rdir/integration_test_results.txt" then deep_archive status=$? @@ -77,23 +89,34 @@ run() { status=1 fi echo "run status: ${status}" + cd "$ddir" || return 1 + echo "--- docker compose ps ---" + docker compose --ansi never --project-name registry ps -a + if $verbose; then + echo "--- docker compose logs ---" + docker compose \ + --ansi never \ + --profile int-registry-batch-loader \ + --project-name registry \ + logs + fi clean # shellcheck disable=SC2086 # because we need to return an int return $status } -if [ $# -gt 1 ] -then - echo "Usage: $0 [--verify]" - exit 1 -fi - -if [ $# -eq 1 ] && [ "$1" != "--verify" ] -then - echo "Error: Invalid argument '$1'" - echo "Usage: $0 [--verify]" - exit 1 -fi +verbose=false +verify=false +for arg in "$@"; do + case "$arg" in + --verbose) verbose=true ;; + --verify) verify=true ;; + *) + echo "Error: Invalid argument '$arg'" + echo "Usage: $0 [--verify] [--verbose]" + exit 1 ;; + esac +done bdir=$(dirname "$(realpath "$0")") rdir=$(realpath "$bdir/../..") @@ -103,6 +126,7 @@ branchname=$(git branch --show-current) branchname=${branchname/issue/api} branchname=${branchname/_/-} tdir=$(mktemp -d) +echo "temporary directory: $tdir" # The EXIT pseudo-signal covers normal exits, errors, and interruptions (Ctrl+C) trap 'rm -rf "$tdir"' EXIT export tdir @@ -116,7 +140,7 @@ fi echo "registry being used" git status reg_gitrev=$(git describe --always --abbrev=40 --dirty='+' --exclude '*') -if [ "$1" == "--verify" ]; then +if $verify; then echo "Running in VERIFY mode..." status=failure cd "$tdir" || exit 1 diff --git a/.github/workflows/last_integration_test.json b/.github/workflows/last_integration_test.json index 45df6e74..0974041e 100644 --- a/.github/workflows/last_integration_test.json +++ b/.github/workflows/last_integration_test.json @@ -1,5 +1,5 @@ { - "api_gitrev": "426d29b016fc118e77855d2807456d1ccc592366", - "reg_gitrev": "d1be5e2574c073227cebac29367521be34f74b4d", + "api_gitrev": "95fda445bd34f646383a7708d2e4fee42201fe85+", + "reg_gitrev": "0fcb3cfb684511ac071739f10aafcf07e6007379", "status": "success" } diff --git a/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/TestParsing.java b/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/TestParsing.java index 36651e39..4c0a6742 100644 --- a/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/TestParsing.java +++ b/lexer/src/test/java/api/pds/nasa/gov/api_search_query_lexer/TestParsing.java @@ -134,8 +134,6 @@ void testFieldExistence() { Assertions.assertNull(listener.strval); Assertions.assertEquals("apple", listener.fields.get(0)); } - - @Test void testParenFieldExistence() { String queryString = "(exists apple)"; From 88c42a190fa6224f82701a0ea83b84895505a0bc Mon Sep 17 00:00:00 2001 From: Thomas Loubrieu Date: Mon, 29 Jun 2026 12:35:32 -0700 Subject: [PATCH 113/137] update integration tests --- .github/workflows/last_integration_test.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/last_integration_test.json b/.github/workflows/last_integration_test.json index 0974041e..a4a41214 100644 --- a/.github/workflows/last_integration_test.json +++ b/.github/workflows/last_integration_test.json @@ -1,5 +1,5 @@ { - "api_gitrev": "95fda445bd34f646383a7708d2e4fee42201fe85+", + "api_gitrev": "ddfb17cf345a6b2d92967dc6af9a103be251a359", "reg_gitrev": "0fcb3cfb684511ac071739f10aafcf07e6007379", "status": "success" } From 4c5f51f352d772e838f6ba40ad6c48e1da81e0e2 Mon Sep 17 00:00:00 2001 From: PDSEN CI Bot Date: Mon, 29 Jun 2026 20:02:16 +0000 Subject: [PATCH 114/137] Update changelog --- CHANGELOG.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c2f8ca1..ea2e7239 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## [«unknown»](https://github.com/NASA-PDS/registry-api/tree/«unknown») (2026-05-26) +## [«unknown»](https://github.com/NASA-PDS/registry-api/tree/«unknown») (2026-06-29) [Full Changelog](https://github.com/NASA-PDS/registry-api/compare/v1.6.2...«unknown») @@ -154,7 +154,6 @@ **Requirements:** - As a user, I want my API request to execute successfully even when the registry contains corrupted documents [\#361](https://github.com/NASA-PDS/registry-api/issues/361) -- As a PDS operator, I want to know the health of the registry API service [\#336](https://github.com/NASA-PDS/registry-api/issues/336) **Defects:** From ef6e1110ac30bccbd1182e63226ca07399db6501 Mon Sep 17 00:00:00 2001 From: Thomas Loubrieu Date: Mon, 29 Jun 2026 20:36:26 -0700 Subject: [PATCH 115/137] updated integration tests --- .github/workflows/last_integration_test.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/last_integration_test.json b/.github/workflows/last_integration_test.json index a4a41214..f1ca7c2b 100644 --- a/.github/workflows/last_integration_test.json +++ b/.github/workflows/last_integration_test.json @@ -1,5 +1,5 @@ { - "api_gitrev": "ddfb17cf345a6b2d92967dc6af9a103be251a359", + "api_gitrev": "a9e0779a96963687d3f1ab64385c9bb86dc0587d", "reg_gitrev": "0fcb3cfb684511ac071739f10aafcf07e6007379", "status": "success" } From 28a62abd285ff1c13fe83b06f974befe1a00c998 Mon Sep 17 00:00:00 2001 From: Thomas Loubrieu Date: Mon, 29 Jun 2026 20:47:31 -0700 Subject: [PATCH 116/137] fix quality review from dependabot, automate actual tests on github action for dependabot PR's --- .github/workflows/branch-cicd.yaml | 6 +++++- .../java/gov/nasa/pds/api/registry/model/SearchUtil.java | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/branch-cicd.yaml b/.github/workflows/branch-cicd.yaml index 26ab2702..f01dde27 100644 --- a/.github/workflows/branch-cicd.yaml +++ b/.github/workflows/branch-cicd.yaml @@ -97,7 +97,11 @@ jobs: - name: ∫ Integration and deep archive tests … hold onto your hats, pardners run: | - .github/workflows/integration_tests.sh --verify + if [[ "${{ github.actor }}" == "dependabot[bot]" ]]; then + .github/workflows/integration_tests.sh + else + .github/workflows/integration_tests.sh --verify + fi ... diff --git a/service/src/main/java/gov/nasa/pds/api/registry/model/SearchUtil.java b/service/src/main/java/gov/nasa/pds/api/registry/model/SearchUtil.java index e099c27a..e6b271f3 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/model/SearchUtil.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/model/SearchUtil.java @@ -22,7 +22,7 @@ public class SearchUtil { private static String fnArch; @Value("${registry.field.name.architecture}") - public void setFnArch(String fnArch) { + static public void setFnArch(String fnArch) { SearchUtil.fnArch = fnArch; } From 93e388bc2053c758b1a04c9f472f3e76ba545bd0 Mon Sep 17 00:00:00 2001 From: Thomas Loubrieu Date: Tue, 30 Jun 2026 05:49:46 -0700 Subject: [PATCH 117/137] update integration test results --- .github/workflows/last_integration_test.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/last_integration_test.json b/.github/workflows/last_integration_test.json index f1ca7c2b..8c03c53f 100644 --- a/.github/workflows/last_integration_test.json +++ b/.github/workflows/last_integration_test.json @@ -1,5 +1,5 @@ { - "api_gitrev": "a9e0779a96963687d3f1ab64385c9bb86dc0587d", + "api_gitrev": "28a62abd285ff1c13fe83b06f974befe1a00c998", "reg_gitrev": "0fcb3cfb684511ac071739f10aafcf07e6007379", "status": "success" } From 5e39a8604d4ffcf70b625186ae48321799ea16f3 Mon Sep 17 00:00:00 2001 From: Thomas Loubrieu Date: Tue, 30 Jun 2026 06:00:19 -0700 Subject: [PATCH 118/137] reorder modifiers to comply with java specification --- .../nasa/pds/api/registry/model/SearchUtil.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/service/src/main/java/gov/nasa/pds/api/registry/model/SearchUtil.java b/service/src/main/java/gov/nasa/pds/api/registry/model/SearchUtil.java index e6b271f3..351c0f29 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/model/SearchUtil.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/model/SearchUtil.java @@ -22,16 +22,16 @@ public class SearchUtil { private static String fnArch; @Value("${registry.field.name.architecture}") - static public void setFnArch(String fnArch) { + public static void setFnArch(String fnArch) { SearchUtil.fnArch = fnArch; } - static public String jsonPropertyToOpenProperty(String jsonProperty) { + public static String jsonPropertyToOpenProperty(String jsonProperty) { if (SearchUtil.fnArch == null || SearchUtil.fnArch.equalsIgnoreCase("flat")) return jsonProperty.replace(".", "/"); return jsonProperty; } - static public String[] jsonPropertyToOpenProperty(String[] jsonProperties) { + public static String[] jsonPropertyToOpenProperty(String[] jsonProperties) { if (jsonProperties != null && jsonProperties.length > 0) { for (int i = 0; i < jsonProperties.length; i++) { jsonProperties[i] = jsonPropertyToOpenProperty(jsonProperties[i]); @@ -40,7 +40,7 @@ static public String[] jsonPropertyToOpenProperty(String[] jsonProperties) { return jsonProperties; } - static public List jsonPropertyToOpenProperty(List jsonProperties) { + public static List jsonPropertyToOpenProperty(List jsonProperties) { if (jsonProperties != null && jsonProperties.size() > 0) { for (int i = 0; i < jsonProperties.size(); i++) { jsonProperties.set(i, jsonPropertyToOpenProperty(jsonProperties.get(i))); @@ -49,13 +49,13 @@ static public List jsonPropertyToOpenProperty(List jsonPropertie return jsonProperties; } - static public String openPropertyToJsonProperty(String openProperty) + public static String openPropertyToJsonProperty(String openProperty) throws UnsupportedSearchProperty { if (SearchUtil.fnArch == null || SearchUtil.fnArch.equalsIgnoreCase("flat")) return openProperty.replace('/', '.'); return openProperty; } - static private void addReference(ArrayList to, String ID, URL baseURL) { + private static void addReference(ArrayList to, String ID, URL baseURL) { Reference reference = new Reference(); reference.setId(ID); @@ -86,7 +86,7 @@ static private void addReference(ArrayList to, String ID, URL baseURL to.add(reference); } - static private PdsProduct addPropertiesFromESEntity(PdsProduct product, EntityProduct ep, + private static PdsProduct addPropertiesFromESEntity(PdsProduct product, EntityProduct ep, URL baseURL) { product.setId(ep.getLidVid()); product.setType(ep.getProductClass()); @@ -155,7 +155,7 @@ static private PdsProduct addPropertiesFromESEntity(PdsProduct product, EntityPr return product; } - static public PdsProduct entityProductToAPIProduct(EntityProduct ep, URL baseURL) { + public static PdsProduct entityProductToAPIProduct(EntityProduct ep, URL baseURL) { log.debug("convert EntityProduct (ep) to API object without XML label"); PdsProduct product = new PdsProduct(); From 129659ce9a49b675e558eef044b7377e9768fcdf Mon Sep 17 00:00:00 2001 From: Thomas Loubrieu Date: Tue, 30 Jun 2026 06:12:50 -0700 Subject: [PATCH 119/137] update test results --- .github/workflows/last_integration_test.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/last_integration_test.json b/.github/workflows/last_integration_test.json index 8c03c53f..c2594125 100644 --- a/.github/workflows/last_integration_test.json +++ b/.github/workflows/last_integration_test.json @@ -1,5 +1,5 @@ { - "api_gitrev": "28a62abd285ff1c13fe83b06f974befe1a00c998", + "api_gitrev": "5e39a8604d4ffcf70b625186ae48321799ea16f3", "reg_gitrev": "0fcb3cfb684511ac071739f10aafcf07e6007379", "status": "success" } From 64e0d0eaa66b8e440f6c3ac4d873613bd171f2ff Mon Sep 17 00:00:00 2001 From: PDSEN CI Bot Date: Tue, 30 Jun 2026 13:20:35 +0000 Subject: [PATCH 120/137] Update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea2e7239..da433810 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## [«unknown»](https://github.com/NASA-PDS/registry-api/tree/«unknown») (2026-06-29) +## [«unknown»](https://github.com/NASA-PDS/registry-api/tree/«unknown») (2026-06-30) [Full Changelog](https://github.com/NASA-PDS/registry-api/compare/v1.6.2...«unknown») From 97e2a8cec5b855837f44f6df6b6c55af80ccc1f1 Mon Sep 17 00:00:00 2001 From: thomas loubrieu <60993872+tloubrieu-jpl@users.noreply.github.com> Date: Thu, 2 Jul 2026 09:18:20 -0700 Subject: [PATCH 121/137] Dependabot 20260630 (#786) * Bump org.apache.maven.plugins:maven-gpg-plugin from 3.0.1 to 3.2.8 Bumps [org.apache.maven.plugins:maven-gpg-plugin](https://github.com/apache/maven-gpg-plugin) from 3.0.1 to 3.2.8. - [Release notes](https://github.com/apache/maven-gpg-plugin/releases) - [Commits](https://github.com/apache/maven-gpg-plugin/compare/maven-gpg-plugin-3.0.1...maven-gpg-plugin-3.2.8) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-gpg-plugin dependency-version: 3.2.8 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Bump org.opensearch.client:opensearch-rest-high-level-client Bumps [org.opensearch.client:opensearch-rest-high-level-client](https://github.com/opensearch-project/OpenSearch) from 3.2.0 to 3.6.0. - [Release notes](https://github.com/opensearch-project/OpenSearch/releases) - [Changelog](https://github.com/opensearch-project/OpenSearch/blob/main/CHANGELOG.md) - [Commits](https://github.com/opensearch-project/OpenSearch/compare/3.2.0...3.6.0) --- updated-dependencies: - dependency-name: org.opensearch.client:opensearch-rest-high-level-client dependency-version: 3.6.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Bump com.github.joschi.jackson:jackson-datatype-threetenbp Bumps [com.github.joschi.jackson:jackson-datatype-threetenbp](https://github.com/joschi/jackson-datatype-threetenbp) from 2.12.5 to 2.18.2. - [Release notes](https://github.com/joschi/jackson-datatype-threetenbp/releases) - [Changelog](https://github.com/joschi/jackson-datatype-threetenbp/blob/master/CHANGELOG.md) - [Commits](https://github.com/joschi/jackson-datatype-threetenbp/compare/jackson-datatype-threetenbp-2.12.5...jackson-datatype-threetenbp-2.18.2) --- updated-dependencies: - dependency-name: com.github.joschi.jackson:jackson-datatype-threetenbp dependency-version: 2.18.2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Bump io.swagger.core.v3:swagger-models from 2.2.8 to 2.2.49 Bumps io.swagger.core.v3:swagger-models from 2.2.8 to 2.2.49. --- updated-dependencies: - dependency-name: io.swagger.core.v3:swagger-models dependency-version: 2.2.49 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Bump joda-time:joda-time from 2.14.0 to 2.14.2 Bumps [joda-time:joda-time](https://github.com/JodaOrg/joda-time) from 2.14.0 to 2.14.2. - [Release notes](https://github.com/JodaOrg/joda-time/releases) - [Changelog](https://github.com/JodaOrg/joda-time/blob/main/RELEASE-NOTES.txt) - [Commits](https://github.com/JodaOrg/joda-time/compare/v2.14.0...v2.14.2) --- updated-dependencies: - dependency-name: joda-time:joda-time dependency-version: 2.14.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Bump org.springframework:spring-webmvc from 6.2.17 to 6.2.18 Bumps [org.springframework:spring-webmvc](https://github.com/spring-projects/spring-framework) from 6.2.17 to 6.2.18. - [Release notes](https://github.com/spring-projects/spring-framework/releases) - [Commits](https://github.com/spring-projects/spring-framework/compare/v6.2.17...v6.2.18) --- updated-dependencies: - dependency-name: org.springframework:spring-webmvc dependency-version: 6.2.18 dependency-type: direct:production ... Signed-off-by: dependabot[bot] * Bump actions/checkout from 6 to 7 Bumps [actions/checkout](https://github.com/actions/checkout) from 6 to 7. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v6...v7) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * Update hashicorp/aws requirement from ~> 6.32.1 to ~> 6.52.0 Updates the requirements on [hashicorp/aws](https://github.com/hashicorp/terraform-provider-aws) to permit the latest version. - [Release notes](https://github.com/hashicorp/terraform-provider-aws/releases) - [Changelog](https://github.com/hashicorp/terraform-provider-aws/blob/main/CHANGELOG.md) - [Commits](https://github.com/hashicorp/terraform-provider-aws/compare/v6.32.1...v6.52.0) --- updated-dependencies: - dependency-name: hashicorp/aws dependency-version: 6.52.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] * Bump actions/cache from 5 to 6 Bumps [actions/cache](https://github.com/actions/cache) from 5 to 6. - [Release notes](https://github.com/actions/cache/releases) - [Changelog](https://github.com/actions/cache/blob/main/RELEASES.md) - [Commits](https://github.com/actions/cache/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/cache dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * Bump dcarbone/install-jq-action from 3 to 4 Bumps [dcarbone/install-jq-action](https://github.com/dcarbone/install-jq-action) from 3 to 4. - [Release notes](https://github.com/dcarbone/install-jq-action/releases) - [Commits](https://github.com/dcarbone/install-jq-action/compare/v3...v4) --- updated-dependencies: - dependency-name: dcarbone/install-jq-action dependency-version: '4' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * update the integration test not to wait with a sleep * update integration test --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Thomas Loubrieu --- .github/workflows/branch-cicd.yaml | 6 +++--- .github/workflows/codeql-analysis.yml | 4 ++-- .github/workflows/integration_tests.sh | 2 +- .github/workflows/last_integration_test.json | 2 +- .github/workflows/secrets-detection.yaml | 2 +- .github/workflows/stable-cicd.yaml | 4 ++-- .github/workflows/unstable-cicd.yaml | 4 ++-- model/pom.xml | 6 +++--- pom.xml | 2 +- service/pom.xml | 6 +++--- terraform/provider.tf | 2 +- 11 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/branch-cicd.yaml b/.github/workflows/branch-cicd.yaml index f01dde27..2184a76f 100644 --- a/.github/workflows/branch-cicd.yaml +++ b/.github/workflows/branch-cicd.yaml @@ -40,14 +40,14 @@ jobs: steps: - name: 💳 Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: lfs: true fetch-depth: 0 token: ${{secrets.ADMIN_GITHUB_TOKEN || github.token}} - name: 💵 Maven Cache - uses: actions/cache@v5 + uses: actions/cache@v6 with: path: ~/.m2/repository # The "key" used to indicate a set of cached files is the operating system runner @@ -92,7 +92,7 @@ jobs: - name: Install jq - uses: dcarbone/install-jq-action@v3 + uses: dcarbone/install-jq-action@v4 - name: ∫ Integration and deep archive tests … hold onto your hats, pardners diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index d996aefe..535d21e6 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -23,7 +23,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: # We must fetch at least the immediate parents so that if this is # a pull request then we can checkout the head. @@ -93,7 +93,7 @@ jobs: steps: - name: 💳 Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: lfs: true fetch-depth: 0 diff --git a/.github/workflows/integration_tests.sh b/.github/workflows/integration_tests.sh index 25f2cdf0..f92e7649 100755 --- a/.github/workflows/integration_tests.sh +++ b/.github/workflows/integration_tests.sh @@ -85,7 +85,7 @@ run() { --ansi never \ --profile int-registry-batch-loader \ --project-name registry \ - run --rm --no-TTY reg-api-integration-test-with-wait \ + run --rm --no-TTY reg-api-integration-test \ 2>&1 | tee "$rdir/integration_test_results.txt" then deep_archive diff --git a/.github/workflows/last_integration_test.json b/.github/workflows/last_integration_test.json index c2594125..368b2ce5 100644 --- a/.github/workflows/last_integration_test.json +++ b/.github/workflows/last_integration_test.json @@ -1,5 +1,5 @@ { - "api_gitrev": "5e39a8604d4ffcf70b625186ae48321799ea16f3", + "api_gitrev": "7ef9c85ff21ba973ebed95c151ca99af24538c4e", "reg_gitrev": "0fcb3cfb684511ac071739f10aafcf07e6007379", "status": "success" } diff --git a/.github/workflows/secrets-detection.yaml b/.github/workflows/secrets-detection.yaml index 92f4d894..cc73adb1 100644 --- a/.github/workflows/secrets-detection.yaml +++ b/.github/workflows/secrets-detection.yaml @@ -13,7 +13,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Install necessary packages run: | diff --git a/.github/workflows/stable-cicd.yaml b/.github/workflows/stable-cicd.yaml index 28fd6c1d..9c12f21a 100644 --- a/.github/workflows/stable-cicd.yaml +++ b/.github/workflows/stable-cicd.yaml @@ -50,14 +50,14 @@ jobs: steps: - name: 💳 Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: lfs: true token: ${{secrets.ADMIN_GITHUB_TOKEN}} fetch-depth: 0 - name: 💵 Maven Cache - uses: actions/cache@v5 + uses: actions/cache@v6 with: path: ~/.m2/repository # The "key" used to indicate a set of cached files is the operating system runner diff --git a/.github/workflows/unstable-cicd.yaml b/.github/workflows/unstable-cicd.yaml index b0fa48ea..a1a3cfdc 100644 --- a/.github/workflows/unstable-cicd.yaml +++ b/.github/workflows/unstable-cicd.yaml @@ -50,14 +50,14 @@ jobs: steps: - name: 💳 Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: lfs: true fetch-depth: 0 token: ${{secrets.ADMIN_GITHUB_TOKEN}} - name: 💵 Maven Cache - uses: actions/cache@v5 + uses: actions/cache@v6 with: path: ~/.m2/repository # The "key" used to indicate a set of cached files is the operating system runner diff --git a/model/pom.xml b/model/pom.xml index 505fb635..8624ade7 100644 --- a/model/pom.xml +++ b/model/pom.xml @@ -187,7 +187,7 @@ com.github.joschi.jackson jackson-datatype-threetenbp - 2.12.5 + 2.18.2 @@ -200,7 +200,7 @@ joda-time joda-time - 2.14.0 + 2.14.2 @@ -223,7 +223,7 @@ io.swagger.core.v3 swagger-models - 2.2.8 + 2.2.49 diff --git a/pom.xml b/pom.xml index 3c62e849..1c8091aa 100644 --- a/pom.xml +++ b/pom.xml @@ -50,7 +50,7 @@ Go through this file line-by-line and replace the template values with your own. Registry API UTF-8 17 - 6.2.17 + 6.2.18 gov.nasa.pds diff --git a/service/pom.xml b/service/pom.xml index 65d07f9e..40de6608 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -174,12 +174,12 @@ com.github.joschi.jackson jackson-datatype-threetenbp - 2.12.5 + 2.18.2 joda-time joda-time - 2.14.0 + 2.14.2 com.sun.xml.bind @@ -245,7 +245,7 @@ org.opensearch.client opensearch-rest-high-level-client - 3.2.0 + 3.6.0 diff --git a/terraform/provider.tf b/terraform/provider.tf index 6f8cb26b..76aae315 100644 --- a/terraform/provider.tf +++ b/terraform/provider.tf @@ -16,7 +16,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "~> 6.32.1" + version = "~> 6.52.0" } } } From 704b48250e05895f1f48f7122405c621bcaea917 Mon Sep 17 00:00:00 2001 From: PDSEN CI Bot Date: Thu, 2 Jul 2026 16:24:09 +0000 Subject: [PATCH 122/137] Update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index da433810..eca8cfbb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## [«unknown»](https://github.com/NASA-PDS/registry-api/tree/«unknown») (2026-06-30) +## [«unknown»](https://github.com/NASA-PDS/registry-api/tree/«unknown») (2026-07-02) [Full Changelog](https://github.com/NASA-PDS/registry-api/compare/v1.6.2...«unknown») From 13dfc35d9f5179173e03d145e1e1c90490cf9ed8 Mon Sep 17 00:00:00 2001 From: Thomas Loubrieu Date: Thu, 2 Jul 2026 13:17:25 -0700 Subject: [PATCH 123/137] match ssm parameter with cloudfront expectation, simplify not useful variables --- terraform/output.tf | 2 +- terraform/variables.tf | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/terraform/output.tf b/terraform/output.tf index b83a149d..6c61b6b1 100644 --- a/terraform/output.tf +++ b/terraform/output.tf @@ -11,7 +11,7 @@ output "load_balancer_domain" { } resource "aws_ssm_parameter" "load_balancer_domain" { - name = "${local.ssm_prefix}/api-load-balancer-domain" + name = "${local.ssm_prefix}/load-balancer-domain" description = "Registry API load balancer domain" type = "String" value = aws_lb.registry-api-lb.dns_name diff --git a/terraform/variables.tf b/terraform/variables.tf index ac65e01e..4e1dc870 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -44,13 +44,10 @@ variable "ecs_task_execution_role" { description = "ECS task execution role" } -<<<<<<< HEAD -======= variable "registry_api_docker_image" { description = "AWS image name for Fargate" } ->>>>>>> develop variable "aws_s3_bucket_logs_id" { description = "AWS S3 bucket with the logs" } @@ -84,13 +81,25 @@ variable "common_tags" { } } +variable "create_github_secret_credentials" { + description = "Whether to create GitHub secret credentials (1) or not (0)" + type = number + default = 0 +} + +# TODO remove as the ECR cache it is used for does not work, +# besides we would like to configure it in a infra module instead of this specific registry-api module variable "github_username" { description = "GitHub username for ECR pull through cache" + default = "" } +# TODO remove as the ECR cache it is used for does not work, +# besides we would like to configure it in a infra module instead of this specific registry-api module variable "github_token" { description = "GitHub personal access token for ECR pull through cache" sensitive = true + default = "" } variable "cloudfront_dns" { From 101d95b59ed75583ed403f1acd4bfce062f3f521 Mon Sep 17 00:00:00 2001 From: Thomas Loubrieu Date: Thu, 2 Jul 2026 13:43:11 -0700 Subject: [PATCH 124/137] update integration test results --- .github/workflows/last_integration_test.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/last_integration_test.json b/.github/workflows/last_integration_test.json index 368b2ce5..f49b7239 100644 --- a/.github/workflows/last_integration_test.json +++ b/.github/workflows/last_integration_test.json @@ -1,5 +1,5 @@ { - "api_gitrev": "7ef9c85ff21ba973ebed95c151ca99af24538c4e", - "reg_gitrev": "0fcb3cfb684511ac071739f10aafcf07e6007379", + "api_gitrev": "13dfc35d9f5179173e03d145e1e1c90490cf9ed8", + "reg_gitrev": "bf1e9a4772c77debcabd256b051d053a7c980917", "status": "success" } From 07b911ce88034658d907de07ef677157a5586d83 Mon Sep 17 00:00:00 2001 From: Jordan Padams Date: Mon, 6 Jul 2026 12:51:30 -0700 Subject: [PATCH 125/137] Run tests --- .github/workflows/last_integration_test.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/last_integration_test.json b/.github/workflows/last_integration_test.json index f49b7239..2e9647c4 100644 --- a/.github/workflows/last_integration_test.json +++ b/.github/workflows/last_integration_test.json @@ -1,5 +1,5 @@ { - "api_gitrev": "13dfc35d9f5179173e03d145e1e1c90490cf9ed8", - "reg_gitrev": "bf1e9a4772c77debcabd256b051d053a7c980917", + "api_gitrev": "101d95b59ed75583ed403f1acd4bfce062f3f521", + "reg_gitrev": "09e5646af87591c0fb20876d1fea9c31498d2a6a", "status": "success" } From ca3d9ecb8b2fb3e6c1f78b19b133468d28737eb2 Mon Sep 17 00:00:00 2001 From: Jordan Padams Date: Mon, 6 Jul 2026 13:45:19 -0700 Subject: [PATCH 126/137] Successful integration test run --- .github/workflows/last_integration_test.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/last_integration_test.json b/.github/workflows/last_integration_test.json index 2e9647c4..180cdd88 100644 --- a/.github/workflows/last_integration_test.json +++ b/.github/workflows/last_integration_test.json @@ -1,5 +1,5 @@ { - "api_gitrev": "101d95b59ed75583ed403f1acd4bfce062f3f521", + "api_gitrev": "07b911ce88034658d907de07ef677157a5586d83", "reg_gitrev": "09e5646af87591c0fb20876d1fea9c31498d2a6a", "status": "success" } From e78080103402006e9ff7ca25209d631e3e9dfe2d Mon Sep 17 00:00:00 2001 From: PDSEN CI Bot Date: Mon, 6 Jul 2026 21:15:22 +0000 Subject: [PATCH 127/137] Update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eca8cfbb..c9e0db4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## [«unknown»](https://github.com/NASA-PDS/registry-api/tree/«unknown») (2026-07-02) +## [«unknown»](https://github.com/NASA-PDS/registry-api/tree/«unknown») (2026-07-06) [Full Changelog](https://github.com/NASA-PDS/registry-api/compare/v1.6.2...«unknown») From 4e64f8e7dddcff076f3ac2bd6b6785d8dccd4706 Mon Sep 17 00:00:00 2001 From: Jordan Padams <33492486+jordanpadams@users.noreply.github.com> Date: Tue, 7 Jul 2026 12:29:38 -0700 Subject: [PATCH 128/137] Update dependabot.yml --- .github/dependabot.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 004f4dc5..7b1a5f73 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -15,6 +15,10 @@ updates: directory: "/" # Location of package manifests schedule: interval: "weekly" + groups: + github-actions: + patterns: + - "*" target-branch: "develop" - package-ecosystem: "docker" # See documentation for possible values From 498591d45b24ae3cdd787083a68c823e815c2b78 Mon Sep 17 00:00:00 2001 From: PDSEN CI Bot Date: Tue, 7 Jul 2026 19:35:27 +0000 Subject: [PATCH 129/137] Update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c9e0db4c..ec558e96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## [«unknown»](https://github.com/NASA-PDS/registry-api/tree/«unknown») (2026-07-06) +## [«unknown»](https://github.com/NASA-PDS/registry-api/tree/«unknown») (2026-07-07) [Full Changelog](https://github.com/NASA-PDS/registry-api/compare/v1.6.2...«unknown») From 9e7846d4c8b32d565e1fbceb9b26841bc360d8b7 Mon Sep 17 00:00:00 2001 From: Thomas Loubrieu Date: Thu, 9 Jul 2026 12:08:03 -0700 Subject: [PATCH 130/137] make ECS service and load balancer security groups distincts --- terraform/main.tf | 5 +++-- terraform/variables.tf | 5 +++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/terraform/main.tf b/terraform/main.tf index b07236f1..c27e6f3c 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -8,7 +8,7 @@ resource "aws_lb" "registry-api-lb" { name = "registry-api-lb" internal = false load_balancer_type = "application" - security_groups = var.aws_fg_security_groups + security_groups = var.aws_lb_security_groups subnets = var.aws_lb_subnets enable_deletion_protection = false @@ -126,7 +126,8 @@ resource "aws_cloudwatch_log_group" "pds-registry-log-group" { # The task definition for app. resource "aws_ecs_task_definition" "pds-registry-ecs-task" { - family = "pds-registry-api-task" + family = "pds-registry-api-task" + skip_destroy = true container_definitions = < Date: Mon, 13 Jul 2026 14:31:12 -0700 Subject: [PATCH 131/137] fix integration test results --- .github/workflows/last_integration_test.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/last_integration_test.json b/.github/workflows/last_integration_test.json index 180cdd88..acaea552 100644 --- a/.github/workflows/last_integration_test.json +++ b/.github/workflows/last_integration_test.json @@ -1,5 +1,5 @@ { - "api_gitrev": "07b911ce88034658d907de07ef677157a5586d83", - "reg_gitrev": "09e5646af87591c0fb20876d1fea9c31498d2a6a", + "api_gitrev": "9e7846d4c8b32d565e1fbceb9b26841bc360d8b7+", + "reg_gitrev": "6bb58c44f99f71e7769cc899a3e094f77fa86bd0", "status": "success" } From a36dc50cbbae011cf3f241881f78dabd9b6aa8ba Mon Sep 17 00:00:00 2001 From: Thomas Loubrieu Date: Tue, 14 Jul 2026 14:54:36 -0700 Subject: [PATCH 132/137] update integration tests --- .github/workflows/last_integration_test.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/last_integration_test.json b/.github/workflows/last_integration_test.json index acaea552..1ab35ec4 100644 --- a/.github/workflows/last_integration_test.json +++ b/.github/workflows/last_integration_test.json @@ -1,5 +1,5 @@ { - "api_gitrev": "9e7846d4c8b32d565e1fbceb9b26841bc360d8b7+", - "reg_gitrev": "6bb58c44f99f71e7769cc899a3e094f77fa86bd0", + "api_gitrev": "74e023e2917ec0e02827ad271ffe045a9a47b0ca", + "reg_gitrev": "ac709d286536173432a316a60957f547e201f658", "status": "success" } From 87d6eb445261f010ee72976b092b6022137982af Mon Sep 17 00:00:00 2001 From: Thomas Loubrieu Date: Tue, 14 Jul 2026 15:48:17 -0700 Subject: [PATCH 133/137] update local integration test results --- .github/workflows/last_integration_test.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/last_integration_test.json b/.github/workflows/last_integration_test.json index 1ab35ec4..5a4d5b84 100644 --- a/.github/workflows/last_integration_test.json +++ b/.github/workflows/last_integration_test.json @@ -1,5 +1,5 @@ { - "api_gitrev": "74e023e2917ec0e02827ad271ffe045a9a47b0ca", + "api_gitrev": "a36dc50cbbae011cf3f241881f78dabd9b6aa8ba+", "reg_gitrev": "ac709d286536173432a316a60957f547e201f658", "status": "success" } From b0711d49e711ff012dca26ff455682b29a5ca99c Mon Sep 17 00:00:00 2001 From: Thomas Loubrieu Date: Wed, 15 Jul 2026 08:36:40 -0700 Subject: [PATCH 134/137] format terraform --- terraform/main.tf | 24 ++++++++++++------------ terraform/output.tf | 12 ++++++------ terraform/variables.tf | 32 ++++++++++++++++---------------- 3 files changed, 34 insertions(+), 34 deletions(-) diff --git a/terraform/main.tf b/terraform/main.tf index c27e6f3c..135f1271 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -36,9 +36,9 @@ resource "aws_lb_target_group" "pds-registry-api-target-group" { } health_check { - enabled = true - path = "/health" - matcher = "200" + enabled = true + path = "/health" + matcher = "200" interval = 300 } @@ -62,7 +62,7 @@ resource "aws_lb_listener_rule" "pds-registry-forward-rule" { action { type = "forward" - target_group_arn = aws_lb_target_group.pds-registry-api-target-group.arn + target_group_arn = aws_lb_target_group.pds-registry-api-target-group.arn } # no condition for now @@ -70,7 +70,7 @@ resource "aws_lb_listener_rule" "pds-registry-forward-rule" { # used for multiple back-end service condition { path_pattern { - values = ["/*"] + values = ["/*"] } } } @@ -87,10 +87,10 @@ resource "aws_secretsmanager_secret" "github_ecr_credentials" { resource "aws_secretsmanager_secret_version" "github_ecr_credentials" { count = var.create_github_secret_credentials - secret_id = aws_secretsmanager_secret.github_ecr_credentials[count.index].id + secret_id = aws_secretsmanager_secret.github_ecr_credentials[count.index].id secret_string = jsonencode({ - username = var.github_username - accessToken = var.github_token + username = var.github_username + accessToken = var.github_token }) } @@ -112,8 +112,8 @@ resource "aws_ecr_pull_through_cache_rule" "ghcr" { } resource "aws_ecr_repository" "ghcr_registry_api" { - name = "ghcr/nasa-pds/registry-api" - tags = var.common_tags + name = "ghcr/nasa-pds/registry-api" + tags = var.common_tags } # Log groups hold logs from our app. @@ -205,8 +205,8 @@ resource "aws_ecs_service" "pds-registry-reg-service" { network_configuration { assign_public_ip = false - security_groups = var.aws_fg_security_groups - subnets = var.aws_fg_subnets + security_groups = var.aws_fg_security_groups + subnets = var.aws_fg_subnets } tags = var.common_tags diff --git a/terraform/output.tf b/terraform/output.tf index 6c61b6b1..737d4bcc 100644 --- a/terraform/output.tf +++ b/terraform/output.tf @@ -1,19 +1,19 @@ locals { module_relative_path = replace(abspath(path.module), "/^.*\\/terraform(\\/|$)/", "") - ssm_prefix = "/pds/${var.component_name}${local.module_relative_path}" + ssm_prefix = "/pds/${var.component_name}${local.module_relative_path}" } output "load_balancer_domain" { description = "Registry API load balancer domain" - value = aws_lb.registry-api-lb.dns_name + value = aws_lb.registry-api-lb.dns_name } resource "aws_ssm_parameter" "load_balancer_domain" { - name = "${local.ssm_prefix}/load-balancer-domain" + name = "${local.ssm_prefix}/load-balancer-domain" description = "Registry API load balancer domain" - type = "String" - value = aws_lb.registry-api-lb.dns_name - tags = var.common_tags + type = "String" + value = aws_lb.registry-api-lb.dns_name + tags = var.common_tags } \ No newline at end of file diff --git a/terraform/variables.tf b/terraform/variables.tf index 04bc1e0e..b2ca0c11 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -1,11 +1,11 @@ variable "node_name_abbr" { description = "Node name abbreviation" - default="en" + default = "en" } variable "aws_region" { description = "AWS Region" - default = "us-west-2" + default = "us-west-2" } variable "spring_boot_args" { @@ -14,7 +14,7 @@ variable "spring_boot_args" { variable "aws_profile" { description = "AWS profile" - default = "" + default = "" } variable "aws_fg_vpc" { @@ -23,22 +23,22 @@ variable "aws_fg_vpc" { variable "aws_fg_security_groups" { description = "AWS Security groups for Fargate" - type = list(string) + type = list(string) } variable "aws_lb_security_groups" { description = "AWS Security groups for Fargate" - type = list(string) + type = list(string) } variable "aws_fg_subnets" { description = "AWS Subnets for Fargate" - type = list(string) + type = list(string) } variable "aws_lb_subnets" { description = "AWS Subnets for the load balancer" - type = list(string) + type = list(string) } variable "ecs_task_role" { @@ -59,12 +59,12 @@ variable "aws_s3_bucket_logs_id" { variable "aws_fg_cpu_units" { description = "CPU Units for fargate" - default = 256 + default = 256 } variable "aws_fg_ram_units" { description = "RAM Units for Fargate" - default = 512 + default = 512 } variable "aws_acm_certificate_arn" { @@ -72,17 +72,17 @@ variable "aws_acm_certificate_arn" { } variable "component_name" { - description = "Component this subcomponents belongs to" - type = string - default = "registry" + description = "Component this subcomponents belongs to" + type = string + default = "registry" } variable "common_tags" { description = "Common tags to apply to all resources" type = map(string) default = { - Project = "registry" - ManagedBy = "terraform" + Project = "registry" + ManagedBy = "terraform" } } @@ -96,7 +96,7 @@ variable "create_github_secret_credentials" { # besides we would like to configure it in a infra module instead of this specific registry-api module variable "github_username" { description = "GitHub username for ECR pull through cache" - default = "" + default = "" } # TODO remove as the ECR cache it is used for does not work, @@ -104,7 +104,7 @@ variable "github_username" { variable "github_token" { description = "GitHub personal access token for ECR pull through cache" sensitive = true - default = "" + default = "" } variable "cloudfront_dns" { From f3b0c7a5ee67ade417098a3d25cb80d3d1b7dd2a Mon Sep 17 00:00:00 2001 From: Jordan Padams Date: Wed, 15 Jul 2026 10:51:04 -0700 Subject: [PATCH 135/137] Run integration tests with sweepers:main --- .github/workflows/last_integration_test.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/last_integration_test.json b/.github/workflows/last_integration_test.json index 5a4d5b84..ef58c8ab 100644 --- a/.github/workflows/last_integration_test.json +++ b/.github/workflows/last_integration_test.json @@ -1,5 +1,5 @@ { - "api_gitrev": "a36dc50cbbae011cf3f241881f78dabd9b6aa8ba+", - "reg_gitrev": "ac709d286536173432a316a60957f547e201f658", + "api_gitrev": "b0711d49e711ff012dca26ff455682b29a5ca99c+", + "reg_gitrev": "dda5e706096b4e50575006f85f82f3c111d05b1f", "status": "success" } From 54c8a2c9d10d43ffab15c23e6ca87e544d5caead Mon Sep 17 00:00:00 2001 From: PDSEN CI Bot Date: Wed, 15 Jul 2026 18:38:07 +0000 Subject: [PATCH 136/137] Update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ec558e96..ce631540 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## [«unknown»](https://github.com/NASA-PDS/registry-api/tree/«unknown») (2026-07-07) +## [«unknown»](https://github.com/NASA-PDS/registry-api/tree/«unknown») (2026-07-15) [Full Changelog](https://github.com/NASA-PDS/registry-api/compare/v1.6.2...«unknown») From 2c4b78ae870bd2644a6a0fee4c3f67aab7eea890 Mon Sep 17 00:00:00 2001 From: PDSEN CI Bot Date: Wed, 15 Jul 2026 18:56:18 +0000 Subject: [PATCH 137/137] Update changelog