From dfc34cef5aaac0de1abb16acdd0f15b2869f911a Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 29 Aug 2025 10:22:23 -0400 Subject: [PATCH 01/12] update query to make Integer value more obvious --- src/main/java/edu/harvard/iq/dataverse/FileMetadata.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/FileMetadata.java b/src/main/java/edu/harvard/iq/dataverse/FileMetadata.java index ca3e2d67263..932bbd60be6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FileMetadata.java +++ b/src/main/java/edu/harvard/iq/dataverse/FileMetadata.java @@ -94,9 +94,10 @@ " )", resultSetMapping = "IdToLongMapping" ) +/* When this mapping was to Long.class, Postgres was still returning an Integer, causing indexing failures - see #11776 */ @SqlResultSetMapping( name = "IdToLongMapping", - columns = @ColumnResult(name = "id", type = Long.class) + columns = @ColumnResult(name = "id", type = Integer.class) ) @Entity public class FileMetadata implements Serializable { From 0689560af1056728268f45e180ede4faec5c304f Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 29 Aug 2025 11:04:04 -0400 Subject: [PATCH 02/12] Make add check object type --- .../iq/dataverse/search/IndexServiceBean.java | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index 4aa1b5abb20..80d59c53017 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -69,6 +69,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Future; @@ -1428,7 +1429,31 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set queryResults = query.getResultList(); + for (Object result : queryResults) { + if (result != null) { + // Ensure we're adding Long objects to the list + if (result instanceof Integer intResult) { + logger.finest("Converted Integer result to Long: " + result); + changedFileMetadataIds.add(Long.valueOf(intResult)); + } else if (result instanceof Long longResult) { + // Already a Long, add directly + logger.finest("Added existing Long to list: " + result); + changedFileMetadataIds.add(longResult); + } else { + // If it's not a Long, convert it to one via String + try { + changedFileMetadataIds.add(Long.valueOf(result.toString())); + logger.finest("Converted non-Long result to Long: " + result + " of type " + result.getClass().getName()); + } catch (NumberFormatException e) { + logger.warning("Could not convert query result to Long: " + result); + } + } + } + } logger.fine( "We are indexing a draft version of a dataset that has a released version. We'll be checking file metadatas if they are exact clones of the released versions."); } else if (datasetVersion.isDraft()) { @@ -1515,6 +1540,8 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set Date: Fri, 29 Aug 2025 11:59:55 -0400 Subject: [PATCH 03/12] add test --- .../harvard/iq/dataverse/api/SearchIT.java | 86 +++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java index a31cc37b814..e991fd1aed8 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java @@ -2328,5 +2328,91 @@ public void testShowCollections() { )); } + + @Test + public void testFileAddedAfterPublicationIsIndexed() { + // Create user + Response createUser = UtilIT.createRandomUser(); + createUser.prettyPrint(); + String username = UtilIT.getUsernameFromResponse(createUser); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + + // Create dataverse + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + createDataverseResponse.prettyPrint(); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + // Create dataset + Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); + createDatasetResponse.prettyPrint(); + Integer datasetId = UtilIT.getDatasetIdFromResponse(createDatasetResponse); + String datasetPersistentId = UtilIT.getDatasetPersistentIdFromResponse(createDatasetResponse); + + // Publish dataverse and dataset + Response publishDataverse = UtilIT.publishDataverseViaSword(dataverseAlias, apiToken); + publishDataverse.prettyPrint(); + publishDataverse.then().assertThat() + .statusCode(OK.getStatusCode()); + + Response publishDataset = UtilIT.publishDatasetViaNativeApi(datasetId, "major", apiToken); + publishDataset.prettyPrint(); + publishDataset.then().assertThat() + .statusCode(OK.getStatusCode()); + + // Verify no files in search results initially + Response searchBeforeFileUpload = UtilIT.search("parentId:" + datasetId, apiToken); + searchBeforeFileUpload.prettyPrint(); + searchBeforeFileUpload.then().assertThat() + .body("data.total_count", CoreMatchers.is(0)) + .statusCode(OK.getStatusCode()); + + // Upload a file after publication + String pathToFile = "src/main/webapp/resources/images/dataverseproject.png"; + Response uploadFileResponse = UtilIT.uploadFileViaNative(datasetId.toString(), pathToFile, apiToken); + uploadFileResponse.prettyPrint(); + uploadFileResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + + // Get file ID from the upload response + Integer fileId = JsonPath.from(uploadFileResponse.getBody().asString()).getInt("data.files[0].dataFile.id"); + + // Wait for indexing to complete + String searchQuery = "parentId:" + fileId; + assertTrue(UtilIT.sleepForSearch(searchQuery, apiToken, "", 1, UtilIT.MAXIMUM_INGEST_LOCK_DURATION), + "Failed test if search exceeds max duration " + searchQuery); + + // Search for the file and verify it's indexed + Response searchAfterFileUpload = UtilIT.search(searchQuery, apiToken); + searchAfterFileUpload.prettyPrint(); + searchAfterFileUpload.then().assertThat() + .body("data.total_count", CoreMatchers.is(1)) + .body("data.items[0].name", is("dataverseproject.png")) + .body("data.items[0].file_content_type", CoreMatchers.is("image/png")) + .statusCode(OK.getStatusCode()); + + // Clean up - delete dataset, dataverse, and user + try { + // Delete the dataset + Response deleteDatasetResponse = UtilIT.deleteDatasetViaNativeApi(datasetId, apiToken); + deleteDatasetResponse.prettyPrint(); + deleteDatasetResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + + // Delete the dataverse + Response deleteDataverseResponse = UtilIT.deleteDataverse(dataverseAlias, apiToken); + deleteDataverseResponse.prettyPrint(); + deleteDataverseResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + + // Delete the user + Response deleteUserResponse = UtilIT.deleteUser(username); + deleteUserResponse.prettyPrint(); + deleteUserResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + } catch (Exception e) { + // Log any cleanup failures but don't fail the test + System.out.println("Error during cleanup: " + e.getMessage()); + } + } } From 0f672862e63f5b6dcf3f3936eb62e759d69213e4 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 29 Aug 2025 12:02:02 -0400 Subject: [PATCH 04/12] release note --- doc/release-notes/11776- index fix.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/release-notes/11776- index fix.md diff --git a/doc/release-notes/11776- index fix.md b/doc/release-notes/11776- index fix.md new file mode 100644 index 00000000000..959b1ca3e95 --- /dev/null +++ b/doc/release-notes/11776- index fix.md @@ -0,0 +1 @@ +A bug, introduced in v6.7, that caused files in draft versions that were added after the initial dataset version was published, has been fixed. \ No newline at end of file From b9d9c350e1073ebc86686c98a848f260b92939e4 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 29 Aug 2025 12:13:59 -0400 Subject: [PATCH 05/12] add comment/warning on unexpected case --- .../edu/harvard/iq/dataverse/search/IndexServiceBean.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index 80d59c53017..f89a88ae05d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -1527,6 +1527,8 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set Date: Fri, 29 Aug 2025 12:21:14 -0400 Subject: [PATCH 06/12] autocomplete typo --- .../java/edu/harvard/iq/dataverse/search/IndexServiceBean.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index f89a88ae05d..1f7f869a35f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -1537,7 +1537,7 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set Date: Fri, 29 Aug 2025 15:07:49 -0400 Subject: [PATCH 07/12] fix test --- src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java index e991fd1aed8..b2158a8985f 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java @@ -2377,7 +2377,7 @@ public void testFileAddedAfterPublicationIsIndexed() { Integer fileId = JsonPath.from(uploadFileResponse.getBody().asString()).getInt("data.files[0].dataFile.id"); // Wait for indexing to complete - String searchQuery = "parentId:" + fileId; + String searchQuery = "entityId:" + fileId; assertTrue(UtilIT.sleepForSearch(searchQuery, apiToken, "", 1, UtilIT.MAXIMUM_INGEST_LOCK_DURATION), "Failed test if search exceeds max duration " + searchQuery); From fe636029e31f355d7367bec113069b6c2665c81b Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 29 Aug 2025 16:40:51 -0400 Subject: [PATCH 08/12] up to superuser to delete dataverse, remove try --- src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java index b2158a8985f..afd6e0d6f33 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java @@ -2391,13 +2391,16 @@ public void testFileAddedAfterPublicationIsIndexed() { .statusCode(OK.getStatusCode()); // Clean up - delete dataset, dataverse, and user - try { + // Delete the dataset Response deleteDatasetResponse = UtilIT.deleteDatasetViaNativeApi(datasetId, apiToken); deleteDatasetResponse.prettyPrint(); deleteDatasetResponse.then().assertThat() .statusCode(OK.getStatusCode()); + Response makeSuperUser = UtilIT.setSuperuserStatus(username, true); + assertEquals(200, makeSuperUser.getStatusCode()); + // Delete the dataverse Response deleteDataverseResponse = UtilIT.deleteDataverse(dataverseAlias, apiToken); deleteDataverseResponse.prettyPrint(); @@ -2409,10 +2412,6 @@ public void testFileAddedAfterPublicationIsIndexed() { deleteUserResponse.prettyPrint(); deleteUserResponse.then().assertThat() .statusCode(OK.getStatusCode()); - } catch (Exception e) { - // Log any cleanup failures but don't fail the test - System.out.println("Error during cleanup: " + e.getMessage()); - } } } From 6a2e094e828eea55090b8ddd7d92c43cf03d943e Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 29 Aug 2025 16:41:16 -0400 Subject: [PATCH 09/12] fix indent --- .../harvard/iq/dataverse/api/SearchIT.java | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java index afd6e0d6f33..af44a3e1a48 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java @@ -2392,26 +2392,26 @@ public void testFileAddedAfterPublicationIsIndexed() { // Clean up - delete dataset, dataverse, and user - // Delete the dataset - Response deleteDatasetResponse = UtilIT.deleteDatasetViaNativeApi(datasetId, apiToken); - deleteDatasetResponse.prettyPrint(); - deleteDatasetResponse.then().assertThat() - .statusCode(OK.getStatusCode()); - - Response makeSuperUser = UtilIT.setSuperuserStatus(username, true); - assertEquals(200, makeSuperUser.getStatusCode()); - - // Delete the dataverse - Response deleteDataverseResponse = UtilIT.deleteDataverse(dataverseAlias, apiToken); - deleteDataverseResponse.prettyPrint(); - deleteDataverseResponse.then().assertThat() - .statusCode(OK.getStatusCode()); - - // Delete the user - Response deleteUserResponse = UtilIT.deleteUser(username); - deleteUserResponse.prettyPrint(); - deleteUserResponse.then().assertThat() - .statusCode(OK.getStatusCode()); + // Delete the dataset + Response deleteDatasetResponse = UtilIT.deleteDatasetViaNativeApi(datasetId, apiToken); + deleteDatasetResponse.prettyPrint(); + deleteDatasetResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + + Response makeSuperUser = UtilIT.setSuperuserStatus(username, true); + assertEquals(200, makeSuperUser.getStatusCode()); + + // Delete the dataverse + Response deleteDataverseResponse = UtilIT.deleteDataverse(dataverseAlias, apiToken); + deleteDataverseResponse.prettyPrint(); + deleteDataverseResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + + // Delete the user + Response deleteUserResponse = UtilIT.deleteUser(username); + deleteUserResponse.prettyPrint(); + deleteUserResponse.then().assertThat() + .statusCode(OK.getStatusCode()); } } From c549bf56da994ad24f17548ec75fba8066381b0c Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 1 Sep 2025 11:34:23 -0400 Subject: [PATCH 10/12] allow delete of published dataset --- src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java index af44a3e1a48..ef1f72b9d78 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java @@ -2391,16 +2391,17 @@ public void testFileAddedAfterPublicationIsIndexed() { .statusCode(OK.getStatusCode()); // Clean up - delete dataset, dataverse, and user - + + //Superuser to delete published dataset + Response makeSuperUser = UtilIT.setSuperuserStatus(username, true); + assertEquals(200, makeSuperUser.getStatusCode()); + // Delete the dataset Response deleteDatasetResponse = UtilIT.deleteDatasetViaNativeApi(datasetId, apiToken); deleteDatasetResponse.prettyPrint(); deleteDatasetResponse.then().assertThat() .statusCode(OK.getStatusCode()); - Response makeSuperUser = UtilIT.setSuperuserStatus(username, true); - assertEquals(200, makeSuperUser.getStatusCode()); - // Delete the dataverse Response deleteDataverseResponse = UtilIT.deleteDataverse(dataverseAlias, apiToken); deleteDataverseResponse.prettyPrint(); From df396e317a8b96082c2c6830fd5ea3bbb377254a Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 2 Sep 2025 10:26:37 -0400 Subject: [PATCH 11/12] try sleep --- src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java index ef1f72b9d78..bd084461398 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java @@ -2402,6 +2402,12 @@ public void testFileAddedAfterPublicationIsIndexed() { deleteDatasetResponse.then().assertThat() .statusCode(OK.getStatusCode()); + try { + // give the bag time to generate + Thread.sleep(3000); + } catch (InterruptedException ex) { + } + // Delete the dataverse Response deleteDataverseResponse = UtilIT.deleteDataverse(dataverseAlias, apiToken); deleteDataverseResponse.prettyPrint(); From ec3c2742fa618cf2c046d719f8bfd33519571ae0 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 2 Sep 2025 11:20:19 -0400 Subject: [PATCH 12/12] use destroy since we have 2 versions --- src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java index bd084461398..ca9e19e1bbf 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java @@ -2397,7 +2397,7 @@ public void testFileAddedAfterPublicationIsIndexed() { assertEquals(200, makeSuperUser.getStatusCode()); // Delete the dataset - Response deleteDatasetResponse = UtilIT.deleteDatasetViaNativeApi(datasetId, apiToken); + Response deleteDatasetResponse = UtilIT.destroyDataset(datasetId, apiToken); deleteDatasetResponse.prettyPrint(); deleteDatasetResponse.then().assertThat() .statusCode(OK.getStatusCode());