Skip to content

Commit

Permalink
Merge pull request IQSS#11180 from vera/fix/dataset-sort-order
Browse files Browse the repository at this point in the history 
Fix sorting of dataset drafts and minor versions when sorting by "newest first"
  • Loading branch information
@ofahimIQSS
ofahimIQSS authored Feb 19, 2025
2 parents e226d86 + 172802b commit 2210d16
Show file tree
Hide file tree
Showing 3 changed files with 215 additions and 24 deletions.
8 changes: 8 additions & 0 deletions doc/release-notes/11178-bug-fix-sort-by-newest-first.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
### Bug fix: Sorting by "newest first"

Fixed an issue where draft versions of datasets were sorted using the release timestamp of their most recent major version.
This caused newer drafts to appear incorrectly alongside their corresponding major version, instead of at the top, when sorted by "newest first".
Sorting now uses the last update timestamp when sorting draft datasets.
The sorting behavior of published dataset versions (major and minor) is unchanged.

**Upgrade instructions**: draft datasets must be reindexed for this fix to take effect.
57 changes: 33 additions & 24 deletions src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -947,32 +947,36 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set<Long
solrInputDocument.addField(SearchFields.CATEGORY_OF_DATAVERSE, dvIndexableCategoryName);
solrInputDocument.addField(SearchFields.IDENTIFIER_OF_DATAVERSE, dvAlias);
solrInputDocument.addField(SearchFields.DATAVERSE_NAME, dvDisplayName);

Date datasetSortByDate = new Date();
Date majorVersionReleaseDate = dataset.getMostRecentMajorVersionReleaseDate();
if (majorVersionReleaseDate != null) {
if (true) {
String msg = "major release date found: " + majorVersionReleaseDate.toString();
logger.fine(msg);

Date datasetSortByDate;
// For now, drafts are indexed using to their last update time, and published versions are indexed using their
// most recent major version release date.
// This means that newly created or edited drafts will show up on the top when sorting by newest, newly
// published major versions will also show up on the top, and newly published minor versions will be shown
// next to their corresponding major version.
if (state.equals(DatasetState.WORKING_COPY)) {
Date lastUpdateTime = indexableDataset.getDatasetVersion().getLastUpdateTime();
if (lastUpdateTime != null) {
logger.fine("using last update time of indexed dataset version: " + lastUpdateTime);
datasetSortByDate = lastUpdateTime;
} else {
logger.fine("can't find last update time, using \"now\"");
datasetSortByDate = new Date();
}
datasetSortByDate = majorVersionReleaseDate;
} else {
if (indexableDataset.getDatasetState().equals(IndexableDataset.DatasetState.WORKING_COPY)) {
solrInputDocument.addField(SearchFields.PUBLICATION_STATUS, UNPUBLISHED_STRING);
} else if (indexableDataset.getDatasetState().equals(IndexableDataset.DatasetState.DEACCESSIONED)) {
solrInputDocument.addField(SearchFields.PUBLICATION_STATUS, DEACCESSIONED_STRING);
}
Date createDate = dataset.getCreateDate();
if (createDate != null) {
if (true) {
String msg = "can't find major release date, using create date: " + createDate;
logger.fine(msg);
}
datasetSortByDate = createDate;
Date majorVersionReleaseDate = dataset.getMostRecentMajorVersionReleaseDate();
if (majorVersionReleaseDate != null) {
logger.fine("major release date found: " + majorVersionReleaseDate.toString());
datasetSortByDate = majorVersionReleaseDate;
} else {
String msg = "can't find major release date or create date, using \"now\"";
logger.info(msg);
datasetSortByDate = new Date();
Date createDate = dataset.getCreateDate();
if (createDate != null) {
logger.fine("can't find major release date, using create date: " + createDate);
datasetSortByDate = createDate;
} else {
logger.fine("can't find major release date or create date, using \"now\"");
datasetSortByDate = new Date();
}
}
}
solrInputDocument.addField(SearchFields.RELEASE_OR_CREATE_DATE, datasetSortByDate);
Expand All @@ -985,7 +989,12 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set<Long
// solrInputDocument.addField(SearchFields.RELEASE_OR_CREATE_DATE,
// dataset.getPublicationDate());
} else if (state.equals(DatasetState.WORKING_COPY)) {
if (dataset.getReleasedVersion() == null) {
solrInputDocument.addField(SearchFields.PUBLICATION_STATUS, UNPUBLISHED_STRING);
}
solrInputDocument.addField(SearchFields.PUBLICATION_STATUS, DRAFT_STRING);
} else if (state.equals(IndexableDataset.DatasetState.DEACCESSIONED)) {
solrInputDocument.addField(SearchFields.PUBLICATION_STATUS, DEACCESSIONED_STRING);
}

addDatasetReleaseDateToSolrDoc(solrInputDocument, dataset);
Expand Down Expand Up @@ -1588,7 +1597,7 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set<Long
}
datafileSolrInputDocument.addField(SearchFields.RELEASE_OR_CREATE_DATE, fileSortByDate);

if (majorVersionReleaseDate == null && !datafile.isHarvested()) {
if (dataset.getReleasedVersion() == null && !datafile.isHarvested()) {
datafileSolrInputDocument.addField(SearchFields.PUBLICATION_STATUS, UNPUBLISHED_STRING);
}

Expand Down
174 changes: 174 additions & 0 deletions src/test/java/edu/harvard/iq/dataverse/api/DataRetrieverApiIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,180 @@ public void testRetrieveMyDataAsJsonString() {
assertEquals(200, deleteUserResponse.getStatusCode());
}

@Test
public void testRetrieveMyDataAsJsonStringSortOrder() {
// Create superuser
Response createSuperUserResponse = UtilIT.createRandomUser();
String superUserIdentifier = UtilIT.getUsernameFromResponse(createSuperUserResponse);
String superUserApiToken = UtilIT.getApiTokenFromResponse(createSuperUserResponse);
Response makeSuperUserResponse = UtilIT.setSuperuserStatus(superUserIdentifier, true);
assertEquals(OK.getStatusCode(), makeSuperUserResponse.getStatusCode());

// Create regular user
Response createUserResponse = UtilIT.createRandomUser();
String userApiToken = UtilIT.getApiTokenFromResponse(createUserResponse);
String userIdentifier = UtilIT.getUsernameFromResponse(createUserResponse);

// Call as regular user with no result
Response myDataEmptyResponse = UtilIT.retrieveMyDataAsJsonString(userApiToken, "", new ArrayList<>(Arrays.asList(6L)));
assertEquals(prettyPrintError("myDataFinder.error.result.role.empty", Arrays.asList("Contributor")), myDataEmptyResponse.prettyPrint());
assertEquals(OK.getStatusCode(), myDataEmptyResponse.getStatusCode());

// Create and publish a dataverse
Response createDataverseResponse = UtilIT.createRandomDataverse(superUserApiToken);
createDataverseResponse.prettyPrint();
String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
Response publishDataverse = UtilIT.publishDataverseViaNativeApi(dataverseAlias, superUserApiToken);
publishDataverse.then().assertThat().statusCode(OK.getStatusCode());

// Allow user to create datasets in dataverse
Response grantRole = UtilIT.grantRoleOnDataverse(dataverseAlias, DataverseRole.DS_CONTRIBUTOR, "@" + userIdentifier, superUserApiToken);
grantRole.prettyPrint();
assertEquals(OK.getStatusCode(), grantRole.getStatusCode());

// As user, create two datasets and submit them for review
Response createDatasetOneResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, userApiToken);
createDatasetOneResponse.prettyPrint();
Integer datasetOneId = UtilIT.getDatasetIdFromResponse(createDatasetOneResponse);
String datasetOnePid = UtilIT.getDatasetPersistentIdFromResponse(createDatasetOneResponse);
UtilIT.sleepForReindex(datasetOneId.toString(), userApiToken, 4);

Response createDatasetTwoResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, userApiToken);
createDatasetTwoResponse.prettyPrint();
Integer datasetTwoId = UtilIT.getDatasetIdFromResponse(createDatasetTwoResponse);
String datasetTwoPid = UtilIT.getDatasetPersistentIdFromResponse(createDatasetTwoResponse);
UtilIT.sleepForReindex(datasetTwoId.toString(), userApiToken, 4);

// Request datasets belonging to user
Response twoDatasetsInReviewResponse = UtilIT.retrieveMyDataAsJsonString(userApiToken, "", new ArrayList<>(Arrays.asList(6L)));
twoDatasetsInReviewResponse.prettyPrint();
assertEquals(OK.getStatusCode(), twoDatasetsInReviewResponse.getStatusCode());
JsonPath jsonPathTwoDatasetsInReview = twoDatasetsInReviewResponse.getBody().jsonPath();
assertEquals(2, jsonPathTwoDatasetsInReview.getInt("data.total_count"));
// Expect newest dataset (dataset 2) first
assertEquals(datasetTwoId, jsonPathTwoDatasetsInReview.getInt("data.items[0].entity_id"));
assertEquals("DRAFT", jsonPathTwoDatasetsInReview.getString("data.items[0].versionState"));
assertEquals(datasetOneId, jsonPathTwoDatasetsInReview.getInt("data.items[1].entity_id"));
assertEquals("DRAFT", jsonPathTwoDatasetsInReview.getString("data.items[1].versionState"));

// Publish dataset 1
Response publishDatasetOne = UtilIT.publishDatasetViaNativeApi(datasetOneId, "major", superUserApiToken);
publishDatasetOne.prettyPrint();
publishDatasetOne.then().assertThat().statusCode(OK.getStatusCode());
UtilIT.sleepForReindex(datasetOneId.toString(), userApiToken, 4);

// Publish dataset 2
Response publishDatasetTwo = UtilIT.publishDatasetViaNativeApi(datasetTwoId, "major", superUserApiToken);
publishDatasetTwo.prettyPrint();
publishDatasetTwo.then().assertThat().statusCode(OK.getStatusCode());
UtilIT.sleepForReindex(datasetTwoId.toString(), userApiToken, 4);

// Request datasets belonging to user
Response twoPublishedDatasetsResponse = UtilIT.retrieveMyDataAsJsonString(userApiToken, "", new ArrayList<>(Arrays.asList(6L)));
twoPublishedDatasetsResponse.prettyPrint();
assertEquals(OK.getStatusCode(), twoPublishedDatasetsResponse.getStatusCode());
JsonPath jsonPathTwoPublishedDatasets = twoPublishedDatasetsResponse.getBody().jsonPath();
assertEquals(2, jsonPathTwoPublishedDatasets.getInt("data.total_count"));
// Expect newest dataset (dataset 2) first
assertEquals(datasetTwoId, jsonPathTwoPublishedDatasets.getInt("data.items[0].entity_id"));
assertEquals("RELEASED", jsonPathTwoPublishedDatasets.getString("data.items[0].versionState"));
assertEquals(datasetOneId, jsonPathTwoPublishedDatasets.getInt("data.items[1].entity_id"));
assertEquals("RELEASED", jsonPathTwoPublishedDatasets.getString("data.items[1].versionState"));

// Create new draft version of dataset 1 by updating metadata
String pathToJsonFilePostPub= "doc/sphinx-guides/source/_static/api/dataset-add-metadata-after-pub.json";
Response addDataToPublishedVersion = UtilIT.addDatasetMetadataViaNative(datasetOnePid, pathToJsonFilePostPub, userApiToken);
addDataToPublishedVersion.prettyPrint();
addDataToPublishedVersion.then().assertThat().statusCode(OK.getStatusCode());
UtilIT.sleepForReindex(datasetOneId.toString(), userApiToken, 4);

// Request datasets belonging to user
Response twoPublishedDatasetsOneDraftResponse = UtilIT.retrieveMyDataAsJsonString(userApiToken, "", new ArrayList<>(Arrays.asList(6L)));
twoPublishedDatasetsOneDraftResponse.prettyPrint();
assertEquals(OK.getStatusCode(), twoPublishedDatasetsOneDraftResponse.getStatusCode());
JsonPath jsonPathTwoPublishedDatasetsOneDraft = twoPublishedDatasetsOneDraftResponse.getBody().jsonPath();
assertEquals(3, jsonPathTwoPublishedDatasetsOneDraft.getInt("data.total_count"));

// Expect newest dataset version (draft of dataset 1) first
assertEquals(datasetOneId, jsonPathTwoPublishedDatasetsOneDraft.getInt("data.items[0].entity_id"));
assertEquals("DRAFT", jsonPathTwoPublishedDatasetsOneDraft.getString("data.items[0].versionState"));
// ...followed by dataset 2 (created after dataset 1)
assertEquals(datasetTwoId, jsonPathTwoPublishedDatasetsOneDraft.getInt("data.items[1].entity_id"));
assertEquals("RELEASED", jsonPathTwoPublishedDatasetsOneDraft.getString("data.items[1].versionState"));
// ...followed by dataset 1 (oldest, created before dataset 2)
assertEquals(datasetOneId, jsonPathTwoPublishedDatasetsOneDraft.getInt("data.items[2].entity_id"));
assertEquals("RELEASED", jsonPathTwoPublishedDatasetsOneDraft.getString("data.items[2].versionState"));

// Create new draft version of dataset 2 by uploading a file
String pathToFile = "src/main/webapp/resources/images/dataverseproject.png";
Response uploadImage = UtilIT.uploadFileViaNative(datasetTwoId.toString(), pathToFile, userApiToken);
uploadImage.prettyPrint();
uploadImage.then().assertThat().statusCode(OK.getStatusCode());
UtilIT.sleepForReindex(datasetTwoId.toString(), userApiToken, 4);

// Request datasets belonging to user
Response twoPublishedDatasetsTwoDraftsResponse = UtilIT.retrieveMyDataAsJsonString(userApiToken, "", new ArrayList<>(Arrays.asList(6L)));
twoPublishedDatasetsTwoDraftsResponse.prettyPrint();
assertEquals(OK.getStatusCode(), twoPublishedDatasetsTwoDraftsResponse.getStatusCode());
JsonPath jsonPathTwoPublishedDatasetsTwoDrafts = twoPublishedDatasetsTwoDraftsResponse.getBody().jsonPath();
assertEquals(4, jsonPathTwoPublishedDatasetsTwoDrafts.getInt("data.total_count"));

// Expect newest dataset version (draft of dataset 2) first
assertEquals(datasetTwoId, jsonPathTwoPublishedDatasetsTwoDrafts.getInt("data.items[0].entity_id"));
assertEquals("DRAFT", jsonPathTwoPublishedDatasetsTwoDrafts.getString("data.items[0].versionState"));
assertEquals(datasetOneId, jsonPathTwoPublishedDatasetsTwoDrafts.getInt("data.items[1].entity_id"));
assertEquals("DRAFT", jsonPathTwoPublishedDatasetsTwoDrafts.getString("data.items[1].versionState"));
assertEquals(datasetTwoId, jsonPathTwoPublishedDatasetsTwoDrafts.getInt("data.items[2].entity_id"));
assertEquals("RELEASED", jsonPathTwoPublishedDatasetsTwoDrafts.getString("data.items[2].versionState"));
assertEquals(datasetOneId, jsonPathTwoPublishedDatasetsTwoDrafts.getInt("data.items[3].entity_id"));
assertEquals("RELEASED", jsonPathTwoPublishedDatasetsTwoDrafts.getString("data.items[3].versionState"));

// Publish minor version of dataset 1
Response publishDatasetOneMinor = UtilIT.publishDatasetViaNativeApi(datasetOneId, "minor", superUserApiToken);
publishDatasetOneMinor.prettyPrint();
publishDatasetOneMinor.then().assertThat().statusCode(OK.getStatusCode());
UtilIT.sleepForReindex(datasetOneId.toString(), userApiToken, 4);

// Request datasets belonging to user
Response oneMinorOneMajorOneDraftDatasetResponse = UtilIT.retrieveMyDataAsJsonString(userApiToken, "", new ArrayList<>(Arrays.asList(6L)));
oneMinorOneMajorOneDraftDatasetResponse.prettyPrint();
assertEquals(OK.getStatusCode(), oneMinorOneMajorOneDraftDatasetResponse.getStatusCode());
JsonPath jsonPathOneMinorOneMajorOneDraftDataset = oneMinorOneMajorOneDraftDatasetResponse.getBody().jsonPath();
assertEquals(3, jsonPathOneMinorOneMajorOneDraftDataset.getInt("data.total_count"));

// Expect minor version of dataset 1 to be sorted last (based on release date of major version)
assertEquals(datasetTwoId, jsonPathOneMinorOneMajorOneDraftDataset.getInt("data.items[0].entity_id"));
assertEquals("DRAFT", jsonPathOneMinorOneMajorOneDraftDataset.getString("data.items[0].versionState"));

assertEquals(datasetTwoId, jsonPathOneMinorOneMajorOneDraftDataset.getInt("data.items[1].entity_id"));
assertEquals("RELEASED", jsonPathOneMinorOneMajorOneDraftDataset.getString("data.items[1].versionState"));

assertEquals(datasetOneId, jsonPathOneMinorOneMajorOneDraftDataset.getInt("data.items[2].entity_id"));
assertEquals("RELEASED", jsonPathOneMinorOneMajorOneDraftDataset.getString("data.items[2].versionState"));
assertEquals(1, jsonPathOneMinorOneMajorOneDraftDataset.getInt("data.items[2].majorVersion"));
assertEquals(1, jsonPathOneMinorOneMajorOneDraftDataset.getInt("data.items[2].minorVersion"));

// Clean up
Response deleteDatasetOneResponse = UtilIT.destroyDataset(datasetOneId, superUserApiToken);
deleteDatasetOneResponse.prettyPrint();
assertEquals(OK.getStatusCode(), deleteDatasetOneResponse.getStatusCode());
Response deleteDatasetTwoResponse = UtilIT.destroyDataset(datasetTwoId, superUserApiToken);
deleteDatasetTwoResponse.prettyPrint();
assertEquals(OK.getStatusCode(), deleteDatasetTwoResponse.getStatusCode());

Response deleteDataverseResponse = UtilIT.deleteDataverse(dataverseAlias, superUserApiToken);
deleteDataverseResponse.prettyPrint();
assertEquals(OK.getStatusCode(), deleteDataverseResponse.getStatusCode());

Response deleteUserResponse = UtilIT.deleteUser(userIdentifier);
deleteUserResponse.prettyPrint();
assertEquals(OK.getStatusCode(), deleteUserResponse.getStatusCode());

Response deleteSuperUserResponse = UtilIT.deleteUser(superUserIdentifier);
deleteSuperUserResponse.prettyPrint();
assertEquals(OK.getStatusCode(), deleteSuperUserResponse.getStatusCode());
}

private static String prettyPrintError(String resourceBundleKey, List<String> params) {
final String errorMessage;
if (params == null || params.isEmpty()) {
Expand Down

0 comments on commit 2210d16

Please sign in to comment.