Merge pull request #184 from kit-data-manager/fix-49-data-entities-without-hasPart

Avoid adding dataEntities which are not being added to the root data entity

Author: Pfeil

src/main/java/edu/kit/datamanager/ro_crate/Crate.java

@@ -24,7 +24,7 @@ public interface Crate {
   /**
    * Read version from the crate descriptor and return it as a class
    * representation.
-   * 
+   * <p>
    * NOTE: If there is no version in the crate, it does not comply with the
    * specification.
    * 
@@ -36,9 +36,9 @@ public interface Crate {
   /**
    * Returns strings indicating the conformance of a crate with other
    * specifications than the RO-Crate version.
-   * 
+   * <p>
    * If you need the crate version too, refer to {@link #getVersion()}.
-   * 
+   * <p>
    * This corresponds technically to all conformsTo values, excluding the RO crate
    * version / specification.
    * 
@@ -71,7 +71,12 @@ public interface Crate {
 
   AbstractEntity getEntityById(java.lang.String id);
 
-  void addDataEntity(DataEntity entity, Boolean toHasPart);
+  /**
+   * Adds a data entity to the crate.
+   *
+   * @param entity the DataEntity to add to this crate.
+   */
+  void addDataEntity(DataEntity entity);
 
   void addContextualEntity(ContextualEntity entity);
 

src/main/java/edu/kit/datamanager/ro_crate/RoCrate.java

@@ -30,7 +30,7 @@
 
 /**
  * The class that represents a single ROCrate.
- *
+ * <p>
  * To build or modify it, use a instance of {@link RoCrateBuilder}. In the case
  * features of RO-Crate DRAFT specifications are needed, refer to
  * {@link BuilderWithDraftFeatures} and its documentation.
@@ -40,13 +40,13 @@
  */
 public class RoCrate implements Crate {
 
-    private final CratePayload roCratePayload;
-    private CrateMetadataContext metadataContext;
-    private CratePreview roCratePreview;
-    private RootDataEntity rootDataEntity;
-    private ContextualEntity jsonDescriptor;
+    protected final CratePayload roCratePayload;
+    protected CrateMetadataContext metadataContext;
+    protected CratePreview roCratePreview;
+    protected RootDataEntity rootDataEntity;
+    protected ContextualEntity jsonDescriptor;
 
-    private Collection<File> untrackedFiles;
+    protected Collection<File> untrackedFiles;
 
     @Override
     public CratePreview getPreview() {
@@ -181,13 +181,19 @@ public AbstractEntity getEntityById(String id) {
         return this.roCratePayload.getEntityById(id);
     }
 
+    /**
+     * {@inheritDoc}
+     * <p>
+     * Note: This will also link the DataEntity to the root node
+     * using the root nodes hasPart property.
+     *
+     * @param entity the DataEntity to add to this crate.
+     */
     @Override
-    public void addDataEntity(DataEntity entity, Boolean toHasPart) {
+    public void addDataEntity(DataEntity entity) {
         this.metadataContext.checkEntity(entity);
         this.roCratePayload.addDataEntity(entity);
-        if (Boolean.TRUE.equals(toHasPart)) {
-            this.rootDataEntity.addToHasPart(entity.getId());
-        }
+        this.rootDataEntity.addToHasPart(entity.getId());
     }
 
     @Override
@@ -327,10 +333,12 @@ public RoCrateBuilder addDescription(String description) {
         }
 
         /**
-         * Adding a data entity to the crate. The important part here is to also
-         * add its id to the RootData Entity hasPart.
+         * Adds a data entity to the crate.
+         * <p>
+         * Note: This will also link the DataEntity to the root node
+         * using the root nodes hasPart property.
          *
-         * @param dataEntity the DataEntity object.
+         * @param dataEntity the DataEntity to add to this crate.
          * @return returns the builder for further usage.
          */
         public RoCrateBuilder addDataEntity(DataEntity dataEntity) {
@@ -414,7 +422,7 @@ public RoCrateBuilder addUntrackedFile(File file) {
         }
 
         /**
-         * Returns a crate with the information from this builder.
+         * @return a crate with the information from this builder.
          */
         public RoCrate build() {
             return new RoCrate(this);
@@ -424,9 +432,9 @@ public RoCrate build() {
     /**
      * Builder for Crates, supporting features which are not in a final
      * specification yet.
-     *
+     * <p>
      * NOTE: This will change the specification version of your crate.
-     *
+     * <p>
      * We only add features we expect to be in the new specification in the end.
      * In case a feature will not make it into the specification, we will mark
      * it as deprecated and remove it in new major versions. If a feature is
@@ -467,7 +475,7 @@ public BuilderWithDraftFeatures(RoCrate crate) {
         /**
          * Indicate this crate also conforms to the given specification, in
          * addition to the version this builder adds.
-         *
+         * <p>
          * This is helpful for profiles or other specifications the crate
          * conforms to. Can be called multiple times to add more specifications.
          *

src/main/java/edu/kit/datamanager/ro_crate/externalproviders/dataentities/ImportFromZenodo.java

@@ -113,7 +113,7 @@ private static void addToCrateFromZotero(String url, Crate crate) {
         if (entity.get("@id").asText().equals(mainId)) {
           var dataEntity = new DataEntity.DataEntityBuilder()
               .setAll((ObjectNode) entity).build();
-          crate.addDataEntity(dataEntity, true);
+          crate.addDataEntity(dataEntity);
         } else {
           // here we have to think of a way to differentiate between data and contextual entities.
           var contextualEntity = new ContextualEntity.ContextualEntityBuilder()

src/main/java/edu/kit/datamanager/ro_crate/reader/RoCrateReader.java

@@ -10,29 +10,40 @@
 import edu.kit.datamanager.ro_crate.entities.contextual.ContextualEntity;
 import edu.kit.datamanager.ro_crate.entities.data.DataEntity;
 import edu.kit.datamanager.ro_crate.entities.data.RootDataEntity;
+import edu.kit.datamanager.ro_crate.special.IdentifierUtils;
 import edu.kit.datamanager.ro_crate.special.JsonUtilFunctions;
-import static edu.kit.datamanager.ro_crate.special.IdentifierUtils.decode;
-import static edu.kit.datamanager.ro_crate.special.IdentifierUtils.isUrl;
 
 import edu.kit.datamanager.ro_crate.validation.JsonSchemaValidation;
 import edu.kit.datamanager.ro_crate.validation.Validator;
-import org.apache.commons.io.FileUtils;
 
 import java.io.File;
-import java.nio.file.Path;
 import java.util.*;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 import java.util.stream.StreamSupport;
 
 /**
  * This class allows reading crates from the outside into the library in order
  * to inspect or modify it.
- * 
+ * <p>
  * The class takes a strategy to support different ways of importing the crates.
  * (from zip, folder, etc.)
  */
 public class RoCrateReader {
 
+  /**
+   * This is a private inner class that shall not be exposed.
+   * **Do not make it public or protected.** It serves only the
+   * purpose of unsafe operations while reading a crate and
+   * may be specific to this implementation.
+   */
+  private static class RoCrateUnsafe extends RoCrate {
+    public void addDataEntityWithoutRootHasPart(DataEntity entity) {
+      this.metadataContext.checkEntity(entity);
+      this.roCratePayload.addDataEntity(entity);
+    }
+  }
+
   /**
    * If the number of JSON entities in the crate is larger than this number,
    * parallelization will be used.
@@ -80,34 +91,37 @@ public RoCrate readCrate(String location) {
     JsonNode context = metadataJson.get(PROP_CONTEXT);
 
     CrateMetadataContext crateContext = new RoCrateMetadataContext(context);
-    RoCrate crate = new RoCrate();
+    RoCrateUnsafe crate = new RoCrateUnsafe();
     crate.setMetadataContext(crateContext);
     JsonNode graph = metadataJson.get(PROP_GRAPH);
 
     if (graph.isArray()) {
-
       moveRootEntitiesFromGraphToCrate(crate, (ArrayNode) graph);
-      for (JsonNode node : graph) {
-        // if the id is in the root hasPart list, we know this entity is a data entity
-        RootDataEntity root = crate.getRootDataEntity();
-        if (root != null && root.hasInHasPart(node.get(PROP_ID).asText())) {
-          // data entity
-          DataEntity.DataEntityBuilder dataEntity = new DataEntity.DataEntityBuilder()
-                  .setAll(node.deepCopy());
-
-          // Handle data entities with corresponding file
-          File loc = checkFolderHasFile(node.get(PROP_ID).asText(), files);
-          if (loc != null) {
-            usedFiles.add(loc.getPath());
-            dataEntity.setLocationWithExceptions(loc.toPath())
-                .setId(loc.getName());
-          }
+      RootDataEntity root = crate.getRootDataEntity();
+      if (root != null) {
+        Set<String> dataEntityIds = getDataEntityIds(root, graph);
+        for (JsonNode entityJson : graph) {
+          String eId = unpackId(entityJson);
+          if (dataEntityIds.contains(eId)) {
+            // data entity
+            DataEntity.DataEntityBuilder dataEntity = new DataEntity.DataEntityBuilder()
+                    .setAll(entityJson.deepCopy());
+
+            // Handle data entities with corresponding file
+            checkFolderHasFile(entityJson.get(PROP_ID).asText(), files).ifPresent(file -> {
+              usedFiles.add(file.getPath());
+              dataEntity.setLocationWithExceptions(file.toPath())
+                      .setId(file.getName());
+            });
 
-          crate.addDataEntity(dataEntity.build(), false);
-        } else {
-          // contextual entity
-          crate.addContextualEntity(
-              new ContextualEntity.ContextualEntityBuilder().setAll(node.deepCopy()).build());
+            crate.addDataEntityWithoutRootHasPart(dataEntity.build());
+          } else {
+            // contextual entity
+            crate.addContextualEntity(
+                    new ContextualEntity.ContextualEntityBuilder()
+                            .setAll(entityJson.deepCopy())
+                            .build());
+          }
         }
       }
     }
@@ -123,18 +137,79 @@ public RoCrate readCrate(String location) {
     return crate;
   }
 
-  protected File checkFolderHasFile(String id, File file) {
-    if (isUrl(id)) return null;
-    Path path = file.toPath().resolve(decode(id).get());
-    if (path.toFile().exists()) {
-      return path.toFile();
+  /**
+   * Extracts graph connections from top to bottom.
+   * <p>
+   * Example: (connections.get(parent) -> children)
+   *
+   * @param graph the ArrayNode with all Entities.
+   * @return the graph connections.
+   */
+  protected Map<String, Set<String>> makeEntityGraph(JsonNode graph) {
+    Map<String, Set<String>> connections = new HashMap<>();
+
+    Map<String, JsonNode> idToNodes = new HashMap<>();
+    StreamSupport.stream(graph.spliterator(), false)
+            .forEach(jsonNode -> idToNodes.put(unpackId(jsonNode), jsonNode));
+
+    for (JsonNode entityNode : graph) {
+      String currentId = unpackId(entityNode);
+      StreamSupport.stream(entityNode.path("hasPart").spliterator(), false)
+              .map(this::unpackId)
+              .map(s -> idToNodes.getOrDefault(s, null))
+              .filter(Objects::nonNull)
+              .forEach(child -> connections.computeIfAbsent(currentId, key -> new HashSet<>())
+                      .add(unpackId(child)));
+      StreamSupport.stream(entityNode.path("isPartOf").spliterator(), false)
+              .map(this::unpackId)
+              .map(s -> idToNodes.getOrDefault(s, null))
+              .filter(Objects::nonNull)
+              .forEach(parent -> connections.computeIfAbsent(unpackId(parent), key -> new HashSet<>())
+                      .add(currentId));
     }
-    return null;
+    return connections;
+  }
+
+  protected Set<String> getDataEntityIds(RootDataEntity root, JsonNode graph) {
+    if (root == null) { return Set.of(); }
+    Map<String, Set<String>> network = makeEntityGraph(graph);
+    Set<String> directDataEntities = new HashSet<>(root.hasPart);
+    return Stream.concat(
+            directDataEntities.stream(),
+            directDataEntities.stream().flatMap(entity -> getDataEntityIdsRecursive(entity, network))
+    ).collect(Collectors.toSet());
+  }
+
+  protected Stream<String> getDataEntityIdsRecursive(
+          String parent,
+          Map<String, Set<String>> network
+  ) {
+    return Stream.concat(
+            Stream.of(parent),
+            network.getOrDefault(parent, new HashSet<>()).stream()
+                    .flatMap(s -> getDataEntityIdsRecursive(s, network))
+                    .filter(Objects::nonNull)
+    );
+  }
+
+  protected String unpackId(JsonNode node) {
+    if (node.isTextual()) {
+      return node.asText();
+    } else /*if (node.isObject())*/ {
+      return node.path(PROP_ID).asText();
+    }
+  }
+
+  protected Optional<File> checkFolderHasFile(String filepathOrId, File folder) {
+    if (IdentifierUtils.isUrl(filepathOrId)) { return Optional.empty(); }
+    return IdentifierUtils.decode(filepathOrId)
+            .map(decoded -> folder.toPath().resolve(decoded).toFile())
+            .filter(File::exists);
   }
 
   /**
    * Moves the descriptor and the root entity from the graph to the crate.
-   * 
+   * <p>
    * Extracts the root data entity and the Metadata File Descriptor from the graph
    * and inserts them into the crate object. It also deletes it from the graph.
    * We will need the root dataset to distinguish between data entities and
@@ -170,7 +245,7 @@ protected void moveRootEntitiesFromGraphToCrate(RoCrate crate, ArrayNode graph)
 
   /**
    * Find the metadata descriptor.
-   * 
+   * <p>
    * Currently prefers algorithm of version 1.1 over the one of 1.2-DRAFT.
    * 
    * @param graph the graph to search the descriptor in.
@@ -200,10 +275,12 @@ protected Optional<JsonNode> getMetadataDescriptor(ArrayNode graph) {
   /**
    * Extracts the root entity from the graph, using the information from the
    * descriptor.
-   * 
+   * <p>
    * Basically implements step 5 of the algorithm described here:
-   * https://www.researchobject.org/ro-crate/1.1/root-data-entity.html#finding-the-root-data-entity
-   * 
+   * <a href="https://www.researchobject.org/ro-crate/1.1/root-data-entity.html#finding-the-root-data-entity">
+   *     https://www.researchobject.org/ro-crate/1.1/root-data-entity.html#finding-the-root-data-entity
+   * </a>
+   *
    * @param graph      the graph from the metadata JSON-LD file
    * @param descriptor the RO-Crate descriptor
    * @return the root entity, if found

src/main/java/edu/kit/datamanager/ro_crate/special/IdentifierUtils.java

@@ -16,7 +16,7 @@
 /**
  * This class defines methods regarding URIs in general, which in RO-Crate
  * context means usually a valid, resolvable URL or a relative file path.
- *
+ * <p>
  * The purpose is to have a simple abstraction where the way e.g. a URL is
  * checked can be changed and tested easily for the whole library.
  */
@@ -28,7 +28,7 @@ private IdentifierUtils() {}
     /**
      * Returns true, if the given String is encoded and can be used as an
      * identifier in RO-Crate.
-     * 
+     * <p>
      * Possible identifiers include:
      * - a uri
      * - a url
@@ -50,7 +50,9 @@ public static boolean isValidUri(String uri) {
      */
     public static boolean isUrl(String uri) {
         try {
-            return asUrl(encode(uri).get()).isPresent();
+            return encode(uri)
+                    .map(decodedUri -> asUrl(decodedUri).isPresent())
+                    .orElse(false);
         } catch (Exception e) {
             return false;
         }