Skip to content

Commit d7126e6

Browse files
SirOibafSirOibaf
authoredJun 13, 2023
[HWORKS-588] Remove inode foreign key from external feature groups (#1441)
1 parent 52b9f86 commit d7126e6

File tree

7 files changed

+37
-59
lines changed

7 files changed

+37
-59
lines changed
 

‎featurestore-ee/src/main/java/io/hops/hopsworks/featurestore/tags/FeatureStoreTagsEEController.java

+6-6
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ public class FeatureStoreTagsEEController implements FeatureStoreTagControllerIf
5858
@Override
5959
public Map<String, String> getAll(Project accessProject, Users user, Featurestore featureStore,
6060
Featuregroup featureGroup)
61-
throws DatasetException, MetadataException {
61+
throws DatasetException, MetadataException, FeaturestoreException {
6262
String path = featuregroupController.getFeatureGroupLocation(featureGroup);
6363
return tagsController.getAll(accessProject, user, path);
6464
}
@@ -122,7 +122,7 @@ public Map<String, String> getAll(Project accessProject, Users user, Featurestor
122122
@Override
123123
public String get(Project accessProject, Users user, Featurestore featureStore, Featuregroup featureGroup,
124124
String name)
125-
throws DatasetException, MetadataException, SchematizedTagException {
125+
throws DatasetException, MetadataException, SchematizedTagException, FeaturestoreException {
126126

127127
String path = featuregroupController.getFeatureGroupLocation(featureGroup);
128128
return tagsController.get(accessProject, user, path, name);
@@ -188,7 +188,7 @@ public String get(Project accessProject, Users user, Featurestore featureStore,
188188
@Override
189189
public AttachTagResult upsert(Project project, Users user, Featurestore featureStore, Featuregroup featureGroup,
190190
String name, String value)
191-
throws MetadataException, SchematizedTagException {
191+
throws MetadataException, SchematizedTagException, FeaturestoreException {
192192

193193
String path = featuregroupController.getFeatureGroupLocation(featureGroup);
194194
return tagsController.upsert(project, user, path, name, value);
@@ -254,7 +254,7 @@ public AttachTagResult upsert(Project project, Users user, Featurestore featureS
254254
@Override
255255
public AttachTagResult upsert(Project project, Users user, Featurestore featureStore, Featuregroup featureGroup,
256256
Map<String, String> newTags)
257-
throws MetadataException, SchematizedTagException {
257+
throws MetadataException, SchematizedTagException, FeaturestoreException {
258258

259259
String path = featuregroupController.getFeatureGroupLocation(featureGroup);
260260
return tagsController.upsert(project, user, path, newTags);
@@ -315,7 +315,7 @@ public AttachTagResult upsert(Project project, Users user, Featurestore featureS
315315
*/
316316
@Override
317317
public void deleteAll(Project accessProject, Users user, Featurestore featureStore, Featuregroup featureGroup)
318-
throws MetadataException, DatasetException {
318+
throws MetadataException, DatasetException, FeaturestoreException {
319319

320320
String path = featuregroupController.getFeatureGroupLocation(featureGroup);
321321
tagsController.deleteAll(accessProject, user, path);
@@ -374,7 +374,7 @@ public void deleteAll(Project accessProject, Users user, Featurestore featureSto
374374
@Override
375375
public void delete(Project accessProject, Users user, Featurestore featureStore, Featuregroup featureGroup,
376376
String name)
377-
throws MetadataException, DatasetException {
377+
throws MetadataException, DatasetException, FeaturestoreException {
378378

379379
String path = featuregroupController.getFeatureGroupLocation(featureGroup);
380380
tagsController.delete(accessProject, user, path, name);

‎hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuregroup/FeatureGroupTagResource.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import io.hops.hopsworks.common.dataset.util.DatasetPath;
2424
import io.hops.hopsworks.common.featurestore.featuregroup.FeaturegroupController;
2525
import io.hops.hopsworks.exceptions.DatasetException;
26+
import io.hops.hopsworks.exceptions.FeaturestoreException;
2627
import io.hops.hopsworks.persistence.entity.dataset.DatasetType;
2728
import io.hops.hopsworks.persistence.entity.featurestore.featuregroup.Featuregroup;
2829

@@ -54,7 +55,7 @@ public void setFeatureGroup(Featuregroup featureGroup) {
5455
}
5556

5657
@Override
57-
protected DatasetPath getDatasetPath() throws DatasetException {
58+
protected DatasetPath getDatasetPath() throws DatasetException, FeaturestoreException {
5859
return datasetHelper.getDatasetPath(project, featuregroupController.getFeatureGroupLocation(featureGroup),
5960
DatasetType.FEATURESTORE);
6061
}

‎hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/tag/FeatureStoreTagResource.java

-1
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,6 @@ public Response putTag(@Context SecurityContext sc,
119119
@ApiParam(value = "Name of the tag", required = true) @PathParam("name") String name,
120120
@ApiParam(value = "Value to set for the tag") String value)
121121
throws MetadataException, SchematizedTagException, DatasetException, FeaturestoreException {
122-
123122
Users user = jwtHelper.getUserPrincipal(sc);
124123
AttachTagResult result = tagController.upsert(user, getDatasetPath(), name, value);
125124
FeatureStoreTagUri tagUri = new FeatureStoreTagUri(uriInfo, featureStore.getId(), getItemType(), getItemId());

‎hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/FeaturegroupController.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -847,7 +847,7 @@ public List<String> getFeatureNames(Featuregroup featuregroup, Project project,
847847
}
848848

849849

850-
public String getFeatureGroupLocation(Featuregroup featureGroup) {
850+
public String getFeatureGroupLocation(Featuregroup featureGroup) throws FeaturestoreException {
851851
// Cached feature groups also have a `location` field.
852852
// the issue is that the host is slightly different due to a configuration of Hive
853853
// so here we resolve only the path based on the indoe
@@ -856,7 +856,7 @@ public String getFeatureGroupLocation(Featuregroup featureGroup) {
856856
} else if (featureGroup.getFeaturegroupType() == FeaturegroupType.STREAM_FEATURE_GROUP) {
857857
return inodeController.getPath(featureGroup.getStreamFeatureGroup().getHiveTbls().getSdId().getInode());
858858
} else {
859-
return inodeController.getPath(featureGroup.getOnDemandFeaturegroup().getInode());
859+
return onDemandFeaturegroupController.getFeatureGroupLocation(featureGroup);
860860
}
861861
}
862862

‎hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/ondemand/OnDemandFeaturegroupController.java

+20-25
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,7 @@
2727
import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorDTO;
2828
import io.hops.hopsworks.common.hdfs.DistributedFileSystemOps;
2929
import io.hops.hopsworks.common.hdfs.DistributedFsService;
30-
import io.hops.hopsworks.common.hdfs.HdfsUsersController;
3130
import io.hops.hopsworks.common.hdfs.Utils;
32-
import io.hops.hopsworks.common.hdfs.inode.InodeController;
3331
import io.hops.hopsworks.exceptions.FeaturestoreException;
3432
import io.hops.hopsworks.exceptions.HopsSecurityException;
3533
import io.hops.hopsworks.exceptions.KafkaException;
@@ -45,7 +43,6 @@
4543
import io.hops.hopsworks.persistence.entity.featurestore.featuregroup.ondemand.OnDemandOption;
4644
import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.FeaturestoreConnector;
4745
import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.FeaturestoreConnectorType;
48-
import io.hops.hopsworks.persistence.entity.hdfs.inode.Inode;
4946
import io.hops.hopsworks.persistence.entity.project.Project;
5047
import io.hops.hopsworks.persistence.entity.user.Users;
5148
import io.hops.hopsworks.restutils.RESTCodes;
@@ -82,10 +79,6 @@ public class OnDemandFeaturegroupController {
8279
@EJB
8380
private DistributedFsService distributedFsService;
8481
@EJB
85-
private HdfsUsersController hdfsUsersController;
86-
@EJB
87-
private InodeController inodeController;
88-
@EJB
8982
private OnlineFeaturegroupController onlineFeatureGroupController;
9083
@EJB
9184
private FeatureGroupInputValidation featureGroupInputValidation;
@@ -132,13 +125,14 @@ public OnDemandFeaturegroup createOnDemandFeaturegroup(Featurestore featurestore
132125
"Data format required when specifying " + connector.getConnectorType() + " storage connectors");
133126
}
134127

128+
createFile(project, user, featurestore, onDemandFeaturegroupDTO);
129+
135130
//Persist on-demand featuregroup
136131
OnDemandFeaturegroup onDemandFeaturegroup = new OnDemandFeaturegroup();
137132
onDemandFeaturegroup.setDescription(onDemandFeaturegroupDTO.getDescription());
138133
onDemandFeaturegroup.setFeaturestoreConnector(connector);
139134
onDemandFeaturegroup.setQuery(onDemandFeaturegroupDTO.getQuery());
140135
onDemandFeaturegroup.setFeatures(convertOnDemandFeatures(onDemandFeaturegroupDTO, onDemandFeaturegroup));
141-
onDemandFeaturegroup.setInode(createFile(project, user, featurestore, onDemandFeaturegroupDTO));
142136
onDemandFeaturegroup.setDataFormat(onDemandFeaturegroupDTO.getDataFormat());
143137
onDemandFeaturegroup.setPath(onDemandFeaturegroupDTO.getPath());
144138

@@ -154,13 +148,15 @@ public OnDemandFeaturegroup createOnDemandFeaturegroup(Featurestore featurestore
154148
}
155149

156150
public OnDemandFeaturegroup createSpineGroup(Featurestore featurestore,
157-
OnDemandFeaturegroupDTO onDemandFeaturegroupDTO, Project project, Users user) throws FeaturestoreException {
151+
OnDemandFeaturegroupDTO onDemandFeaturegroupDTO,
152+
Project project, Users user) throws FeaturestoreException {
153+
createFile(project, user, featurestore, onDemandFeaturegroupDTO);
154+
158155
OnDemandFeaturegroup onDemandFeaturegroup = new OnDemandFeaturegroup();
159156
onDemandFeaturegroup.setDescription(onDemandFeaturegroupDTO.getDescription());
160157
onDemandFeaturegroup.setFeatures(convertOnDemandFeatures(onDemandFeaturegroupDTO, onDemandFeaturegroup));
161158
onDemandFeaturegroup.setSpine(onDemandFeaturegroupDTO.getSpine());
162-
onDemandFeaturegroup.setInode(createFile(project, user, featurestore, onDemandFeaturegroupDTO));
163-
159+
164160
onDemandFeaturegroupFacade.persist(onDemandFeaturegroup);
165161
return onDemandFeaturegroup;
166162
}
@@ -224,8 +220,6 @@ public void updateOnDemandFeaturegroupMetadata(Project project, Users user, Feat
224220
}
225221
// finally merge in database
226222
onDemandFeaturegroupFacade.updateMetadata(onDemandFeaturegroup);
227-
228-
229223
}
230224

231225
private void updateOnDemandFeatures(OnDemandFeaturegroup onDemandFeaturegroup,
@@ -276,14 +270,11 @@ public void verifySchemaUnchangedAndValid(Collection<OnDemandFeature> previousSc
276270
*/
277271
public void removeOnDemandFeaturegroup(Featurestore featurestore, Featuregroup featuregroup,
278272
Project project, Users user) throws FeaturestoreException {
279-
String username = hdfsUsersController.getHdfsUserName(project, user);
280-
DistributedFileSystemOps udfso = null;
281-
282-
// this is here for old feature groups that don't have a file
283273
onDemandFeaturegroupFacade.remove(featuregroup.getOnDemandFeaturegroup());
284274

275+
DistributedFileSystemOps udfso = null;
285276
try {
286-
udfso = distributedFsService.getDfsOps(username);
277+
udfso = distributedFsService.getDfsOps(project, user);
287278
udfso.rm(getFilePath(featurestore, featuregroup.getName(), featuregroup.getVersion()), false);
288279
} catch (IOException | URISyntaxException e) {
289280
throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.COULD_NOT_DELETE_ON_DEMAND_FEATUREGROUP,
@@ -307,25 +298,29 @@ private List<OnDemandFeature> convertOnDemandFeatures(OnDemandFeaturegroupDTO on
307298
return features;
308299
}
309300

310-
private Inode createFile(Project project, Users user, Featurestore featurestore,
301+
private void createFile(Project project, Users user, Featurestore featurestore,
311302
OnDemandFeaturegroupDTO onDemandFeaturegroupDTO) throws FeaturestoreException {
312-
String username = hdfsUsersController.getHdfsUserName(project, user);
313-
314-
Path path = null;
315303
DistributedFileSystemOps udfso = null;
316304
try {
317-
path = getFilePath(featurestore, onDemandFeaturegroupDTO.getName(), onDemandFeaturegroupDTO.getVersion());
305+
Path path = getFilePath(featurestore, onDemandFeaturegroupDTO.getName(), onDemandFeaturegroupDTO.getVersion());
318306

319-
udfso = distributedFsService.getDfsOps(username);
307+
udfso = distributedFsService.getDfsOps(project, user);
320308
udfso.touchz(path);
321309
} catch (IOException | URISyntaxException e) {
322310
throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.COULD_NOT_CREATE_ON_DEMAND_FEATUREGROUP,
323311
Level.SEVERE, "Error creating the placeholder file", e.getMessage(), e);
324312
} finally {
325313
distributedFsService.closeDfsClient(udfso);
326314
}
315+
}
327316

328-
return inodeController.getInodeAtPath(path.toString());
317+
public String getFeatureGroupLocation(Featuregroup featuregroup) throws FeaturestoreException {
318+
try {
319+
return getFilePath(featuregroup.getFeaturestore(), featuregroup.getName(), featuregroup.getVersion()).toString();
320+
} catch (URISyntaxException e) {
321+
throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.COULD_NOT_GET_FEATURE_GROUP_METADATA,
322+
Level.SEVERE, "", e.getMessage(), e);
323+
}
329324
}
330325

331326
private Path getFilePath(Featurestore featurestore, String name, Integer version) throws URISyntaxException {

‎hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/tag/FeatureStoreTagControllerIface.java

+6-6
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@
3333
public interface FeatureStoreTagControllerIface {
3434

3535
Map<String, String> getAll(Project project, Users user, Featurestore featureStore, Featuregroup featureGroup)
36-
throws DatasetException, MetadataException;
36+
throws DatasetException, MetadataException, FeaturestoreException;
3737

3838
Map<String, String> getAll(Project project, Users user, Featurestore featureStore, TrainingDataset trainingDataset)
3939
throws DatasetException, MetadataException, SchematizedTagException;
@@ -42,7 +42,7 @@ Map<String, String> getAll(Project project, Users user, Featurestore featureStor
4242
throws DatasetException, MetadataException, SchematizedTagException, FeaturestoreException;
4343

4444
String get(Project project, Users user, Featurestore featureStore, Featuregroup featureGroup, String name)
45-
throws DatasetException, MetadataException, SchematizedTagException;
45+
throws DatasetException, MetadataException, SchematizedTagException, FeaturestoreException;
4646

4747
String get(Project project, Users user, Featurestore featureStore, TrainingDataset trainingDataset, String name)
4848
throws DatasetException, MetadataException, SchematizedTagException;
@@ -52,7 +52,7 @@ String get(Project project, Users user, Featurestore featureStore, FeatureView f
5252

5353
AttachTagResult upsert(Project project, Users user, Featurestore featureStore, Featuregroup featureGroup,
5454
String name, String value)
55-
throws MetadataException, SchematizedTagException;
55+
throws MetadataException, SchematizedTagException, FeaturestoreException;
5656

5757
AttachTagResult upsert(Project project, Users user, Featurestore featureStore, TrainingDataset trainingDataset,
5858
String name, String value)
@@ -64,7 +64,7 @@ AttachTagResult upsert(Project project, Users user, Featurestore featureStore, F
6464

6565
AttachTagResult upsert(Project project, Users user, Featurestore featureStore, Featuregroup featureGroup,
6666
Map<String, String> tags)
67-
throws MetadataException, SchematizedTagException;
67+
throws MetadataException, SchematizedTagException, FeaturestoreException;
6868

6969
AttachTagResult upsert(Project project, Users user, Featurestore featureStore, TrainingDataset trainingDataset,
7070
Map<String, String> tags)
@@ -75,7 +75,7 @@ AttachTagResult upsert(Project project, Users user, Featurestore featureStore, F
7575
throws MetadataException, SchematizedTagException, FeaturestoreException;
7676

7777
void deleteAll(Project project, Users user, Featurestore featureStore, Featuregroup featureGroup)
78-
throws DatasetException, MetadataException;
78+
throws DatasetException, MetadataException, FeaturestoreException;
7979

8080
void deleteAll(Project project, Users user, Featurestore featureStore, TrainingDataset trainingDataset)
8181
throws DatasetException, MetadataException, SchematizedTagException;
@@ -84,7 +84,7 @@ void deleteAll(Project project, Users user, Featurestore featureStore, FeatureVi
8484
throws DatasetException, MetadataException, SchematizedTagException, FeaturestoreException;
8585

8686
void delete(Project project, Users user, Featurestore featureStore, Featuregroup featureGroup, String name)
87-
throws DatasetException, MetadataException;
87+
throws DatasetException, MetadataException, FeaturestoreException;
8888

8989
void delete(Project project, Users user, Featurestore featureStore, TrainingDataset trainingDataset, String name)
9090
throws DatasetException, MetadataException, SchematizedTagException;

‎hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/featuregroup/ondemand/OnDemandFeaturegroup.java

+1-18
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717
package io.hops.hopsworks.persistence.entity.featurestore.featuregroup.ondemand;
1818

1919
import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.FeaturestoreConnector;
20-
import io.hops.hopsworks.persistence.entity.hdfs.inode.Inode;
2120

2221
import javax.persistence.Basic;
2322
import javax.persistence.CascadeType;
@@ -29,11 +28,9 @@
2928
import javax.persistence.GenerationType;
3029
import javax.persistence.Id;
3130
import javax.persistence.JoinColumn;
32-
import javax.persistence.JoinColumns;
3331
import javax.persistence.NamedQueries;
3432
import javax.persistence.NamedQuery;
3533
import javax.persistence.OneToMany;
36-
import javax.persistence.OneToOne;
3734
import javax.persistence.Table;
3835
import javax.xml.bind.annotation.XmlRootElement;
3936
import java.io.Serializable;
@@ -78,13 +75,7 @@ public class OnDemandFeaturegroup implements Serializable {
7875

7976
@Column(name = "spine")
8077
private boolean spine;
81-
@JoinColumns({
82-
@JoinColumn(name = "inode_pid", referencedColumnName = "parent_id"),
83-
@JoinColumn(name = "inode_name", referencedColumnName = "name"),
84-
@JoinColumn(name = "partition_id", referencedColumnName = "partition_id")})
85-
@OneToOne(optional = false)
86-
private Inode inode;
87-
78+
8879
public static long getSerialVersionUID() {
8980
return serialVersionUID;
9081
}
@@ -125,14 +116,6 @@ public Integer getId() {
125116
return id;
126117
}
127118

128-
public Inode getInode() {
129-
return inode;
130-
}
131-
132-
public void setInode(Inode inode) {
133-
this.inode = inode;
134-
}
135-
136119
public OnDemandDataFormat getDataFormat() {
137120
return dataFormat;
138121
}

0 commit comments

Comments
 (0)