Skip to content

Commit 1ad26c3

Browse files
bubriksbubriks
committed
[FSTORE-1406] Delete Hive table only if it exists when deleting feature group (#1796)
* init * fix null exception if path doesnt exist * add test * fix getSchema * minor improvement
1 parent 2e9f2a1 commit 1ad26c3

File tree

3 files changed

+53
-17
lines changed

3 files changed

+53
-17
lines changed

‎hopsworks-IT/src/test/ruby/spec/featuregroup_spec.rb

+17
Original file line numberDiff line numberDiff line change
@@ -384,6 +384,23 @@
384384
expect_status_details(200)
385385
end
386386

387+
it "should be able to delete a cached featuregroup from the featurestore if hive table was already deleted" do
388+
project = get_project
389+
featurestore_id = get_featurestore_id(project.id)
390+
json_result, featuregroup_name = create_cached_featuregroup(project.id, featurestore_id)
391+
parsed_json = JSON.parse(json_result)
392+
expect_status_details(201)
393+
394+
# delete feature group offlinefs directory
395+
delete "#{ENV['HOPSWORKS_API']}/project/#{project.id.to_s}/dataset//apps/hive/warehouse/#{project.projectname.downcase}_featurestore.db/#{parsed_json["name"]}_#{parsed_json["version"]}"
396+
expect_status_details(204)
397+
398+
featuregroup_id = parsed_json["id"]
399+
delete_featuregroup_endpoint = "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/featuregroups/" + featuregroup_id.to_s
400+
delete delete_featuregroup_endpoint
401+
expect_status_details(200)
402+
end
403+
387404
it "should be able to clear the contents of a cached featuregroup in the featurestore" do
388405
project = get_project
389406
featurestore_id = get_featurestore_id(project.id)

‎hopsworks-common/src/main/java/io/hops/hopsworks/common/commands/featurestore/search/SearchFSCommandLogger.java

+22-16
Original file line numberDiff line numberDiff line change
@@ -64,82 +64,82 @@ public long count() {
6464

6565
public void create(Featuregroup featureGroup) throws FeaturestoreException {
6666
SearchFSCommand command = getCommand(featureGroup, SearchFSCommandOp.CREATE);
67-
commandFacade.persistAndFlush(command);
67+
persistCommand(command);
6868
}
6969

7070
public void create(FeatureView featureView) throws FeaturestoreException {
7171
SearchFSCommand command = getCommand(featureView, SearchFSCommandOp.CREATE);
72-
commandFacade.persistAndFlush(command);
72+
persistCommand(command);
7373
}
7474

7575
public void create(TrainingDataset trainingDataset) throws FeaturestoreException {
7676
SearchFSCommand command = getCommand(trainingDataset, SearchFSCommandOp.CREATE);
77-
commandFacade.persistAndFlush(command);
77+
persistCommand(command);
7878
}
7979

8080
public void updateMetadata(Featuregroup featureGroup) throws FeaturestoreException {
8181
SearchFSCommand command = getCommand(featureGroup, SearchFSCommandOp.UPDATE_METADATA);
82-
commandFacade.persistAndFlush(command);
82+
persistCommand(command);
8383
}
8484

8585
public void updateMetadata(FeatureView featureView) throws FeaturestoreException {
8686
SearchFSCommand command = getCommand(featureView, SearchFSCommandOp.UPDATE_METADATA);
87-
commandFacade.persistAndFlush(command);
87+
persistCommand(command);
8888
}
8989

9090
public void updateMetadata(TrainingDataset trainingDataset) throws FeaturestoreException {
9191
SearchFSCommand command = getCommand(trainingDataset, SearchFSCommandOp.UPDATE_METADATA);
92-
commandFacade.persistAndFlush(command);
92+
persistCommand(command);
9393
}
9494

9595
public void updateKeywords(Featuregroup featureGroup) throws FeaturestoreException {
9696
SearchFSCommand command = getCommand(featureGroup, SearchFSCommandOp.UPDATE_KEYWORDS);
97-
commandFacade.persistAndFlush(command);
97+
persistCommand(command);
9898
}
9999

100100
public void updateKeywords(FeatureView featureView) throws FeaturestoreException {
101101
SearchFSCommand command = getCommand(featureView, SearchFSCommandOp.UPDATE_KEYWORDS);
102-
commandFacade.persistAndFlush(command);
102+
persistCommand(command);
103103
}
104104

105105
public void updateKeywords(TrainingDataset trainingDataset) throws FeaturestoreException {
106106
SearchFSCommand command = getCommand(trainingDataset, SearchFSCommandOp.UPDATE_KEYWORDS);
107-
commandFacade.persistAndFlush(command);
107+
persistCommand(command);
108108
}
109109

110110
public void updateTags(Featuregroup featureGroup) throws FeaturestoreException {
111111
SearchFSCommand command = getCommand(featureGroup, SearchFSCommandOp.UPDATE_TAGS);
112-
commandFacade.persistAndFlush(command);
112+
persistCommand(command);
113113
}
114114

115115
public void updateTags(FeatureView featureView) throws FeaturestoreException {
116116
SearchFSCommand command = getCommand(featureView, SearchFSCommandOp.UPDATE_TAGS);
117-
commandFacade.persistAndFlush(command);
117+
persistCommand(command);
118118
}
119119

120120
public void updateTags(TrainingDataset trainingDataset) throws FeaturestoreException {
121121
SearchFSCommand command = getCommand(trainingDataset, SearchFSCommandOp.UPDATE_TAGS);
122-
commandFacade.persistAndFlush(command);
122+
persistCommand(command);
123123
}
124124

125125
public void delete(Featuregroup featuregroup) throws FeaturestoreException {
126126
SearchFSCommand command = getCommand(featuregroup, SearchFSCommandOp.DELETE_ARTIFACT);
127-
commandFacade.persistAndFlush(command);
127+
persistCommand(command);
128128
}
129129

130130
public void delete(FeatureView featureView) throws FeaturestoreException {
131131
SearchFSCommand command = getCommand(featureView, SearchFSCommandOp.DELETE_ARTIFACT);
132-
commandFacade.persistAndFlush(command);
132+
persistCommand(command);
133133
}
134134

135135
public void delete(TrainingDataset trainingDataset) throws FeaturestoreException {
136136
SearchFSCommand command = getCommand(trainingDataset, SearchFSCommandOp.DELETE_ARTIFACT);
137-
commandFacade.persistAndFlush(command);
137+
persistCommand(command);
138138
}
139139

140140
public void delete(Project project) {
141141
SearchFSCommand command = getCommand(project, SearchFSCommandOp.DELETE_PROJECT);
142-
commandFacade.persistAndFlush(command);
142+
persistCommand(command);
143143
}
144144

145145
private SearchFSCommand getCommand(Featuregroup featureGroup, SearchFSCommandOp op) throws FeaturestoreException {
@@ -207,4 +207,10 @@ private SearchFSCommand getCommand(Project p, SearchFSCommandOp op) {
207207
command.setOp(op);
208208
return command;
209209
}
210+
211+
private void persistCommand(SearchFSCommand command) {
212+
if (command != null) {
213+
commandFacade.persistAndFlush(command);
214+
}
215+
}
210216
}

‎hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/cached/OfflineFeatureGroupController.java

+14-1
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
import org.apache.hadoop.hive.metastore.api.AddDefaultConstraintRequest;
3434
import org.apache.hadoop.hive.metastore.api.DefaultConstraintsRequest;
3535
import org.apache.hadoop.hive.metastore.api.FieldSchema;
36+
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
3637
import org.apache.hadoop.hive.metastore.api.SQLDefaultConstraint;
3738
import org.apache.hadoop.hive.metastore.api.SerDeInfo;
3839
import org.apache.hadoop.hive.metastore.api.SkewedInfo;
@@ -182,6 +183,8 @@ public void alterHiveTableFeatures(Featurestore featurestore, String tableName,
182183

183184
public List<FeatureGroupFeatureDTO> getSchema(Featurestore featurestore, String tableName,
184185
Project project, Users user) throws FeaturestoreException {
186+
List<FeatureGroupFeatureDTO> featureSchema = new ArrayList<>();
187+
185188
String dbName = featurestoreController.getOfflineFeaturestoreDbName(featurestore.getProject());
186189
ThriftHiveMetastore.Client client = getMetaStoreClient(project, user);
187190
Table table;
@@ -191,14 +194,22 @@ public List<FeatureGroupFeatureDTO> getSchema(Featurestore featurestore, String
191194
table = getTable(client, dbName, tableName);
192195
schema = getFields(client, dbName, tableName);
193196
defaultConstraints = getDefaultConstraints(client, "hive", dbName, tableName);
197+
} catch (FeaturestoreException e) {
198+
if (e.getCause() instanceof NoSuchObjectException) {
199+
// If the hive table doesn't exist return the feature group without features
200+
// (Otherwise the user would not be able to get/delete the broken feature group in the UI/HSFS,
201+
// since he would receive 500 responses to GET requests)
202+
LOGGER.log(Level.SEVERE, "Feature group Hive table does not exist", e);
203+
return featureSchema;
204+
}
205+
throw e;
194206
} finally {
195207
hiveController.finalizeMetastoreOperation(project, user, client);
196208
}
197209

198210
// Setup a map of constraint values for easy access
199211
Map<String, String> defaultConstraintsMap = defaultConstraints.stream()
200212
.collect(Collectors.toMap(SQLDefaultConstraint::getColumn_name, SQLDefaultConstraint::getDefault_value));
201-
List<FeatureGroupFeatureDTO> featureSchema = new ArrayList<>();
202213

203214
// Partitions are listed first
204215
for (FieldSchema fieldSchema : table.getPartitionKeys()) {
@@ -295,6 +306,8 @@ public void dropFeatureGroup(String dbName, String tableName, Project project, U
295306
try {
296307
client = hiveController.openUserMetastoreClient(project, user);
297308
client.drop_table(dbName, tableName, true);
309+
} catch (NoSuchObjectException e) {
310+
LOGGER.log(Level.INFO, "Hive table being deleted does not exist", e);
298311
} catch (TException e) {
299312
throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.COULD_NOT_DELETE_FEATUREGROUP, Level.SEVERE,
300313
"Error dropping feature group in the Hive Metastore: " + e.getMessage(), e.getMessage(), e);

0 commit comments

Comments
 (0)