Skip to content

Commit b94f853

Browse files
committed
[FSTORE_612] Support for feature monitoring (#1692)
Co-authored-by: Robin Andersson @robzor92 Co-authored-by: Victor Jouffrey @vatj Co-authored-by: Kenneth Mak @kennethmhc Co-authored-by: Dhananjay Mukhedkar @dhananjay-mk Co-authored-by: Ehsan Heydari @ehsan-github
1 parent 8d1964d commit b94f853

File tree

109 files changed

+10112
-1184
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

109 files changed

+10112
-1184
lines changed

hopsworks-IT/src/test/ruby/spec/feature_monitoring_spec.rb

+1,026
Large diffs are not rendered by default.

hopsworks-IT/src/test/ruby/spec/feature_store_activity_spec.rb

+391-364
Large diffs are not rendered by default.

hopsworks-IT/src/test/ruby/spec/feature_group_alert_spec.rb hopsworks-IT/src/test/ruby/spec/feature_store_alert_spec.rb

+79-19
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,32 @@
1-
=begin
2-
This file is part of Hopsworks
3-
Copyright (C) 2021, Logical Clocks AB. All rights reserved
1+
# This file is part of Hopsworks
2+
# Copyright (C) 2024, Hopsworks AB. All rights reserved
3+
#
4+
# Hopsworks is free software: you can redistribute it and/or modify it under the terms of
5+
# the GNU Affero General Public License as published by the Free Software Foundation,
6+
# either version 3 of the License, or (at your option) any later version.
7+
#
8+
# Hopsworks is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9+
# without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
10+
# PURPOSE. See the GNU Affero General Public License for more details.
11+
#
12+
# You should have received a copy of the GNU Affero General Public License along with this program.
13+
# If not, see <https://www.gnu.org/licenses/>.
14+
#
415

5-
Hopsworks is free software: you can redistribute it and/or modify it under the terms of
6-
the GNU Affero General Public License as published by the Free Software Foundation,
7-
either version 3 of the License, or (at your option) any later version.
16+
describe "On #{ENV['OS']}" do
817

9-
Hopsworks is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
10-
without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
11-
PURPOSE. See the GNU Affero General Public License for more details.
18+
before :all do
19+
# ensure feature monitoring is enabled
20+
@enable_feature_monitoring = getVar('enable_feature_monitoring')
21+
setVar('enable_feature_monitoring', "true")
22+
end
1223

13-
You should have received a copy of the GNU Affero General Public License along with this program.
14-
If not, see <https://www.gnu.org/licenses/>.
15-
=end
24+
after :all do
25+
# revert feature monitoring flag
26+
setVar('enable_feature_monitoring', @enable_feature_monitoring[:value])
27+
clean_all_test_projects(spec: "fg_alert")
28+
end
1629

17-
describe "On #{ENV['OS']}" do
18-
after(:all) {clean_all_test_projects(spec: "fg_alert")}
1930
describe 'Alert' do
2031
context 'without authentication' do
2132
before :all do
@@ -31,17 +42,25 @@
3142
create_fg_alert(@project, @featuregroup, get_fg_alert_success(@project))
3243
expect_status_details(401)
3344
end
45+
it "should fail to create feature monitoring status" do
46+
create_fg_alert(@project, @featuregroup, get_fm_alert_success(@project))
47+
expect_status_details(401)
48+
end
3449
end
3550
context 'with authentication' do
3651
before :all do
3752
with_valid_project
3853
@featuregroup = with_valid_fg(@project)
54+
json_result = create_feature_view_from_feature_group(@project.id, get_featurestore_id(@project.id), @featuregroup)
55+
@feature_view = JSON.parse(json_result)
3956
create_fg_alerts(@project, @featuregroup)
57+
create_fm_alerts(@project, @featuregroup, @feature_view)
58+
4059
end
4160
it "should get" do
4261
get_fg_alerts(@project, @featuregroup)
4362
expect_status_details(200)
44-
expect(json_body[:count]).to eq(2)
63+
expect(json_body[:count]).to eq(3)
4564
end
4665
it "should update" do
4766
get_fg_alerts(@project, @featuregroup)
@@ -68,7 +87,7 @@
6887
expect_status_details(201)
6988
check_route_created(@project, alert[:receiver], alert[:status], fg: @featuregroup)
7089
get_fg_alerts(@project, @featuregroup)
71-
expect(json_body[:count]).to eq(3)
90+
expect(json_body[:count]).to eq(4)
7291
end
7392
it "should fail to create duplicate" do
7493
create_fg_alert(@project, @featuregroup, get_fg_alert_warning(@project))
@@ -81,7 +100,48 @@
81100
expect_status_details(204)
82101
check_route_deleted(@project, alert[:receiver], alert[:status], fg: @featuregroup)
83102
get_fg_alerts(@project, @featuregroup)
84-
expect(json_body[:count]).to eq(2)
103+
expect(json_body[:count]).to eq(3)
104+
end
105+
it "should create alert with feature monitoring status" do
106+
alert_data = get_fm_alert_success(@project)
107+
json_result = create_fg_alert(@project, @featuregroup, alert_data)
108+
parsed_alert_json = JSON.parse(json_result)
109+
expect_status_details(201)
110+
check_route_created_fm(@project, alert_data[:receiver], alert_data[:status], fg: @featuregroup)
111+
expect(parsed_alert_json['status']).to eql(alert_data[:status])
112+
expect(parsed_alert_json['receiver']).to eql(alert_data[:receiver])
113+
expect(parsed_alert_json['severity']).to eql(alert_data[:severity])
114+
expect(parsed_alert_json['featureGroupId']).to eql(@featuregroup["id"])
115+
end
116+
it "should create feature view monitoring alert" do
117+
alert_data = get_fm_alert_success(@project)
118+
json_result = create_feature_view_alert(@project, @feature_view, alert_data)
119+
parsed_alert_json = JSON.parse(json_result)
120+
expect_status_details(201)
121+
expect(parsed_alert_json['status']).to eql(alert_data[:status])
122+
expect(parsed_alert_json['receiver']).to eql(alert_data[:receiver])
123+
expect(parsed_alert_json['severity']).to eql(alert_data[:severity])
124+
expect(parsed_alert_json['featureViewName']).to eql(@feature_view["name"])
125+
expect(parsed_alert_json['featureViewVersion']).to eql(@feature_view["version"])
126+
check_route_created_fm(@project, alert_data[:receiver], alert_data[:status], fv: @feature_view)
127+
end
128+
it "should get and update feature view monitoring alert" do
129+
get_featureview_alerts(@project, @feature_view)
130+
expect_status_details(200)
131+
alert = json_body[:items].detect { |a| a[:status] == "FEATURE_MONITOR_SHIFT_UNDETECTED" && a[:featureViewId]
132+
.present? }
133+
receiver_original = alert[:receiver]
134+
alert[:receiver] = "#{@project[:projectname]}__slack1"
135+
json_result = update_featureview_alert(@project, @feature_view, alert[:id], alert)
136+
expect_status_details(200)
137+
parsed_updated_alert = JSON.parse(json_result)
138+
expect(parsed_updated_alert['status']).to eql(alert[:status])
139+
expect(parsed_updated_alert['receiver']).to eql(alert[:receiver])
140+
expect(parsed_updated_alert['severity']).to eql(alert[:severity])
141+
expect(parsed_updated_alert['featureViewName']).to eql(@feature_view["name"])
142+
expect(parsed_updated_alert['featureViewVersion']).to eql(@feature_view["version"])
143+
check_route_created_fm(@project, alert[:receiver], alert[:status], fv: @feature_view)
144+
check_route_deleted_fm(@project, receiver_original, alert[:status], fv: @feature_view)
85145
end
86146
it "should cleanup receivers and routes when deleting project" do
87147
delete_project(@project)
@@ -102,7 +162,7 @@
102162
featuregroup = with_valid_fg(project)
103163
create_fg_alerts_global(project, featuregroup)
104164
get_fg_alerts(project, featuregroup)
105-
expect(json_body[:count]).to eq(3)
165+
expect(json_body[:count]).to eq(5)
106166
alert_receiver = AlertReceiver.where("name LIKE '#{project[:projectname]}__%'")
107167
expect(alert_receiver.length()).to eq(0)
108168
get_routes_admin()
@@ -161,4 +221,4 @@
161221
end
162222
end
163223
end
164-
end
224+
end

hopsworks-IT/src/test/ruby/spec/featurestore_statistics_spec.rb

+38
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,14 @@
4646
commit_metadata = {commitDateString:20201025231125,commitTime:1603667485000,rowsInserted:4,rowsUpdated:0,rowsDeleted:0}
4747
json_result = commit_cached_featuregroup(@project[:id], @featurestore_id, @stream_feature_group["id"], commit_metadata: commit_metadata)
4848
expect_status_details(200)
49+
# feature view - cached feature group - no time travel
50+
json_result = create_feature_view_from_feature_group(@project[:id], @featurestore_id, @cached_feature_group)
51+
expect_status_details(201)
52+
@cached_feature_view = JSON.parse(json_result)
53+
# feature view - stream feature group - time travel
54+
json_result = create_feature_view_from_feature_group(@project[:id], @featurestore_id, @stream_feature_group)
55+
expect_status_details(201)
56+
@stream_feature_view = JSON.parse(json_result)
4957
# training datasets - with and without splits
5058
all_metadata = create_featureview_training_dataset_from_project(@project)
5159
@training_dataset = all_metadata["response"]
@@ -99,6 +107,36 @@
99107
# all query parameter combinations (including window times) are covered in the unit tests
100108
end
101109

110+
# feature view - left feature group - no time travel (for feature monitoring)
111+
112+
it "should be able to add statistics as a commit to a feature view with a left feature group with time travel disabled (feature monitoring)" do
113+
create_statistics_commit_fv(@project[:id], @featurestore_id, @cached_feature_view["name"], @cached_feature_view["version"])
114+
expect_status_details(200)
115+
end
116+
117+
it "should fail to add statistics as a commit to a feature view with a left feature group with time travel disabled and window times (feature monitoring)" do
118+
create_statistics_commit_fv(@project[:id], @featurestore_id, @cached_feature_view["name"], @cached_feature_view["version"], computation_time: 1597903688010, window_start_commit_time: 1597903688000, window_end_commit_time: 1597903688010)
119+
expect_status_details(400, error_code: 270229)
120+
# all query parameter combinations (including window times) are covered in the unit tests
121+
end
122+
123+
# feature view - left stream feature group - time travel enable (for feature monitoring)
124+
125+
it "should be able to add statistics as a commit to a feature view with a left stream feature group (feature monitoring)" do
126+
create_statistics_commit_fv(@project[:id], @featurestore_id, @stream_feature_view["name"], @stream_feature_view["version"])
127+
expect_status_details(200)
128+
end
129+
130+
it "should be able to add statistics as a commit to a feature view with a left stream feature group with window times (feature monitoring)" do
131+
# on two commits
132+
create_statistics_commit_fv(@project[:id], @featurestore_id, @stream_feature_view["name"], @stream_feature_view["version"], computation_time: 1603667485000, window_start_commit_time: 1603577485000, window_end_commit_time: 1603667485000)
133+
expect_status_details(200)
134+
# on a single commit
135+
create_statistics_commit_fv(@project[:id], @featurestore_id, @stream_feature_view["name"], @stream_feature_view["version"], computation_time: 1603667485000, window_start_commit_time: 1603667485000, window_end_commit_time: 1603667485000)
136+
expect_status_details(200)
137+
# all query parameter combinations (including window times) are covered in the unit tests
138+
end
139+
102140
# training dataset
103141

104142
it "should be able to add statistics as a commit to a training dataset" do

hopsworks-IT/src/test/ruby/spec/helpers/alert_helper.rb

+30-1
Original file line numberDiff line numberDiff line change
@@ -299,6 +299,10 @@ def with_receivers(project)
299299
pagerdutyConfigs: [create_pager_duty_config]))
300300
create_receivers_checked(project, create_receiver(project, name: "#{project[:projectname]}__slack1",
301301
slackConfigs: [create_slack_config]))
302+
create_receivers_checked(project, create_receiver(project, name: "#{project[:projectname]}__pagerduty1",
303+
pagerdutyConfigs: [create_pager_duty_config]))
304+
create_receivers_checked(project, create_receiver(project, name: "#{project[:projectname]}__slack2",
305+
slackConfigs: [create_slack_config]))
302306
end
303307

304308
def get_alerts(project, query: "")
@@ -582,6 +586,16 @@ def create_route_match_query(project, receiver, status, job: nil, fg: nil)
582586
return query
583587
end
584588

589+
def create_fm_route_match_query(project, receiver, status, fg:nil, fv: nil)
590+
query = "?match=status:#{status}"
591+
query = receiver.start_with?("global-receiver__") ?
592+
"#{query}&match=type:global-alert-#{receiver.partition('__')[2]}" :
593+
"#{query}&match=project:#{project[:projectname]}&match=type:project-alert"
594+
query = fv ? "#{query}&match=featureViewName:#{fv['name']}" : query
595+
query = fg ? "#{query}&match=featureGroup:#{fg['name']}" : query
596+
return query
597+
end
598+
585599
def check_route_created(project, receiver, status, job: nil, fg: nil)
586600
query = create_route_match_query(project, receiver, status, job: job, fg: fg)
587601
get_routes_by_receiver(project, receiver, query: query)
@@ -603,4 +617,19 @@ def check_route_deleted(project, receiver, status, job: nil, fg: nil)
603617
expect_status_details(400)
604618
end
605619

606-
end
620+
def check_route_created_fm(project, receiver, status, fg: nil, fv: nil)
621+
query = create_fm_route_match_query(project, receiver, status, fg: fg, fv: fv)
622+
get_routes_by_receiver(project, receiver, query:query)
623+
expect_status_details(200)
624+
check_backup_contains_route(json_body)
625+
check_alert_receiver_created(receiver)
626+
expect(json_body[:receiver]).to eq(receiver)
627+
end
628+
629+
def check_route_deleted_fm(project, receiver, status, fg: nil, fv: nil)
630+
query = create_fm_route_match_query(project, receiver, status, fg: fg, fv: fv)
631+
get_routes_by_receiver(project, receiver, query: query)
632+
expect_status_details(400)
633+
end
634+
635+
end

hopsworks-IT/src/test/ruby/spec/helpers/feature_group_alert_helper.rb

+48-3
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,14 @@
1616

1717
module FeatureGroupAlertHelper
1818
@@fg_alert_resource = "#{ENV['HOPSWORKS_API']}/project/%{projectId}/featurestores/%{fsId}/featuregroups/%{fgId}/alerts"
19-
2019
@@alert_success = {"status": "SUCCESS", "receiver": "global-receiver__email", "severity": "INFO"}
2120
@@alert_warning = {"status": "WARNING", "receiver": "global-receiver__slack", "severity": "WARNING"}
2221
@@alert_failed = {"status": "FAILURE", "receiver": "global-receiver__pagerduty", "severity": "CRITICAL"}
23-
22+
# feature monitor specific
23+
@@featureview_alert_resource = "#{ENV['HOPSWORKS_API']}/project/%{projectId}/featurestores/%{fsId}/featureview/%{name}/version/%{version}/alerts"
24+
@@fm_alert_success = {"status": "FEATURE_MONITOR_SHIFT_UNDETECTED", "receiver": "global-receiver__email", "severity":
25+
"INFO"}
26+
@@fm_alert_failed = {"status": "FEATURE_MONITOR_SHIFT_DETECTED", "receiver": "global-receiver__pagerduty", "severity": "CRITICAL"}
2427
def get_fg_alert_success(project)
2528
success = @@alert_success.clone
2629
success[:receiver] = "#{project[:projectname]}__email"
@@ -80,11 +83,53 @@ def create_fg_alerts_global(project, featuregroup)
8083
create_fg_alert(project, featuregroup, @@alert_success.clone)
8184
create_fg_alert(project, featuregroup, @@alert_warning.clone)
8285
create_fg_alert(project, featuregroup, @@alert_failed.clone)
86+
create_fg_alert(project, featuregroup, @@fm_alert_failed.clone)
87+
create_fg_alert(project, featuregroup, @@fm_alert_success.clone)
8388
end
8489

8590
def with_valid_fg(project)
8691
featurestore_id = get_featurestore_id(project[:id])
8792
json_result, featuregroup_name = create_cached_featuregroup(project[:id], featurestore_id)
8893
return JSON.parse(json_result)
8994
end
90-
end
95+
96+
def get_fm_alert_failure(project)
97+
failed = @@fm_alert_failed.clone
98+
failed[:receiver] = "#{project[:projectname]}__pagerduty"
99+
return failed
100+
end
101+
102+
def get_fm_alert_success(project)
103+
success = @@fm_alert_success.clone
104+
success[:receiver] = "#{project[:projectname]}__email"
105+
return success
106+
end
107+
108+
def create_fm_alerts(project, featuregroup, featureview)
109+
create_fg_alert(project, featuregroup, get_fm_alert_failure(project))
110+
create_feature_view_alert(project, featureview, get_fm_alert_failure(project))
111+
end
112+
113+
def create_feature_view_alert(project, featureview, alert)
114+
post "#{@@featureview_alert_resource}" % { projectId: project[:id],
115+
fsId: featureview["featurestoreId"],
116+
name: featureview["name"],
117+
version: featureview["version"]}, alert.to_json
118+
end
119+
120+
def update_featureview_alert(project, featureview, id, alert)
121+
put "#{@@featureview_alert_resource}/#{id}" % { projectId: project[:id],
122+
fsId: featureview["featurestoreId"],
123+
name: featureview["name"],
124+
version: featureview["version"]}, alert.to_json
125+
end
126+
127+
def get_featureview_alerts(project, featureview)
128+
get "#{@@featureview_alert_resource}" % {projectId: project[:id],
129+
fsId: featureview["featurestoreId"],
130+
name: featureview["name"],
131+
version: featureview["version"]}
132+
end
133+
134+
135+
end

0 commit comments

Comments
 (0)