Skip to content

Commit 457a665

Browse files
maskarbmaskarb
authored
[COST-3607] key metrics (#128)
1 parent 06c6096 commit 457a665

10 files changed

+468
-1
lines changed

‎kokudaily/reports.py

+55-1
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,24 @@
3131
+ relativedelta(months=1),
3232
"provider_types": ["OCP"], # MUST be a list!
3333
}
34+
THIS_MONTH_PARAMS = {
35+
"start_time": datetime.datetime.now().replace(
36+
day=1,
37+
hour=0,
38+
minute=0,
39+
second=0,
40+
microsecond=0,
41+
tzinfo=UTC,
42+
),
43+
"end_time": datetime.datetime.now().replace(
44+
day=1,
45+
hour=0,
46+
minute=0,
47+
second=0,
48+
microsecond=0,
49+
tzinfo=UTC,
50+
) + relativedelta(months=1)
51+
}
3452

3553
REQUIRES = [
3654
{
@@ -60,12 +78,33 @@
6078
"frequency": "daily",
6179
"sql_parameters": {},
6280
},
81+
{
82+
"file": "sql/key_metrics/cust_cloud_costs_data_setup.sql",
83+
"status": "",
84+
"frequency": "daily",
85+
"sql_parameters": THIS_MONTH_PARAMS
86+
},
87+
{
88+
"file": "sql/key_metrics/cust_openshift_costs_data_setup.sql",
89+
"status": "",
90+
"frequency": "daily",
91+
"sql_parameters": THIS_MONTH_PARAMS
92+
},
93+
{
94+
"file": "sql/key_metrics/cust_openshift_infra_data_setup.sql",
95+
"status": "",
96+
"frequency": "daily",
97+
"sql_parameters": THIS_MONTH_PARAMS
98+
},
6399
],
64100
"teardown": [
65101
{"file": "sql/cust_cost_model_report_teardown.sql", "status": ""},
66102
{"file": "sql/cust_node_report_teardown.sql", "status": ""},
67103
{"file": "sql/cust_size_report_teardown.sql", "status": ""},
68104
{"file": "sql/cust_tag_report_teardown.sql", "status": ""},
105+
{"file": "sql/key_metrics/cust_cloud_costs_data_teardown.sql", "status": ""},
106+
{"file": "sql/key_metrics/cust_openshift_costs_data_teardown.sql", "status": ""},
107+
{"file": "sql/key_metrics/cust_openshift_infra_data_teardown.sql", "status": ""},
69108
],
70109
}
71110
]
@@ -94,7 +133,22 @@
94133
"cust_tag_report": {
95134
"file": "sql/cust_tag_report.sql",
96135
"target": "engineering",
97-
}
136+
},
137+
"customer_km_cloud_costs": {
138+
"file": "sql/key_metrics/cust_cloud_costs_data.sql",
139+
"target": "marketing",
140+
"sql_parameters": THIS_MONTH_PARAMS
141+
},
142+
"customer_km_openshift_costs": {
143+
"file": "sql/key_metrics/cust_openshift_costs_data.sql",
144+
"target": "marketing",
145+
"sql_parameters": THIS_MONTH_PARAMS
146+
},
147+
"customer_km_openshift_infra": {
148+
"file": "sql/key_metrics/cust_openshift_infra_data.sql",
149+
"target": "marketing",
150+
"sql_parameters": THIS_MONTH_PARAMS
151+
},
98152
}
99153

100154
REPORTS = {

‎kokudaily/sql/key_metrics/cust_cloud_costs_data.sql

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
select * from __cust_cloud_cost_report;

‎kokudaily/sql/key_metrics/cust_cloud_costs_data_setup.sql

+135
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
DROP TABLE IF EXISTS __cust_cloud_cost_report;
2+
3+
-- create temp table for results
4+
CREATE TEMPORARY TABLE IF NOT EXISTS __cust_cloud_cost_report (
5+
id serial,
6+
aws_unblended_cost numeric(33, 2),
7+
aws_calculated_amortized_cost numeric(33, 2),
8+
azure_pretax_cost numeric(33, 2),
9+
gcp_unblended_cost numeric(33, 2),
10+
gcp_total numeric(33, 2),
11+
oci_cost numeric(33, 2)
12+
);
13+
14+
-- loop construct
15+
DO $BODY$
16+
DECLARE
17+
schema_rec record;
18+
stmt_tmpl text = '
19+
INSERT INTO __cust_cloud_cost_report (
20+
aws_unblended_cost,
21+
aws_calculated_amortized_cost,
22+
azure_pretax_cost,
23+
gcp_unblended_cost,
24+
gcp_total,
25+
oci_cost)
26+
WITH aws_costs_currencies AS (
27+
SELECT
28+
''%%1$s'' AS "customer",
29+
SUM(unblended_cost) AS "unblended_cost",
30+
SUM(calculated_amortized_cost) AS "calculated_amortized_cost",
31+
currency_code AS "currency"
32+
FROM
33+
%%1$s.reporting_aws_cost_summary_p
34+
WHERE
35+
usage_start >= ''%%2$s''::date
36+
AND usage_start < ''%%3$s''::date
37+
GROUP BY currency
38+
),
39+
aws_costs AS (
40+
SELECT
41+
''%%1$s'' AS "customer",
42+
SUM(unblended_cost * pae.exchange_rate) AS "aws_unblended_cost",
43+
SUM(calculated_amortized_cost * pae.exchange_rate) AS "aws_calculated_amortized_cost"
44+
FROM aws_costs_currencies acc
45+
JOIN public.api_exchangerates pae ON LOWER(pae.currency_type)=LOWER(acc.currency)
46+
),
47+
azure_costs_currencies AS (
48+
SELECT
49+
''%%1$s'' AS "customer",
50+
SUM(pretax_cost) AS "pretax_cost",
51+
currency AS "currency"
52+
FROM
53+
%%1$s.reporting_azure_cost_summary_p
54+
WHERE
55+
usage_start >= ''%%2$s''::date
56+
AND usage_start < ''%%3$s''::date
57+
GROUP BY currency
58+
),
59+
azure_costs AS (
60+
SELECT
61+
''%%1$s'' AS "customer",
62+
SUM(pretax_cost * pae.exchange_rate) AS "azure_pretax_cost"
63+
FROM azure_costs_currencies azcc
64+
JOIN public.api_exchangerates pae ON LOWER(pae.currency_type)=LOWER(azcc.currency)
65+
),
66+
gcp_costs_currencies AS (
67+
SELECT
68+
''%%1$s'' AS "customer",
69+
SUM(unblended_cost) AS "unblended_cost",
70+
SUM(credit_amount) AS "credit_amount",
71+
currency AS "currency"
72+
FROM
73+
%%1$s.reporting_gcp_cost_summary_p
74+
WHERE
75+
usage_start >= ''%%2$s''::date
76+
AND usage_start < ''%%3$s''::date
77+
GROUP BY currency
78+
),
79+
gcp_costs AS (
80+
SELECT
81+
''%%1$s'' AS "customer",
82+
SUM(unblended_cost * pae.exchange_rate) AS "gcp_unblended_cost",
83+
SUM(unblended_cost * pae.exchange_rate + credit_amount * pae.exchange_rate) AS "gcp_total"
84+
FROM gcp_costs_currencies gcc
85+
JOIN public.api_exchangerates pae ON LOWER(pae.currency_type)=LOWER(gcc.currency)
86+
),
87+
oci_costs_currencies AS (
88+
SELECT
89+
''%%1$s'' AS "customer",
90+
SUM(cost) AS "cost",
91+
currency AS "currency"
92+
FROM
93+
%%1$s.reporting_oci_cost_summary_p
94+
WHERE
95+
usage_start >= ''%%2$s''::date
96+
AND usage_start < ''%%3$s''::date
97+
GROUP BY currency
98+
),
99+
oci_costs AS (
100+
SELECT
101+
''%%1$s'' AS "customer",
102+
SUM(cost * pae.exchange_rate) AS "oci_cost"
103+
FROM oci_costs_currencies oc
104+
JOIN public.api_exchangerates pae ON LOWER(pae.currency_type)=LOWER(oc.currency)
105+
)
106+
SELECT
107+
-- awc.customer AS "customer", -- customer is used for grouping, but left off report for anonymity
108+
awc.aws_unblended_cost AS "aws_unblended_cost",
109+
awc.aws_calculated_amortized_cost AS "aws_calculated_amortized_cost",
110+
azc.azure_pretax_cost AS "azure_pretax_cost",
111+
gc.gcp_unblended_cost AS "gcp_unblended_cost",
112+
gc.gcp_total AS "gcp_total",
113+
oc.oci_cost AS "oci_cost"
114+
FROM
115+
aws_costs awc
116+
JOIN azure_costs azc ON awc.customer = azc.customer
117+
JOIN gcp_costs gc ON awc.customer = gc.customer
118+
JOIN oci_costs oc ON awc.customer = oc.customer
119+
';
120+
BEGIN
121+
FOR schema_rec IN SELECT DISTINCT
122+
t.schema_name
123+
FROM
124+
public.api_tenant t
125+
JOIN pg_namespace n ON n.nspname = t.schema_name
126+
JOIN public.api_customer c ON c.schema_name = t.schema_name
127+
WHERE
128+
t.schema_name ~ '^acct'
129+
OR t.schema_name ~ '^org'
130+
ORDER BY
131+
t.schema_name
132+
LOOP
133+
EXECUTE format(stmt_tmpl, schema_rec.schema_name, %(start_time)s, %(end_time)s);
134+
END LOOP;
135+
END $BODY$ LANGUAGE plpgsql;

‎kokudaily/sql/key_metrics/cust_cloud_costs_data_teardown.sql

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
drop table if exists __cust_cloud_cost_report;

‎kokudaily/sql/key_metrics/cust_openshift_costs_data.sql

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
select * from __cust_openshift_cost_report;

‎kokudaily/sql/key_metrics/cust_openshift_costs_data_setup.sql

+144
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
DROP TABLE IF EXISTS __cust_openshift_cost_report;
2+
3+
-- create temp table for results
4+
CREATE TEMPORARY TABLE IF NOT EXISTS __cust_openshift_cost_report (
5+
id serial,
6+
total_infrastructure_raw_cost numeric(33, 2),
7+
total_cost_model_costs numeric(33, 2),
8+
infra_total_cost_model numeric(33, 2),
9+
infra_cost_model_cpu_cost numeric(33, 2),
10+
infra_cost_model_memory_cost numeric(33, 2),
11+
infra_cost_model_volume_cost numeric(33, 2),
12+
sup_total_cost_model numeric(33, 2),
13+
sup_cost_model_cpu_cost numeric(33, 2),
14+
sup_cost_model_memory_cost numeric(33, 2),
15+
sup_cost_model_volume_cost numeric(33, 2)
16+
);
17+
18+
-- loop construct
19+
DO $BODY$
20+
DECLARE
21+
schema_rec record;
22+
stmt_tmpl text = '
23+
INSERT INTO __cust_openshift_cost_report (
24+
total_infrastructure_raw_cost,
25+
total_cost_model_costs,
26+
infra_total_cost_model,
27+
infra_cost_model_cpu_cost,
28+
infra_cost_model_memory_cost,
29+
infra_cost_model_volume_cost,
30+
sup_total_cost_model,
31+
sup_cost_model_cpu_cost,
32+
sup_cost_model_memory_cost,
33+
sup_cost_model_volume_cost
34+
)
35+
WITH infra_raw_currencies AS (
36+
SELECT
37+
''%%1$s'' AS "customer",
38+
SUM(infrastructure_raw_cost) AS "infrastructure_raw_cost",
39+
raw_currency AS "currency"
40+
FROM
41+
%%1$s.reporting_ocp_cost_summary_p
42+
WHERE
43+
usage_start >= ''%%2$s''::date
44+
AND usage_start < ''%%3$s''::date
45+
GROUP BY raw_currency
46+
),
47+
infra_raw AS (
48+
SELECT
49+
''%%1$s'' AS "customer",
50+
SUM(infrastructure_raw_cost * pae.exchange_rate) AS "infrastructure_raw_cost"
51+
FROM infra_raw_currencies irc
52+
JOIN public.api_exchangerates pae ON LOWER(pae.currency_type)=LOWER(irc.currency)
53+
),
54+
infra_costs_grouped_by_source AS (
55+
SELECT
56+
''%%1$s'' AS "customer",
57+
SUM(cost_model_cpu_cost) AS "cost_model_cpu_cost",
58+
SUM(cost_model_memory_cost) AS "cost_model_memory_cost",
59+
SUM(cost_model_volume_cost) AS "cost_model_volume_cost",
60+
SUM(cost_model_cpu_cost+cost_model_memory_cost+cost_model_volume_cost) AS "cost_model_total",
61+
source_uuid AS "provider_uuid",
62+
currency AS "currency"
63+
FROM
64+
%%1$s.reporting_ocp_cost_summary_p
65+
JOIN %%1$s.cost_model_map cmm ON cmm.provider_uuid = source_uuid
66+
JOIN %%1$s.cost_model cm ON cm.uuid = cmm.cost_model_id
67+
WHERE
68+
usage_start >= ''%%2$s''::date
69+
AND usage_start < ''%%3$s''::date
70+
AND cost_model_rate_type=''Infrastructure''
71+
GROUP BY source_uuid, currency
72+
),
73+
infra_costs AS (
74+
SELECT
75+
''%%1$s'' AS "customer",
76+
SUM(cost_model_cpu_cost * pae.exchange_rate) AS "infra_cost_model_cpu_cost",
77+
SUM(cost_model_memory_cost * pae.exchange_rate) AS "infra_cost_model_memory_cost",
78+
SUM(cost_model_volume_cost * pae.exchange_rate) AS "infra_cost_model_volume_cost",
79+
SUM(cost_model_total * pae.exchange_rate) AS "infra_total_cost_model"
80+
FROM infra_costs_grouped_by_source icgbs
81+
JOIN public.api_exchangerates pae ON LOWER(pae.currency_type)=LOWER(icgbs.currency)
82+
),
83+
sup_costs_grouped_by_source AS (
84+
SELECT
85+
''%%1$s'' AS "customer",
86+
SUM(cost_model_cpu_cost) AS "cost_model_cpu_cost",
87+
SUM(cost_model_memory_cost) AS "cost_model_memory_cost",
88+
SUM(cost_model_volume_cost) AS "cost_model_volume_cost",
89+
SUM(cost_model_cpu_cost+cost_model_memory_cost+cost_model_volume_cost) AS "cost_model_total",
90+
source_uuid AS "provider_uuid",
91+
currency AS "currency"
92+
FROM
93+
%%1$s.reporting_ocp_cost_summary_p
94+
JOIN %%1$s.cost_model_map cmm ON cmm.provider_uuid = source_uuid
95+
JOIN %%1$s.cost_model cm ON cm.uuid = cmm.cost_model_id
96+
WHERE
97+
usage_start >= ''%%2$s''::date
98+
AND usage_start < ''%%3$s''::date
99+
AND cost_model_rate_type=''Supplementary''
100+
GROUP BY source_uuid, currency
101+
),
102+
sup_costs AS (
103+
SELECT
104+
''%%1$s'' AS "customer",
105+
SUM(cost_model_cpu_cost * pae.exchange_rate) AS "sup_cost_model_cpu_cost",
106+
SUM(cost_model_memory_cost * pae.exchange_rate) AS "sup_cost_model_memory_cost",
107+
SUM(cost_model_volume_cost * pae.exchange_rate) AS "sup_cost_model_volume_cost",
108+
SUM(cost_model_total * pae.exchange_rate) AS "sup_total_cost_model"
109+
FROM sup_costs_grouped_by_source scgbs
110+
JOIN public.api_exchangerates pae ON LOWER(pae.currency_type)=LOWER(scgbs.currency)
111+
)
112+
SELECT
113+
-- ir.customer AS "customer", -- customer is used for grouping, but left off report for anonymity
114+
COALESCE(ir.infrastructure_raw_cost, 0) AS "total_infrastructure_raw_cost",
115+
ic.infra_total_cost_model+sc.sup_total_cost_model AS "total_cost_model_costs",
116+
ic.infra_total_cost_model AS "infra_total_cost_model",
117+
ic.infra_cost_model_cpu_cost AS "infra_cost_model_cpu_cost",
118+
ic.infra_cost_model_memory_cost AS "infra_cost_model_memory_cost",
119+
ic.infra_cost_model_volume_cost AS "infra_cost_model_volume_cost",
120+
sc.sup_total_cost_model AS "sup_total_cost_model",
121+
sc.sup_cost_model_cpu_cost AS "sup_cost_model_cpu_cost",
122+
sc.sup_cost_model_memory_cost AS "sup_cost_model_memory_cost",
123+
sc.sup_cost_model_volume_cost AS "sup_cost_model_volume_cost"
124+
FROM
125+
infra_raw ir
126+
JOIN infra_costs ic ON ir.customer = ic.customer
127+
JOIN sup_costs sc ON ir.customer = sc.customer
128+
';
129+
BEGIN
130+
FOR schema_rec IN SELECT DISTINCT
131+
t.schema_name
132+
FROM
133+
public.api_tenant t
134+
JOIN pg_namespace n ON n.nspname = t.schema_name
135+
JOIN public.api_customer c ON c.schema_name = t.schema_name
136+
WHERE
137+
t.schema_name ~ '^acct'
138+
OR t.schema_name ~ '^org'
139+
ORDER BY
140+
t.schema_name
141+
LOOP
142+
EXECUTE format(stmt_tmpl, schema_rec.schema_name, %(start_time)s, %(end_time)s);
143+
END LOOP;
144+
END $BODY$ LANGUAGE plpgsql;

‎kokudaily/sql/key_metrics/cust_openshift_costs_data_teardown.sql

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
drop table if exists __cust_openshift_cost_report;

‎kokudaily/sql/key_metrics/cust_openshift_infra_data.sql

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
select * from __cust_openshift_infra_report;

0 commit comments

Comments
 (0)