From a38007f7d6726de267e2bfbc60da79a03defddd4 Mon Sep 17 00:00:00 2001 From: Chance Zibolski Date: Wed, 23 Oct 2019 10:27:56 -0700 Subject: [PATCH 1/3] charts,manifests: Fix HiveTable partitions partitionSpec validation partitionSpec is an object/map containing arbitrary key/value string pairs, so the validation was previously wrong as type array doesn't work this this. --- charts/metering-ansible-operator/templates/crds/hive.crd.yaml | 4 +++- .../ocp-testing/metering-ansible-operator/hive.crd.yaml | 4 +++- manifests/deploy/ocp-testing/olm/bundle/4.3/hive.crd.yaml | 4 +++- .../deploy/openshift/metering-ansible-operator/hive.crd.yaml | 4 +++- manifests/deploy/openshift/olm/bundle/4.3/hive.crd.yaml | 4 +++- manifests/deploy/openshift/telemeter/list.yaml | 4 +++- .../deploy/upstream/metering-ansible-operator/hive.crd.yaml | 4 +++- manifests/deploy/upstream/olm/bundle/4.3/hive.crd.yaml | 4 +++- 8 files changed, 24 insertions(+), 8 deletions(-) diff --git a/charts/metering-ansible-operator/templates/crds/hive.crd.yaml b/charts/metering-ansible-operator/templates/crds/hive.crd.yaml index f9349b5e1..315b10c1b 100644 --- a/charts/metering-ansible-operator/templates/crds/hive.crd.yaml +++ b/charts/metering-ansible-operator/templates/crds/hive.crd.yaml @@ -200,11 +200,13 @@ spec: - location properties: partitionSpec: - type: array + type: object description: | PartitionSpec is a map containing string keys and values, where each key is expected to be the name of a partition column, and the value is the value of the partition column. + additionalProperties: + type: string location: type: string description: | diff --git a/manifests/deploy/ocp-testing/metering-ansible-operator/hive.crd.yaml b/manifests/deploy/ocp-testing/metering-ansible-operator/hive.crd.yaml index 95968ac1f..fc88b7b51 100644 --- a/manifests/deploy/ocp-testing/metering-ansible-operator/hive.crd.yaml +++ b/manifests/deploy/ocp-testing/metering-ansible-operator/hive.crd.yaml @@ -200,11 +200,13 @@ spec: - location properties: partitionSpec: - type: array + type: object description: | PartitionSpec is a map containing string keys and values, where each key is expected to be the name of a partition column, and the value is the value of the partition column. + additionalProperties: + type: string location: type: string description: | diff --git a/manifests/deploy/ocp-testing/olm/bundle/4.3/hive.crd.yaml b/manifests/deploy/ocp-testing/olm/bundle/4.3/hive.crd.yaml index 95968ac1f..fc88b7b51 100644 --- a/manifests/deploy/ocp-testing/olm/bundle/4.3/hive.crd.yaml +++ b/manifests/deploy/ocp-testing/olm/bundle/4.3/hive.crd.yaml @@ -200,11 +200,13 @@ spec: - location properties: partitionSpec: - type: array + type: object description: | PartitionSpec is a map containing string keys and values, where each key is expected to be the name of a partition column, and the value is the value of the partition column. + additionalProperties: + type: string location: type: string description: | diff --git a/manifests/deploy/openshift/metering-ansible-operator/hive.crd.yaml b/manifests/deploy/openshift/metering-ansible-operator/hive.crd.yaml index 95968ac1f..fc88b7b51 100644 --- a/manifests/deploy/openshift/metering-ansible-operator/hive.crd.yaml +++ b/manifests/deploy/openshift/metering-ansible-operator/hive.crd.yaml @@ -200,11 +200,13 @@ spec: - location properties: partitionSpec: - type: array + type: object description: | PartitionSpec is a map containing string keys and values, where each key is expected to be the name of a partition column, and the value is the value of the partition column. + additionalProperties: + type: string location: type: string description: | diff --git a/manifests/deploy/openshift/olm/bundle/4.3/hive.crd.yaml b/manifests/deploy/openshift/olm/bundle/4.3/hive.crd.yaml index 95968ac1f..fc88b7b51 100644 --- a/manifests/deploy/openshift/olm/bundle/4.3/hive.crd.yaml +++ b/manifests/deploy/openshift/olm/bundle/4.3/hive.crd.yaml @@ -200,11 +200,13 @@ spec: - location properties: partitionSpec: - type: array + type: object description: | PartitionSpec is a map containing string keys and values, where each key is expected to be the name of a partition column, and the value is the value of the partition column. + additionalProperties: + type: string location: type: string description: | diff --git a/manifests/deploy/openshift/telemeter/list.yaml b/manifests/deploy/openshift/telemeter/list.yaml index 61a0e6ee9..7945e11be 100644 --- a/manifests/deploy/openshift/telemeter/list.yaml +++ b/manifests/deploy/openshift/telemeter/list.yaml @@ -210,11 +210,13 @@ objects: minLength: 1 type: string partitionSpec: + additionalProperties: + type: string description: | PartitionSpec is a map containing string keys and values, where each key is expected to be the name of a partition column, and the value is the value of the partition column. - type: array + type: object required: - partitionSpec - location diff --git a/manifests/deploy/upstream/metering-ansible-operator/hive.crd.yaml b/manifests/deploy/upstream/metering-ansible-operator/hive.crd.yaml index 95968ac1f..fc88b7b51 100644 --- a/manifests/deploy/upstream/metering-ansible-operator/hive.crd.yaml +++ b/manifests/deploy/upstream/metering-ansible-operator/hive.crd.yaml @@ -200,11 +200,13 @@ spec: - location properties: partitionSpec: - type: array + type: object description: | PartitionSpec is a map containing string keys and values, where each key is expected to be the name of a partition column, and the value is the value of the partition column. + additionalProperties: + type: string location: type: string description: | diff --git a/manifests/deploy/upstream/olm/bundle/4.3/hive.crd.yaml b/manifests/deploy/upstream/olm/bundle/4.3/hive.crd.yaml index 95968ac1f..fc88b7b51 100644 --- a/manifests/deploy/upstream/olm/bundle/4.3/hive.crd.yaml +++ b/manifests/deploy/upstream/olm/bundle/4.3/hive.crd.yaml @@ -200,11 +200,13 @@ spec: - location properties: partitionSpec: - type: array + type: object description: | PartitionSpec is a map containing string keys and values, where each key is expected to be the name of a partition column, and the value is the value of the partition column. + additionalProperties: + type: string location: type: string description: | From 093af8541353f68a5b726bc739f726bfba5319e0 Mon Sep 17 00:00:00 2001 From: Chance Zibolski Date: Wed, 23 Oct 2019 10:29:42 -0700 Subject: [PATCH 2/3] pkg/operator: Fix AWS partition values column names --- pkg/operator/aws_usage_hive.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pkg/operator/aws_usage_hive.go b/pkg/operator/aws_usage_hive.go index 6d51b2000..1813b9113 100644 --- a/pkg/operator/aws_usage_hive.go +++ b/pkg/operator/aws_usage_hive.go @@ -26,12 +26,15 @@ WITH SERDEPROPERTIES ( "timestamp.formats" = "yyyy-MM-dd'T'HH:mm:ssZ" ) ` + + billingPeriodStartPartitionColumnName = "billing_period_start" + billingPeriodEndPartitionColumnName = "billing_period_end" ) var ( AWSUsageHivePartitions = []hive.Column{ - {Name: "billing_period_start", Type: "string"}, - {Name: "billing_period_end", Type: "string"}, + {Name: billingPeriodStartPartitionColumnName, Type: "string"}, + {Name: billingPeriodEndPartitionColumnName, Type: "string"}, } ) @@ -151,8 +154,8 @@ func getDesiredPartitions(bucket string, manifests []*aws.Manifest) ([]metering. p := metering.HiveTablePartition{ Location: location, PartitionSpec: hive.PartitionSpec{ - "start": start, - "end": end, + billingPeriodStartPartitionColumnName: start, + billingPeriodEndPartitionColumnName: end, }, } desiredPartitions = append(desiredPartitions, p) From cdb26b515acbaf149d35466a885f75ef6200a541 Mon Sep 17 00:00:00 2001 From: Chance Zibolski Date: Wed, 23 Oct 2019 11:13:10 -0700 Subject: [PATCH 3/3] pkg/operator/reporting: Fix quoting of partition values and partition location --- pkg/operator/reporting/db.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/operator/reporting/db.go b/pkg/operator/reporting/db.go index 184842626..ef8e91b11 100644 --- a/pkg/operator/reporting/db.go +++ b/pkg/operator/reporting/db.go @@ -52,7 +52,7 @@ func (m *HiveManager) AddPartition(tableName string, partitionColumns []hive.Col partitionSpecStr := FmtPartitionSpec(partitionColumns, partition.PartitionSpec) locationStr := "" if partition.Location != "" { - locationStr = "LOCATION " + partition.Location + locationStr = fmt.Sprintf("LOCATION '%s'", partition.Location) } _, err := m.execer.Exec(fmt.Sprintf("ALTER TABLE %s ADD IF NOT EXISTS PARTITION (%s) %s", tableName, partitionSpecStr, locationStr)) return err @@ -70,7 +70,7 @@ func FmtPartitionSpec(partitionColumns []hive.Column, partSpec hive.PartitionSpe val := partSpec[col.Name] // Quote strings if strings.ToLower(col.Type) == "string" { - val = "`" + val + "`" + val = fmt.Sprintf("'%s'", val) } partitionVals = append(partitionVals, fmt.Sprintf("`%s`=%s", col.Name, val)) }