Skip to content

Make backup file names configurable #2895

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions charts/postgres-operator/crds/operatorconfigurations.yaml
Original file line number Diff line number Diff line change
@@ -551,6 +551,9 @@ spec:
default: "30 00 * * *"
logical_backup_cronjob_environment_secret:
type: string
logical_backup_filename_date_format:
type: string
default: "+%s"
debug:
type: object
properties:
2 changes: 2 additions & 0 deletions charts/postgres-operator/values.yaml
Original file line number Diff line number Diff line change
@@ -392,6 +392,8 @@ configLogicalBackup:
logical_backup_schedule: "30 00 * * *"
# secret to be used as reference for env variables in cronjob
logical_backup_cronjob_environment_secret: ""
# backup filename date format
logical_backup_filename_date_format: ""

# automate creation of human users with teams API service
configTeamsApi:
3 changes: 3 additions & 0 deletions docs/reference/operator_parameters.md
Original file line number Diff line number Diff line change
@@ -876,6 +876,9 @@ grouped under the `logical_backup` key.
* **logical_backup_cronjob_environment_secret**
Reference to a Kubernetes secret, which keys will be added as environment variables to the cronjob. Default: ""

* **logical_backup_filename_date_format**
Date format to use for the logical backup filename. Uses date linux utility. Default: "+%s"

## Debugging the operator

Options to aid debugging of the operator itself. Grouped under the `debug` key.
7 changes: 4 additions & 3 deletions logical-backup/dump.sh
Original file line number Diff line number Diff line change
@@ -10,6 +10,7 @@ ALL_DB_SIZE_QUERY="select sum(pg_database_size(datname)::numeric) from pg_databa
PG_BIN=$PG_DIR/$PG_VERSION/bin
DUMP_SIZE_COEFF=5
ERRORCOUNT=0
TIMESTAMP=$(eval date $LOGICAL_BACKUP_FILENAME_DATE_FORMAT)

TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
KUBERNETES_SERVICE_PORT=${KUBERNETES_SERVICE_PORT:-443}
@@ -45,7 +46,7 @@ function compress {
}

function az_upload {
PATH_TO_BACKUP=$LOGICAL_BACKUP_S3_BUCKET"/"$LOGICAL_BACKUP_S3_BUCKET_PREFIX"/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/"$(date +%s).sql.gz
PATH_TO_BACKUP=$LOGICAL_BACKUP_S3_BUCKET"/"$LOGICAL_BACKUP_S3_BUCKET_PREFIX"/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/"$TIMESTAMP.sql.gz

az storage blob upload --file "$1" --account-name "$LOGICAL_BACKUP_AZURE_STORAGE_ACCOUNT_NAME" --account-key "$LOGICAL_BACKUP_AZURE_STORAGE_ACCOUNT_KEY" -c "$LOGICAL_BACKUP_AZURE_STORAGE_CONTAINER" -n "$PATH_TO_BACKUP"
}
@@ -107,7 +108,7 @@ function aws_upload {
# mimic bucket setup from Spilo
# to keep logical backups at the same path as WAL
# NB: $LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX already contains the leading "/" when set by the Postgres Operator
PATH_TO_BACKUP=s3://$LOGICAL_BACKUP_S3_BUCKET"/"$LOGICAL_BACKUP_S3_BUCKET_PREFIX"/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/"$(date +%s).sql.gz
PATH_TO_BACKUP=s3://$LOGICAL_BACKUP_S3_BUCKET"/"$LOGICAL_BACKUP_S3_BUCKET_PREFIX"/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/"$TIMESTAMP.sql.gz

args=()

@@ -120,7 +121,7 @@ function aws_upload {
}

function gcs_upload {
PATH_TO_BACKUP=gs://$LOGICAL_BACKUP_S3_BUCKET"/"$LOGICAL_BACKUP_S3_BUCKET_PREFIX"/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/"$(date +%s).sql.gz
PATH_TO_BACKUP=gs://$LOGICAL_BACKUP_S3_BUCKET"/"$LOGICAL_BACKUP_S3_BUCKET_PREFIX"/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/"$TIMESTAMP.sql.gz

gsutil -o Credentials:gs_service_key_file=$LOGICAL_BACKUP_GOOGLE_APPLICATION_CREDENTIALS cp - "$PATH_TO_BACKUP"
}
1 change: 1 addition & 0 deletions manifests/configmap.yaml
Original file line number Diff line number Diff line change
@@ -101,6 +101,7 @@ data:
logical_backup_s3_sse: "AES256"
logical_backup_s3_retention_time: ""
logical_backup_schedule: "30 00 * * *"
logical_backup_filename_date_format: "+%s"
major_version_upgrade_mode: "manual"
# major_version_upgrade_team_allow_list: ""
master_dns_name_format: "{cluster}.{namespace}.{hostedzone}"
3 changes: 3 additions & 0 deletions pkg/apis/acid.zalan.do/v1/crds.go
Original file line number Diff line number Diff line change
@@ -1800,6 +1800,9 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{
"logical_backup_cronjob_environment_secret": {
Type: "string",
},
"logical_backup_filename_date_format": {
Type: "string",
},
},
},
"debug": {
1 change: 1 addition & 0 deletions pkg/apis/acid.zalan.do/v1/operator_configuration_type.go
Original file line number Diff line number Diff line change
@@ -244,6 +244,7 @@ type OperatorLogicalBackupConfiguration struct {
MemoryRequest string `json:"logical_backup_memory_request,omitempty"`
CPULimit string `json:"logical_backup_cpu_limit,omitempty"`
MemoryLimit string `json:"logical_backup_memory_limit,omitempty"`
FilenameDateFormat string `json:"logical_backup_filename_date_format,omitempty"`
}

// PatroniConfiguration defines configuration for Patroni
1 change: 1 addition & 0 deletions pkg/cluster/cluster_test.go
Original file line number Diff line number Diff line change
@@ -1611,6 +1611,7 @@ func TestCompareLogicalBackupJob(t *testing.T) {
LogicalBackupS3SSE: "aws:kms",
LogicalBackupS3RetentionTime: "3 months",
LogicalBackupCronjobEnvironmentSecret: "",
LogicalBackupFilenameDateFormat: "+%s",
},
},
}, client, pg, logger, eventRecorder)
4 changes: 4 additions & 0 deletions pkg/cluster/k8sres.go
Original file line number Diff line number Diff line change
@@ -2486,6 +2486,10 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar {
Name: "LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX",
Value: getBucketScopeSuffix(string(c.Postgresql.GetUID())),
},
{
Name: "LOGICAL_BACKUP_FILENAME_DATE_FORMAT",
Value: c.OpConfig.LogicalBackup.LogicalBackupFilenameDateFormat,
},
}

switch backupProvider {
32 changes: 19 additions & 13 deletions pkg/cluster/k8sres_test.go
Original file line number Diff line number Diff line change
@@ -3784,21 +3784,26 @@ func TestGenerateLogicalBackupPodEnvVars(t *testing.T) {
},
{
envIndex: 13,
envVarConstant: "LOGICAL_BACKUP_FILENAME_DATE_FORMAT",
envVarValue: "+%s",
},
{
envIndex: 14,
envVarConstant: "LOGICAL_BACKUP_S3_REGION",
envVarValue: "eu-central-1",
},
{
envIndex: 14,
envIndex: 15,
envVarConstant: "LOGICAL_BACKUP_S3_ENDPOINT",
envVarValue: "",
},
{
envIndex: 15,
envIndex: 16,
envVarConstant: "LOGICAL_BACKUP_S3_SSE",
envVarValue: "",
},
{
envIndex: 16,
envIndex: 17,
envVarConstant: "LOGICAL_BACKUP_S3_RETENTION_TIME",
envVarValue: "1 month",
},
@@ -3811,7 +3816,7 @@ func TestGenerateLogicalBackupPodEnvVars(t *testing.T) {
envVarValue: "gcs",
},
{
envIndex: 13,
envIndex: 14,
envVarConstant: "LOGICAL_BACKUP_GOOGLE_APPLICATION_CREDENTIALS",
envVarValue: "some-path-to-credentials",
},
@@ -3824,25 +3829,25 @@ func TestGenerateLogicalBackupPodEnvVars(t *testing.T) {
envVarValue: "az",
},
{
envIndex: 13,
envIndex: 14,
envVarConstant: "LOGICAL_BACKUP_AZURE_STORAGE_ACCOUNT_NAME",
envVarValue: "some-azure-storage-account-name",
},
{
envIndex: 14,
envIndex: 15,
envVarConstant: "LOGICAL_BACKUP_AZURE_STORAGE_CONTAINER",
envVarValue: "some-azure-storage-container",
},
{
envIndex: 15,
envIndex: 16,
envVarConstant: "LOGICAL_BACKUP_AZURE_STORAGE_ACCOUNT_KEY",
envVarValue: "some-azure-storage-account-key",
},
}

expectedLogicalBackupRetentionTime := []ExpectedValue{
{
envIndex: 16,
envIndex: 17,
envVarConstant: "LOGICAL_BACKUP_S3_RETENTION_TIME",
envVarValue: "3 months",
},
@@ -3858,11 +3863,12 @@ func TestGenerateLogicalBackupPodEnvVars(t *testing.T) {
subTest: "logical backup with provider: s3",
opConfig: config.Config{
LogicalBackup: config.LogicalBackup{
LogicalBackupProvider: "s3",
LogicalBackupS3Bucket: dummyBucket,
LogicalBackupS3BucketPrefix: "spilo",
LogicalBackupS3Region: "eu-central-1",
LogicalBackupS3RetentionTime: "1 month",
LogicalBackupProvider: "s3",
LogicalBackupS3Bucket: dummyBucket,
LogicalBackupS3BucketPrefix: "spilo",
LogicalBackupFilenameDateFormat: "+%s",
LogicalBackupS3Region: "eu-central-1",
LogicalBackupS3RetentionTime: "1 month",
},
},
expectedValues: expectedLogicalBackupS3Bucket,
1 change: 1 addition & 0 deletions pkg/controller/operator_config.go
Original file line number Diff line number Diff line change
@@ -200,6 +200,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
result.LogicalBackupMemoryRequest = fromCRD.LogicalBackup.MemoryRequest
result.LogicalBackupCPULimit = fromCRD.LogicalBackup.CPULimit
result.LogicalBackupMemoryLimit = fromCRD.LogicalBackup.MemoryLimit
result.LogicalBackupFilenameDateFormat = fromCRD.LogicalBackup.FilenameDateFormat

// debug config
result.DebugLogging = fromCRD.OperatorDebug.DebugLogging
1 change: 1 addition & 0 deletions pkg/util/config/config.go
Original file line number Diff line number Diff line change
@@ -129,6 +129,7 @@ type LogicalBackup struct {
LogicalBackupSchedule string `name:"logical_backup_schedule" default:"30 00 * * *"`
LogicalBackupDockerImage string `name:"logical_backup_docker_image" default:"ghcr.io/zalando/postgres-operator/logical-backup:v1.14.0"`
LogicalBackupProvider string `name:"logical_backup_provider" default:"s3"`
LogicalBackupFilenameDateFormat string `name:"logical_backup_filename_date_format"`
LogicalBackupAzureStorageAccountName string `name:"logical_backup_azure_storage_account_name" default:""`
LogicalBackupAzureStorageContainer string `name:"logical_backup_azure_storage_container" default:""`
LogicalBackupAzureStorageAccountKey string `name:"logical_backup_azure_storage_account_key" default:""`