allow using file:// and cloud storage to manage packages

api/v1alpha1/pulsarpackage_types.go

@@ -32,6 +32,7 @@ type PulsarPackageSpec struct {
 	PackageURL string `json:"packageURL"`
 
 	// FileURL is the download-able URL of the package from http or https protocol
+	// Support cloud storage providers: AWS S3 (s3://), Google Cloud Storage (gs://), Azure Blob Storage (azblob://)
 	// +kubebuilder:validation:Required
 	FileURL string `json:"fileURL"`
 
@@ -50,6 +51,12 @@ type PulsarPackageSpec struct {
 	// +kubebuilder:validation:Enum=CleanUpAfterDeletion;KeepAfterDeletion
 	// +optional
 	LifecyclePolicy PulsarResourceLifeCyclePolicy `json:"lifecyclePolicy,omitempty"`
+
+	// SyncPolicy represents the sync policy of the package, including Always, IfNotPresent, Never
+	// Defaults to Always if @latest tag is used in the package URL, or IfNotPresent otherwise
+	// +kubebuilder:validation:Enum=Always;IfNotPresent;Never
+	// +optional
+	SyncPolicy PulsarPackageSyncPolicy `json:"syncPolicy,omitempty"`
 }
 
 // PulsarPackageStatus defines the observed state of PulsarPackage
@@ -97,6 +104,19 @@ type PulsarPackageList struct {
 	Items           []PulsarPackage `json:"items"`
 }
 
+// PulsarPackageSyncPolicy represents the sync policy of the package, including Always, IfNotPresent, Never
+// +enum
+type PulsarPackageSyncPolicy string
+
+const (
+	// PulsarPackageSyncAlways means that kubelet always attempts to pull the latest image. Container will fail If the pull fails.
+	PulsarPackageSyncAlways PulsarPackageSyncPolicy = "Always"
+	// PulsarPackageSyncNever means that kubelet never pulls an image, but only uses a local image. Container will fail if the image isn't present
+	PulsarPackageSyncNever PulsarPackageSyncPolicy = "Never"
+	// PulsarPackageSyncIfNotPresent means that kubelet pulls if the image isn't present on disk. Container will fail if the image isn't present and the pull fails.
+	PulsarPackageSyncIfNotPresent PulsarPackageSyncPolicy = "IfNotPresent"
+)
+
 func init() {
 	SchemeBuilder.Register(&PulsarPackage{}, &PulsarPackageList{})
 }

charts/pulsar-resources-operator/crds/resource.streamnative.io_pulsarpackages.yaml

@@ -94,8 +94,9 @@ spec:
               description:
                 type: string
               fileURL:
-                description: FileURL is the download-able URL of the package from
-                  http or https protocol
+                description: |-
+                  FileURL is the download-able URL of the package from http or https protocol
+                  Support cloud storage providers: AWS S3 (s3://), Google Cloud Storage (gs://), Azure Blob Storage (azblob://)
                 type: string
               lifecyclePolicy:
                 description: |-
@@ -114,6 +115,15 @@ spec:
                 additionalProperties:
                   type: string
                 type: object
+              syncPolicy:
+                description: |-
+                  SyncPolicy represents the sync policy of the package, including Always, IfNotPresent, Never
+                  Defaults to Always if @latest tag is used in the package URL, or IfNotPresent otherwise
+                enum:
+                - Always
+                - IfNotPresent
+                - Never
+                type: string
             required:
             - connectionRef
             - fileURL

charts/pulsar-resources-operator/templates/cloud-storage-secrets.yaml

@@ -0,0 +1,65 @@
+# Copyright 2025 StreamNative
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{{- if .Values.cloudStorage.s3.enabled }}
+{{- if .Values.cloudStorage.s3.credentials.create }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ .Values.cloudStorage.s3.credentials.secretName }}
+  namespace: {{ include "pulsar-resources-operator.namespace" . }}
+  labels:
+    {{- include "pulsar-resources-operator.labels" . | nindent 4 }}
+type: Opaque
+data:
+  access-key-id: {{ .Values.cloudStorage.s3.credentials.accessKeyId | b64enc }}
+  secret-access-key: {{ .Values.cloudStorage.s3.credentials.secretAccessKey | b64enc }}
+---
+{{- end }}
+{{- end }}
+
+{{- if and .Values.cloudStorage.gcs.enabled (not .Values.cloudStorage.gcs.serviceAccount.useWorkloadIdentity) }}
+{{- if .Values.cloudStorage.gcs.serviceAccount.key.create }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ .Values.cloudStorage.gcs.serviceAccount.key.secretName }}
+  namespace: {{ include "pulsar-resources-operator.namespace" . }}
+  labels:
+    {{- include "pulsar-resources-operator.labels" . | nindent 4 }}
+type: Opaque
+data:
+  key.json: {{ .Values.cloudStorage.gcs.serviceAccount.key.json | b64enc }}
+---
+{{- end }}
+{{- end }}
+
+{{- if .Values.cloudStorage.azure.enabled }}
+{{- if .Values.cloudStorage.azure.credentials.create }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ .Values.cloudStorage.azure.credentials.secretName }}
+  namespace: {{ include "pulsar-resources-operator.namespace" . }}
+  labels:
+    {{- include "pulsar-resources-operator.labels" . | nindent 4 }}
+type: Opaque
+data:
+  {{- if .Values.cloudStorage.azure.credentials.useAccountKey }}
+  storage-key: {{ .Values.cloudStorage.azure.credentials.accountKey | b64enc }}
+  {{- else }}
+  sas-token: {{ .Values.cloudStorage.azure.credentials.sasToken | b64enc }}
+  {{- end }}
+{{- end }}
+{{- end }} 

charts/pulsar-resources-operator/templates/deployment.yaml

@@ -38,6 +38,14 @@ spec:
         {{- toYaml . | nindent 8 }}
         {{- end }}
     spec:
+      {{- if or .Values.cloudStorage.gcs.enabled .Values.cloudStorage.s3.enabled .Values.cloudStorage.azure.enabled }}
+      volumes:
+        {{- if and .Values.cloudStorage.gcs.enabled (not .Values.cloudStorage.gcs.serviceAccount.useWorkloadIdentity) }}
+        - name: gcs-credentials
+          secret:
+            secretName: {{ .Values.cloudStorage.gcs.serviceAccount.key.secretName }}
+        {{- end }}
+      {{- end }}
       imagePullSecrets:
         {{- toYaml .Values.imagePullSecrets | nindent 8 }}
       serviceAccountName: {{ include "pulsar-resources-operator.serviceAccountName" . }}
@@ -52,16 +60,57 @@ spec:
         - --retry-count={{ .Values.features.retryCount | default 5 }}
         command:
         - /manager
-        {{- if .Values.features.alwaysUpdatePulsarResource }}
         env:
+        {{- if .Values.features.alwaysUpdatePulsarResource }}
         - name: ALWAYS_UPDATE_PULSAR_RESOURCE
           value: "true"
         {{- end }}
+        {{- if .Values.cloudStorage.s3.enabled }}
+        - name: AWS_REGION
+          value: {{ .Values.cloudStorage.s3.region | quote }}
+        - name: AWS_ACCESS_KEY_ID
+          valueFrom:
+            secretKeyRef:
+              name: {{ .Values.cloudStorage.s3.credentials.secretName }}
+              key: access-key-id
+        - name: AWS_SECRET_ACCESS_KEY
+          valueFrom:
+            secretKeyRef:
+              name: {{ .Values.cloudStorage.s3.credentials.secretName }}
+              key: secret-access-key
+        {{- end }}
+        {{- if and .Values.cloudStorage.gcs.enabled (not .Values.cloudStorage.gcs.serviceAccount.useWorkloadIdentity) }}
+        - name: GOOGLE_APPLICATION_CREDENTIALS
+          value: {{ printf "%s/key.json" .Values.cloudStorage.gcs.serviceAccount.key.mountPath }}
+        {{- end }}
+        {{- if .Values.cloudStorage.azure.enabled }}
+        - name: AZURE_STORAGE_ACCOUNT
+          value: {{ .Values.cloudStorage.azure.accountName | quote }}
+        {{- if .Values.cloudStorage.azure.credentials.useAccountKey }}
+        - name: AZURE_STORAGE_KEY
+          valueFrom:
+            secretKeyRef:
+              name: {{ .Values.cloudStorage.azure.credentials.secretName }}
+              key: storage-key
+        {{- else }}
+        - name: AZURE_STORAGE_SAS_TOKEN
+          valueFrom:
+            secretKeyRef:
+              name: {{ .Values.cloudStorage.azure.credentials.secretName }}
+              key: sas-token
+        {{- end }}
+        {{- end }}
         name: manager
         securityContext:
           {{- toYaml .Values.securityContext | nindent 10 }}
         image: "{{ .Values.image.manager.registry }}/{{ .Values.image.manager.repository }}:{{ .Values.image.manager.tag | default .Chart.AppVersion }}"
         imagePullPolicy: {{ .Values.image.pullPolicy }}
+        {{- if and .Values.cloudStorage.gcs.enabled (not .Values.cloudStorage.gcs.serviceAccount.useWorkloadIdentity) }}
+        volumeMounts:
+        - name: gcs-credentials
+          mountPath: {{ .Values.cloudStorage.gcs.serviceAccount.key.mountPath }}
+          readOnly: true
+        {{- end }}
         livenessProbe:
           httpGet:
             path: /healthz

charts/pulsar-resources-operator/templates/serviceaccount.yaml

@@ -21,8 +21,11 @@ metadata:
   namespace: {{ include "pulsar-resources-operator.namespace" .}}
   labels:
     {{- include "pulsar-resources-operator.labels" . | nindent 4 }}
-  {{- with .Values.serviceAccount.annotations }}
   annotations:
+    {{- if and .Values.cloudStorage.gcs.enabled .Values.cloudStorage.gcs.serviceAccount.useWorkloadIdentity }}
+    iam.gke.io/gcp-service-account: {{ .Values.cloudStorage.gcs.serviceAccount.name }}
+    {{- end }}
+    {{- with .Values.serviceAccount.annotations }}
     {{- toYaml . | nindent 4 }}
-  {{- end }}
+    {{- end }}
 {{- end }}