Skip to content

Commit 7f58147

Browse files
bsctlbsctl
authored
Merge pull request #20 from /issues/17
feat: add scheduling constraints to Cloud Controller Manager
2 parents 661643e + 4e89fe4 commit 7f58147

File tree

5 files changed

+103
-37
lines changed

5 files changed

+103
-37
lines changed

‎README.md

+17-19
Original file line numberDiff line numberDiff line change
@@ -91,11 +91,9 @@ This command installs the Cluster Autoscaler and configures it to manage the wor
9191

9292
## Prerequisites
9393

94-
- Kubernetes 1.28+
9594
- Kamaji installed and configured
96-
- Cluster API with vSphere provider
97-
- IPAM provider (optional)
98-
- Helm 3.x
95+
- Cluster API vSphere provider installed and configured
96+
- Cluster API IPAM provider installed and configure (optional)
9997
- Access to vSphere environment
10098

10199
## Installation
@@ -106,7 +104,7 @@ helm repo add clastix https://clastix.github.io/charts
106104
helm repo update
107105

108106
# Install with custom values
109-
helm install cluster-name clastix/capi-kamaji-vsphere -f my-values.yaml
107+
helm install cluster-name clastix/capi-kamaji-vsphere -f values.yaml
110108
```
111109

112110
## Credentials Management
@@ -134,7 +132,7 @@ metadata:
134132
cluster.x-k8s.io/cluster-name: "cluster-name"
135133
stringData:
136134
username: "administrator@vsphere.local"
137-
password: "YOUR_PASSWORD"
135+
password: "password"
138136
EOF
139137
```
140138

@@ -152,14 +150,13 @@ stringData:
152150
vsphere.conf: |
153151
global:
154152
port: 443
155-
insecure-flag: false
156-
password: "YOUR_PASSWORD"
153+
insecureFlag: false
154+
password: "password"
157155
user: "administrator@vsphere.local"
158-
thumbprint: "YOUR_VCENTER_THUMBPRINT"
159156
vcenter:
160157
vcenter.example.com:
161158
datacenters:
162-
- "YOUR_DATACENTER"
159+
- "datacenter-name"
163160
server: "vcenter.example.com"
164161
EOF
165162
```
@@ -177,11 +174,9 @@ kind: Secret
177174
metadata:
178175
name: vsphere-secret
179176
namespace: capv-system
180-
labels:
181-
cluster.x-k8s.io/cluster-name: "cluster-name"
182177
stringData:
183178
username: "administrator@vsphere.local"
184-
password: "YOUR_PASSWORD"
179+
password: "password"
185180
EOF
186181
```
187182

@@ -199,9 +194,9 @@ spec:
199194
allowedNamespaces:
200195
selector:
201196
matchLabels: {} # allow all namespaces
197+
EOF
202198
```
203199

204-
205200
```yaml
206201
# Create the vsphere-config-secret for Cloud Controller Manager
207202
cat <<EOF | kubectl apply -f -
@@ -217,13 +212,12 @@ stringData:
217212
global:
218213
port: 443
219214
insecure-flag: false
220-
password: "YOUR_PASSWORD"
215+
password: "password"
221216
user: "administrator@vsphere.local"
222-
thumbprint: "YOUR_VCENTER_THUMBPRINT"
223217
vcenter:
224218
vcenter.example.com:
225219
datacenters:
226-
- "YOUR_DATACENTER"
220+
- "datacenter-name"
227221
server: "vcenter.example.com"
228222
EOF
229223
```
@@ -253,7 +247,9 @@ nodePools:
253247
- name: default
254248
template: "ubuntu-2204-kube-v1.32.0"
255249
vSphereCloudControllerManager:
256-
version: "v1.32.0"
250+
image:
251+
tag: "v1.32.0"
252+
257253

258254
# Apply upgrade
259255
helm upgrade cluster-name ./cluster-api-kamaji-vsphere -f values.yaml
@@ -297,13 +293,15 @@ kubectl wait --for=delete vspheremachines -l cluster.x-k8s.io/cluster-name=clust
297293
helm uninstall cluster-name
298294
```
299295

300-
If nodes taints are not removed:
296+
If nodes taints are not removed, check Cloud Controller Manager logs:
301297

302298
```bash
303299
# Check CPI Controller logs
304300
kubectl logs -l component=cloud-controller-manager
305301
```
306302

303+
Most of the time the issue is related to authentication issues with vSphere credentials. Check the secret used by the `VSphereClusterIdentity` or `VSphereCluster` and ensure that the credentials are correct.
304+
307305
## Configuration
308306

309307
See the values you can override [here](charts/capi-kamaji-vsphere/README.md).

‎charts/capi-kamaji-vsphere/README.md

+16-4
Original file line numberDiff line numberDiff line change
@@ -72,13 +72,14 @@ A Helm chart for deploying a Kamaji Tenant Cluster on vSphere using Cluster API
7272
| nodePools[0].resourcePool | string | `"*/Resources"` | VSphere resource pool to use |
7373
| nodePools[0].staticRoutes | list | `[]` | Static network routes if required |
7474
| nodePools[0].storagePolicyName | string | `""` | VSphere storage policy to use |
75-
| nodePools[0].template | string | `"ubuntu-2204-kube-v1.32.0"` | VSphere template to clone |
75+
| nodePools[0].template | string | `"ubuntu-2404-kube-v1.32.0"` | VSphere template to clone |
7676
| nodePools[0].users | list | `[{"lockPassword":true,"name":"ubuntu","shell":"/bin/bash","sshAuthorizedKeys":[],"sudo":"ALL=(ALL) NOPASSWD:ALL"}]` | users to create on machines |
7777
| nodePools[0].users[0] | object | `{"lockPassword":true,"name":"ubuntu","shell":"/bin/bash","sshAuthorizedKeys":[],"sudo":"ALL=(ALL) NOPASSWD:ALL"}` | username to create |
7878
| nodePools[0].users[0].lockPassword | bool | `true` | Lock login password |
7979
| nodePools[0].users[0].shell | string | `"/bin/bash"` | SSH shell to use |
8080
| nodePools[0].users[0].sshAuthorizedKeys | list | `[]` | SSH public key to add |
8181
| nodePools[0].users[0].sudo | string | `"ALL=(ALL) NOPASSWD:ALL"` | sudoers configuration |
82+
| vSphere.caFile | string | `""` | Path to the CA file if it has been mounted into the pod. |
8283
| vSphere.dataCenter | string | `"datacenter"` | Datacenter to use |
8384
| vSphere.identityRef | object | `{"name":"vsphere-secret","type":"Secret"}` | VSphere Identity Management |
8485
| vSphere.identityRef.name | string | `"vsphere-secret"` | Specifies the name of the VSphereClusterIdentity or Secret |
@@ -87,9 +88,20 @@ A Helm chart for deploying a Kamaji Tenant Cluster on vSphere using Cluster API
8788
| vSphere.port | int | `443` | VSphere server port |
8889
| vSphere.server | string | `"server.sample.org"` | VSphere server dns name or address |
8990
| vSphere.tlsThumbprint | string | `""` | VSphere https TLS thumbprint |
90-
| vSphereCloudControllerManager.enabled | bool | `true` | Installs vsphere-cloud-controller-manager on the management cluster |
91-
| vSphereCloudControllerManager.secret.name | string | `"vsphere-config-secret"` | The name of an existing Secret for vSphere. |
92-
| vSphereCloudControllerManager.version | string | `"v1.32.0"` | Version of the vsphere-cloud-controller-manager to install. The major and minor versions of releases should be equivalent to the compatible upstream Kubernetes release. |
91+
| vSphereCloudControllerManager.additionalMetadata.annotations | object | `{}` | |
92+
| vSphereCloudControllerManager.additionalMetadata.labels | object | `{}` | |
93+
| vSphereCloudControllerManager.affinity | object | `{}` | Affinity scheduling rules |
94+
| vSphereCloudControllerManager.image | object | `{"pullPolicy":"IfNotPresent","registry":"registry.k8s.io","repository":"cloud-pv-vsphere/cloud-provider-vsphere","tag":"v1.32.0"}` | Image of the vsphere-cloud-controller-manager to install. |
95+
| vSphereCloudControllerManager.image.pullPolicy | string | `"IfNotPresent"` | Set the image pull policy. |
96+
| vSphereCloudControllerManager.image.registry | string | `"registry.k8s.io"` | Set the image registry |
97+
| vSphereCloudControllerManager.image.repository | string | `"cloud-pv-vsphere/cloud-provider-vsphere"` | Set the image repository |
98+
| vSphereCloudControllerManager.image.tag | string | `"v1.32.0"` | The tag should be equivalent to the upstream Kubernetes version. |
99+
| vSphereCloudControllerManager.nodeSelector | object | `{"kubernetes.io/os":"linux"}` | NodeSelector for scheduling |
100+
| vSphereCloudControllerManager.podAdditionalMetadata | object | `{"annotations":{},"labels":{}}` | Pods Additional metadata as labels and annotations |
101+
| vSphereCloudControllerManager.secret.name | string | `"vsphere-config-secret"` | The name of an existing Secret to access the vCenter API |
102+
| vSphereCloudControllerManager.tolerations | list | `[]` | Tolerations for scheduling |
103+
| vSphereCloudControllerManager.topology | object | `{"region":"k8s-region","zone":"k8s-zone"}` | Labels for toplogy regions and zones |
104+
| vSphereCloudControllerManager.topologySpreadConstraints | list | `[]` | TopologySpreadConstraints for scheduling |
93105

94106
----------------------------------------------
95107
Autogenerated from chart metadata using [helm-docs v1.11.0](https://github.com/norwoodj/helm-docs/releases/v1.11.0)

‎charts/capi-kamaji-vsphere/templates/_helpers.tpl

+1-4
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,4 @@
1313
{{- .Values.vSphereCloudControllerManager.secret.name | default "vsphere-config-secret" -}}
1414
{{- end -}}
1515

16-
{{/* CSI vSphere config secret name used by CSI */}}
17-
{{- define "cluster-api-kamaji-vsphere.csi-config-secret-name" -}}
18-
{{- .Values.vSphereStorageControllerManager.secret.name | default "csi-config-secret" -}}
19-
{{- end -}}
16+

‎charts/capi-kamaji-vsphere/templates/cpi-manifests.yaml

+33-4
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
1-
{{- if .Values.vSphereCloudControllerManager.enabled }}
2-
---
31
kind: ServiceAccount
42
apiVersion: v1
53
metadata:
@@ -36,6 +34,13 @@ metadata:
3634
labels:
3735
component: cloud-controller-manager
3836
cluster.x-k8s.io/cluster-name: {{ include "cluster-api-kamaji-vsphere.cluster-name" . | quote }}
37+
{{- with .Values.vSphereCloudControllerManager.additionalMetadata.labels }}
38+
{{- toYaml . | nindent 4 }}
39+
{{- end }}
40+
annotations:
41+
{{- with .Values.vSphereCloudControllerManager.additionalMetadata.annotations }}
42+
{{- toYaml . | nindent 4 }}
43+
{{- end }}
3944
name: {{ include "cluster-api-kamaji-vsphere.cluster-name" . }}-vsphere-cloud-controller-manager
4045
namespace: {{ .Release.Namespace }}
4146
spec:
@@ -48,10 +53,18 @@ spec:
4853
labels:
4954
component: cloud-controller-manager
5055
app: {{ include "cluster-api-kamaji-vsphere.cluster-name" . }}-vsphere-cloud-controller-manager
56+
{{- with .Values.vSphereCloudControllerManager.podAdditionalMetadata.labels }}
57+
{{- toYaml . | nindent 8 }}
58+
{{- end }}
59+
annotations:
60+
{{- with .Values.vSphereCloudControllerManager.podAdditionalMetadata.annotations }}
61+
{{- toYaml . | nindent 8 }}
62+
{{- end }}
5163
spec:
5264
containers:
5365
- name: vsphere-cloud-controller-manager
54-
image: registry.k8s.io/cloud-pv-vsphere/cloud-provider-vsphere:{{ .Values.vSphereCloudControllerManager.version }}
66+
image: {{ .Values.vSphereCloudControllerManager.image.registry }}/{{ .Values.vSphereCloudControllerManager.image.repository }}:{{ .Values.vSphereCloudControllerManager.image.tag }}
67+
imagePullPolicy: {{ .Values.vSphereCloudControllerManager.image.pullPolicy }}
5568
args:
5669
- --v=2
5770
- --cloud-config=/etc/cloud/vsphere.conf
@@ -71,6 +84,22 @@ spec:
7184
hostNetwork: false
7285
securityContext:
7386
runAsUser: 1001
87+
affinity:
88+
{{- with .Values.vSphereCloudControllerManager.affinity }}
89+
{{- toYaml . | nindent 6 }}
90+
{{- end }}
91+
tolerations:
92+
{{- with .Values.vSphereCloudControllerManager.tolerations }}
93+
{{- toYaml . | nindent 6 }}
94+
{{- end }}
95+
nodeSelector:
96+
{{- with .Values.vSphereCloudControllerManager.nodeSelector }}
97+
{{- toYaml . | nindent 6 }}
98+
{{- end }}
99+
topologySpreadConstraints:
100+
{{- with .Values.vSphereCloudControllerManager.topologySpreadConstraints }}
101+
{{- toYaml . | nindent 6 }}
102+
{{- end }}
74103
serviceAccountName: {{ include "cluster-api-kamaji-vsphere.cluster-name" . }}-cloud-controller-manager
75104
volumes:
76105
- name: vsphere-config-volume
@@ -79,4 +108,4 @@ spec:
79108
- name: '{{ include "cluster-api-kamaji-vsphere.cluster-name" . }}-admin-kubeconfig'
80109
secret:
81110
secretName: '{{ include "cluster-api-kamaji-vsphere.cluster-name" . }}-admin-kubeconfig'
82-
{{- end }}
111+

‎charts/capi-kamaji-vsphere/values.yaml

+36-6
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ vSphere:
99
insecure: false
1010
# -- VSphere https TLS thumbprint
1111
tlsThumbprint: ""
12+
# -- Path to the CA file if it has been mounted into the pod.
13+
caFile: ""
1214
# -- VSphere Identity Management
1315
identityRef:
1416
# -- Specifies whether use VSphereClusterIdentity or Secret
@@ -124,7 +126,7 @@ nodePools:
124126
# -- VSphere storage policy to use
125127
storagePolicyName: ""
126128
# -- VSphere template to clone
127-
template: "ubuntu-2204-kube-v1.32.0"
129+
template: "ubuntu-2404-kube-v1.32.0"
128130
# -- VSphere network for VMs and CSI
129131
network: "network"
130132
# -- Number of worker VMs instances
@@ -186,10 +188,38 @@ nodePools:
186188
# -- Taints to add to the node pool when joining the cluster
187189
# taints: "node.kubernetes.io/node:NoSchedule"
188190
vSphereCloudControllerManager:
189-
# -- Installs vsphere-cloud-controller-manager on the management cluster
190-
enabled: true
191+
# -- Labels for toplogy regions and zones
192+
topology:
193+
region: "k8s-region"
194+
zone: "k8s-zone"
195+
# -- Image of the vsphere-cloud-controller-manager to install.
196+
image:
197+
# -- Set the image registry
198+
registry: registry.k8s.io
199+
# -- Set the image repository
200+
repository: cloud-pv-vsphere/cloud-provider-vsphere
201+
# -- The tag should be equivalent to the upstream Kubernetes version.
202+
tag: v1.32.0
203+
# -- Set the image pull policy.
204+
pullPolicy: IfNotPresent
191205
secret:
192-
# -- The name of an existing Secret for vSphere.
206+
# -- The name of an existing Secret to access the vCenter API
193207
name: vsphere-config-secret
194-
# -- Version of the vsphere-cloud-controller-manager to install. The major and minor versions of releases should be equivalent to the compatible upstream Kubernetes release.
195-
version: v1.32.0
208+
# -- Additional metadata as labels and annotations
209+
additionalMetadata:
210+
labels: {}
211+
annotations: {}
212+
# -- Pods Additional metadata as labels and annotations
213+
podAdditionalMetadata:
214+
labels: {}
215+
annotations: {}
216+
# -- Affinity scheduling rules
217+
affinity: {}
218+
# -- Tolerations for scheduling
219+
tolerations: []
220+
# -- NodeSelector for scheduling
221+
nodeSelector:
222+
kubernetes.io/os: linux
223+
# -- TopologySpreadConstraints for scheduling
224+
topologySpreadConstraints: []
225+

0 commit comments

Comments
 (0)