Skip to content

Commit 79436d9

Browse files
tormath1tormath1
committed
test: add Flatcar e2e test
Signed-off-by: Mathieu Tortuyaux <mtortuyaux@microsoft.com>
1 parent 00ec75a commit 79436d9

File tree

4 files changed

+297
-0
lines changed

4 files changed

+297
-0
lines changed

‎test/e2e/config/gcp-ci.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ providers:
7373
- sourcePath: "${PWD}/test/e2e/data/infrastructure-gcp/cluster-template-ci-gke-autopilot.yaml"
7474
- sourcePath: "${PWD}/test/e2e/data/infrastructure-gcp/cluster-template-ci-gke-custom-subnet.yaml"
7575
- sourcePath: "${PWD}/test/e2e/data/infrastructure-gcp/cluster-template-ci-with-internal-lb.yaml"
76+
- sourcePath: "${PWD}/test/e2e/data/infrastructure-gcp/cluster-template-ci-flatcar.yaml"
7677

7778
variables:
7879
KUBERNETES_VERSION: "${KUBERNETES_VERSION:-v1.30.5}"
@@ -86,6 +87,10 @@ variables:
8687
KUBERNETES_VERSION_UPGRADE_TO: "${KUBERNETES_VERSION_UPGRADE_TO:-v1.30.5}"
8788
KUBERNETES_VERSION_UPGRADE_FROM: "${KUBERNETES_VERSION_UPGRADE_FROM:-v1.29.5}"
8889
EXP_CLUSTER_RESOURCE_SET: "true"
90+
# Required for Ignition based images (e.g Flatcar)
91+
EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION: "true"
92+
# TODO: Set this based on latest available image
93+
FLATCAR_IMAGE_ID: "projects/kinvolk-public/global/images/flatcar-stable-4081-2-0"
8994
CLUSTER_TOPOLOGY: "true"
9095
# Cluster Addons
9196
CNI: "${PWD}/test/e2e/data/cni/calico/calico.yaml"

‎test/e2e/data/infrastructure-gcp/cluster-template-ci-flatcar.yaml

Lines changed: 245 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,245 @@
1+
---
2+
apiVersion: cluster.x-k8s.io/v1beta1
3+
kind: Cluster
4+
metadata:
5+
name: "${CLUSTER_NAME}"
6+
spec:
7+
clusterNetwork:
8+
pods:
9+
cidrBlocks: ["192.168.0.0/16"]
10+
infrastructureRef:
11+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
12+
kind: GCPCluster
13+
name: "${CLUSTER_NAME}"
14+
controlPlaneRef:
15+
kind: KubeadmControlPlane
16+
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
17+
name: "${CLUSTER_NAME}-control-plane"
18+
---
19+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
20+
kind: GCPCluster
21+
metadata:
22+
name: "${CLUSTER_NAME}"
23+
spec:
24+
project: "${GCP_PROJECT}"
25+
region: "${GCP_REGION}"
26+
network:
27+
name: "${GCP_NETWORK_NAME}"
28+
---
29+
kind: KubeadmControlPlane
30+
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
31+
metadata:
32+
name: "${CLUSTER_NAME}-control-plane"
33+
spec:
34+
replicas: ${CONTROL_PLANE_MACHINE_COUNT}
35+
machineTemplate:
36+
infrastructureRef:
37+
kind: GCPMachineTemplate
38+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
39+
name: "${CLUSTER_NAME}-control-plane"
40+
kubeadmConfigSpec:
41+
files: []
42+
format: ignition
43+
ignition:
44+
containerLinuxConfig:
45+
additionalConfig: |
46+
storage:
47+
links:
48+
- path: /etc/extensions/kubernetes.raw
49+
hard: false
50+
target: /opt/extensions/kubernetes/kubernetes-${KUBERNETES_VERSION}-x86-64.raw
51+
files:
52+
- path: /etc/sysupdate.kubernetes.d/kubernetes-${KUBERNETES_VERSION%.*}.conf
53+
mode: 0644
54+
contents:
55+
remote:
56+
url: https://github.com/flatcar/sysext-bakery/releases/download/latest/kubernetes-${KUBERNETES_VERSION%.*}.conf
57+
- path: /etc/sysupdate.d/noop.conf
58+
mode: 0644
59+
contents:
60+
remote:
61+
url: https://github.com/flatcar/sysext-bakery/releases/download/latest/noop.conf
62+
- path: /opt/extensions/kubernetes/kubernetes-${KUBERNETES_VERSION}-x86-64.raw
63+
contents:
64+
remote:
65+
url: https://github.com/flatcar/sysext-bakery/releases/download/latest/kubernetes-${KUBERNETES_VERSION}-x86-64.raw
66+
systemd:
67+
units:
68+
- name: systemd-sysupdate.service
69+
dropins:
70+
- name: kubernetes.conf
71+
contents: |
72+
[Service]
73+
ExecStartPre=/usr/bin/sh -c "readlink --canonicalize /etc/extensions/kubernetes.raw > /tmp/kubernetes"
74+
ExecStartPre=/usr/lib/systemd/systemd-sysupdate -C kubernetes update
75+
ExecStartPost=/usr/bin/sh -c "readlink --canonicalize /etc/extensions/kubernetes.raw > /tmp/kubernetes-new"
76+
ExecStartPost=/usr/bin/sh -c "if ! cmp --silent /tmp/kubernetes /tmp/kubernetes-new; then touch /run/reboot-required; fi"
77+
- name: update-engine.service
78+
# Set this to 'false' if you want to enable Flatcar auto-update
79+
mask: ${FLATCAR_DISABLE_AUTO_UPDATE:=true}
80+
- name: locksmithd.service
81+
# NOTE: To coordinate the node reboot in this context, we recommend to use Kured.
82+
mask: true
83+
- name: systemd-sysupdate.timer
84+
# Set this to 'true' if you want to enable the Kubernetes auto-update.
85+
# NOTE: Only patches version will be pulled.
86+
enabled: ${FLATCAR_DISABLE_AUTO_UPDATE:=true}
87+
- name: coreos-metadata-sshkeys@.service
88+
enabled: true
89+
- name: kubeadm.service
90+
enabled: true
91+
dropins:
92+
- name: 10-flatcar.conf
93+
contents: |
94+
[Unit]
95+
Requires=containerd.service coreos-metadata.service
96+
After=containerd.service coreos-metadata.service
97+
[Service]
98+
EnvironmentFile=/run/metadata/flatcar
99+
initConfiguration:
100+
nodeRegistration:
101+
name: $${COREOS_GCE_HOSTNAME}
102+
kubeletExtraArgs:
103+
cloud-provider: gce
104+
feature-gates: "DisableCloudProviders=false,DisableKubeletCloudCredentialProviders=false"
105+
clusterConfiguration:
106+
apiServer:
107+
timeoutForControlPlane: 20m
108+
extraArgs:
109+
cloud-provider: gce
110+
feature-gates: "DisableCloudProviders=false,DisableKubeletCloudCredentialProviders=false"
111+
controllerManager:
112+
extraArgs:
113+
cloud-provider: gce
114+
feature-gates: "DisableCloudProviders=false,DisableKubeletCloudCredentialProviders=false"
115+
allocate-node-cidrs: "false"
116+
joinConfiguration:
117+
nodeRegistration:
118+
name: $${COREOS_GCE_HOSTNAME}
119+
kubeletExtraArgs:
120+
cloud-provider: gce
121+
feature-gates: "DisableCloudProviders=false,DisableKubeletCloudCredentialProviders=false"
122+
preKubeadmCommands:
123+
- export COREOS_GCE_HOSTNAME=$${COREOS_GCE_HOSTNAME%%.*}
124+
- envsubst < /etc/kubeadm.yml > /etc/kubeadm.yml.tmp
125+
- mv /etc/kubeadm.yml.tmp /etc/kubeadm.yml
126+
version: "${KUBERNETES_VERSION}"
127+
---
128+
kind: GCPMachineTemplate
129+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
130+
metadata:
131+
name: "${CLUSTER_NAME}-control-plane"
132+
spec:
133+
template:
134+
spec:
135+
instanceType: "${GCP_CONTROL_PLANE_MACHINE_TYPE}"
136+
image: "${IMAGE_ID}"
137+
---
138+
apiVersion: cluster.x-k8s.io/v1beta1
139+
kind: MachineDeployment
140+
metadata:
141+
name: "${CLUSTER_NAME}-md-0"
142+
spec:
143+
clusterName: "${CLUSTER_NAME}"
144+
replicas: ${WORKER_MACHINE_COUNT}
145+
selector:
146+
matchLabels:
147+
template:
148+
spec:
149+
clusterName: "${CLUSTER_NAME}"
150+
version: "${KUBERNETES_VERSION}"
151+
bootstrap:
152+
configRef:
153+
name: "${CLUSTER_NAME}-md-0"
154+
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
155+
kind: KubeadmConfigTemplate
156+
infrastructureRef:
157+
name: "${CLUSTER_NAME}-md-0"
158+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
159+
kind: GCPMachineTemplate
160+
---
161+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
162+
kind: GCPMachineTemplate
163+
metadata:
164+
name: "${CLUSTER_NAME}-md-0"
165+
spec:
166+
template:
167+
spec:
168+
instanceType: "${GCP_NODE_MACHINE_TYPE}"
169+
image: "${IMAGE_ID}"
170+
---
171+
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
172+
kind: KubeadmConfigTemplate
173+
metadata:
174+
name: "${CLUSTER_NAME}-md-0"
175+
spec:
176+
template:
177+
spec:
178+
files: []
179+
format: ignition
180+
ignition:
181+
containerLinuxConfig:
182+
additionalConfig: |
183+
storage:
184+
links:
185+
- path: /etc/extensions/kubernetes.raw
186+
hard: false
187+
target: /opt/extensions/kubernetes/kubernetes-${KUBERNETES_VERSION}-x86-64.raw
188+
files:
189+
- path: /etc/sysupdate.kubernetes.d/kubernetes-${KUBERNETES_VERSION%.*}.conf
190+
mode: 0644
191+
contents:
192+
remote:
193+
url: https://github.com/flatcar/sysext-bakery/releases/download/latest/kubernetes-${KUBERNETES_VERSION%.*}.conf
194+
- path: /etc/sysupdate.d/noop.conf
195+
mode: 0644
196+
contents:
197+
remote:
198+
url: https://github.com/flatcar/sysext-bakery/releases/download/latest/noop.conf
199+
- path: /opt/extensions/kubernetes/kubernetes-${KUBERNETES_VERSION}-x86-64.raw
200+
contents:
201+
remote:
202+
url: https://github.com/flatcar/sysext-bakery/releases/download/latest/kubernetes-${KUBERNETES_VERSION}-x86-64.raw
203+
systemd:
204+
units:
205+
- name: systemd-sysupdate.service
206+
dropins:
207+
- name: kubernetes.conf
208+
contents: |
209+
[Service]
210+
ExecStartPre=/usr/bin/sh -c "readlink --canonicalize /etc/extensions/kubernetes.raw > /tmp/kubernetes"
211+
ExecStartPre=/usr/lib/systemd/systemd-sysupdate -C kubernetes update
212+
ExecStartPost=/usr/bin/sh -c "readlink --canonicalize /etc/extensions/kubernetes.raw > /tmp/kubernetes-new"
213+
ExecStartPost=/usr/bin/sh -c "if ! cmp --silent /tmp/kubernetes /tmp/kubernetes-new; then touch /run/reboot-required; fi"
214+
- name: update-engine.service
215+
# Set this to 'false' if you want to enable Flatcar auto-update
216+
mask: ${FLATCAR_DISABLE_AUTO_UPDATE:=true}
217+
- name: locksmithd.service
218+
# NOTE: To coordinate the node reboot in this context, we recommend to use Kured.
219+
mask: true
220+
- name: systemd-sysupdate.timer
221+
# Set this to 'true' if you want to enable the Kubernetes auto-update.
222+
# NOTE: Only patches version will be pulled.
223+
enabled: ${FLATCAR_DISABLE_AUTO_UPDATE:=true}
224+
- name: coreos-metadata-sshkeys@.service
225+
enabled: true
226+
- name: kubeadm.service
227+
enabled: true
228+
dropins:
229+
- name: 10-flatcar.conf
230+
contents: |
231+
[Unit]
232+
Requires=containerd.service coreos-metadata.service
233+
After=containerd.service coreos-metadata.service
234+
[Service]
235+
EnvironmentFile=/run/metadata/flatcar
236+
joinConfiguration:
237+
nodeRegistration:
238+
name: $${COREOS_GCE_HOSTNAME}
239+
kubeletExtraArgs:
240+
cloud-provider: gce
241+
feature-gates: "DisableCloudProviders=false,DisableKubeletCloudCredentialProviders=false"
242+
preKubeadmCommands:
243+
- export COREOS_GCE_HOSTNAME=$${COREOS_GCE_HOSTNAME%%.*}
244+
- envsubst < /etc/kubeadm.yml > /etc/kubeadm.yml.tmp
245+
- mv /etc/kubeadm.yml.tmp /etc/kubeadm.yml

‎test/e2e/e2e_test.go

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -206,4 +206,50 @@ var _ = Describe("Workload cluster creation", func() {
206206
}, result)
207207
})
208208
})
209+
210+
Context("Creating a single Flatcar control-plane cluster", func() {
211+
It("Should create a cluster with 1 Flatcar worker node and can be scaled", func() {
212+
By("Initializes with 1 Flatcar worker node")
213+
clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{
214+
ClusterProxy: bootstrapClusterProxy,
215+
ConfigCluster: clusterctl.ConfigClusterInput{
216+
LogFolder: clusterctlLogFolder,
217+
ClusterctlConfigPath: clusterctlConfigPath,
218+
KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(),
219+
InfrastructureProvider: clusterctl.DefaultInfrastructureProvider,
220+
Flavor: "ci-flatcar",
221+
Namespace: namespace.Name,
222+
ClusterName: clusterName,
223+
KubernetesVersion: e2eConfig.GetVariable(KubernetesVersion),
224+
ControlPlaneMachineCount: ptr.To[int64](1),
225+
WorkerMachineCount: ptr.To[int64](1),
226+
ClusterctlVariables: map[string]string{"IMAGE_ID": e2eConfig.GetVariable(FlatcarImageID)},
227+
},
228+
WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"),
229+
WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),
230+
WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-worker-nodes"),
231+
}, result)
232+
233+
By("Scaling Flatcar worker node to 3")
234+
clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{
235+
ClusterProxy: bootstrapClusterProxy,
236+
ConfigCluster: clusterctl.ConfigClusterInput{
237+
LogFolder: clusterctlLogFolder,
238+
ClusterctlConfigPath: clusterctlConfigPath,
239+
KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(),
240+
InfrastructureProvider: clusterctl.DefaultInfrastructureProvider,
241+
Flavor: "ci-flatcar",
242+
Namespace: namespace.Name,
243+
ClusterName: clusterName,
244+
KubernetesVersion: e2eConfig.GetVariable(KubernetesVersion),
245+
ControlPlaneMachineCount: ptr.To[int64](1),
246+
WorkerMachineCount: ptr.To[int64](3),
247+
ClusterctlVariables: map[string]string{"IMAGE_ID": e2eConfig.GetVariable(FlatcarImageID)},
248+
},
249+
WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"),
250+
WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),
251+
WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-worker-nodes"),
252+
}, result)
253+
})
254+
})
209255
})

‎test/e2e/suite_test.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ import (
4343
)
4444

4545
const (
46+
FlatcarImageID = "FLATCAR_IMAGE_ID"
4647
KubernetesVersion = "KUBERNETES_VERSION"
4748
KubernetesVersionManagement = "KUBERNETES_VERSION_MANAGEMENT"
4849

0 commit comments

Comments
 (0)