Refactor Cilium CNI installation

README.md

@@ -119,7 +119,7 @@ Note:
 - Network Plugin
   - [cni-plugins](https://github.com/containernetworking/plugins) 1.4.1
   - [calico](https://github.com/projectcalico/calico) 3.29.3
-  - [cilium](https://github.com/cilium/cilium) 1.15.9
+  - [cilium](https://github.com/cilium/cilium) 1.17.2
   - [flannel](https://github.com/flannel-io/flannel) 0.22.0
   - [kube-ovn](https://github.com/alauda/kube-ovn) 1.12.21
   - [kube-router](https://github.com/cloudnativelabs/kube-router) 2.1.1

docs/CNI/cilium.md

@@ -233,7 +233,7 @@ cilium_operator_extra_volume_mounts:
 ## Choose Cilium version
 
 ```yml
-cilium_version: "1.15.9"
+cilium_version: "1.17.2"
 ```
 
 ## Add variable to config

inventory/sample/group_vars/k8s_cluster/k8s-net-cilium.yml

@@ -1,5 +1,5 @@
 ---
-# cilium_version: "1.15.9"
+# cilium_version: "1.17.2"
 
 # Log-level
 # cilium_debug: false

roles/kubespray-defaults/defaults/main/download.yml

@@ -113,7 +113,7 @@ flannel_cni_version: 1.1.2
 weave_version: 2.8.7
 cni_version: "{{ (cni_binary_checksums['amd64'] | dict2items)[0].key }}"
 
-cilium_version: "1.15.9"
+cilium_version: "1.17.2"
 cilium_cli_version: "{{ (ciliumcli_binary_checksums['amd64'] | dict2items)[0].key }}"
 cilium_enable_hubble: false
 
@@ -261,13 +261,13 @@ cilium_operator_image_tag: "v{{ cilium_version }}"
 cilium_hubble_relay_image_repo: "{{ quay_image_repo }}/cilium/hubble-relay"
 cilium_hubble_relay_image_tag: "v{{ cilium_version }}"
 cilium_hubble_certgen_image_repo: "{{ quay_image_repo }}/cilium/certgen"
-cilium_hubble_certgen_image_tag: "v0.1.8"
+cilium_hubble_certgen_image_tag: "v0.2.1"
 cilium_hubble_ui_image_repo: "{{ quay_image_repo }}/cilium/hubble-ui"
-cilium_hubble_ui_image_tag: "v0.11.0"
+cilium_hubble_ui_image_tag: "v0.13.2"
 cilium_hubble_ui_backend_image_repo: "{{ quay_image_repo }}/cilium/hubble-ui-backend"
-cilium_hubble_ui_backend_image_tag: "v0.11.0"
-cilium_hubble_envoy_image_repo: "{{ docker_image_repo }}/envoyproxy/envoy"
-cilium_hubble_envoy_image_tag: "v1.22.5"
+cilium_hubble_ui_backend_image_tag: "v0.13.2"
+cilium_hubble_envoy_image_repo: "{{ quay_image_repo }}/cilium/cilium-envoy"
+cilium_hubble_envoy_image_tag: "v1.32.5-1744305768-f9ddca7dcd91f7ca25a505560e655c47d3dec2cf"
 kube_ovn_container_image_repo: "{{ docker_image_repo }}/kubeovn/kube-ovn"
 kube_ovn_container_image_tag: "v{{ kube_ovn_version }}"
 kube_ovn_vpc_container_image_repo: "{{ docker_image_repo }}/kubeovn/vpc-nat-gateway"

roles/network_plugin/cilium/defaults/main.yml

@@ -1,17 +1,17 @@
 ---
-cilium_min_version_required: "1.10"
+cilium_min_version_required: "1.15"
 # Log-level
 cilium_debug: false
 
-cilium_mtu: ""
+cilium_mtu: "0"
 cilium_enable_ipv4: "{{ ipv4_stack }}"
 cilium_enable_ipv6: "{{ ipv6_stack }}"
 
 # Enable l2 announcement from cilium to replace Metallb Ref: https://docs.cilium.io/en/v1.14/network/l2-announcements/
 cilium_l2announcements: false
 
 # Cilium agent health port
-cilium_agent_health_port: "{%- if cilium_version is version('1.11.6', '>=') -%}9879{%- else -%}9876{%- endif -%}"
+cilium_agent_health_port: "9879"
 
 # Identity allocation mode selects how identities are shared between cilium
 # nodes by setting how they are stored. The options are "crd" or "kvstore".
@@ -26,7 +26,7 @@ cilium_agent_health_port: "{%- if cilium_version is version('1.11.6', '>=') -%}9
 #   - --synchronize-k8s-nodes
 #   - --identity-allocation-mode=kvstore
 #   - Ref: https://docs.cilium.io/en/stable/internals/cilium_operator/#kvstore-operations
-cilium_identity_allocation_mode: kvstore
+cilium_identity_allocation_mode: crd
 
 # Etcd SSL dirs
 cilium_cert_dir: /etc/cilium/certs
@@ -55,20 +55,14 @@ cilium_enable_prometheus: false
 cilium_enable_portmap: false
 # Monitor aggregation level (none/low/medium/maximum)
 cilium_monitor_aggregation: medium
-# Kube Proxy Replacement mode (strict/partial)
-cilium_kube_proxy_replacement: partial
+# Kube Proxy Replacement mode (true/false)
+cilium_kube_proxy_replacement: false
 
 # If upgrading from Cilium < 1.5, you may want to override some of these options
 # to prevent service disruptions. See also:
 # http://docs.cilium.io/en/stable/install/upgrade/#changes-that-may-require-action
 cilium_preallocate_bpf_maps: false
 
-# `cilium_tofqdns_enable_poller` is deprecated in 1.8, removed in 1.9
-cilium_tofqdns_enable_poller: false
-
-# `cilium_enable_legacy_services` is deprecated in 1.6, removed in 1.9
-cilium_enable_legacy_services: false
-
 # Deploy cilium even if kube_network_plugin is not cilium.
 # This enables to deploy cilium alongside another CNI to replace kube-proxy.
 cilium_deploy_additionally: false
@@ -104,7 +98,7 @@ cilium_encryption_enabled: false
 cilium_encryption_type: "ipsec"
 
 # Enable encryption for pure node to node traffic.
-# This option is only effective when `cilium_encryption_type` is set to `ipsec`.
+# This option is only effective when `cilium_encryption_type` is set to `wireguard`.
 cilium_ipsec_node_encryption: false
 
 # If your kernel or distribution does not support WireGuard, Cilium agent can be configured to fall back on the user-space implementation.
@@ -119,6 +113,7 @@ cilium_wireguard_userspace_fallback: false
 # In case they select the Pod at egress, then the bandwidth enforcement will be disabled for those Pods.
 # Bandwidth Manager requires a v5.1.x or more recent Linux kernel.
 cilium_enable_bandwidth_manager: false
+cilium_enable_bandwidth_manager_bbr: false
 
 # IP Masquerade Agent
 # https://docs.cilium.io/en/stable/concepts/networking/masquerading/
@@ -141,6 +136,7 @@ cilium_non_masquerade_cidrs:
 ### Indicates whether to masquerade traffic to the link local prefix.
 ### If the masqLinkLocal is not set or set to false, then 169.254.0.0/16 is appended to the non-masquerade CIDRs list.
 cilium_masq_link_local: false
+cilium_masq_link_local_ipv6: false
 ### A time interval at which the agent attempts to reload config from disk
 cilium_ip_masq_resync_interval: 60s
 
@@ -149,10 +145,10 @@ cilium_ip_masq_resync_interval: 60s
 cilium_enable_hubble: false
 ### Enable Hubble-ui
 cilium_enable_hubble_ui: "{{ cilium_enable_hubble }}"
-### Enable Hubble Metrics
+### Enable Hubble Metrics (deprecated)
 cilium_enable_hubble_metrics: false
 ### if cilium_enable_hubble_metrics: true
-cilium_hubble_metrics: {}
+cilium_hubble_metrics: []
 # - dns
 # - drop
 # - tcp
@@ -194,7 +190,8 @@ cilium_ipam_mode: cluster-pool
 
 
 # Extra arguments for the Cilium agent
-cilium_agent_custom_args: []
+cilium_agent_custom_args: [] # deprecated
+cilium_agent_extra_args: []
 
 # For adding and mounting extra volumes to the cilium agent
 cilium_agent_extra_volumes: []
@@ -218,9 +215,15 @@ cilium_operator_extra_volumes: []
 cilium_operator_extra_volume_mounts: []
 
 # Extra arguments for the Cilium Operator
-cilium_operator_custom_args: []
+cilium_operator_custom_args: [] # deprecated
+cilium_operator_extra_args: []
 
+# Unique ID of the cluster. Must be unique across all connected
+# clusters and in the range of 1 to 255. Only required for Cluster Mesh,
+# may be 0 if Cluster Mesh is not used.
+cilium_cluster_id: 0
 # Name of the cluster. Only relevant when building a mesh of clusters.
+# The "default" name cannot be used if the Cluster ID is different from 0.
 cilium_cluster_name: default
 
 # Make Cilium take ownership over the `/etc/cni/net.d` directory on the node, renaming all non-Cilium CNI configurations to `*.cilium_bak`.
@@ -263,7 +266,7 @@ cilium_enable_bpf_masquerade: false
 # host stack (true) or directly and more efficiently out of BPF (false) if
 # the kernel supports it. The latter has the implication that it will also
 # bypass netfilter in the host namespace.
-cilium_enable_host_legacy_routing: true
+cilium_enable_host_legacy_routing: false
 
 # -- Enable use of the remote node identity.
 # ref: https://docs.cilium.io/en/v1.7/install/upgrade/#configmap-remote-node-identity
@@ -307,9 +310,9 @@ cilium_rolling_restart_wait_retries_count: 30
 cilium_rolling_restart_wait_retries_delay_seconds: 10
 
 # Cilium changed the default metrics exporter ports in 1.12
-cilium_agent_scrape_port: "{{ cilium_version is version('1.12', '>=') | ternary('9962', '9090') }}"
-cilium_operator_scrape_port: "{{ cilium_version is version('1.12', '>=') | ternary('9963', '6942') }}"
-cilium_hubble_scrape_port: "{{ cilium_version is version('1.12', '>=') | ternary('9965', '9091') }}"
+cilium_agent_scrape_port: "9962"
+cilium_operator_scrape_port: "9963"
+cilium_hubble_scrape_port: "9965"
 
 # Cilium certgen args for generate certificate for hubble mTLS
 cilium_certgen_args:
@@ -328,23 +331,5 @@ cilium_certgen_args:
   hubble-relay-client-cert-secret-name: hubble-relay-client-certs
   hubble-relay-server-cert-generate: false
 
-# A list of extra rules variables to add to clusterrole for cilium operator, formatted like:
-#   cilium_clusterrole_rules_operator_extra_vars:
-#     - apiGroups:
-#       - '""'
-#       resources:
-#       - pods
-#       verbs:
-#       - delete
-#     - apiGroups:
-#       - '""'
-#       resources:
-#       - nodes
-#       verbs:
-#       - list
-#       - watch
-#       resourceNames:
-#       - toto
-cilium_clusterrole_rules_operator_extra_vars: []
 cilium_enable_host_firewall: false
 cilium_policy_audit_mode: false

roles/network_plugin/cilium/tasks/apply.yml

@@ -1,14 +1,7 @@
 ---
-- name: Cilium | Start Resources
-  kube:
-    name: "{{ item.item.name }}"
-    namespace: "kube-system"
-    kubectl: "{{ bin_dir }}/kubectl"
-    resource: "{{ item.item.type }}"
-    filename: "{{ kube_config_dir }}/{{ item.item.name }}-{{ item.item.file }}"
-    state: "latest"
-  loop: "{{ cilium_node_manifests.results }}"
-  when: inventory_hostname == groups['kube_control_plane'][0] and not item is skipped
+- name: Cilium | Install
+  command: "{{ bin_dir }}/cilium install --version {{ cilium_version }} -f {{ kube_config_dir }}/values.yaml"
+  when: inventory_hostname == groups['kube_control_plane'][0]
 
 - name: Cilium | Wait for pods to run
   command: "{{ kubectl }} -n kube-system get pods -l k8s-app=cilium -o jsonpath='{.items[?(@.status.containerStatuses[0].ready==false)].metadata.name}'"  # noqa literal-compare
@@ -19,19 +12,6 @@
   failed_when: false
   when: inventory_hostname == groups['kube_control_plane'][0]
 
-- name: Cilium | Hubble install
-  kube:
-    name: "{{ item.item.name }}"
-    namespace: "kube-system"
-    kubectl: "{{ bin_dir }}/kubectl"
-    resource: "{{ item.item.type }}"
-    filename: "{{ kube_config_dir }}/addons/hubble/{{ item.item.name }}-{{ item.item.file }}"
-    state: "latest"
-  loop: "{{ cilium_hubble_manifests.results }}"
-  when:
-    - inventory_hostname == groups['kube_control_plane'][0] and not item is skipped
-    - cilium_enable_hubble and cilium_hubble_install
-
 - name: Cilium | Wait for CiliumLoadBalancerIPPool CRD to be present
   command: "{{ kubectl }} wait --for condition=established --timeout=60s crd/ciliumloadbalancerippools.cilium.io"
   register: cillium_lbippool_crd_ready

roles/network_plugin/cilium/tasks/check.yml

@@ -48,7 +48,7 @@
     msg: "cilium_encryption_type must be either 'ipsec' or 'wireguard'"
   when: cilium_encryption_enabled
 
-- name: Stop if cilium_version is < 1.10.0
+- name: Stop if cilium_version is < {{ cilium_min_version_required }}
   assert:
     that: cilium_version is version(cilium_min_version_required, '>=')
     msg: "cilium_version is too low. Minimum version {{ cilium_min_version_required }}"

roles/network_plugin/cilium/tasks/install.yml

@@ -30,64 +30,20 @@
   when:
     - cilium_identity_allocation_mode == "kvstore"
 
-- name: Cilium | Create hubble dir
-  file:
-    path: "{{ kube_config_dir }}/addons/hubble"
-    state: directory
-    owner: root
-    group: root
-    mode: "0755"
-  when:
-    - inventory_hostname == groups['kube_control_plane'][0]
-    - cilium_hubble_install
-
-- name: Cilium | Create Cilium node manifests
+- name: Cilium | Enable portmap addon
   template:
-    src: "{{ item.name }}/{{ item.file }}.j2"
-    dest: "{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}"
+    src: 000-cilium-portmap.conflist.j2
+    dest: /etc/cni/net.d/000-cilium-portmap.conflist
     mode: "0644"
-  loop:
-    - {name: cilium, file: config.yml, type: cm}
-    - {name: cilium-operator, file: crb.yml, type: clusterrolebinding}
-    - {name: cilium-operator, file: cr.yml, type: clusterrole}
-    - {name: cilium, file: crb.yml, type: clusterrolebinding}
-    - {name: cilium, file: cr.yml, type: clusterrole}
-    - {name: cilium, file: secret.yml, type: secret, when: "{{ cilium_encryption_enabled and cilium_encryption_type == 'ipsec' }}"}
-    - {name: cilium, file: ds.yml, type: ds}
-    - {name: cilium-operator, file: deploy.yml, type: deploy}
-    - {name: cilium-operator, file: sa.yml, type: sa}
-    - {name: cilium, file: sa.yml, type: sa}
-  register: cilium_node_manifests
-  when:
-    - ('kube_control_plane' in group_names)
-    - item.when | default(True) | bool
+  when: cilium_enable_portmap
 
-- name: Cilium | Create Cilium Hubble manifests
+- name: Cilium | Render values
   template:
-    src: "{{ item.name }}/{{ item.file }}.j2"
-    dest: "{{ kube_config_dir }}/addons/hubble/{{ item.name }}-{{ item.file }}"
+    src: values.yaml.j2
+    dest: "{{ kube_config_dir }}/values.yaml"
     mode: "0644"
-  loop:
-    - {name: hubble, file: config.yml, type: cm}
-    - {name: hubble, file: crb.yml, type: clusterrolebinding}
-    - {name: hubble, file: cr.yml, type: clusterrole}
-    - {name: hubble, file: cronjob.yml, type: cronjob, when: "{{ cilium_hubble_tls_generate }}"}
-    - {name: hubble, file: deploy.yml, type: deploy}
-    - {name: hubble, file: job.yml, type: job, when: "{{ cilium_hubble_tls_generate }}"}
-    - {name: hubble, file: sa.yml, type: sa}
-    - {name: hubble, file: service.yml, type: service}
-  register: cilium_hubble_manifests
   when:
     - inventory_hostname == groups['kube_control_plane'][0]
-    - cilium_enable_hubble and cilium_hubble_install
-    - item.when | default(True) | bool
-
-- name: Cilium | Enable portmap addon
-  template:
-    src: 000-cilium-portmap.conflist.j2
-    dest: /etc/cni/net.d/000-cilium-portmap.conflist
-    mode: "0644"
-  when: cilium_enable_portmap
 
 - name: Cilium | Copy Ciliumcli binary from download dir
   copy: