add environment-specific cilium profiles

ansible/README.md

@@ -57,15 +57,24 @@ For the optional Cilium path, set these variables in the inventory:
 
 ```yaml
 cilium_enabled: true
+cilium_profile: k3d
 k3d_config_source: "{{ playbook_dir }}/../local-quickstart/k3d-cilium-cluster-config.yaml"
 cilium_k8s_service_host: 192.168.60.252
 cilium_k8s_service_port: 0
 ```
 
-The Cilium role refuses to proceed if a kube-proxy DaemonSet is present. It
-derives the dynamically published k3d API port from the new cluster's
-kubeconfig, passes the API endpoint explicitly to Helm, and waits for the
-Cilium DaemonSet.
+The Cilium role selects the shared values from
+`deployments/cilium/values/common.yaml` and the profile from
+`deployments/cilium/values/profiles/{{ cilium_profile }}.yaml`. Supported
+profiles are `k3d` and `flatcar-k3s`. It refuses to proceed if a kube-proxy
+DaemonSet is present, derives the dynamically published k3d API port from the
+new cluster's kubeconfig, passes the API endpoint explicitly to Helm, and waits
+for the Cilium DaemonSet.
+
+Use `cilium_profile: flatcar-k3s` for a dedicated VM deployment. Review the
+profile's routing assumptions and provide a stable control-plane API address.
+The dedicated VM profile defaults to Geneve tunneling. Native routing should
+only be selected when the VM network routes the pod CIDR between nodes.
 
 The playbook does not create Slack or Discord credentials. Those will be added
 through a separate opt-in Ansible secret toggle so ordinary bootstrap remains

ansible/bootstrap-flatcar-k3d.yml

@@ -7,6 +7,7 @@
   vars:
     argocd_enabled: true
     cilium_enabled: false
+    cilium_profile: k3d
 
   pre_tasks:
     - name: Read host operating system identity

ansible/inventory.flatcar-k3d.example.yml

@@ -8,6 +8,7 @@ all:
           ansible_ssh_private_key_file: ~/.ssh/REPLACE_WITH_KEY
           ansible_become_method: sudo
           cilium_enabled: false
+          cilium_profile: k3d
           cilium_k8s_service_host: REPLACE_WITH_REACHABLE_API_IP
           # Set to zero for k3d so Ansible derives the published API port from
           # the kubeconfig created during cluster bootstrap.

ansible/roles/cilium_bootstrap/defaults/main.yml

@@ -2,7 +2,9 @@
 cilium_enabled: false
 cilium_namespace: kube-system
 cilium_chart_version: 1.20.0
-cilium_values_file: "{{ k3d_workspace }}/deployments/cilium/cilium-values.yaml"
+cilium_profile: k3d
+cilium_common_values_file: "{{ k3d_workspace }}/deployments/cilium/values/common.yaml"
+cilium_profile_values_file: "{{ k3d_workspace }}/deployments/cilium/values/profiles/{{ cilium_profile }}.yaml"
 cilium_k8s_service_host: ""
 # k3d publishes the API on a dynamically allocated host port. A value of zero
 # makes the role derive that port from the kubeconfig created by k3d.

ansible/roles/cilium_bootstrap/tasks/main.yml

@@ -19,6 +19,12 @@
       - cilium_k8s_service_port | int > 0
     fail_msg: Set cilium_k8s_service_host and cilium_k8s_service_port for kube-proxy replacement.
 
+- name: Require a supported Cilium profile
+  ansible.builtin.assert:
+    that:
+      - cilium_profile in ['k3d', 'flatcar-k3s']
+    fail_msg: cilium_profile must be k3d or flatcar-k3s.
+
 - name: Verify kube-proxy is disabled
   ansible.builtin.raw: "{{ k3d_tool_dir }}/kubectl -n kube-system get daemonset kube-proxy"
   become_user: "{{ k3d_user }}"
@@ -47,7 +53,8 @@
     {{ k3d_tool_dir }}/helm upgrade --install cilium cilium/cilium
     --namespace {{ cilium_namespace }}
     --version {{ cilium_chart_version }}
-    --values {{ cilium_values_file }}
+    --values {{ cilium_common_values_file }}
+    --values {{ cilium_profile_values_file }}
     --set kubeProxyReplacement=true
     --set k8sServiceHost={{ cilium_k8s_service_host }}
     --set k8sServicePort={{ cilium_k8s_service_port }}

deployments/argocd/README.md

@@ -36,6 +36,13 @@ The initial admin secret is for local bootstrap only. A later hardening phase
 should replace this with SSO/RBAC and remove the bootstrap credential.
 
 For a kube-proxy-free Cilium cluster, install Cilium before ArgoCD using the
-profile in `local-quickstart/cilium.md` or the Flatcar cloud bootstrap. The
-Cilium Application is stored under `deployments/argocd/optional-apps/` and is not watched
-by the default root app until the cluster is ready for it.
+profile in `local-quickstart/cilium.md` or the Flatcar bootstrap. The Cilium
+Applications are stored under `deployments/argocd/optional-apps/` and are not
+watched by the default root app until the cluster is ready for them.
+
+Select `cilium-k3d.yaml` for the nested k3d profile or
+`cilium-flatcar-k3s.yaml` for dedicated Flatcar K3s VMs. The profile values are
+composed from `deployments/cilium/values/common.yaml` and the matching profile
+overlay. Ansible owns the initial install when kube-proxy is disabled. ArgoCD
+can own subsequent upgrades after bootstrap, but Ansible and ArgoCD should not
+manage the same release with different values at the same time.

deployments/argocd/optional-apps/cilium-flatcar-k3s.yaml

@@ -0,0 +1,26 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: cilium-flatcar-k3s
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "-4"
+spec:
+  project: default
+  sources:
+    - repoURL: https://helm.cilium.io/
+      chart: cilium
+      targetRevision: 1.20.0
+      helm:
+        valueFiles:
+          - $values/deployments/cilium/values/common.yaml
+          - $values/deployments/cilium/values/profiles/flatcar-k3s.yaml
+    - repoURL: https://git.swaphb.com/swaphb/kubernetes-security-baseline-lab.git
+      targetRevision: main
+      ref: values
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: kube-system
+  # Replace the control-plane endpoint in the profile before applying.
+  # Keep this Application outside the default root app-of-apps until the
+  # kube-proxy-free bootstrap is complete.

deployments/argocd/optional-apps/cilium-k3d.yaml

@@ -1,7 +1,7 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
-  name: cilium
+  name: cilium-k3d
   namespace: argocd
   annotations:
     argocd.argoproj.io/sync-wave: "-4"
@@ -13,12 +13,13 @@ spec:
       targetRevision: 1.20.0
       helm:
         valueFiles:
-          - $values/deployments/cilium/cilium-values.yaml
+          - $values/deployments/cilium/values/common.yaml
+          - $values/deployments/cilium/values/profiles/k3d.yaml
     - repoURL: https://git.swaphb.com/swaphb/kubernetes-security-baseline-lab.git
       targetRevision: main
       ref: values
   destination:
     server: https://kubernetes.default.svc
     namespace: kube-system
-  # Apply this Application manually only after bootstrapping a kube-proxy-free
-  # cluster and replacing the API endpoint placeholder.
+  # Apply only after Ansible bootstraps Cilium and confirms the endpoint values.
+  # This Application is intentionally outside the default root app-of-apps.

deployments/cilium/README.md

@@ -0,0 +1,60 @@
+# Cilium deployment profiles
+
+The repository uses one shared Cilium baseline with separate networking
+profiles for nested k3d and dedicated VM clusters.
+
+## Profile layout
+
+```text
+deployments/cilium/
+├── values/
+│   ├── common.yaml
+│   └── profiles/
+│       ├── k3d.yaml
+│       └── flatcar-k3s.yaml
+└── README.md
+```
+
+`common.yaml` owns settings that should remain consistent across environments,
+including kube-proxy replacement, IPAM, Hubble, and the Cilium operator.
+Profile files own routing, masquerading, socket load-balancing, and endpoint
+values that depend on the node topology.
+
+## k3d profile
+
+The k3d profile uses native routing because the k3d nodes share a Docker bridge
+on the Flatcar host. It explicitly sets the pod CIDR required by Cilium native
+routing, enables BPF masquerading, and limits socket load-balancing to the host
+namespace. These settings address the extra network layer introduced by nested
+Docker containers.
+
+Ansible installs this profile before ArgoCD when kube-proxy is disabled. The
+published k3d API port is dynamic, so Ansible supplies the API host and port at
+runtime. This prevents a workstation-specific port from being committed to
+Git.
+
+## Dedicated Flatcar K3s profile
+
+The VM profile uses Geneve tunneling as the portable default. Native routing can
+be enabled when the cloud or virtualization network routes the Kubernetes pod
+CIDR between nodes. In that case, set `routingMode: native`, configure the
+native routing CIDR, and verify the required underlay routes before rollout.
+
+The VM profile does not enable the nested k3d socket load-balancing workaround.
+Replace `REPLACE_WITH_CONTROL_PLANE_API_IP` before using the profile directly
+with ArgoCD, or provide the endpoint through Ansible.
+
+## Ownership and bootstrap order
+
+When kube-proxy is disabled, Cilium must be installed before ordinary cluster
+workloads because it provides the CNI and service datapath. Ansible owns this
+initial bootstrap. ArgoCD may then manage the installed Cilium release for
+ongoing declarative upgrades.
+
+Ansible and ArgoCD should not continuously manage the same Helm release with
+different values. Select one profile and keep the bootstrap values and ArgoCD
+values aligned.
+
+The Cilium kube-proxy-free documentation describes the required API endpoint
+settings and the native and tunnel routing choices:
+<https://docs.cilium.io/en/stable/network/kubernetes/kubeproxy-free/>

deployments/cilium/cilium-values.yaml

@@ -1,37 +0,0 @@
-# Cilium is the CNI, network-policy engine, and eBPF service datapath for the
-# production-style K3s profile. Set this endpoint before bootstrapping a
-# cluster because kube-proxy is intentionally disabled.
-kubeProxyReplacement: true
-k8sServiceHost: "127.0.0.1"
-k8sServicePort: 6443
-
-ipam:
-  mode: kubernetes
-
-# k3d nodes share a directly reachable Docker bridge. Native routing avoids
-# nested VXLAN service paths that can prevent pod access to ClusterIP services.
-routingMode: native
-autoDirectNodeRoutes: true
-# K3s allocates pod addresses from this cluster-wide CIDR. Native routing
-# requires the CIDR so Cilium can distinguish pod traffic from underlay traffic.
-ipv4NativeRoutingCIDR: 10.42.0.0/16
-
-# The k3d nodes run as nested containers on the Flatcar host. eBPF masquerade
-# keeps return traffic for cross-node service backends inside the Cilium path.
-bpf:
-  masquerade: true
-
-# Keep socket load-balancing in the node namespace for the nested k3d profile.
-# Pod traffic is handled by the eBPF service datapath instead.
-socketLB:
-  hostNamespaceOnly: true
-
-hubble:
-  enabled: true
-  relay:
-    enabled: true
-  ui:
-    enabled: true
-
-operator:
-  replicas: 1

deployments/cilium/values/common.yaml

@@ -0,0 +1,16 @@
+# Shared Cilium settings for every cluster profile.
+# Environment-specific routing and API endpoint settings live under profiles/.
+kubeProxyReplacement: true
+
+ipam:
+  mode: kubernetes
+
+hubble:
+  enabled: true
+  relay:
+    enabled: true
+  ui:
+    enabled: true
+
+operator:
+  replicas: 1

deployments/cilium/values/profiles/flatcar-k3s.yaml

@@ -0,0 +1,14 @@
+# Dedicated VM profile.
+# Geneve tunneling is the portable default when the VM network does not have
+# explicit routes for the Kubernetes pod CIDR between nodes.
+routingMode: tunnel
+tunnelProtocol: geneve
+
+# Dedicated VMs do not need the nested-container socket-LB workaround.
+socketLB:
+  hostNamespaceOnly: false
+
+# Set these for an ArgoCD-managed installation, or override them through
+# Ansible when the control-plane address is environment-specific.
+k8sServiceHost: "REPLACE_WITH_CONTROL_PLANE_API_IP"
+k8sServicePort: 6443

deployments/cilium/values/profiles/k3d.yaml

@@ -0,0 +1,21 @@
+# Nested k3d profile.
+# k3d nodes are Docker containers sharing a directly reachable bridge network.
+# Native routing avoids the nested VXLAN service path that previously blocked
+# pod access to ClusterIP services.
+routingMode: native
+autoDirectNodeRoutes: true
+
+# K3s allocates pod addresses from this cluster-wide CIDR. Native routing
+# requires the CIDR so Cilium can distinguish pod traffic from underlay traffic.
+ipv4NativeRoutingCIDR: 10.42.0.0/16
+
+# Nested nodes need BPF masquerading for return traffic across the Docker
+# bridge. Keep socket load-balancing in the node namespace for this profile.
+bpf:
+  masquerade: true
+
+socketLB:
+  hostNamespaceOnly: true
+
+# Ansible supplies k8sServiceHost and k8sServicePort during bootstrap because
+# the published workstation port is dynamically allocated by k3d.

local-quickstart/cilium.md

@@ -16,22 +16,26 @@ Kubernetes service VIP because kube-proxy is disabled during bootstrap.
 ```bash
 kubectl config current-context
 kubectl get nodes -o wide
-cp deployments/cilium/cilium-values.yaml /tmp/cilium-values.yaml
-# Replace REPLACE_WITH_API_SERVER_DNS_OR_PRIVATE_IP with the reachable API endpoint.
+# The profile is composed from common settings and the k3d overlay.
+# Replace the endpoint values in a temporary file if installing manually.
+helm_values_dir=deployments/cilium/values
 helm repo add cilium https://helm.cilium.io/
 helm repo update
 helm upgrade --install cilium cilium/cilium \
   --namespace kube-system \
   --version 1.20.0 \
-  --values /tmp/cilium-values.yaml
+  --values "$helm_values_dir/common.yaml" \
+  --values "$helm_values_dir/profiles/k3d.yaml" \
+  --set k8sServiceHost=REPLACE_WITH_API_SERVER_DNS_OR_PRIVATE_IP \
+  --set k8sServicePort=REPLACE_WITH_API_SERVER_PORT
 kubectl -n kube-system rollout status daemonset/cilium --timeout=300s
 kubectl -n kube-system get pods -l k8s-app=cilium
 ```
 
 Install ArgoCD only after Cilium is Ready. Apply
-`deployments/argocd/optional-apps/cilium.yaml` manually after the endpoint is configured;
-it is intentionally outside the root app-of-apps watched directory until the
-cluster has been bootstrapped without kube-proxy.
+`deployments/argocd/optional-apps/cilium-k3d.yaml` manually only after the
+endpoint is configured. It is intentionally outside the root app-of-apps
+watched directory until the cluster has been bootstrapped without kube-proxy.
 
 Delete the profile with: