diff --git a/ansible/README.md b/ansible/README.md index f5cf183..2edcdd8 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -57,15 +57,24 @@ For the optional Cilium path, set these variables in the inventory: ```yaml cilium_enabled: true +cilium_profile: k3d k3d_config_source: "{{ playbook_dir }}/../local-quickstart/k3d-cilium-cluster-config.yaml" cilium_k8s_service_host: 192.168.60.252 cilium_k8s_service_port: 0 ``` -The Cilium role refuses to proceed if a kube-proxy DaemonSet is present. It -derives the dynamically published k3d API port from the new cluster's -kubeconfig, passes the API endpoint explicitly to Helm, and waits for the -Cilium DaemonSet. +The Cilium role selects the shared values from +`deployments/cilium/values/common.yaml` and the profile from +`deployments/cilium/values/profiles/{{ cilium_profile }}.yaml`. Supported +profiles are `k3d` and `flatcar-k3s`. It refuses to proceed if a kube-proxy +DaemonSet is present, derives the dynamically published k3d API port from the +new cluster's kubeconfig, passes the API endpoint explicitly to Helm, and waits +for the Cilium DaemonSet. + +Use `cilium_profile: flatcar-k3s` for a dedicated VM deployment. Review the +profile's routing assumptions and provide a stable control-plane API address. +The dedicated VM profile defaults to Geneve tunneling. Native routing should +only be selected when the VM network routes the pod CIDR between nodes. The playbook does not create Slack or Discord credentials. Those will be added through a separate opt-in Ansible secret toggle so ordinary bootstrap remains diff --git a/ansible/bootstrap-flatcar-k3d.yml b/ansible/bootstrap-flatcar-k3d.yml index fc8e410..e94048e 100644 --- a/ansible/bootstrap-flatcar-k3d.yml +++ b/ansible/bootstrap-flatcar-k3d.yml @@ -7,6 +7,7 @@ vars: argocd_enabled: true cilium_enabled: false + cilium_profile: k3d pre_tasks: - name: Read host operating system identity diff --git a/ansible/inventory.flatcar-k3d.example.yml b/ansible/inventory.flatcar-k3d.example.yml index 9751fdd..0d6429f 100644 --- a/ansible/inventory.flatcar-k3d.example.yml +++ b/ansible/inventory.flatcar-k3d.example.yml @@ -8,6 +8,7 @@ all: ansible_ssh_private_key_file: ~/.ssh/REPLACE_WITH_KEY ansible_become_method: sudo cilium_enabled: false + cilium_profile: k3d cilium_k8s_service_host: REPLACE_WITH_REACHABLE_API_IP # Set to zero for k3d so Ansible derives the published API port from # the kubeconfig created during cluster bootstrap. diff --git a/ansible/roles/cilium_bootstrap/defaults/main.yml b/ansible/roles/cilium_bootstrap/defaults/main.yml index 9d07e3c..45ce0d0 100644 --- a/ansible/roles/cilium_bootstrap/defaults/main.yml +++ b/ansible/roles/cilium_bootstrap/defaults/main.yml @@ -2,7 +2,9 @@ cilium_enabled: false cilium_namespace: kube-system cilium_chart_version: 1.20.0 -cilium_values_file: "{{ k3d_workspace }}/deployments/cilium/cilium-values.yaml" +cilium_profile: k3d +cilium_common_values_file: "{{ k3d_workspace }}/deployments/cilium/values/common.yaml" +cilium_profile_values_file: "{{ k3d_workspace }}/deployments/cilium/values/profiles/{{ cilium_profile }}.yaml" cilium_k8s_service_host: "" # k3d publishes the API on a dynamically allocated host port. A value of zero # makes the role derive that port from the kubeconfig created by k3d. diff --git a/ansible/roles/cilium_bootstrap/tasks/main.yml b/ansible/roles/cilium_bootstrap/tasks/main.yml index c1209c1..446f7db 100644 --- a/ansible/roles/cilium_bootstrap/tasks/main.yml +++ b/ansible/roles/cilium_bootstrap/tasks/main.yml @@ -19,6 +19,12 @@ - cilium_k8s_service_port | int > 0 fail_msg: Set cilium_k8s_service_host and cilium_k8s_service_port for kube-proxy replacement. +- name: Require a supported Cilium profile + ansible.builtin.assert: + that: + - cilium_profile in ['k3d', 'flatcar-k3s'] + fail_msg: cilium_profile must be k3d or flatcar-k3s. + - name: Verify kube-proxy is disabled ansible.builtin.raw: "{{ k3d_tool_dir }}/kubectl -n kube-system get daemonset kube-proxy" become_user: "{{ k3d_user }}" @@ -47,7 +53,8 @@ {{ k3d_tool_dir }}/helm upgrade --install cilium cilium/cilium --namespace {{ cilium_namespace }} --version {{ cilium_chart_version }} - --values {{ cilium_values_file }} + --values {{ cilium_common_values_file }} + --values {{ cilium_profile_values_file }} --set kubeProxyReplacement=true --set k8sServiceHost={{ cilium_k8s_service_host }} --set k8sServicePort={{ cilium_k8s_service_port }} diff --git a/deployments/argocd/README.md b/deployments/argocd/README.md index 13ae81d..4232282 100644 --- a/deployments/argocd/README.md +++ b/deployments/argocd/README.md @@ -36,6 +36,13 @@ The initial admin secret is for local bootstrap only. A later hardening phase should replace this with SSO/RBAC and remove the bootstrap credential. For a kube-proxy-free Cilium cluster, install Cilium before ArgoCD using the -profile in `local-quickstart/cilium.md` or the Flatcar cloud bootstrap. The -Cilium Application is stored under `deployments/argocd/optional-apps/` and is not watched -by the default root app until the cluster is ready for it. +profile in `local-quickstart/cilium.md` or the Flatcar bootstrap. The Cilium +Applications are stored under `deployments/argocd/optional-apps/` and are not +watched by the default root app until the cluster is ready for them. + +Select `cilium-k3d.yaml` for the nested k3d profile or +`cilium-flatcar-k3s.yaml` for dedicated Flatcar K3s VMs. The profile values are +composed from `deployments/cilium/values/common.yaml` and the matching profile +overlay. Ansible owns the initial install when kube-proxy is disabled. ArgoCD +can own subsequent upgrades after bootstrap, but Ansible and ArgoCD should not +manage the same release with different values at the same time. diff --git a/deployments/argocd/optional-apps/cilium-flatcar-k3s.yaml b/deployments/argocd/optional-apps/cilium-flatcar-k3s.yaml new file mode 100644 index 0000000..1d1aacc --- /dev/null +++ b/deployments/argocd/optional-apps/cilium-flatcar-k3s.yaml @@ -0,0 +1,26 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: cilium-flatcar-k3s + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "-4" +spec: + project: default + sources: + - repoURL: https://helm.cilium.io/ + chart: cilium + targetRevision: 1.20.0 + helm: + valueFiles: + - $values/deployments/cilium/values/common.yaml + - $values/deployments/cilium/values/profiles/flatcar-k3s.yaml + - repoURL: https://git.swaphb.com/swaphb/kubernetes-security-baseline-lab.git + targetRevision: main + ref: values + destination: + server: https://kubernetes.default.svc + namespace: kube-system + # Replace the control-plane endpoint in the profile before applying. + # Keep this Application outside the default root app-of-apps until the + # kube-proxy-free bootstrap is complete. diff --git a/deployments/argocd/optional-apps/cilium.yaml b/deployments/argocd/optional-apps/cilium-k3d.yaml similarity index 62% rename from deployments/argocd/optional-apps/cilium.yaml rename to deployments/argocd/optional-apps/cilium-k3d.yaml index 516a3d7..7032b8c 100644 --- a/deployments/argocd/optional-apps/cilium.yaml +++ b/deployments/argocd/optional-apps/cilium-k3d.yaml @@ -1,7 +1,7 @@ apiVersion: argoproj.io/v1alpha1 kind: Application metadata: - name: cilium + name: cilium-k3d namespace: argocd annotations: argocd.argoproj.io/sync-wave: "-4" @@ -13,12 +13,13 @@ spec: targetRevision: 1.20.0 helm: valueFiles: - - $values/deployments/cilium/cilium-values.yaml + - $values/deployments/cilium/values/common.yaml + - $values/deployments/cilium/values/profiles/k3d.yaml - repoURL: https://git.swaphb.com/swaphb/kubernetes-security-baseline-lab.git targetRevision: main ref: values destination: server: https://kubernetes.default.svc namespace: kube-system - # Apply this Application manually only after bootstrapping a kube-proxy-free - # cluster and replacing the API endpoint placeholder. + # Apply only after Ansible bootstraps Cilium and confirms the endpoint values. + # This Application is intentionally outside the default root app-of-apps. diff --git a/deployments/cilium/README.md b/deployments/cilium/README.md new file mode 100644 index 0000000..05d60f5 --- /dev/null +++ b/deployments/cilium/README.md @@ -0,0 +1,60 @@ +# Cilium deployment profiles + +The repository uses one shared Cilium baseline with separate networking +profiles for nested k3d and dedicated VM clusters. + +## Profile layout + +```text +deployments/cilium/ +├── values/ +│ ├── common.yaml +│ └── profiles/ +│ ├── k3d.yaml +│ └── flatcar-k3s.yaml +└── README.md +``` + +`common.yaml` owns settings that should remain consistent across environments, +including kube-proxy replacement, IPAM, Hubble, and the Cilium operator. +Profile files own routing, masquerading, socket load-balancing, and endpoint +values that depend on the node topology. + +## k3d profile + +The k3d profile uses native routing because the k3d nodes share a Docker bridge +on the Flatcar host. It explicitly sets the pod CIDR required by Cilium native +routing, enables BPF masquerading, and limits socket load-balancing to the host +namespace. These settings address the extra network layer introduced by nested +Docker containers. + +Ansible installs this profile before ArgoCD when kube-proxy is disabled. The +published k3d API port is dynamic, so Ansible supplies the API host and port at +runtime. This prevents a workstation-specific port from being committed to +Git. + +## Dedicated Flatcar K3s profile + +The VM profile uses Geneve tunneling as the portable default. Native routing can +be enabled when the cloud or virtualization network routes the Kubernetes pod +CIDR between nodes. In that case, set `routingMode: native`, configure the +native routing CIDR, and verify the required underlay routes before rollout. + +The VM profile does not enable the nested k3d socket load-balancing workaround. +Replace `REPLACE_WITH_CONTROL_PLANE_API_IP` before using the profile directly +with ArgoCD, or provide the endpoint through Ansible. + +## Ownership and bootstrap order + +When kube-proxy is disabled, Cilium must be installed before ordinary cluster +workloads because it provides the CNI and service datapath. Ansible owns this +initial bootstrap. ArgoCD may then manage the installed Cilium release for +ongoing declarative upgrades. + +Ansible and ArgoCD should not continuously manage the same Helm release with +different values. Select one profile and keep the bootstrap values and ArgoCD +values aligned. + +The Cilium kube-proxy-free documentation describes the required API endpoint +settings and the native and tunnel routing choices: + diff --git a/deployments/cilium/cilium-values.yaml b/deployments/cilium/cilium-values.yaml deleted file mode 100644 index 700dd13..0000000 --- a/deployments/cilium/cilium-values.yaml +++ /dev/null @@ -1,37 +0,0 @@ -# Cilium is the CNI, network-policy engine, and eBPF service datapath for the -# production-style K3s profile. Set this endpoint before bootstrapping a -# cluster because kube-proxy is intentionally disabled. -kubeProxyReplacement: true -k8sServiceHost: "127.0.0.1" -k8sServicePort: 6443 - -ipam: - mode: kubernetes - -# k3d nodes share a directly reachable Docker bridge. Native routing avoids -# nested VXLAN service paths that can prevent pod access to ClusterIP services. -routingMode: native -autoDirectNodeRoutes: true -# K3s allocates pod addresses from this cluster-wide CIDR. Native routing -# requires the CIDR so Cilium can distinguish pod traffic from underlay traffic. -ipv4NativeRoutingCIDR: 10.42.0.0/16 - -# The k3d nodes run as nested containers on the Flatcar host. eBPF masquerade -# keeps return traffic for cross-node service backends inside the Cilium path. -bpf: - masquerade: true - -# Keep socket load-balancing in the node namespace for the nested k3d profile. -# Pod traffic is handled by the eBPF service datapath instead. -socketLB: - hostNamespaceOnly: true - -hubble: - enabled: true - relay: - enabled: true - ui: - enabled: true - -operator: - replicas: 1 diff --git a/deployments/cilium/values/common.yaml b/deployments/cilium/values/common.yaml new file mode 100644 index 0000000..ee65471 --- /dev/null +++ b/deployments/cilium/values/common.yaml @@ -0,0 +1,16 @@ +# Shared Cilium settings for every cluster profile. +# Environment-specific routing and API endpoint settings live under profiles/. +kubeProxyReplacement: true + +ipam: + mode: kubernetes + +hubble: + enabled: true + relay: + enabled: true + ui: + enabled: true + +operator: + replicas: 1 diff --git a/deployments/cilium/values/profiles/flatcar-k3s.yaml b/deployments/cilium/values/profiles/flatcar-k3s.yaml new file mode 100644 index 0000000..b2f7a79 --- /dev/null +++ b/deployments/cilium/values/profiles/flatcar-k3s.yaml @@ -0,0 +1,14 @@ +# Dedicated VM profile. +# Geneve tunneling is the portable default when the VM network does not have +# explicit routes for the Kubernetes pod CIDR between nodes. +routingMode: tunnel +tunnelProtocol: geneve + +# Dedicated VMs do not need the nested-container socket-LB workaround. +socketLB: + hostNamespaceOnly: false + +# Set these for an ArgoCD-managed installation, or override them through +# Ansible when the control-plane address is environment-specific. +k8sServiceHost: "REPLACE_WITH_CONTROL_PLANE_API_IP" +k8sServicePort: 6443 diff --git a/deployments/cilium/values/profiles/k3d.yaml b/deployments/cilium/values/profiles/k3d.yaml new file mode 100644 index 0000000..27cfc83 --- /dev/null +++ b/deployments/cilium/values/profiles/k3d.yaml @@ -0,0 +1,21 @@ +# Nested k3d profile. +# k3d nodes are Docker containers sharing a directly reachable bridge network. +# Native routing avoids the nested VXLAN service path that previously blocked +# pod access to ClusterIP services. +routingMode: native +autoDirectNodeRoutes: true + +# K3s allocates pod addresses from this cluster-wide CIDR. Native routing +# requires the CIDR so Cilium can distinguish pod traffic from underlay traffic. +ipv4NativeRoutingCIDR: 10.42.0.0/16 + +# Nested nodes need BPF masquerading for return traffic across the Docker +# bridge. Keep socket load-balancing in the node namespace for this profile. +bpf: + masquerade: true + +socketLB: + hostNamespaceOnly: true + +# Ansible supplies k8sServiceHost and k8sServicePort during bootstrap because +# the published workstation port is dynamically allocated by k3d. diff --git a/local-quickstart/cilium.md b/local-quickstart/cilium.md index 224b514..cd9e880 100644 --- a/local-quickstart/cilium.md +++ b/local-quickstart/cilium.md @@ -16,22 +16,26 @@ Kubernetes service VIP because kube-proxy is disabled during bootstrap. ```bash kubectl config current-context kubectl get nodes -o wide -cp deployments/cilium/cilium-values.yaml /tmp/cilium-values.yaml -# Replace REPLACE_WITH_API_SERVER_DNS_OR_PRIVATE_IP with the reachable API endpoint. +# The profile is composed from common settings and the k3d overlay. +# Replace the endpoint values in a temporary file if installing manually. +helm_values_dir=deployments/cilium/values helm repo add cilium https://helm.cilium.io/ helm repo update helm upgrade --install cilium cilium/cilium \ --namespace kube-system \ --version 1.20.0 \ - --values /tmp/cilium-values.yaml + --values "$helm_values_dir/common.yaml" \ + --values "$helm_values_dir/profiles/k3d.yaml" \ + --set k8sServiceHost=REPLACE_WITH_API_SERVER_DNS_OR_PRIVATE_IP \ + --set k8sServicePort=REPLACE_WITH_API_SERVER_PORT kubectl -n kube-system rollout status daemonset/cilium --timeout=300s kubectl -n kube-system get pods -l k8s-app=cilium ``` Install ArgoCD only after Cilium is Ready. Apply -`deployments/argocd/optional-apps/cilium.yaml` manually after the endpoint is configured; -it is intentionally outside the root app-of-apps watched directory until the -cluster has been bootstrapped without kube-proxy. +`deployments/argocd/optional-apps/cilium-k3d.yaml` manually only after the +endpoint is configured. It is intentionally outside the root app-of-apps +watched directory until the cluster has been bootstrapped without kube-proxy. Delete the profile with: