From fc786ec63cb35cac99475fae5572a80badca8de3 Mon Sep 17 00:00:00 2001 From: swaphb Date: Mon, 10 Aug 2026 17:19:18 -0400 Subject: [PATCH] Add Flatcar and Cilium kube-proxy-free architecture --- README.md | 13 ++- ansible/bootstrap-k3s.yml | 93 +++++++++++-------- argocd/README.md | 5 + argocd/optional-apps/cilium.yaml | 25 +++++ cilium/cilium-values.yaml | 23 +++++ local-quickstart/cilium.md | 40 ++++++++ .../k3d-cilium-cluster-config.yaml | 32 +++++++ terraform/cloud-cluster/README.md | 8 +- .../cloud-cluster/terraform.tfvars.example | 3 +- terraform/cloud-cluster/variables.tf | 3 +- 10 files changed, 194 insertions(+), 51 deletions(-) create mode 100644 argocd/optional-apps/cilium.yaml create mode 100644 cilium/cilium-values.yaml create mode 100644 local-quickstart/cilium.md create mode 100644 local-quickstart/k3d-cilium-cluster-config.yaml diff --git a/README.md b/README.md index e1b52a0..7700a3a 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ # Kubernetes Security Baseline An auditable, GitOps-managed Kubernetes security baseline: local k3d for a -fast reproducible demo, optional Terraform + Ansible for a cloud K3s host, CIS -Benchmark checks in CI, Kyverno admission policies, and Falco runtime -detection. +fast reproducible demo, optional Flatcar plus Terraform and Ansible for a cloud +K3s host, Cilium eBPF networking, CIS Benchmark checks in CI, Kyverno +admission policies, and Falco runtime detection. ## Phase 1: provision a local cluster @@ -22,8 +22,10 @@ Delete the lab with `k3d cluster delete security-baseline`. ## Production-style option `terraform/cloud-cluster/` and `ansible/bootstrap-k3s.yml` provide an optional, -documented AWS path. It requires your own AWS credentials, an existing EC2 key -pair, an Ubuntu AMI ID for the chosen region, and a tightly scoped admin CIDR. +documented AWS path. It uses Flatcar as the host OS, disables Flannel and +kube-proxy, and installs Cilium as the CNI and eBPF service datapath. It +requires your own AWS credentials, an existing EC2 key pair, a Flatcar AMI ID +for the chosen region, and a tightly scoped admin CIDR. It is not needed for the portfolio demo and is not run in CI. ## Build status @@ -31,6 +33,7 @@ It is not needed for the portfolio demo and is not run in CI. - [x] Phase 1: local k3d definition and optional Terraform/Ansible path - [ ] Phase 2: ArgoCD app-of-apps deployment (scaffolded; runtime verification pending) - [x] Phase 3: kube-bench CI scan +- [ ] Architecture revision: Flatcar hosts and Cilium kube-proxy replacement (configured; runtime verification pending) - [ ] Phase 4: Kyverno policy set and CIS mapping - [ ] Phase 5: Falco rules and webhook alerting - [ ] Phase 6: test workloads and evidence capture diff --git a/ansible/bootstrap-k3s.yml b/ansible/bootstrap-k3s.yml index 2b28d9b..93c262e 100644 --- a/ansible/bootstrap-k3s.yml +++ b/ansible/bootstrap-k3s.yml @@ -1,5 +1,5 @@ --- -- name: Harden host and install K3s +- name: Configure Flatcar host and install K3s hosts: k3s_servers become: true gather_facts: true @@ -7,44 +7,22 @@ vars: k3s_version: "v1.30.6+k3s1" k3s_install_url: "https://get.k3s.io" + k3s_api_server_port: 6443 pre_tasks: - - name: Require a supported Ubuntu release + - name: Require Flatcar Linux ansible.builtin.assert: that: - - ansible_distribution == "Ubuntu" - - ansible_distribution_version is version("22.04", ">=") - fail_msg: "This playbook expects Ubuntu 22.04 or newer." + - ansible_distribution == "Flatcar" + fail_msg: "This playbook expects Flatcar Linux. Use the Ubuntu fallback playbook if needed." tasks: - - name: Install host security prerequisites - ansible.builtin.apt: - name: - - apparmor - - apparmor-utils - - curl - - ca-certificates - - unattended-upgrades - state: present - update_cache: true - - - name: Ensure unattended security upgrades are enabled - ansible.builtin.service: - name: unattended-upgrades - state: started - enabled: true - - - name: Disable swap for Kubernetes node + - name: Disable swap if a swap device is present ansible.builtin.command: swapoff -a changed_when: false + failed_when: false - - name: Remove swap entries from fstab - ansible.builtin.replace: - path: /etc/fstab - regexp: '^([^#].*\sswap\s+.*)$' - replace: '# Disabled for Kubernetes: \1' - - - name: Set restrictive SSH password authentication + - name: Set restrictive SSH authentication ansible.builtin.copy: dest: /etc/ssh/sshd_config.d/60-k8s-baseline.conf owner: root @@ -53,23 +31,62 @@ content: | PasswordAuthentication no PermitRootLogin no - notify: Restart ssh + notify: Restart sshd + + - name: Create K3s configuration directory + ansible.builtin.file: + path: /etc/rancher/k3s + state: directory + owner: root + group: root + mode: '0750' + + - name: Configure K3s for Cilium kube-proxy replacement + ansible.builtin.copy: + dest: /etc/rancher/k3s/config.yaml + owner: root + group: root + mode: '0600' + content: | + write-kubeconfig-mode: "0600" + flannel-backend: none + disable-network-policy: true + disable-kube-proxy: true + disable: + - traefik + https-listen-port: {{ k3s_api_server_port }} + + - name: Download pinned K3s installer + ansible.builtin.get_url: + url: "{{ k3s_install_url }}" + dest: /var/lib/rancher/k3s-install.sh + owner: root + group: root + mode: '0700' - name: Install pinned K3s server - ansible.builtin.shell: | - curl -sfL {{ k3s_install_url }} | INSTALL_K3S_VERSION={{ k3s_version }} sh -s - server --write-kubeconfig-mode 600 - args: + ansible.builtin.command: + cmd: /var/lib/rancher/k3s-install.sh server creates: /usr/local/bin/k3s + environment: + INSTALL_K3S_VERSION: "{{ k3s_version }}" - name: Enable and start K3s - ansible.builtin.service: + ansible.builtin.systemd: name: k3s state: started enabled: true + daemon_reload: true + + - name: Wait for K3s API port + ansible.builtin.wait_for: + host: 127.0.0.1 + port: "{{ k3s_api_server_port }}" + timeout: 120 handlers: - - name: Restart ssh - ansible.builtin.service: - name: ssh + - name: Restart sshd + ansible.builtin.systemd: + name: sshd state: restarted diff --git a/argocd/README.md b/argocd/README.md index 3dcaff6..70da6f5 100644 --- a/argocd/README.md +++ b/argocd/README.md @@ -34,3 +34,8 @@ kubectl -n argocd get secret argocd-initial-admin-secret \ The initial admin secret is for local bootstrap only. A later hardening phase should replace this with SSO/RBAC and remove the bootstrap credential. + +For a kube-proxy-free Cilium cluster, install Cilium before ArgoCD using the +profile in `local-quickstart/cilium.md` or the Flatcar cloud bootstrap. The +Cilium Application is stored under `argocd/optional-apps/` and is not watched +by the default root app until the cluster is ready for it. diff --git a/argocd/optional-apps/cilium.yaml b/argocd/optional-apps/cilium.yaml new file mode 100644 index 0000000..e0314c2 --- /dev/null +++ b/argocd/optional-apps/cilium.yaml @@ -0,0 +1,25 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: cilium + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "-4" +spec: + project: default + sources: + - repoURL: https://helm.cilium.io/ + chart: cilium + targetRevision: 1.20.0 + helm: + valueFiles: + - $values/cilium/cilium-values.yaml + - repoURL: https://git.swaphb.com/swaphb/kubernetes-security-baseline-lab.git + targetRevision: main + ref: values + destination: + server: https://kubernetes.default.svc + namespace: kube-system + # Apply this Application manually only after bootstrapping a kube-proxy-free + # cluster and replacing the API endpoint placeholder. + diff --git a/cilium/cilium-values.yaml b/cilium/cilium-values.yaml new file mode 100644 index 0000000..3b20808 --- /dev/null +++ b/cilium/cilium-values.yaml @@ -0,0 +1,23 @@ +# Cilium is the CNI, network-policy engine, and eBPF service datapath for the +# production-style K3s profile. Set this endpoint before bootstrapping a +# cluster because kube-proxy is intentionally disabled. +kubeProxyReplacement: true +k8sServiceHost: "127.0.0.1" +k8sServicePort: 6443 + +ipam: + mode: kubernetes + +routingMode: tunnel +tunnelProtocol: vxlan + +hubble: + enabled: true + relay: + enabled: true + ui: + enabled: true + +operator: + replicas: 1 + diff --git a/local-quickstart/cilium.md b/local-quickstart/cilium.md new file mode 100644 index 0000000..bc8244c --- /dev/null +++ b/local-quickstart/cilium.md @@ -0,0 +1,40 @@ +# Cilium local profile + +The standard k3d profile is the default quick start. This profile is for +demonstrating Cilium with kube-proxy replacement. + +Create it with: + +```bash +k3d cluster create --config local-quickstart/k3d-cilium-cluster-config.yaml +``` + +Before installing Cilium, set the API server endpoint in a temporary values +file. The endpoint must be reachable directly from every node. Do not use the +Kubernetes service VIP because kube-proxy is disabled during bootstrap. + +```bash +kubectl config current-context +kubectl get nodes -o wide +cp cilium/cilium-values.yaml /tmp/cilium-values.yaml +# Replace REPLACE_WITH_API_SERVER_DNS_OR_PRIVATE_IP with the reachable API endpoint. +helm repo add cilium https://helm.cilium.io/ +helm repo update +helm upgrade --install cilium cilium/cilium \ + --namespace kube-system \ + --version 1.20.0 \ + --values /tmp/cilium-values.yaml +kubectl -n kube-system rollout status daemonset/cilium --timeout=300s +kubectl -n kube-system get pods -l k8s-app=cilium +``` + +Install ArgoCD only after Cilium is Ready. Apply +`argocd/optional-apps/cilium.yaml` manually after the endpoint is configured; +it is intentionally outside the root app-of-apps watched directory until the +cluster has been bootstrapped without kube-proxy. + +Delete the profile with: + +```bash +k3d cluster delete security-baseline-cilium +``` diff --git a/local-quickstart/k3d-cilium-cluster-config.yaml b/local-quickstart/k3d-cilium-cluster-config.yaml new file mode 100644 index 0000000..4d7a572 --- /dev/null +++ b/local-quickstart/k3d-cilium-cluster-config.yaml @@ -0,0 +1,32 @@ +apiVersion: k3d.io/v1alpha5 +kind: Simple +metadata: + name: security-baseline-cilium + +# This is an advanced local profile. The standard k3d profile remains the +# lowest-friction quick start and keeps Flannel enabled. +servers: 1 +agents: 2 + +options: + k3s: + extraArgs: + - arg: --flannel-backend=none + nodeFilters: + - server:* + - arg: --disable-network-policy + nodeFilters: + - server:* + - arg: --disable-kube-proxy + nodeFilters: + - server:* + - arg: --disable=traefik + nodeFilters: + - server:* + k3d: + wait: true + timeout: 180s + kubeconfig: + updateDefaultKubeconfig: true + switchCurrentContext: true + diff --git a/terraform/cloud-cluster/README.md b/terraform/cloud-cluster/README.md index f2e27ad..f14828f 100644 --- a/terraform/cloud-cluster/README.md +++ b/terraform/cloud-cluster/README.md @@ -1,10 +1,10 @@ # Optional production-style path -This directory provisions one encrypted Ubuntu EC2 host with a deliberately +This directory provisions one encrypted Flatcar EC2 host with a deliberately small network surface. It is a scaffold for the documented cloud path; the local k3d path remains the primary, cost-free portfolio demo. -The AWS account, region, AMI, SSH key pair, and administrator CIDR are inputs, +The AWS account, region, Flatcar AMI, SSH key pair, and administrator CIDR are inputs, not repository values. Do not commit a real `terraform.tfvars` file or private keys. The security group intentionally exposes SSH and the Kubernetes API only to `admin_cidr`; add any public application ports explicitly when needed. @@ -21,7 +21,7 @@ terraform apply terraform output -raw public_ip ``` -Then run `ansible/bootstrap-k3s.yml` against the output IP. `terraform destroy` +Then run `ansible/bootstrap-k3s.yml` against the output IP. Configure the +Cilium API endpoint before installing Cilium. `terraform destroy` removes the lab resources when finished. This path is not required for the portfolio demo and is not invoked by CI. - diff --git a/terraform/cloud-cluster/terraform.tfvars.example b/terraform/cloud-cluster/terraform.tfvars.example index 7522a9c..6a9b0da 100644 --- a/terraform/cloud-cluster/terraform.tfvars.example +++ b/terraform/cloud-cluster/terraform.tfvars.example @@ -3,5 +3,4 @@ availability_zone = null instance_type = "t3.medium" ssh_key_name = "replace-with-existing-key-pair" admin_cidr = "203.0.113.10/32" -ami_id = "replace-with-ubuntu-22.04-ami-for-your-region" - +ami_id = "replace-with-flatcar-stable-ami-for-your-region" diff --git a/terraform/cloud-cluster/variables.tf b/terraform/cloud-cluster/variables.tf index 3f7bae1..c119f06 100644 --- a/terraform/cloud-cluster/variables.tf +++ b/terraform/cloud-cluster/variables.tf @@ -27,7 +27,6 @@ variable "admin_cidr" { } variable "ami_id" { - description = "Ubuntu 22.04 LTS AMI ID for the selected region." + description = "Flatcar Stable AMI ID for the selected region and architecture." type = string } -