diff --git a/ansible/README.md b/ansible/README.md index 7d13533..f5cf183 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -32,8 +32,14 @@ ansible-playbook \ The playbook installs pinned k3d, kubectl, and Helm binaries, starts Docker, creates the k3d cluster, writes the core user's kubeconfig, fetches a copy into -the local `artifacts/` directory, installs ArgoCD from a pinned official -manifest, and applies the GitOps root Application from `deployments/argocd/`. +the local `artifacts/` directory, merges that config into the operator's +default `~/.kube/config`, selects the new context as current, installs ArgoCD +from a pinned official manifest, and applies the GitOps root Application from +`deployments/argocd/`. The merge preserves existing contexts and the playbook +verifies the selected context before continuing. + +Set `kubeconfig_import_enabled: false` when the fetched artifact should not +change the operator's default kubeconfig. By default the playbook copies the current repository from the Ansible controller to the host. For a private repository, set @@ -41,22 +47,35 @@ controller to the host. For a private repository, set `bootstrap_repo_ssh_private_key` through Ansible Vault. The private key task is marked `no_log` and is removed after the checkout. +When k3d exports its kubeconfig, it may use `0.0.0.0` as the API server host. +Ansible rewrites that host to `kubeconfig_server_host`, which defaults to the +Flatcar host address, while preserving the dynamically assigned API port. The +rewritten endpoint is verified before the config is merged into the default +kubeconfig. + For the optional Cilium path, set these variables in the inventory: ```yaml cilium_enabled: true k3d_config_source: "{{ playbook_dir }}/../local-quickstart/k3d-cilium-cluster-config.yaml" cilium_k8s_service_host: 192.168.60.252 -cilium_k8s_service_port: 6443 +cilium_k8s_service_port: 0 ``` The Cilium role refuses to proceed if a kube-proxy DaemonSet is present. It -passes the API endpoint explicitly to Helm and waits for the Cilium DaemonSet. +derives the dynamically published k3d API port from the new cluster's +kubeconfig, passes the API endpoint explicitly to Helm, and waits for the +Cilium DaemonSet. The playbook does not create Slack or Discord credentials. Those will be added through a separate opt-in Ansible secret toggle so ordinary bootstrap remains credential-free. +If CoreDNS cannot resolve external names from a nested Docker network, set +`k3d_dns_servers` to DNS servers reachable from the Flatcar host. Ansible +creates the supported K3s `coredns-custom` ConfigMap and restarts CoreDNS so +ArgoCD can resolve the Git server and Falcosidekick can resolve its webhook. + Validate the playbook before connecting to a host: ```bash diff --git a/ansible/inventory.flatcar-k3d.example.yml b/ansible/inventory.flatcar-k3d.example.yml index 4cd9a5a..9751fdd 100644 --- a/ansible/inventory.flatcar-k3d.example.yml +++ b/ansible/inventory.flatcar-k3d.example.yml @@ -9,4 +9,11 @@ all: ansible_become_method: sudo cilium_enabled: false cilium_k8s_service_host: REPLACE_WITH_REACHABLE_API_IP - cilium_k8s_service_port: 6443 + # Set to zero for k3d so Ansible derives the published API port from + # the kubeconfig created during cluster bootstrap. + cilium_k8s_service_port: 0 + # Optional. Use resolvers reachable from the Flatcar network when + # Docker's nested bridge resolver cannot answer external names. + # k3d_dns_servers: + # - REPLACE_WITH_PRIMARY_DNS + # - REPLACE_WITH_SECONDARY_DNS diff --git a/ansible/roles/cilium_bootstrap/defaults/main.yml b/ansible/roles/cilium_bootstrap/defaults/main.yml index 7f9e233..9d07e3c 100644 --- a/ansible/roles/cilium_bootstrap/defaults/main.yml +++ b/ansible/roles/cilium_bootstrap/defaults/main.yml @@ -4,4 +4,6 @@ cilium_namespace: kube-system cilium_chart_version: 1.20.0 cilium_values_file: "{{ k3d_workspace }}/deployments/cilium/cilium-values.yaml" cilium_k8s_service_host: "" -cilium_k8s_service_port: 6443 +# k3d publishes the API on a dynamically allocated host port. A value of zero +# makes the role derive that port from the kubeconfig created by k3d. +cilium_k8s_service_port: 0 diff --git a/ansible/roles/cilium_bootstrap/tasks/main.yml b/ansible/roles/cilium_bootstrap/tasks/main.yml index c1d9eb0..c1209c1 100644 --- a/ansible/roles/cilium_bootstrap/tasks/main.yml +++ b/ansible/roles/cilium_bootstrap/tasks/main.yml @@ -1,4 +1,17 @@ --- +- name: Discover the k3d API server port from the active kubeconfig + ansible.builtin.raw: >- + awk -F: '/server:/{print $4; exit}' /home/{{ k3d_user }}/.kube/config + become_user: "{{ k3d_user }}" + register: cilium_discovered_api_port + changed_when: false + when: cilium_k8s_service_port | int == 0 + +- name: Use the discovered k3d API server port + ansible.builtin.set_fact: + cilium_k8s_service_port: "{{ cilium_discovered_api_port.stdout | trim }}" + when: cilium_k8s_service_port | int == 0 + - name: Require a reachable Kubernetes API endpoint ansible.builtin.assert: that: @@ -46,3 +59,14 @@ {{ k3d_tool_dir }}/kubectl -n {{ cilium_namespace }} rollout status daemonset/cilium --timeout=300s become_user: "{{ k3d_user }}" + +- name: Verify all Kubernetes nodes become Ready after Cilium starts + ansible.builtin.raw: "{{ k3d_tool_dir }}/kubectl get nodes --no-headers" + become_user: "{{ k3d_user }}" + register: cilium_ready_nodes + changed_when: false + retries: 18 + delay: 10 + until: + - cilium_ready_nodes.rc == 0 + - cilium_ready_nodes.stdout is search(' Ready ') diff --git a/ansible/roles/flatcar_k3d_host/defaults/main.yml b/ansible/roles/flatcar_k3d_host/defaults/main.yml index f6bde84..1c212a5 100644 --- a/ansible/roles/flatcar_k3d_host/defaults/main.yml +++ b/ansible/roles/flatcar_k3d_host/defaults/main.yml @@ -21,3 +21,12 @@ bootstrap_repo_source: "{{ playbook_dir }}/.." kubeconfig_fetch_enabled: true kubeconfig_artifact_dir: "{{ playbook_dir }}/../artifacts" +kubeconfig_import_enabled: true +kubeconfig_default_path: "{{ lookup('env', 'HOME') }}/.kube/config" +kubeconfig_context_name: "k3d-{{ k3d_cluster_name }}" +kubeconfig_server_host: "{{ ansible_host }}" + +# Optional upstream DNS servers for nested k3d hosts. Docker may expose an +# unreachable bridge resolver to CoreDNS, so provide the host's real resolvers +# when pods must resolve external Git or webhook endpoints. +k3d_dns_servers: [] diff --git a/ansible/roles/flatcar_k3d_host/tasks/main.yml b/ansible/roles/flatcar_k3d_host/tasks/main.yml index 3dad29b..f5ac6fe 100644 --- a/ansible/roles/flatcar_k3d_host/tasks/main.yml +++ b/ansible/roles/flatcar_k3d_host/tasks/main.yml @@ -103,6 +103,17 @@ register: k3d_cluster_list changed_when: false +- name: Configure the Cilium profile API certificate SAN + ansible.builtin.raw: >- + sed -i + -e 's/K3D_EXTERNAL_API_SERVER_IP/{{ kubeconfig_server_host }}/g' + -e 's/K3D_INTERNAL_API_SERVER_IP/{{ cilium_k8s_service_host }}/g' + {{ k3d_workspace }}/local-quickstart/{{ k3d_config_source | basename }} + when: + - cilium_enabled | bool + - cilium_k8s_service_host | default('') | length > 0 + - k3d_cluster_name not in k3d_cluster_list.stdout + - name: Create the k3d cluster ansible.builtin.raw: >- {{ k3d_tool_dir }}/k3d cluster create @@ -117,17 +128,47 @@ > /home/core/.kube/config && chmod 0600 /home/core/.kube/config become_user: "{{ k3d_user }}" -- name: Verify Kubernetes nodes are Ready +- name: Verify Kubernetes nodes are Ready for the standard profile ansible.builtin.raw: "{{ k3d_tool_dir }}/kubectl get nodes --no-headers" become_user: "{{ k3d_user }}" register: k3d_nodes changed_when: false + when: not (cilium_enabled | bool) retries: 12 delay: 10 until: - k3d_nodes.rc == 0 - k3d_nodes.stdout is search(' Ready ') +- name: Verify Kubernetes API and node objects for the Cilium profile + ansible.builtin.raw: "{{ k3d_tool_dir }}/kubectl get nodes --no-headers" + become_user: "{{ k3d_user }}" + register: k3d_nodes_cilium + changed_when: false + when: cilium_enabled | bool + retries: 12 + delay: 10 + until: + - k3d_nodes_cilium.rc == 0 + - k3d_nodes_cilium.stdout | trim | length > 0 + +- name: Configure reachable upstream DNS servers for CoreDNS + ansible.builtin.raw: >- + {{ k3d_tool_dir }}/kubectl -n kube-system create configmap coredns-custom + --from-literal=upstream.override='forward . {{ (k3d_dns_servers | string | from_yaml) | join(' ') }}' + --dry-run=client -o yaml | + {{ k3d_tool_dir }}/kubectl apply -f - + become_user: "{{ k3d_user }}" + when: k3d_dns_servers | length > 0 + +- name: Restart CoreDNS after changing upstream resolvers + ansible.builtin.raw: >- + {{ k3d_tool_dir }}/kubectl -n kube-system rollout restart deployment/coredns && + {{ k3d_tool_dir }}/kubectl -n kube-system rollout status deployment/coredns + --timeout=120s + become_user: "{{ k3d_user }}" + when: k3d_dns_servers | length > 0 + - name: Ensure local kubeconfig artifact directory exists ansible.builtin.file: path: "{{ kubeconfig_artifact_dir }}" @@ -144,6 +185,122 @@ dest: "{{ kubeconfig_artifact_dir }}/{{ inventory_hostname }}-kubeconfig" mode: pull archive: true + rsync_opts: + - "--checksum" delegate_to: localhost become: false when: kubeconfig_fetch_enabled | bool + +- name: Replace the k3d wildcard API address in the local artifact + ansible.builtin.replace: + path: "{{ kubeconfig_artifact_dir }}/{{ inventory_hostname }}-kubeconfig" + regexp: '(?m)^(\s+server:\s+https://)0\.0\.0\.0(:[0-9]+\s*)$' + replace: '\g<1>{{ kubeconfig_server_host }}\g<2>' + delegate_to: localhost + become: false + when: + - kubeconfig_fetch_enabled | bool + - kubeconfig_import_enabled | bool + +- name: Read the rewritten local kubeconfig server endpoint + ansible.builtin.command: + cmd: >- + kubectl config view + --kubeconfig {{ kubeconfig_artifact_dir }}/{{ inventory_hostname }}-kubeconfig + --minify -o jsonpath={.clusters[0].cluster.server} + register: rewritten_kubeconfig_server + changed_when: false + delegate_to: localhost + become: false + when: + - kubeconfig_fetch_enabled | bool + - kubeconfig_import_enabled | bool + +- name: Require a reachable host address in the local kubeconfig artifact + ansible.builtin.assert: + that: + - rewritten_kubeconfig_server.stdout | trim is match('^https://' ~ (kubeconfig_server_host | regex_escape) ~ ':[0-9]+$') + - rewritten_kubeconfig_server.stdout | trim is not search('0\.0\.0\.0') + fail_msg: >- + The fetched kubeconfig still points to an unusable API address: + {{ rewritten_kubeconfig_server.stdout | trim }} + delegate_to: localhost + become: false + when: + - kubeconfig_fetch_enabled | bool + - kubeconfig_import_enabled | bool + +- name: Ensure the default kubeconfig directory exists + ansible.builtin.file: + path: "{{ kubeconfig_default_path | dirname }}" + state: directory + mode: '0700' + delegate_to: localhost + become: false + run_once: true + when: + - kubeconfig_fetch_enabled | bool + - kubeconfig_import_enabled | bool + +- name: Merge the fetched kubeconfig into the operator default + ansible.builtin.command: + cmd: kubectl config view --flatten + environment: + # Put the freshly fetched artifact first so it overrides an older entry + # with the same cluster and context name. + KUBECONFIG: "{{ kubeconfig_artifact_dir }}/{{ inventory_hostname }}-kubeconfig:{{ kubeconfig_default_path }}" + register: merged_kubeconfig + changed_when: false + delegate_to: localhost + become: false + no_log: true + when: + - kubeconfig_fetch_enabled | bool + - kubeconfig_import_enabled | bool + +- name: Install the merged kubeconfig as the operator default + ansible.builtin.copy: + content: "{{ merged_kubeconfig.stdout }}" + dest: "{{ kubeconfig_default_path }}" + mode: '0600' + delegate_to: localhost + become: false + no_log: true + when: + - kubeconfig_fetch_enabled | bool + - kubeconfig_import_enabled | bool + +- name: Select the bootstrapped cluster context + ansible.builtin.command: + cmd: kubectl config use-context {{ kubeconfig_context_name }} --kubeconfig {{ kubeconfig_default_path }} + register: selected_kubeconfig_context + changed_when: false + delegate_to: localhost + become: false + when: + - kubeconfig_fetch_enabled | bool + - kubeconfig_import_enabled | bool + +- name: Verify the selected kubeconfig context + ansible.builtin.command: + cmd: kubectl config current-context --kubeconfig {{ kubeconfig_default_path }} + register: selected_kubeconfig_context_check + changed_when: false + delegate_to: localhost + become: false + when: + - kubeconfig_fetch_enabled | bool + - kubeconfig_import_enabled | bool + +- name: Require the expected bootstrapped context to be current + ansible.builtin.assert: + that: + - selected_kubeconfig_context_check.stdout | trim == kubeconfig_context_name + fail_msg: >- + The default kubeconfig context was not changed to + {{ kubeconfig_context_name }}. + delegate_to: localhost + become: false + when: + - kubeconfig_fetch_enabled | bool + - kubeconfig_import_enabled | bool diff --git a/deployments/cilium/cilium-values.yaml b/deployments/cilium/cilium-values.yaml index 3b20808..700dd13 100644 --- a/deployments/cilium/cilium-values.yaml +++ b/deployments/cilium/cilium-values.yaml @@ -8,8 +8,23 @@ k8sServicePort: 6443 ipam: mode: kubernetes -routingMode: tunnel -tunnelProtocol: vxlan +# k3d nodes share a directly reachable Docker bridge. Native routing avoids +# nested VXLAN service paths that can prevent pod access to ClusterIP services. +routingMode: native +autoDirectNodeRoutes: true +# K3s allocates pod addresses from this cluster-wide CIDR. Native routing +# requires the CIDR so Cilium can distinguish pod traffic from underlay traffic. +ipv4NativeRoutingCIDR: 10.42.0.0/16 + +# The k3d nodes run as nested containers on the Flatcar host. eBPF masquerade +# keeps return traffic for cross-node service backends inside the Cilium path. +bpf: + masquerade: true + +# Keep socket load-balancing in the node namespace for the nested k3d profile. +# Pod traffic is handled by the eBPF service datapath instead. +socketLB: + hostNamespaceOnly: true hubble: enabled: true @@ -20,4 +35,3 @@ hubble: operator: replicas: 1 - diff --git a/local-quickstart/k3d-cilium-cluster-config.yaml b/local-quickstart/k3d-cilium-cluster-config.yaml index 4d7a572..585df59 100644 --- a/local-quickstart/k3d-cilium-cluster-config.yaml +++ b/local-quickstart/k3d-cilium-cluster-config.yaml @@ -20,6 +20,14 @@ options: - arg: --disable-kube-proxy nodeFilters: - server:* + # Include both the external Flatcar address and the internal k3d server + # address so workstation and in-cluster API clients pass TLS validation. + - arg: --tls-san=K3D_EXTERNAL_API_SERVER_IP + nodeFilters: + - server:* + - arg: --tls-san=K3D_INTERNAL_API_SERVER_IP + nodeFilters: + - server:* - arg: --disable=traefik nodeFilters: - server:* @@ -29,4 +37,3 @@ options: kubeconfig: updateDefaultKubeconfig: true switchCurrentContext: true -