fix cilium routing and cluster dns

ansible/README.md

@@ -32,8 +32,14 @@ ansible-playbook \
 
 The playbook installs pinned k3d, kubectl, and Helm binaries, starts Docker,
 creates the k3d cluster, writes the core user's kubeconfig, fetches a copy into
-the local `artifacts/` directory, installs ArgoCD from a pinned official
+the local `artifacts/` directory, merges that config into the operator's
-manifest, and applies the GitOps root Application from `deployments/argocd/`.
+default `~/.kube/config`, selects the new context as current, installs ArgoCD
+from a pinned official manifest, and applies the GitOps root Application from
+`deployments/argocd/`. The merge preserves existing contexts and the playbook
+verifies the selected context before continuing.
+
+Set `kubeconfig_import_enabled: false` when the fetched artifact should not
+change the operator's default kubeconfig.
 
 By default the playbook copies the current repository from the Ansible
 controller to the host. For a private repository, set
@@ -41,22 +47,35 @@ controller to the host. For a private repository, set
 `bootstrap_repo_ssh_private_key` through Ansible Vault. The private key task is
 marked `no_log` and is removed after the checkout.
 
+When k3d exports its kubeconfig, it may use `0.0.0.0` as the API server host.
+Ansible rewrites that host to `kubeconfig_server_host`, which defaults to the
+Flatcar host address, while preserving the dynamically assigned API port. The
+rewritten endpoint is verified before the config is merged into the default
+kubeconfig.
+
 For the optional Cilium path, set these variables in the inventory:
 
 ```yaml
 cilium_enabled: true
 k3d_config_source: "{{ playbook_dir }}/../local-quickstart/k3d-cilium-cluster-config.yaml"
 cilium_k8s_service_host: 192.168.60.252
-cilium_k8s_service_port: 6443
+cilium_k8s_service_port: 0
 ```
 
 The Cilium role refuses to proceed if a kube-proxy DaemonSet is present. It
-passes the API endpoint explicitly to Helm and waits for the Cilium DaemonSet.
+derives the dynamically published k3d API port from the new cluster's
+kubeconfig, passes the API endpoint explicitly to Helm, and waits for the
+Cilium DaemonSet.
 
 The playbook does not create Slack or Discord credentials. Those will be added
 through a separate opt-in Ansible secret toggle so ordinary bootstrap remains
 credential-free.
 
+If CoreDNS cannot resolve external names from a nested Docker network, set
+`k3d_dns_servers` to DNS servers reachable from the Flatcar host. Ansible
+creates the supported K3s `coredns-custom` ConfigMap and restarts CoreDNS so
+ArgoCD can resolve the Git server and Falcosidekick can resolve its webhook.
+
 Validate the playbook before connecting to a host:
 
 ```bash

ansible/inventory.flatcar-k3d.example.yml

@@ -9,4 +9,11 @@ all:
           ansible_become_method: sudo
           cilium_enabled: false
           cilium_k8s_service_host: REPLACE_WITH_REACHABLE_API_IP
-          cilium_k8s_service_port: 6443
+          # Set to zero for k3d so Ansible derives the published API port from
+          # the kubeconfig created during cluster bootstrap.
+          cilium_k8s_service_port: 0
+          # Optional. Use resolvers reachable from the Flatcar network when
+          # Docker's nested bridge resolver cannot answer external names.
+          # k3d_dns_servers:
+          #   - REPLACE_WITH_PRIMARY_DNS
+          #   - REPLACE_WITH_SECONDARY_DNS

ansible/roles/cilium_bootstrap/defaults/main.yml

@@ -4,4 +4,6 @@ cilium_namespace: kube-system
 cilium_chart_version: 1.20.0
 cilium_values_file: "{{ k3d_workspace }}/deployments/cilium/cilium-values.yaml"
 cilium_k8s_service_host: ""
-cilium_k8s_service_port: 6443
+# k3d publishes the API on a dynamically allocated host port. A value of zero
+# makes the role derive that port from the kubeconfig created by k3d.
+cilium_k8s_service_port: 0

ansible/roles/cilium_bootstrap/tasks/main.yml

@@ -1,4 +1,17 @@
 ---
+- name: Discover the k3d API server port from the active kubeconfig
+  ansible.builtin.raw: >-
+    awk -F: '/server:/{print $4; exit}' /home/{{ k3d_user }}/.kube/config
+  become_user: "{{ k3d_user }}"
+  register: cilium_discovered_api_port
+  changed_when: false
+  when: cilium_k8s_service_port | int == 0
+
+- name: Use the discovered k3d API server port
+  ansible.builtin.set_fact:
+    cilium_k8s_service_port: "{{ cilium_discovered_api_port.stdout | trim }}"
+  when: cilium_k8s_service_port | int == 0
+
 - name: Require a reachable Kubernetes API endpoint
   ansible.builtin.assert:
     that:
@@ -46,3 +59,14 @@
     {{ k3d_tool_dir }}/kubectl -n {{ cilium_namespace }} rollout status
     daemonset/cilium --timeout=300s
   become_user: "{{ k3d_user }}"
+
+- name: Verify all Kubernetes nodes become Ready after Cilium starts
+  ansible.builtin.raw: "{{ k3d_tool_dir }}/kubectl get nodes --no-headers"
+  become_user: "{{ k3d_user }}"
+  register: cilium_ready_nodes
+  changed_when: false
+  retries: 18
+  delay: 10
+  until:
+    - cilium_ready_nodes.rc == 0
+    - cilium_ready_nodes.stdout is search(' Ready ')

ansible/roles/flatcar_k3d_host/defaults/main.yml

@@ -21,3 +21,12 @@ bootstrap_repo_source: "{{ playbook_dir }}/.."
 
 kubeconfig_fetch_enabled: true
 kubeconfig_artifact_dir: "{{ playbook_dir }}/../artifacts"
+kubeconfig_import_enabled: true
+kubeconfig_default_path: "{{ lookup('env', 'HOME') }}/.kube/config"
+kubeconfig_context_name: "k3d-{{ k3d_cluster_name }}"
+kubeconfig_server_host: "{{ ansible_host }}"
+
+# Optional upstream DNS servers for nested k3d hosts. Docker may expose an
+# unreachable bridge resolver to CoreDNS, so provide the host's real resolvers
+# when pods must resolve external Git or webhook endpoints.
+k3d_dns_servers: []

ansible/roles/flatcar_k3d_host/tasks/main.yml

@@ -103,6 +103,17 @@
   register: k3d_cluster_list
   changed_when: false
 
+- name: Configure the Cilium profile API certificate SAN
+  ansible.builtin.raw: >-
+    sed -i
+    -e 's/K3D_EXTERNAL_API_SERVER_IP/{{ kubeconfig_server_host }}/g'
+    -e 's/K3D_INTERNAL_API_SERVER_IP/{{ cilium_k8s_service_host }}/g'
+    {{ k3d_workspace }}/local-quickstart/{{ k3d_config_source | basename }}
+  when:
+    - cilium_enabled | bool
+    - cilium_k8s_service_host | default('') | length > 0
+    - k3d_cluster_name not in k3d_cluster_list.stdout
+
 - name: Create the k3d cluster
   ansible.builtin.raw: >-
     {{ k3d_tool_dir }}/k3d cluster create
@@ -117,17 +128,47 @@
     > /home/core/.kube/config && chmod 0600 /home/core/.kube/config
   become_user: "{{ k3d_user }}"
 
-- name: Verify Kubernetes nodes are Ready
+- name: Verify Kubernetes nodes are Ready for the standard profile
   ansible.builtin.raw: "{{ k3d_tool_dir }}/kubectl get nodes --no-headers"
   become_user: "{{ k3d_user }}"
   register: k3d_nodes
   changed_when: false
+  when: not (cilium_enabled | bool)
   retries: 12
   delay: 10
   until:
     - k3d_nodes.rc == 0
     - k3d_nodes.stdout is search(' Ready ')
 
+- name: Verify Kubernetes API and node objects for the Cilium profile
+  ansible.builtin.raw: "{{ k3d_tool_dir }}/kubectl get nodes --no-headers"
+  become_user: "{{ k3d_user }}"
+  register: k3d_nodes_cilium
+  changed_when: false
+  when: cilium_enabled | bool
+  retries: 12
+  delay: 10
+  until:
+    - k3d_nodes_cilium.rc == 0
+    - k3d_nodes_cilium.stdout | trim | length > 0
+
+- name: Configure reachable upstream DNS servers for CoreDNS
+  ansible.builtin.raw: >-
+    {{ k3d_tool_dir }}/kubectl -n kube-system create configmap coredns-custom
+    --from-literal=upstream.override='forward . {{ (k3d_dns_servers | string | from_yaml) | join(' ') }}'
+    --dry-run=client -o yaml |
+    {{ k3d_tool_dir }}/kubectl apply -f -
+  become_user: "{{ k3d_user }}"
+  when: k3d_dns_servers | length > 0
+
+- name: Restart CoreDNS after changing upstream resolvers
+  ansible.builtin.raw: >-
+    {{ k3d_tool_dir }}/kubectl -n kube-system rollout restart deployment/coredns &&
+    {{ k3d_tool_dir }}/kubectl -n kube-system rollout status deployment/coredns
+    --timeout=120s
+  become_user: "{{ k3d_user }}"
+  when: k3d_dns_servers | length > 0
+
 - name: Ensure local kubeconfig artifact directory exists
   ansible.builtin.file:
     path: "{{ kubeconfig_artifact_dir }}"
@@ -144,6 +185,122 @@
     dest: "{{ kubeconfig_artifact_dir }}/{{ inventory_hostname }}-kubeconfig"
     mode: pull
     archive: true
+    rsync_opts:
+      - "--checksum"
   delegate_to: localhost
   become: false
   when: kubeconfig_fetch_enabled | bool
+
+- name: Replace the k3d wildcard API address in the local artifact
+  ansible.builtin.replace:
+    path: "{{ kubeconfig_artifact_dir }}/{{ inventory_hostname }}-kubeconfig"
+    regexp: '(?m)^(\s+server:\s+https://)0\.0\.0\.0(:[0-9]+\s*)$'
+    replace: '\g<1>{{ kubeconfig_server_host }}\g<2>'
+  delegate_to: localhost
+  become: false
+  when:
+    - kubeconfig_fetch_enabled | bool
+    - kubeconfig_import_enabled | bool
+
+- name: Read the rewritten local kubeconfig server endpoint
+  ansible.builtin.command:
+    cmd: >-
+      kubectl config view
+      --kubeconfig {{ kubeconfig_artifact_dir }}/{{ inventory_hostname }}-kubeconfig
+      --minify -o jsonpath={.clusters[0].cluster.server}
+  register: rewritten_kubeconfig_server
+  changed_when: false
+  delegate_to: localhost
+  become: false
+  when:
+    - kubeconfig_fetch_enabled | bool
+    - kubeconfig_import_enabled | bool
+
+- name: Require a reachable host address in the local kubeconfig artifact
+  ansible.builtin.assert:
+    that:
+      - rewritten_kubeconfig_server.stdout | trim is match('^https://' ~ (kubeconfig_server_host | regex_escape) ~ ':[0-9]+$')
+      - rewritten_kubeconfig_server.stdout | trim is not search('0\.0\.0\.0')
+    fail_msg: >-
+      The fetched kubeconfig still points to an unusable API address:
+      {{ rewritten_kubeconfig_server.stdout | trim }}
+  delegate_to: localhost
+  become: false
+  when:
+    - kubeconfig_fetch_enabled | bool
+    - kubeconfig_import_enabled | bool
+
+- name: Ensure the default kubeconfig directory exists
+  ansible.builtin.file:
+    path: "{{ kubeconfig_default_path | dirname }}"
+    state: directory
+    mode: '0700'
+  delegate_to: localhost
+  become: false
+  run_once: true
+  when:
+    - kubeconfig_fetch_enabled | bool
+    - kubeconfig_import_enabled | bool
+
+- name: Merge the fetched kubeconfig into the operator default
+  ansible.builtin.command:
+    cmd: kubectl config view --flatten
+  environment:
+    # Put the freshly fetched artifact first so it overrides an older entry
+    # with the same cluster and context name.
+    KUBECONFIG: "{{ kubeconfig_artifact_dir }}/{{ inventory_hostname }}-kubeconfig:{{ kubeconfig_default_path }}"
+  register: merged_kubeconfig
+  changed_when: false
+  delegate_to: localhost
+  become: false
+  no_log: true
+  when:
+    - kubeconfig_fetch_enabled | bool
+    - kubeconfig_import_enabled | bool
+
+- name: Install the merged kubeconfig as the operator default
+  ansible.builtin.copy:
+    content: "{{ merged_kubeconfig.stdout }}"
+    dest: "{{ kubeconfig_default_path }}"
+    mode: '0600'
+  delegate_to: localhost
+  become: false
+  no_log: true
+  when:
+    - kubeconfig_fetch_enabled | bool
+    - kubeconfig_import_enabled | bool
+
+- name: Select the bootstrapped cluster context
+  ansible.builtin.command:
+    cmd: kubectl config use-context {{ kubeconfig_context_name }} --kubeconfig {{ kubeconfig_default_path }}
+  register: selected_kubeconfig_context
+  changed_when: false
+  delegate_to: localhost
+  become: false
+  when:
+    - kubeconfig_fetch_enabled | bool
+    - kubeconfig_import_enabled | bool
+
+- name: Verify the selected kubeconfig context
+  ansible.builtin.command:
+    cmd: kubectl config current-context --kubeconfig {{ kubeconfig_default_path }}
+  register: selected_kubeconfig_context_check
+  changed_when: false
+  delegate_to: localhost
+  become: false
+  when:
+    - kubeconfig_fetch_enabled | bool
+    - kubeconfig_import_enabled | bool
+
+- name: Require the expected bootstrapped context to be current
+  ansible.builtin.assert:
+    that:
+      - selected_kubeconfig_context_check.stdout | trim == kubeconfig_context_name
+    fail_msg: >-
+      The default kubeconfig context was not changed to
+      {{ kubeconfig_context_name }}.
+  delegate_to: localhost
+  become: false
+  when:
+    - kubeconfig_fetch_enabled | bool
+    - kubeconfig_import_enabled | bool

deployments/cilium/cilium-values.yaml

@@ -8,8 +8,23 @@ k8sServicePort: 6443
 ipam:
   mode: kubernetes
 
-routingMode: tunnel
+# k3d nodes share a directly reachable Docker bridge. Native routing avoids
-tunnelProtocol: vxlan
+# nested VXLAN service paths that can prevent pod access to ClusterIP services.
+routingMode: native
+autoDirectNodeRoutes: true
+# K3s allocates pod addresses from this cluster-wide CIDR. Native routing
+# requires the CIDR so Cilium can distinguish pod traffic from underlay traffic.
+ipv4NativeRoutingCIDR: 10.42.0.0/16
+
+# The k3d nodes run as nested containers on the Flatcar host. eBPF masquerade
+# keeps return traffic for cross-node service backends inside the Cilium path.
+bpf:
+  masquerade: true
+
+# Keep socket load-balancing in the node namespace for the nested k3d profile.
+# Pod traffic is handled by the eBPF service datapath instead.
+socketLB:
+  hostNamespaceOnly: true
 
 hubble:
   enabled: true
@@ -20,4 +35,3 @@ hubble:
 
 operator:
   replicas: 1
-

local-quickstart/k3d-cilium-cluster-config.yaml

@@ -20,6 +20,14 @@ options:
       - arg: --disable-kube-proxy
         nodeFilters:
           - server:*
+      # Include both the external Flatcar address and the internal k3d server
+      # address so workstation and in-cluster API clients pass TLS validation.
+      - arg: --tls-san=K3D_EXTERNAL_API_SERVER_IP
+        nodeFilters:
+          - server:*
+      - arg: --tls-san=K3D_INTERNAL_API_SERVER_IP
+        nodeFilters:
+          - server:*
       - arg: --disable=traefik
         nodeFilters:
           - server:*
@@ -29,4 +37,3 @@ options:
   kubeconfig:
     updateDefaultKubeconfig: true
     switchCurrentContext: true
-