fix cilium routing and cluster dns

ansible/roles/flatcar_k3d_host/defaults/main.yml

@@ -21,3 +21,12 @@ bootstrap_repo_source: "{{ playbook_dir }}/.."
 
 kubeconfig_fetch_enabled: true
 kubeconfig_artifact_dir: "{{ playbook_dir }}/../artifacts"
+kubeconfig_import_enabled: true
+kubeconfig_default_path: "{{ lookup('env', 'HOME') }}/.kube/config"
+kubeconfig_context_name: "k3d-{{ k3d_cluster_name }}"
+kubeconfig_server_host: "{{ ansible_host }}"
+
+# Optional upstream DNS servers for nested k3d hosts. Docker may expose an
+# unreachable bridge resolver to CoreDNS, so provide the host's real resolvers
+# when pods must resolve external Git or webhook endpoints.
+k3d_dns_servers: []

ansible/roles/flatcar_k3d_host/tasks/main.yml

@@ -103,6 +103,17 @@
   register: k3d_cluster_list
   changed_when: false
 
+- name: Configure the Cilium profile API certificate SAN
+  ansible.builtin.raw: >-
+    sed -i
+    -e 's/K3D_EXTERNAL_API_SERVER_IP/{{ kubeconfig_server_host }}/g'
+    -e 's/K3D_INTERNAL_API_SERVER_IP/{{ cilium_k8s_service_host }}/g'
+    {{ k3d_workspace }}/local-quickstart/{{ k3d_config_source | basename }}
+  when:
+    - cilium_enabled | bool
+    - cilium_k8s_service_host | default('') | length > 0
+    - k3d_cluster_name not in k3d_cluster_list.stdout
+
 - name: Create the k3d cluster
   ansible.builtin.raw: >-
     {{ k3d_tool_dir }}/k3d cluster create
@@ -117,17 +128,47 @@
     > /home/core/.kube/config && chmod 0600 /home/core/.kube/config
   become_user: "{{ k3d_user }}"
 
-- name: Verify Kubernetes nodes are Ready
+- name: Verify Kubernetes nodes are Ready for the standard profile
   ansible.builtin.raw: "{{ k3d_tool_dir }}/kubectl get nodes --no-headers"
   become_user: "{{ k3d_user }}"
   register: k3d_nodes
   changed_when: false
+  when: not (cilium_enabled | bool)
   retries: 12
   delay: 10
   until:
     - k3d_nodes.rc == 0
     - k3d_nodes.stdout is search(' Ready ')
 
+- name: Verify Kubernetes API and node objects for the Cilium profile
+  ansible.builtin.raw: "{{ k3d_tool_dir }}/kubectl get nodes --no-headers"
+  become_user: "{{ k3d_user }}"
+  register: k3d_nodes_cilium
+  changed_when: false
+  when: cilium_enabled | bool
+  retries: 12
+  delay: 10
+  until:
+    - k3d_nodes_cilium.rc == 0
+    - k3d_nodes_cilium.stdout | trim | length > 0
+
+- name: Configure reachable upstream DNS servers for CoreDNS
+  ansible.builtin.raw: >-
+    {{ k3d_tool_dir }}/kubectl -n kube-system create configmap coredns-custom
+    --from-literal=upstream.override='forward . {{ (k3d_dns_servers | string | from_yaml) | join(' ') }}'
+    --dry-run=client -o yaml |
+    {{ k3d_tool_dir }}/kubectl apply -f -
+  become_user: "{{ k3d_user }}"
+  when: k3d_dns_servers | length > 0
+
+- name: Restart CoreDNS after changing upstream resolvers
+  ansible.builtin.raw: >-
+    {{ k3d_tool_dir }}/kubectl -n kube-system rollout restart deployment/coredns &&
+    {{ k3d_tool_dir }}/kubectl -n kube-system rollout status deployment/coredns
+    --timeout=120s
+  become_user: "{{ k3d_user }}"
+  when: k3d_dns_servers | length > 0
+
 - name: Ensure local kubeconfig artifact directory exists
   ansible.builtin.file:
     path: "{{ kubeconfig_artifact_dir }}"
@@ -144,6 +185,122 @@
     dest: "{{ kubeconfig_artifact_dir }}/{{ inventory_hostname }}-kubeconfig"
     mode: pull
     archive: true
+    rsync_opts:
+      - "--checksum"
   delegate_to: localhost
   become: false
   when: kubeconfig_fetch_enabled | bool
+
+- name: Replace the k3d wildcard API address in the local artifact
+  ansible.builtin.replace:
+    path: "{{ kubeconfig_artifact_dir }}/{{ inventory_hostname }}-kubeconfig"
+    regexp: '(?m)^(\s+server:\s+https://)0\.0\.0\.0(:[0-9]+\s*)$'
+    replace: '\g<1>{{ kubeconfig_server_host }}\g<2>'
+  delegate_to: localhost
+  become: false
+  when:
+    - kubeconfig_fetch_enabled | bool
+    - kubeconfig_import_enabled | bool
+
+- name: Read the rewritten local kubeconfig server endpoint
+  ansible.builtin.command:
+    cmd: >-
+      kubectl config view
+      --kubeconfig {{ kubeconfig_artifact_dir }}/{{ inventory_hostname }}-kubeconfig
+      --minify -o jsonpath={.clusters[0].cluster.server}
+  register: rewritten_kubeconfig_server
+  changed_when: false
+  delegate_to: localhost
+  become: false
+  when:
+    - kubeconfig_fetch_enabled | bool
+    - kubeconfig_import_enabled | bool
+
+- name: Require a reachable host address in the local kubeconfig artifact
+  ansible.builtin.assert:
+    that:
+      - rewritten_kubeconfig_server.stdout | trim is match('^https://' ~ (kubeconfig_server_host | regex_escape) ~ ':[0-9]+$')
+      - rewritten_kubeconfig_server.stdout | trim is not search('0\.0\.0\.0')
+    fail_msg: >-
+      The fetched kubeconfig still points to an unusable API address:
+      {{ rewritten_kubeconfig_server.stdout | trim }}
+  delegate_to: localhost
+  become: false
+  when:
+    - kubeconfig_fetch_enabled | bool
+    - kubeconfig_import_enabled | bool
+
+- name: Ensure the default kubeconfig directory exists
+  ansible.builtin.file:
+    path: "{{ kubeconfig_default_path | dirname }}"
+    state: directory
+    mode: '0700'
+  delegate_to: localhost
+  become: false
+  run_once: true
+  when:
+    - kubeconfig_fetch_enabled | bool
+    - kubeconfig_import_enabled | bool
+
+- name: Merge the fetched kubeconfig into the operator default
+  ansible.builtin.command:
+    cmd: kubectl config view --flatten
+  environment:
+    # Put the freshly fetched artifact first so it overrides an older entry
+    # with the same cluster and context name.
+    KUBECONFIG: "{{ kubeconfig_artifact_dir }}/{{ inventory_hostname }}-kubeconfig:{{ kubeconfig_default_path }}"
+  register: merged_kubeconfig
+  changed_when: false
+  delegate_to: localhost
+  become: false
+  no_log: true
+  when:
+    - kubeconfig_fetch_enabled | bool
+    - kubeconfig_import_enabled | bool
+
+- name: Install the merged kubeconfig as the operator default
+  ansible.builtin.copy:
+    content: "{{ merged_kubeconfig.stdout }}"
+    dest: "{{ kubeconfig_default_path }}"
+    mode: '0600'
+  delegate_to: localhost
+  become: false
+  no_log: true
+  when:
+    - kubeconfig_fetch_enabled | bool
+    - kubeconfig_import_enabled | bool
+
+- name: Select the bootstrapped cluster context
+  ansible.builtin.command:
+    cmd: kubectl config use-context {{ kubeconfig_context_name }} --kubeconfig {{ kubeconfig_default_path }}
+  register: selected_kubeconfig_context
+  changed_when: false
+  delegate_to: localhost
+  become: false
+  when:
+    - kubeconfig_fetch_enabled | bool
+    - kubeconfig_import_enabled | bool
+
+- name: Verify the selected kubeconfig context
+  ansible.builtin.command:
+    cmd: kubectl config current-context --kubeconfig {{ kubeconfig_default_path }}
+  register: selected_kubeconfig_context_check
+  changed_when: false
+  delegate_to: localhost
+  become: false
+  when:
+    - kubeconfig_fetch_enabled | bool
+    - kubeconfig_import_enabled | bool
+
+- name: Require the expected bootstrapped context to be current
+  ansible.builtin.assert:
+    that:
+      - selected_kubeconfig_context_check.stdout | trim == kubeconfig_context_name
+    fail_msg: >-
+      The default kubeconfig context was not changed to
+      {{ kubeconfig_context_name }}.
+  delegate_to: localhost
+  become: false
+  when:
+    - kubeconfig_fetch_enabled | bool
+    - kubeconfig_import_enabled | bool