make falco alerting opt in

ansible/README.md

@@ -76,9 +76,23 @@ profile's routing assumptions and provide a stable control-plane API address.
 The dedicated VM profile defaults to Geneve tunneling. Native routing should
 only be selected when the VM network routes the pod CIDR between nodes.
 
-The playbook does not create Slack or Discord credentials. Those will be added
-through a separate opt-in Ansible secret toggle so ordinary bootstrap remains
-credential-free.
+The playbook does not create Slack or Discord credentials by default. The
+opt-in `falco_alerting` role enables Slack forwarding only when supplied with a
+Vault-protected webhook value.
+
+Store these variables in an Ansible Vault file:
+
+```yaml
+falco_alerting_enabled: true
+falco_slack_webhook_url: https://hooks.slack.com/services/REPLACE/ME
+```
+
+Then run the bootstrap with `--ask-vault-pass -e
+@ansible/vars/falco-alerting.vault.yml`. The role creates the
+`falco-alerting` Secret without logging its value, applies the optional
+Falcosidekick ArgoCD Application, and lets ArgoCD reconcile the deployment. If
+the toggle is false, the role removes the optional Application and related
+resources.
 
 If CoreDNS cannot resolve external names from a nested Docker network, set
 `k3d_dns_servers` to DNS servers reachable from the Flatcar host. Ansible

ansible/bootstrap-flatcar-k3d.yml

@@ -8,6 +8,7 @@
     argocd_enabled: true
     cilium_enabled: false
     cilium_profile: k3d
+    falco_alerting_enabled: false
 
   pre_tasks:
     - name: Read host operating system identity
@@ -21,3 +22,4 @@
       when: cilium_enabled | bool
     - role: argocd_bootstrap
       when: argocd_enabled | bool
+    - role: falco_alerting

ansible/roles/falco_alerting/defaults/main.yml

@@ -0,0 +1,4 @@
+---
+falco_alerting_enabled: false
+falco_slack_webhook_url: ""
+falco_alerting_application_manifest: "{{ k3d_workspace }}/deployments/argocd/optional-apps/falco-alerting.yaml"

ansible/roles/falco_alerting/tasks/main.yml

@@ -0,0 +1,59 @@
+---
+- name: Validate Falco alerting inputs
+  ansible.builtin.assert:
+    that:
+      - not (falco_alerting_enabled | bool) or falco_slack_webhook_url | length > 0
+    fail_msg: falco_slack_webhook_url is required when falco_alerting_enabled is true.
+
+- name: Ensure the Falco namespace exists for alerting resources
+  ansible.builtin.raw: >-
+    {{ k3d_tool_dir }}/kubectl create namespace falco
+    --dry-run=client -o yaml |
+    {{ k3d_tool_dir }}/kubectl apply -f -
+  become_user: "{{ k3d_user }}"
+  when: falco_alerting_enabled | bool
+
+- name: Create or update the Falco alerting Secret
+  ansible.builtin.raw: >-
+    {{ k3d_tool_dir }}/kubectl -n falco create secret generic falco-alerting
+    --from-literal=webhook-url='{{ falco_slack_webhook_url }}'
+    --dry-run=client -o yaml |
+    {{ k3d_tool_dir }}/kubectl apply --server-side --force-conflicts -f -
+  become_user: "{{ k3d_user }}"
+  no_log: true
+  when: falco_alerting_enabled | bool
+
+- name: Apply the optional Falco alerting ArgoCD Application
+  ansible.builtin.raw: >-
+    {{ k3d_tool_dir }}/kubectl apply --server-side --force-conflicts
+    -f {{ falco_alerting_application_manifest }}
+  become_user: "{{ k3d_user }}"
+  when: falco_alerting_enabled | bool
+
+- name: Remove the optional Falco alerting ArgoCD Application when disabled
+  ansible.builtin.raw: >-
+    {{ k3d_tool_dir }}/kubectl -n argocd delete application falco-alerting
+    --ignore-not-found=true --wait=true
+  become_user: "{{ k3d_user }}"
+  when: not (falco_alerting_enabled | bool)
+
+- name: Remove the Falcosidekick Component when alerting is disabled
+  ansible.builtin.raw: >-
+    {{ k3d_tool_dir }}/kubectl -n falco delete component falcosidekick
+    --ignore-not-found=true --wait=true
+  become_user: "{{ k3d_user }}"
+  when: not (falco_alerting_enabled | bool)
+
+- name: Remove the Falcosidekick output Config when alerting is disabled
+  ansible.builtin.raw: >-
+    {{ k3d_tool_dir }}/kubectl -n falco delete config falco-falcosidekick-output
+    --ignore-not-found=true --wait=true
+  become_user: "{{ k3d_user }}"
+  when: not (falco_alerting_enabled | bool)
+
+- name: Remove the Falco alerting Secret when alerting is disabled
+  ansible.builtin.raw: >-
+    {{ k3d_tool_dir }}/kubectl -n falco delete secret falco-alerting
+    --ignore-not-found=true --wait=true
+  become_user: "{{ k3d_user }}"
+  when: not (falco_alerting_enabled | bool)

deployments/argocd/optional-apps/falco-alerting.yaml

@@ -0,0 +1,23 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: falco-alerting
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "2"
+spec:
+  project: default
+  source:
+    repoURL: https://git.swaphb.com/swaphb/kubernetes-security-baseline-lab.git
+    targetRevision: main
+    path: deployments/falco/alerting-resources
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: falco
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+      - ServerSideApply=true

deployments/falco/README.md

@@ -1,10 +1,17 @@
-# Falco alerting secret
+# Falco alerting and runtime troubleshooting
+
+The default Falco deployment runs Falco and its custom rules without
+Falcosidekick. Slack forwarding is opt-in because a webhook is a credential
+and should not be required for local cluster bootstrap.
+
+## Optional Slack alerting
 
 Falco alert delivery must use a Kubernetes Secret. Do not commit a Slack
 Incoming Webhook URL, Slack token, Discord webhook URL, or generated Secret
 manifest to this repository.
 
-The expected Secret contract for the Git-managed Falcosidekick configuration is:
+When alerting is enabled, the Ansible role creates this Secret contract. The
+Secret is never stored in Git:
 
 ```yaml
 apiVersion: v1
@@ -17,23 +24,34 @@ stringData:
   webhook-url: https://hooks.slack.com/services/REPLACE/ME
 ```
 
-The example value is a placeholder. Create the Secret directly on the cluster
-with the real Slack Incoming Webhook URL:
+Store the real value in an Ansible Vault file:
 
-```bash
-kubectl create namespace falco --dry-run=client -o yaml | kubectl apply -f -
-kubectl create secret generic falco-alerting \
-  --namespace falco \
-  --from-literal=webhook-url='https://hooks.slack.com/services/REPLACE/ME' \
-  --dry-run=client -o yaml | kubectl apply -f -
+```yaml
+falco_alerting_enabled: true
+falco_slack_webhook_url: https://hooks.slack.com/services/REPLACE/ME
 ```
 
-The `webhook-url` key is injected into Falcosidekick as `SLACK_WEBHOOKURL`.
-Falcosidekick sends only `WARNING` and higher priority events to Slack through
-`SLACK_MINIMUMPRIORITY`. Lower-priority events remain available in Falco logs.
-The Secret is intentionally not included in the Falco
-Kustomization because ArgoCD must not manage or render the credential from
-Git.
+Run the bootstrap with the encrypted file:
+
+```bash
+ansible-vault encrypt ansible/vars/falco-alerting.vault.yml
+ansible-playbook \
+  -i ansible/inventory.flatcar-k3d.yml \
+  ansible/bootstrap-flatcar-k3d.yml \
+  --ask-vault-pass \
+  -e @ansible/vars/falco-alerting.vault.yml
+```
+
+The Ansible role creates the Secret without logging its value and applies the
+optional `falco-alerting` ArgoCD Application only when both the toggle and
+webhook value are supplied. When disabled, it removes the optional Application,
+Component, output Config, and Secret. The default Falco Application does not
+include Falcosidekick resources.
+
+The `webhook-url` key is injected into Falcosidekick as
+`SLACK_WEBHOOKURL`. Falcosidekick sends only `WARNING` and higher priority
+events to Slack through `SLACK_MINIMUMPRIORITY`. Lower-priority events remain
+available in Falco logs.
 
 Verify only the Secret name and key, never the value:
 
@@ -43,12 +61,20 @@ kubectl get secret falco-alerting -n falco \
 ```
 
 Avoid printing Secret data in shared terminals or CI logs. To rotate the
-webhook, update the Secret in place with the creation command and restart the
-Falco workloads after the alerting integration is configured.
+webhook, update the Vault value and rerun Ansible.
 
-The repository verifies Falco runtime detections locally. Webhook delivery is
-enabled by the Git-managed Falcosidekick resources after the Secret exists, and
-still requires an interactive Slack or Discord credential for end-to-end testing.
+## Falco inotify behavior
+
+The Falco configuration disables `watch_config_files`. Nested k3d nodes share
+one Flatcar kernel, and Falco 0.44.1 can fail during startup while initializing
+its inotify watcher on one node. GitOps changes are applied through resource
+reconciliation and pod rollout instead of Falco hot reload. This is a
+reliability trade-off for the nested local profile and should be revisited when
+testing on separate production-style VMs.
+
+The upstream Falco documentation describes configuration file watching and the
+manual reload behavior when watching is disabled:
+<https://falco.org/docs/setup/packages/>
 
 For runtime alert interpretation, see
 [`docs/runtime-detection.md`](../../docs/runtime-detection.md). The local k3d

deployments/falco/alerting-resources/falcosidekick-component.yaml

@@ -4,7 +4,7 @@ metadata:
   name: falcosidekick
   namespace: falco
   annotations:
-    # Deploy the event forwarder after the Falco instance and its CRDs exist.
+    # Deploy the event forwarder only when the alerting Secret is enabled.
     argocd.argoproj.io/sync-wave: "2"
 spec:
   component:
@@ -20,7 +20,6 @@ spec:
                 secretKeyRef:
                   name: falco-alerting
                   key: webhook-url
-            # Keep low-priority Falco events in cluster logs, but do not send
-            # NOTICE-level activity to the Slack alert channel.
+            # Keep low-priority events in Falco logs instead of Slack.
             - name: SLACK_MINIMUMPRIORITY
               value: warning

deployments/falco/alerting-resources/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: falco
+resources:
+  - falcosidekick-component.yaml
+  - falcosidekick-config.yaml

deployments/falco/operator-resources/falco-config.yaml

@@ -10,3 +10,6 @@ spec:
   config:
     json_output: true
     json_include_output_property: true
+    # Avoid the Falco inotify watcher in nested k3d nodes. Configuration and
+    # rule changes are applied through GitOps reconciliation and pod rollout.
+    watch_config_files: false

deployments/falco/operator-resources/kustomization.yaml

@@ -7,8 +7,6 @@ resources:
   - falco-config.yaml
   - container-plugin.yaml
   - custom-rulesfile.yaml
-  - falcosidekick-component.yaml
-  - falcosidekick-config.yaml
 
 configMapGenerator:
   - name: falco-custom-rules