From 6e7f5b39fbc380fe6cf9466a0bf8f8840712ac5e Mon Sep 17 00:00:00 2001 From: swaphb Date: Sun, 16 Aug 2026 21:36:09 -0400 Subject: [PATCH] make falco alerting opt in --- ansible/README.md | 20 +++++- ansible/bootstrap-flatcar-k3d.yml | 2 + .../roles/falco_alerting/defaults/main.yml | 4 ++ ansible/roles/falco_alerting/tasks/main.yml | 59 ++++++++++++++++ .../argocd/optional-apps/falco-alerting.yaml | 23 +++++++ deployments/falco/README.md | 68 +++++++++++++------ .../falcosidekick-component.yaml | 5 +- .../falcosidekick-config.yaml | 0 .../alerting-resources/kustomization.yaml | 6 ++ .../operator-resources/falco-config.yaml | 3 + .../operator-resources/kustomization.yaml | 2 - 11 files changed, 163 insertions(+), 29 deletions(-) create mode 100644 ansible/roles/falco_alerting/defaults/main.yml create mode 100644 ansible/roles/falco_alerting/tasks/main.yml create mode 100644 deployments/argocd/optional-apps/falco-alerting.yaml rename deployments/falco/{operator-resources => alerting-resources}/falcosidekick-component.yaml (71%) rename deployments/falco/{operator-resources => alerting-resources}/falcosidekick-config.yaml (100%) create mode 100644 deployments/falco/alerting-resources/kustomization.yaml diff --git a/ansible/README.md b/ansible/README.md index 2edcdd8..48d34d5 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -76,9 +76,23 @@ profile's routing assumptions and provide a stable control-plane API address. The dedicated VM profile defaults to Geneve tunneling. Native routing should only be selected when the VM network routes the pod CIDR between nodes. -The playbook does not create Slack or Discord credentials. Those will be added -through a separate opt-in Ansible secret toggle so ordinary bootstrap remains -credential-free. +The playbook does not create Slack or Discord credentials by default. The +opt-in `falco_alerting` role enables Slack forwarding only when supplied with a +Vault-protected webhook value. + +Store these variables in an Ansible Vault file: + +```yaml +falco_alerting_enabled: true +falco_slack_webhook_url: https://hooks.slack.com/services/REPLACE/ME +``` + +Then run the bootstrap with `--ask-vault-pass -e +@ansible/vars/falco-alerting.vault.yml`. The role creates the +`falco-alerting` Secret without logging its value, applies the optional +Falcosidekick ArgoCD Application, and lets ArgoCD reconcile the deployment. If +the toggle is false, the role removes the optional Application and related +resources. If CoreDNS cannot resolve external names from a nested Docker network, set `k3d_dns_servers` to DNS servers reachable from the Flatcar host. Ansible diff --git a/ansible/bootstrap-flatcar-k3d.yml b/ansible/bootstrap-flatcar-k3d.yml index e94048e..a4b43c4 100644 --- a/ansible/bootstrap-flatcar-k3d.yml +++ b/ansible/bootstrap-flatcar-k3d.yml @@ -8,6 +8,7 @@ argocd_enabled: true cilium_enabled: false cilium_profile: k3d + falco_alerting_enabled: false pre_tasks: - name: Read host operating system identity @@ -21,3 +22,4 @@ when: cilium_enabled | bool - role: argocd_bootstrap when: argocd_enabled | bool + - role: falco_alerting diff --git a/ansible/roles/falco_alerting/defaults/main.yml b/ansible/roles/falco_alerting/defaults/main.yml new file mode 100644 index 0000000..a861d33 --- /dev/null +++ b/ansible/roles/falco_alerting/defaults/main.yml @@ -0,0 +1,4 @@ +--- +falco_alerting_enabled: false +falco_slack_webhook_url: "" +falco_alerting_application_manifest: "{{ k3d_workspace }}/deployments/argocd/optional-apps/falco-alerting.yaml" diff --git a/ansible/roles/falco_alerting/tasks/main.yml b/ansible/roles/falco_alerting/tasks/main.yml new file mode 100644 index 0000000..5e99070 --- /dev/null +++ b/ansible/roles/falco_alerting/tasks/main.yml @@ -0,0 +1,59 @@ +--- +- name: Validate Falco alerting inputs + ansible.builtin.assert: + that: + - not (falco_alerting_enabled | bool) or falco_slack_webhook_url | length > 0 + fail_msg: falco_slack_webhook_url is required when falco_alerting_enabled is true. + +- name: Ensure the Falco namespace exists for alerting resources + ansible.builtin.raw: >- + {{ k3d_tool_dir }}/kubectl create namespace falco + --dry-run=client -o yaml | + {{ k3d_tool_dir }}/kubectl apply -f - + become_user: "{{ k3d_user }}" + when: falco_alerting_enabled | bool + +- name: Create or update the Falco alerting Secret + ansible.builtin.raw: >- + {{ k3d_tool_dir }}/kubectl -n falco create secret generic falco-alerting + --from-literal=webhook-url='{{ falco_slack_webhook_url }}' + --dry-run=client -o yaml | + {{ k3d_tool_dir }}/kubectl apply --server-side --force-conflicts -f - + become_user: "{{ k3d_user }}" + no_log: true + when: falco_alerting_enabled | bool + +- name: Apply the optional Falco alerting ArgoCD Application + ansible.builtin.raw: >- + {{ k3d_tool_dir }}/kubectl apply --server-side --force-conflicts + -f {{ falco_alerting_application_manifest }} + become_user: "{{ k3d_user }}" + when: falco_alerting_enabled | bool + +- name: Remove the optional Falco alerting ArgoCD Application when disabled + ansible.builtin.raw: >- + {{ k3d_tool_dir }}/kubectl -n argocd delete application falco-alerting + --ignore-not-found=true --wait=true + become_user: "{{ k3d_user }}" + when: not (falco_alerting_enabled | bool) + +- name: Remove the Falcosidekick Component when alerting is disabled + ansible.builtin.raw: >- + {{ k3d_tool_dir }}/kubectl -n falco delete component falcosidekick + --ignore-not-found=true --wait=true + become_user: "{{ k3d_user }}" + when: not (falco_alerting_enabled | bool) + +- name: Remove the Falcosidekick output Config when alerting is disabled + ansible.builtin.raw: >- + {{ k3d_tool_dir }}/kubectl -n falco delete config falco-falcosidekick-output + --ignore-not-found=true --wait=true + become_user: "{{ k3d_user }}" + when: not (falco_alerting_enabled | bool) + +- name: Remove the Falco alerting Secret when alerting is disabled + ansible.builtin.raw: >- + {{ k3d_tool_dir }}/kubectl -n falco delete secret falco-alerting + --ignore-not-found=true --wait=true + become_user: "{{ k3d_user }}" + when: not (falco_alerting_enabled | bool) diff --git a/deployments/argocd/optional-apps/falco-alerting.yaml b/deployments/argocd/optional-apps/falco-alerting.yaml new file mode 100644 index 0000000..8e51d96 --- /dev/null +++ b/deployments/argocd/optional-apps/falco-alerting.yaml @@ -0,0 +1,23 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: falco-alerting + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "2" +spec: + project: default + source: + repoURL: https://git.swaphb.com/swaphb/kubernetes-security-baseline-lab.git + targetRevision: main + path: deployments/falco/alerting-resources + destination: + server: https://kubernetes.default.svc + namespace: falco + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true diff --git a/deployments/falco/README.md b/deployments/falco/README.md index 89f7d12..06284b6 100644 --- a/deployments/falco/README.md +++ b/deployments/falco/README.md @@ -1,10 +1,17 @@ -# Falco alerting secret +# Falco alerting and runtime troubleshooting + +The default Falco deployment runs Falco and its custom rules without +Falcosidekick. Slack forwarding is opt-in because a webhook is a credential +and should not be required for local cluster bootstrap. + +## Optional Slack alerting Falco alert delivery must use a Kubernetes Secret. Do not commit a Slack Incoming Webhook URL, Slack token, Discord webhook URL, or generated Secret manifest to this repository. -The expected Secret contract for the Git-managed Falcosidekick configuration is: +When alerting is enabled, the Ansible role creates this Secret contract. The +Secret is never stored in Git: ```yaml apiVersion: v1 @@ -17,23 +24,34 @@ stringData: webhook-url: https://hooks.slack.com/services/REPLACE/ME ``` -The example value is a placeholder. Create the Secret directly on the cluster -with the real Slack Incoming Webhook URL: +Store the real value in an Ansible Vault file: -```bash -kubectl create namespace falco --dry-run=client -o yaml | kubectl apply -f - -kubectl create secret generic falco-alerting \ - --namespace falco \ - --from-literal=webhook-url='https://hooks.slack.com/services/REPLACE/ME' \ - --dry-run=client -o yaml | kubectl apply -f - +```yaml +falco_alerting_enabled: true +falco_slack_webhook_url: https://hooks.slack.com/services/REPLACE/ME ``` -The `webhook-url` key is injected into Falcosidekick as `SLACK_WEBHOOKURL`. -Falcosidekick sends only `WARNING` and higher priority events to Slack through -`SLACK_MINIMUMPRIORITY`. Lower-priority events remain available in Falco logs. -The Secret is intentionally not included in the Falco -Kustomization because ArgoCD must not manage or render the credential from -Git. +Run the bootstrap with the encrypted file: + +```bash +ansible-vault encrypt ansible/vars/falco-alerting.vault.yml +ansible-playbook \ + -i ansible/inventory.flatcar-k3d.yml \ + ansible/bootstrap-flatcar-k3d.yml \ + --ask-vault-pass \ + -e @ansible/vars/falco-alerting.vault.yml +``` + +The Ansible role creates the Secret without logging its value and applies the +optional `falco-alerting` ArgoCD Application only when both the toggle and +webhook value are supplied. When disabled, it removes the optional Application, +Component, output Config, and Secret. The default Falco Application does not +include Falcosidekick resources. + +The `webhook-url` key is injected into Falcosidekick as +`SLACK_WEBHOOKURL`. Falcosidekick sends only `WARNING` and higher priority +events to Slack through `SLACK_MINIMUMPRIORITY`. Lower-priority events remain +available in Falco logs. Verify only the Secret name and key, never the value: @@ -43,12 +61,20 @@ kubectl get secret falco-alerting -n falco \ ``` Avoid printing Secret data in shared terminals or CI logs. To rotate the -webhook, update the Secret in place with the creation command and restart the -Falco workloads after the alerting integration is configured. +webhook, update the Vault value and rerun Ansible. -The repository verifies Falco runtime detections locally. Webhook delivery is -enabled by the Git-managed Falcosidekick resources after the Secret exists, and -still requires an interactive Slack or Discord credential for end-to-end testing. +## Falco inotify behavior + +The Falco configuration disables `watch_config_files`. Nested k3d nodes share +one Flatcar kernel, and Falco 0.44.1 can fail during startup while initializing +its inotify watcher on one node. GitOps changes are applied through resource +reconciliation and pod rollout instead of Falco hot reload. This is a +reliability trade-off for the nested local profile and should be revisited when +testing on separate production-style VMs. + +The upstream Falco documentation describes configuration file watching and the +manual reload behavior when watching is disabled: + For runtime alert interpretation, see [`docs/runtime-detection.md`](../../docs/runtime-detection.md). The local k3d diff --git a/deployments/falco/operator-resources/falcosidekick-component.yaml b/deployments/falco/alerting-resources/falcosidekick-component.yaml similarity index 71% rename from deployments/falco/operator-resources/falcosidekick-component.yaml rename to deployments/falco/alerting-resources/falcosidekick-component.yaml index 7b3a5f2..ca7f278 100644 --- a/deployments/falco/operator-resources/falcosidekick-component.yaml +++ b/deployments/falco/alerting-resources/falcosidekick-component.yaml @@ -4,7 +4,7 @@ metadata: name: falcosidekick namespace: falco annotations: - # Deploy the event forwarder after the Falco instance and its CRDs exist. + # Deploy the event forwarder only when the alerting Secret is enabled. argocd.argoproj.io/sync-wave: "2" spec: component: @@ -20,7 +20,6 @@ spec: secretKeyRef: name: falco-alerting key: webhook-url - # Keep low-priority Falco events in cluster logs, but do not send - # NOTICE-level activity to the Slack alert channel. + # Keep low-priority events in Falco logs instead of Slack. - name: SLACK_MINIMUMPRIORITY value: warning diff --git a/deployments/falco/operator-resources/falcosidekick-config.yaml b/deployments/falco/alerting-resources/falcosidekick-config.yaml similarity index 100% rename from deployments/falco/operator-resources/falcosidekick-config.yaml rename to deployments/falco/alerting-resources/falcosidekick-config.yaml diff --git a/deployments/falco/alerting-resources/kustomization.yaml b/deployments/falco/alerting-resources/kustomization.yaml new file mode 100644 index 0000000..1966fbd --- /dev/null +++ b/deployments/falco/alerting-resources/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: falco +resources: + - falcosidekick-component.yaml + - falcosidekick-config.yaml diff --git a/deployments/falco/operator-resources/falco-config.yaml b/deployments/falco/operator-resources/falco-config.yaml index 10a134a..eaca9dd 100644 --- a/deployments/falco/operator-resources/falco-config.yaml +++ b/deployments/falco/operator-resources/falco-config.yaml @@ -10,3 +10,6 @@ spec: config: json_output: true json_include_output_property: true + # Avoid the Falco inotify watcher in nested k3d nodes. Configuration and + # rule changes are applied through GitOps reconciliation and pod rollout. + watch_config_files: false diff --git a/deployments/falco/operator-resources/kustomization.yaml b/deployments/falco/operator-resources/kustomization.yaml index 8e42246..0080cb5 100644 --- a/deployments/falco/operator-resources/kustomization.yaml +++ b/deployments/falco/operator-resources/kustomization.yaml @@ -7,8 +7,6 @@ resources: - falco-config.yaml - container-plugin.yaml - custom-rulesfile.yaml - - falcosidekick-component.yaml - - falcosidekick-config.yaml configMapGenerator: - name: falco-custom-rules