OCPBUGS-61792: Collect coredumps on all nodes during CI runs

ci-operator/config/openshift/release/openshift-release-master__ci-4.20-upgrade-from-stable-4.19.yaml

@@ -97,6 +97,12 @@ tests:
     observers:
       enable:
       - observers-resource-watch
+    post:
+    - chain: gather-core-dump
+    - chain: ipi-azure-post
+    pre:
+    - ref: enable-node-coredumps
+    - chain: ipi-azure-pre-stableinitial
     workflow: openshift-upgrade-azure-ovn
   timeout: 5h30m0s
 - as: e2e-aws-ovn-uwm

ci-operator/config/openshift/router/openshift-router-master.yaml

@@ -61,6 +61,12 @@ tests:
   skip_if_only_changed: ^docs/|\.md$|^(?:.*/)?(?:\.gitignore|OWNERS|OWNERS_ALIASES|PROJECT|LICENSE)$
   steps:
     cluster_profile: gcp-3
+    post:
+    - chain: gather-core-dump
+    - chain: ipi-gcp-post
+    pre:
+    - ref: enable-node-coredumps
+    - chain: ipi-gcp-pre
     workflow: openshift-e2e-gcp
 - as: e2e-metal-ipi-ovn-ipv6
   cluster: build05
@@ -72,6 +78,12 @@ tests:
       DEVSCRIPTS_CONFIG: |
         IP_STACK=v6
         NETWORK_TYPE=OVNKubernetes
+    post:
+    - chain: gather-core-dump
+    - chain: baremetalds-ofcir-post
+    pre:
+    - ref: enable-node-coredumps
+    - chain: baremetalds-ofcir-pre
     workflow: baremetalds-e2e
 - as: e2e-metal-ipi-ovn-dualstack
   cluster: build05
@@ -83,6 +95,12 @@ tests:
       DEVSCRIPTS_CONFIG: |
         IP_STACK=v4v6
         NETWORK_TYPE=OVNKubernetes
+    post:
+    - chain: gather-core-dump
+    - chain: baremetalds-ofcir-post
+    pre:
+    - ref: enable-node-coredumps
+    - chain: baremetalds-ofcir-pre
     workflow: baremetalds-e2e
 - as: e2e-aws-serial
   skip_if_only_changed: ^docs/|\.md$|^(?:.*/)?(?:\.gitignore|OWNERS|OWNERS_ALIASES|PROJECT|LICENSE)$
@@ -100,6 +118,12 @@ tests:
       DEVSCRIPTS_CONFIG: |
         IP_STACK=v4v6
         NETWORK_TYPE=OVNKubernetes
+    post:
+    - chain: gather-core-dump
+    - chain: baremetalds-ofcir-post
+    pre:
+    - ref: enable-node-coredumps
+    - chain: baremetalds-ofcir-pre
     test:
     - as: baremetalds-e2e-conf-router
       commands: |
@@ -148,6 +172,12 @@ tests:
   skip_if_only_changed: ^docs/|\.md$|^(?:.*/)?(?:\.gitignore|OWNERS|OWNERS_ALIASES|PROJECT|LICENSE)$
   steps:
     cluster_profile: azure4
+    post:
+    - chain: gather-core-dump
+    - chain: ipi-azure-post
+    pre:
+    - ref: enable-node-coredumps
+    - chain: ipi-azure-pre
     workflow: openshift-upgrade-azure
 - always_run: false
   as: perfscale-aws-ingress-perf
@@ -162,8 +192,10 @@ tests:
       OPENSHIFT_INFRA_NODE_INSTANCE_TYPE: c5.4xlarge
       SET_ENV_BY_PLATFORM: custom
     post:
+    - chain: gather-core-dump
     - chain: ipi-aws-post
     pre:
+    - ref: enable-node-coredumps
     - chain: ipi-aws-pre
     - chain: create-infra-move-ingress-monitoring-registry
     test:
@@ -183,8 +215,10 @@ tests:
       OPENSHIFT_INFRA_NODE_INSTANCE_TYPE: c5.4xlarge
       SET_ENV_BY_PLATFORM: custom
     post:
+    - chain: gather-core-dump
     - chain: ipi-aws-post
     pre:
+    - ref: enable-node-coredumps
     - chain: ipi-aws-pre
     - ref: fips-check
     - chain: create-infra-move-ingress-monitoring-registry

ci-operator/step-registry/enable-node-coredumps/OWNERS

@@ -0,0 +1,23 @@
+approvers:
+  - knobunc
+  - Miciah
+  - candita
+  - rfredette
+  - alebedev87
+  - gcs278
+  - Thealisyed
+  - grzpiotrowski
+  - rikatz
+  - bentito
+options: {}
+reviewers:
+  - knobunc
+  - Miciah
+  - candita
+  - rfredette
+  - alebedev87
+  - gcs278
+  - Thealisyed
+  - grzpiotrowski
+  - rikatz
+  - bentito

ci-operator/step-registry/enable-node-coredumps/enable-node-coredumps-commands.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# Create a machine config that installs a systemd unit on nodes. The systemd unit configures the nodes to save any
+# coredumps that are generated, which will be collected during the gather-extra step.
+
+echo "Creating manifests to enable coredump collection on nodes"
+
+for role in master worker; do
+cat > "${SHARED_DIR}/manifest_enable_node_coredumps_machineconfig_${role}.yaml" <<EOF
+apiVersion: machineconfiguration.openshift.io/v1
+kind: MachineConfig
+metadata:
+  labels:
+    machineconfiguration.openshift.io/role: $role
+  name: enable-node-coredumps-${role}
+spec:
+  config:
+    ignition:
+      version: 3.2.0
+    systemd:
+      units:
+      - contents: |
+          [Unit]
+          After=multi-user.target
+
+          [Service]
+          Type=oneshot
+          ExecStart=sysctl -w fs.suid_dumpable=1
+
+          [Install]
+          WantedBy=multi-user.target
+        enabled: true
+        name: enable-node-coredumps.service
+EOF
+echo "manifest_enable_node_coredumps_machineconfig_${role}.yaml"
+echo "---------------------------------------------"
+cat ${SHARED_DIR}/manifest_enable_node_coredumps_machineconfig_${role}.yaml
+done

ci-operator/step-registry/enable-node-coredumps/enable-node-coredumps-ref.metadata.json

@@ -0,0 +1,29 @@
+{
+	"path": "enable-node-coredumps/enable-node-coredumps-ref.yaml",
+	"owners": {
+		"approvers": [
+			"knobunc",
+			"Miciah",
+			"candita",
+			"rfredette",
+			"alebedev87",
+			"gcs278",
+			"Thealisyed",
+			"grzpiotrowski",
+			"rikatz",
+			"bentito"
+		],
+		"reviewers": [
+			"knobunc",
+			"Miciah",
+			"candita",
+			"rfredette",
+			"alebedev87",
+			"gcs278",
+			"Thealisyed",
+			"grzpiotrowski",
+			"rikatz",
+			"bentito"
+		]
+	}
+}

ci-operator/step-registry/enable-node-coredumps/enable-node-coredumps-ref.yaml

@@ -0,0 +1,18 @@
+ref:
+  as: enable-node-coredumps
+  from_image:
+   namespace: origin
+   name: centos
+   tag: '8'
+  commands: enable-node-coredumps-commands.sh
+  resources:
+    requests:
+      cpu: 10m
+      memory: 100Mi
+  documentation: |-
+    The coredump service configures nodes to save all generated coredumps. This is useful for debugging failures of
+    components running in privileged pods, such as router pods.
+    The service is deployed by injecting an installer manifest containing a MachineConfig. This contains one systemd
+    unit, which sets the sysctl variable fs.suid_dumpable to 1, instructing systemd-coredump to save any coredumps it
+    encounters into /var/lib/systemd/coredump/ . Those coredump files can then be collected using the gather-core-dump
+    chain.