#!/bin/bash set -e -u SELF="${0##*/}" mode='crash_kernel' address= port='666' interface= driver= output='/var/lib/crash' size='128M' main () { parse_args "$@" "setup_${mode}" } parse_args () { local temp temp="$(getopt -n "$SELF" -o 'hca:p:i:d:o:s:v' -l 'help,collector,address:,port:,interface:,driver:,output:,size:,verbose' -- "$@")" || die "Terminating" eval set -- "$temp" while [ -n "$1" ] do case "$1" in -h|--help) usage 0 ;; -c|--collector) mode='collector' ; shift ;; -a|--address) address="$2" ; shift 2 ;; -p|--port) port="$2" ; shift 2 ;; -i|--interface) interface="$2" ; shift 2 ;; -d|--driver) interface="$2" ; shift 2 ;; -o|--output) output="$2" ; shift 2 ;; -s|--size) size="$2" ; shift 2 ;; -v|--verbose) set -x ; shift ;; --) shift ; break ;; *) die "Internal error!" ;; esac done CRASHHOSTNAME="${1:-}" } usage () { echo "Usage: $SELF [options] (--collector | )" echo " Either setup host as crash collector, which installs a service listening for incoming connections," echo " or setup local host to send memory dumps on crashes to given host." echo echo "Options:" echo " -c, --collector" echo " -o, --output [default: '${output}']" echo echo " -a, --address " echo " -i, --interface " echo " -d, --driver " echo " -s, --size [default: '${size}']" echo echo " -p, --port [default: '${port}']" echo " -v, --verbose" echo " -h, --help" exit ${1:-0} } setup_collector () { echo "Setting up as crash collector..." # Create user and service for collecting data from the network: adduser --system crash --home /var/lib/crash create_file "${output}/collector.sh" <<__SH__ #!/bin/sh umask 0177 exec cat >"${output}/core-\$(date +%s).gz" __SH__ update-inetd --add "${port}\tstream\ttcp\tnowait\tcrash\t${output}/collector.sh" # Make sure the port is allowed by the firewall: ucr set "security/packetfilter/tcp/${port}/all=ACCEPT" service univention-firewall restart } setup_crash_kernel () { echo "Setting up crash kernel..." detect_hypervisor detect_network install_kexec create_initramfs_hook create_initramfs_modules create_initramfs_script create_initramfs setup_grub setup_kernel setup_boot finish } detect_hypervisor () { local hypervisor="$(cat /sys/hypervisor/type 2>/dev/null || :)" echo "Hypervisor: $hypervisor" case "$hypervisor" in xen) [ -n "${FORCE_XEN:-}" ] || die "Xen is not supported. (Xen implements its own mechanism to capture crashed domains. Support for kexec was only added later. If you want to try anyway, set FORCE_XEN=1)" ;; esac } detect_network () { local brif sys declare -a pif=() [ -n "$CRASHHOSTNAME" ] || die "Missing address" : ${address:=$(dig +short +search "$CRASHHOSTNAME" | tail -n 1)} echo " using IP ${address:?Failed to resolve "$CRASHHOSTNAME"}" : ${interface:=$(ip route get "$address" | sed -rne 's,.* dev (\w+) +src .*,\1,p')} echo " using network interface ${interface:?Network interface not found; use '-i '}" sys="/sys/class/net/$interface/brif" if [ -d "$sys" ] then echo " bridge interface $interface detected - looking for physical interfaces..." for brif in "$sys"/* do brif="${brif##*/}" [ -e "/sys/class/net/$brif/device/driver" ] || continue echo " found $brif" pif+=("$brif") done case "${#pif[@]}" in 0) die "No physical interface found; use '-i '" ;; 1) ;; *) die "More than one interface found; use '-i '" ;; esac else pif="$interface" fi : ${driver:=$(basename "$(readlink "/sys/class/net/$pif/device/driver")")} echo " using network driver ${driver:?Network driver not found; use '-d '}" } install_kexec () { which kexec >/dev/null 2>/dev/null && return ucr set repository/online/unmaintained=yes apt-get -qq update apt-get -qq install kexec-tools sed -i -e '/^LOAD_KEXEC=/s/true/false/' /etc/default/kexec kexec -u || : } create_initramfs_hook () { create_file /etc/initramfs-tools/hooks/crash <<'__SH__' #!/bin/sh PREREQ= prereqs () { echo "$PREREQ" } case "$1" in prereqs) prereqs exit 0 ;; esac . /usr/share/initramfs-tools/hook-functions copy_exec /bin/ip copy_exec /bin/nc __SH__ } create_initramfs_modules () { grep -qFe "$driver" /etc/initramfs-tools/modules && return echo "$driver" >>/etc/initramfs-tools/modules update_initramfs=true } create_initramfs_script () { create_file /etc/initramfs-tools/scripts/crash <<__SH__ #!/bin/sh exec >/dev/console 2>/dev/console set -x trap 'sh /dev/console 2>/dev/console' EXIT echo "Loading network driver '$driver'..." modprobe "$driver" sleep 5 cd /sys/class/net for iface in * do echo "Trying network interface '\$iface'..." [ -e "\$iface/device/driver" ] || continue ip addr add "$(ip addr show dev "$interface" | sed -rne 's,^.*inet ([0-9./]+) brd.*,\1,p')" dev "\$iface" ip link set "\$iface" up ip route add default via "$(ip route list | sed -rne 's,^default via ([0-9./]+) dev .*,\1,p')" echo "Dumping crash to '$address'..." gzip -c /proc/sysrq-trigger break } echo "Failed." ip addr flush dev "\$iface" done __SH__ } create_initramfs () { "$update_initramfs" || return # kernel + initramfs + uncompressed <= 128 MB ! ucr set initramfs/modules=dep update-initramfs -u } setup_grub () { grub=$(ucr get grub/append) case "$grub" in *crashkernel=*) ;; *) ucr set grub/append="$grub crashkernel=${size}" ;; # 384M-2G:64M,2G-:128M esac ucr set grub/gfxmode= grub/gfxpayload= grub/bootsplash=nosplash grub/loglevel=7 grub/quiet=no } setup_kernel () { create_file /etc/sysctl.d/crash.conf 0644 <<__CONF__ # panic when hung task is detected kernel.hung_task_panic=0 # panic on NMIs from I/O kernel.panic_on_io_nmi=0 # panic on oops or kernel bug detection kernel.panic_on_oops=1 # panic on NMIs from memory or unknown kernel.panic_on_unrecovered_nmi=1 # panic when soft lockups are detected kernel.softlockup_panic=0 # panic when out-of-memory happens vm.panic_on_oom=0 __CONF__ } setup_boot () { create_file /etc/rc.local '' FORCE <<'__SH__' #!/bin/sh -e # # rc.local # # This script is executed at the end of each multiuser runlevel. # Make sure that the script will "exit 0" on success or any other # value on error. # # In order to enable or disable this script just change the execution # bits. # # By default this script does nothing. load_crash () { [ "$(cat /sys/kernel/kexec_crash_loaded)" = 0 ] || return console=--reset-vga args='1 irqpoll maxcpus=1 reset_devices boot=crash debug break=init nosplash' set -- $(cat /proc/cmdline) while [ $# -ge 1 ] do case "$1" in blacklist=*) args="${args} $1" ;; boot=*) ;; BOOTIF=*) args="${args} $1" ;; BOOT_IMAGE=*) ;; break|break=*) ;; console=ttyS*) args="${args} $1" console=--console-serial ;; console=*) args="${args} $1" ;; crashkernel=*) ;; drop_capabilities=*) ;; fastboot|forcefsck|fsckfix) ;; loglevel=*) args="${args} $1" ;; netconsole=*) args="${args} $1" ;; nfsroot=*|ip=*) ;; panic=*) ;; quiet|verbose|debug|debug=*) ;; resume=*|resume_offset=*|noresume) ;; root=*|rootflags=*|rootfstype=*|rootdelay=*) ;; ro|rw) ;; splash|nosplash) ;; *) echo "Unknown arg '$1'" ;; esac shift done kexec -p /boot/vmlinuz-`uname -r` --initrd /boot/initrd.img-`uname -r` --append="$args" $console cat /sys/kernel/kexec_crash_loaded } load_crash : __SH__ } finish () { if [ "$(cat /sys/kernel/kexec_crash_size)" = 0 ] then echo "First 'reboot'; after that test with 'echo c >/proc/sysrq-trigger'" else if [ "$(cat /sys/kernel/kexec_crash_loaded)" != 0 ] then kexec -p -u fi /etc/rc.local echo "Test with 'echo c >/proc/sysrq-trigger'" fi } update_initramfs=false create_file () { [ -z "${3:-}" ] && [ -f "$1" ] && return 0 cat >"$1" chmod "${2:-0755}" "$1" case "$1" in */initramfs-tools/*) update_initramfs=true ;; esac } die () { echo "${0##*/}: $*" >&2 exit 1 } main "$@"