iiab/scripts/iiab-diagnostics

#!/bin/bash

# Collect IIAB diagnostic info into 1 file for easy online/offline circulation!
# PLEASE SEE /opt/iiab/iiab/scripts/iiab-diagnostics.README.md OR ONLINE HERE:
# https://github.com/iiab/iiab/blob/master/scripts/iiab-diagnostics.README.md

IIAB_RELEASE=$(cat /etc/iiab/iiab.env | grep IIAB_RELEASE | cut -d'=' -f2)
OS_VER=$(cat /etc/iiab/iiab.env | grep OS_VER | cut -d'=' -f2)
YMDT=$(date +%F_%T_%Z)

git config --global --add safe.directory /opt/iiab/iiab    # Nec below, if non-root
HASH1=$(cd /opt/iiab/iiab; git log --pretty=format:'%H' -n 1)    # --pretty=format:'%h' (8 chars)
BRANCH1=$(cd /opt/iiab/iiab; git branch --show-current)
REMOTE_URL1=$(cd /opt/iiab/iiab; git config remote.$(git config branch.$BRANCH1.remote).url)
PR_COUNT1=$(cd /opt/iiab/iiab; git log "$(git describe --tags --abbrev=0)..HEAD" --oneline --grep='Merge pull request' | wc -l)
TAG_COMMITS1=$(cd /opt/iiab/iiab; git describe --tags | sed 's/-[^-]*$//' | sed 's/-\([[:digit:]][[:digit:]]*\)$/ (\1 commits)/')
git config --global --add safe.directory /opt/iiab/iiab-admin-console    # Nec below, if non-root
HASH2=$(cd /opt/iiab/iiab-admin-console; git log --pretty=format:'%H' -n 1)
BRANCH2=$(cd /opt/iiab/iiab-admin-console; git branch --show-current)
REMOTE_URL2=$(cd /opt/iiab/iiab-admin-console; git config remote.$(git config branch.$BRANCH2.remote).url)
PR_COUNT2=$(cd /opt/iiab/iiab-admin-console; git log "$(git describe --tags --abbrev=0)..HEAD" --oneline --grep='Merge pull request' | wc -l)
TAG_COMMITS2=$(cd /opt/iiab/iiab-admin-console; git describe --tags | sed 's/-[^-]*$//' | sed 's/-\([[:digit:]][[:digit:]]*\)$/ (\1 commits)/')

echo -e "\nGathers IIAB diagnostics into 1 file, to accelerate troubleshooting.  USAGE:"
echo
echo -e "   iiab-diagnostics"
echo -e "   sudo iiab-diagnostics                                # USE 'sudo' FOR MORE"
echo -e "   sudo iiab-diagnostics PATH/FILE1 PATH/FILE2 ...      # COMPLETE RESULTS !!"
echo
echo -ne "Can you provide a \e[1mshort public nickname:\e[0m (no spaces!) "
read nickname < /dev/tty
if [[ $nickname == "" ]]; then
    nickname="NONAME"
fi

# Build up a meaningful shared filename for DEV / IMPLEM / LEARNING team(s)
outfile=/etc/iiab/diag/${IIAB_RELEASE}_${OS_VER}_${YMDT}_$nickname

# System "snapshots" (time-stamped output from this 'iiab-diagnostics' command)
# will be stored in globally-writable directory /etc/iiab/diag as created by
# roles/0-init/tasks/main.yml.  A bit like system logs, but only on request.

function cat_file_raw() {    # $1 = path/filename; $2 = # of lines, for tail
    if [ -f "$1" ]; then
        ls -l "$1" >> $outfile
        if [ ! -s "$1" ]; then
            echo >> $outfile
            echo "FILE EXISTS BUT IS EMPTY!" >> $outfile
        elif [ $# -eq 1 ]; then
            echo >> $outfile
            # Redact (mask) most passwords from /etc/iiab/local_vars.yml, /etc/hostapd/hostapd.conf, /etc/wpa_supplicant/wpa_supplicant.conf, /etc/netplan/*, /etc/network/interfaces, /etc/network/interfaces.d/*, /etc/NetworkManager/system-connections/* ETC -- not much to worry about in /etc/iiab/iiab.ini (' = ')
            cat "$1" | sed 's/^\(\s*[[:alnum:]#_-]*\(psk\|passphrase\|password\|wep-key[0-3]\):\).*/\1 [REDACTED]/; s/^\(\s*[[:alnum:]#_-]*\(psk\|passphrase\|password\|wep-key[0-3]\)[= \t]\).*/\1[REDACTED]/' | iconv -t UTF-8//IGNORE >> $outfile
        else    # e.g. last 100 lines, maximum
            echo "                        ...ITS LAST $2 LINES FOLLOW..." >> $outfile
            echo >> $outfile
            tail -$2 "$1" | sed 's/^\(\s*[[:alnum:]#_-]*\(psk\|passphrase\|password\|wep-key[0-3]\):\).*/\1 [REDACTED]/; s/^\(\s*[[:alnum:]#_-]*\(psk\|passphrase\|password\|wep-key[0-3]\)[= \t]\).*/\1[REDACTED]/' | iconv -t UTF-8//IGNORE >> $outfile
        fi
        echo >> $outfile
    elif [ -h "$1" ]; then
        ls -l "$1" >> $outfile
        echo >> $outfile
        echo "SYMLINK DOES NOT LEAD TO A REGULAR FILE!" >> $outfile
        echo >> $outfile
    elif [ -d "$1" ]; then
        ls -ld "$1" >> $outfile
        echo >> $outfile
        echo "THIS IS A DIRECTORY NOT A FILE!" >> $outfile
        echo >> $outfile
    else
        echo "FILE DOES NOT EXIST: $1" >> $outfile
    fi
}

function cat_file() {
    echo "     $1"
    echo "=IIAB==========================================================================" >> $outfile
    cat_file_raw "$1"
}

function cat_dir() {
    echo "     $1"
    echo "=IIAB==========================================================================" >> $outfile
    if [ -d "$1" ]; then
        echo "DIRECTORY $1 FILES WILL FOLLOW...IF THEY EXIST" >> $outfile
        shopt -s nullglob    # To avoid looping over empty directories
        for f in "$1"/*; do
            echo "-IIAB--------------------------------------------------------------------------" >> $outfile
            cat_file_raw "$f" 100
        done
    else
        echo "DIRECTORY DOES NOT EXIST: $1" >> $outfile
    fi
}

function cat_cmd() {    # $1 = command + params, $2 = explanation
    echo "     $1  # $2"
    echo "=IIAB==========================================================================" >> $outfile
    cmd=$(echo "$1" | sed 's/^\s*\(\S\S*\)\b.*$/\1/')    # Keep command on left; Drop params on right (NEC b/c 'command -v' interprets every word on the line!)
    #pth=$(command -v $cmd | sed 's/[^/]*$//')           # Keep only path on left; Drop command & params on right
    path_cmd=$(command -v $cmd)                                   # Use canonical path on left (would drop params on right, but over-interpret each word as a cmd!)
    spc_params=$(echo "$1" | sed 's/^\s*\S\S*\s*/ /;s/\s*$//')    # Drop command on left; Keep a single space + params on right; RTrim
    #spc_params=$(echo "$1" | sed 's/^\s*\S*//;s/\s*$//;s/^\s\s*/ /')    # LTrim + drop original path + command on left; RTrim; Compress whitespace in between
    #spc_params=$(echo "$1" | sed 's/^[[:blank:]]*[^[:blank:]]*//;s/[[:blank:]]*$//;s/^[[:blank:]][[:blank:]]*/ /')    # Equivalent (POSIX compliant)
    if [[ $2 == "" ]]; then
	echo "COMMAND: $path_cmd$spc_params" >> $outfile
    else
	echo "COMMAND: $path_cmd$spc_params    # $2" >> $outfile
    fi
    echo >> $outfile
    if [[ $path_cmd == "" ]]; then
        echo "COMMAND NOT FOUND: $1" >> $outfile
    else
        bash -c "$1" >> $outfile    # Works with | (pipes) and 'ls -l /lib/firmware/brcm/*43455*' etc!
	#(exec $1 >> $outfile)      # Works with | (pipes) and 'ls -l /lib/firmware/brcm/*43455*' etc!  Subshell needed (parens) as exec then exits entire shell.
	#eval $1 >> $outfile              # Should be identical to below, i.e. insufficient -- "eval" combine ARGs into a single string.
        #$(echo "eval $1") >> $outfile    # "eval" works with | (pipes) per https://stackoverflow.com/a/7184782 BUT globbing like 'ls -l /lib/firmware/brcm/*43455*' FAILS to output lines w/ filenames that contain spaces (ugly IFS issues!)
    fi
    echo >> $outfile
}

function cat_tail() {    # $1 = path/filename; $2 = # of lines, for tail
    echo "     $1"
    echo "=IIAB==========================================================================" >> $outfile
    cat_file_raw "$1" $2    # e.g. last 100 lines, maximum
}

# START BUILDING UP THE FILE THAT'LL CONTAIN THE DIAGNOSTICS!
echo -e "\nCompiling diagnostics..."

echo -e "\n  0. Filename Header + Git Hashes + Raspberry Pi Model + OS"
echo "This is: $outfile" >> $outfile
echo >> $outfile
echo -e "\n\n\n\n0. GIT INFO + RASPBERRY PI MODEL + OS" >> $outfile
echo >> $outfile
echo "iiab commit:                       $HASH1" >> $outfile
echo "     remote:                       $REMOTE_URL1" >> $outfile
echo "     branch:                       $BRANCH1" >> $outfile
printf "%4s merged PR's since recent tag: $TAG_COMMITS1\n" $PR_COUNT1 >> $outfile
echo >> $outfile
echo "iiab-admin-console commit:         $HASH2" >> $outfile
echo "     remote:                       $REMOTE_URL2" >> $outfile
echo "     branch:                       $BRANCH2" >> $outfile
printf "%4s merged PR's since recent tag: $TAG_COMMITS2\n" $PR_COUNT2 >> $outfile
echo >> $outfile
cat_file /etc/iiab/pr-list-pulled
cat_file /proc/device-tree/model    # Should be identical to /sys/firmware/devicetree/base/model
cat_file /etc/rpi-issue
echo "-IIAB-EXPLANATION-OF-THE-ABOVE-------------------------------------------------" >> $outfile
echo >> $outfile
if [ -f /etc/rpi-issue ]; then
    echo "stage2 = Raspberry Pi OS Lite" >> $outfile
    echo "stage4 = Raspberry Pi OS with desktop" >> $outfile
    echo "stage5 = Raspberry Pi OS with desktop + recommended software" >> $outfile
    echo >> $outfile
    echo "SEE https://github.com/RPi-Distro/pi-gen#stage-anatomy" >> $outfile
else
    echo "(This is NOT Raspberry Pi OS!)" >> $outfile
fi
echo >> $outfile
cat_file /etc/issue.net
cat_file /etc/debian_version
cat_cmd 'dpkg --print-architecture' 'RaspiOS-on-PC shows: i386'
cat_cmd 'dpkg --print-foreign-architectures' 'RaspiOS-on-PC shows: amd64'
cat_cmd 'systemctl is-active display-manager.service' 'Graphical Desktop?'
cat_cmd 'grep "^openvpn_" /etc/iiab/local_vars.yml'

echo -e '\n\n  1. Files Specially Requested: (from "iiab-diagnostics PATH/FILE1 PATH/FILE2")\n'
echo -e '\n\n\n\n1. FILES SPECIALLY REQUESTED (FROM "iiab-diagnostics PATH/FILE1 PATH/FILE2")\n' >> $outfile
for f in "$@"; do
    cat_file $f
done

if [ $# -eq 0 ]; then
    echo -e "  2. Regular Files:\n"
else
    echo -e "\n  2. Regular Files:\n"
fi
echo -e "\n\n\n\n2. REGULAR FILES\n" >> $outfile
#cat_file /dev/sda                    # Device "file" test
#cat_file /nonsense                   # Non-existence test
#cat_file /opt/iiab/iiab              # Directory test
#cat_file /tmp/empty-file             # Empty file test
#cat_file /usr/bin/iiab-support-on    # Symlink test
cat_file /.iiab-image
cat_file /etc/iiab/iiab.env
cat_file /etc/iiab/iiab.ini
cat_file /etc/iiab/local_vars.yml      # Redacts most passwords above
cat_file /etc/iiab/iiab_state.yml
cat_file /etc/resolv.conf
cat_file /etc/network/interfaces
cat_file /etc/hostapd/hostapd.conf                  # Redacts most passwords above
cat_file /etc/wpa_supplicant/wpa_supplicant.conf    # Redacts most passwords above
cat_file /library/www/html/home/menu.json

# Record all Ansible variables: SLOW! OUTPUT TOO LARGE?
#pushd /opt/iiab/iiab > /dev/null
#./runrole all-vars /tmp/all-ansible-vars
#popd > /dev/null
#cat_file /tmp/all-ansible-vars

echo -e "\n  3. Content of Directories: (1-level deep)\n"
echo -e "\n\n\n\n3. CONTENT OF DIRECTORIES (1-LEVEL DEEP)\n" >> $outfile
cat_dir /etc/network/interfaces.d
cat_dir /etc/systemd/network
cat_dir /etc/NetworkManager/system-connections    # Redacts most passwords above
cat_dir /etc/netplan    # Redacts most passwords above
#cat_dir /etc/sysconfig/network-scripts/if-cfg*    # No longer common
#cat_dir /etc/network    # Above file /etc/network/interfaces suffices

echo -e "\n  4. Output of Commands:\n"
echo -e "\n\n\n\n\n4. OUTPUT OF COMMANDS\n" >> $outfile
cat_cmd 'uname -a' 'Linux kernel'
cat_cmd 'free' 'RAM memory'
cat_cmd 'lscpu' 'CPU details'
cat_cmd 'df -h' 'Disk usage'
cat_cmd 'df -ah' 'Disk usage detail'
cat_cmd 'lsblk' 'Partition mount points'
cat_cmd 'blkid' 'Mount point details'
cat_cmd 'ip addr' 'Network interfaces'
cat_cmd 'ifconfig' 'Network interfaces (old view)'
cat_cmd 'ip route' 'Routing table'
cat_cmd 'netstat -rn' 'Routing table (old view)'
cat_cmd 'bridge -d link' 'Bridge for LAN side'
cat_cmd 'sudo netstat -natp' 'Ports/Services in use'
cat_cmd 'systemctl status dnsmasq' 'Is dnsmasq running?'
cat_cmd 'sudo journalctl -b 0 -u dnsmasq' 'dnsmasq log'
cat_cmd 'networkctl' 'systemd-networkd status'
cat_cmd 'nmcli d' 'NetworkManager status'
cat_cmd 'sudo journalctl -b 0 -u networkd-dispatcher' 'networkd-dispatcher log'
cat_cmd 'rfkill list' 'Are WiFi and Bluetooth interfaces blocked?'
cat_cmd 'iw reg get' 'Detected WiFi country code / legal frequencies'
cat_cmd 'iw dev' 'List wireless interfaces'
cat_cmd 'iw list' 'List capabilities of all wireless devices'
cat_cmd 'systemctl status hostapd' 'Downstream Wi-Fi: Is hostapd running?'
cat_cmd 'ls -l /etc/wpa_supplicant' 'Upstream Wi-Fi'
cat_cmd 'ps -AH' 'Process hierarchy: staging of hostapd & wpa_supplicant?'
cat_cmd 'dmesg | grep brcm' 'Diagnostic messages: RPi Wi-Fi firmware'
cat_cmd 'lspci -nn' 'Devices on PCI buses'
cat_cmd 'ls -l /lib/firmware/brcm/*43430*' 'RPi Zero W & 3 WiFi firmware'
cat_cmd 'ls -l /lib/firmware/brcm/*43455*' 'RPi 3 B+ & 4 WiFi firmware'
cat_cmd 'env' 'Environment variables'
cat_cmd '/opt/iiab/kiwix/bin/kiwix-serve --version' 'kiwix-tools'
cat_cmd 'journalctl -t IIAB-CMDSRV' 'Admin Console CMDSRV log'
#cat_cmd 'ansible localhost -m setup 2>/dev/null' 'All Ansible facts'    # For cleaner scraping of Ansible vars, consider "./runrole all-vars /tmp/all-ansible-vars" 27-31 lines above?

echo -e "\n  5. Firewall Rules:\n"
echo -e "\n\n\n\n5. FIREWALL RULES\n" >> $outfile
#cat_file /usr/bin/iiab-gen-iptables
cat_cmd 'sudo iptables-save' 'Firewall rules'

echo -e "\n  6. Log Files: (e.g. last 100 lines of each)\n"
echo -e "\n\n\n\n6. LOG FILES (e.g. LAST 100 LINES OF EACH)\n" >> $outfile
cat_cmd 'grep -B2 "SEE ERROR ABOVE" /opt/iiab/iiab/*.log' 'for skip_role_on_error'
cat_tail /opt/iiab/iiab/iiab-install.log 100
cat_tail /opt/iiab/iiab/iiab-configure.log 100
cat_tail /opt/iiab/iiab/iiab-debug.log 100
cat_tail /opt/iiab/iiab/iiab-network.log 100
cat_tail /opt/iiab/iiab-admin-console/admin-install.log 100
cat_tail /var/log/messages 100
cat_tail /var/log/syslog 100

linecount=$(wc -l $outfile | sed 's/\s.*$//')
sizecount=$(du -h $outfile | sed 's/\s.*$//')
echo -e "\n\e[32m\e[1mCOMPLETE! Your diagnostics file ($sizecount, $linecount lines) is:"
echo
echo -e "   $outfile\e[0m"

#if [ "$1" == "-y" ]; then
#    ans="y"    # if user ran "iiab-diganostics -y" to avoid interactive prompt
#else
echo
echo -ne "\e[42;1mPublish it to a web pastebin? [Y/n]\e[0m "
read -n 1 -r ans < /dev/tty
echo
#fi

echo -e "\e[1m"
#if [ "$ans" == "" ] || [ "$ans" == "y" ] || [ "$ans" == "Y" ]; then
if ! [[ $ans =~ ^[nN]$ ]]; then
    echo -ne "PUBLISHING TO URL... "
    #pastebinit -b dpaste.com < $outfile
    pastebinit -b sprunge.us < $outfile    # Run 'pastebinit -l' to list other possible pastebin site URLs
else
    echo -e "If you later decide to publish it, run:"
    echo
    echo -e "   pastebinit -b sprunge.us < $outfile"
fi
echo -e "\e[0m"