Merge pull request #1763 from holta/iiab-diagnostics

WIP: iiab-diagnostics friendly community tooling, to build relationships between dev teams, implementers & educators
2025-03-09 15:40:17 +00:00 · 2019-06-23 05:35:55 -04:00 · 2019-06-23 05:35:55 -04:00 · f05f1a3067
commit f05f1a3067
parent ace5321020 1d4b39a905
7 changed files with 255 additions and 140 deletions
--- a/roles/0-init/tasks/main.yml
+++ b/roles/0-init/tasks/main.yml
@ -1,5 +1,11 @@
 # Initialize

+- name: Create symlink /usr/bin/iiab-diagnostics
+  file:
+    src: "{{ iiab_dir }}/scripts/iiab-diagnostics"
+    dest: /usr/bin/iiab-diagnostics
+    state: link
+
 - name: ...IS BEGINNING ============================================
  stat:
    path: "{{ iiab_env_file }}"
--- a/roles/1-prep/templates/iiab-network-reset
+++ b/roles/1-prep/templates/iiab-network-reset
@ -1,137 +0,0 @@
-#!/bin/bash
-# if called w/ parameter, skip the reset, create diagnostic package w/ param as file name
-SCRIPTDIR=$(cd `dirname $0` pwd)
-diagnose_name=
-if [ $# -ne 0 ]; then
-   basket=$1
-   diagnose_name=$1
-else
-   basket=netlog.$$
-fi
-
-# collect all the network info in one place
-mkdir -p /tmp/$basket
-cat << EOF > /tmp/script2overview
-#!/bin/bash
-# generate the body overview part diagnostic package about network
-echo "=========================================================="
-for f in \$(ls  /etc/sysconfig/network-scripts/ifcfg-*|gawk '{printf(" %s",\$1)}'); do
-   echo
-   echo \$f
-   cat \$f
-done
-echo 
-echo "=========================================================="
-echo ifconfig
-ifconfig
-echo
-echo "=========================================================="
-echo ip addr
-ip addr
-echo
-echo "=========================================================="
-echo "brctl show"
-brctl show
-echo
-echo "=========================================================="
-echo "/etc/resolv.conf"
-cat /etc/resolv.conf
-echo
-echo "=========================================================="
-echo "cat {{ iiab_ini_file }}"
-cat {{ iiab_ini_file }}
-echo
-echo "=========================================================="
-echo "routing table"
-netstat -rn
-echo
-echo "=========================================================="
-echo "install log -- last 50 lines"
-tail -50 /opt/schoolserver/iiab/iiab-install.log
-echo
-echo "=========================================================="
-echo "iiab-network log -- last 50 lines"
-if [ -f /opt/schoolserver/iiab/iiab-network.log ]; then
-tail -50 /opt/schoolserver/iiab/iiab-network.log
-else 
-  echo no iiab-network.log
-fi
-echo
-echo "=========================================================="
-cat /etc/fedora-release | grep 18
-if [ \$? -eq 0 ]; then
-  echo "nmcli conn list"
-  nmcli conn list 3>&2
-else
-  echo "nmcli conn show"
-  nmcli conn show 3>&2
-fi
-echo
-echo "=========================================================="
-echo nmcli dev wifi list
-nmcli dev wifi list
-EOF
-chmod 755 /tmp/script2overview
-/tmp/script2overview > /tmp/$basket/overview
-
-if [ -f /opt/schoolserver/iiab/iiab-network.log ]; then
-   cp /opt/schoolserver/iiab/iiab-network.log /tmp/$basket
-else
-  touch /tmp/$basket/no_iiab-network.log
-fi
-
-if [ -f /etc/sysconfig/iiab_domain_name ];then
-  cp -p /etc/sysconfig/iiab_domain_name /tmp/$basket
-else 
-  touch /tmp/$basket/iiab_domain_name_not_set
-fi
-
-if [ -f /etc/sysconfig/iiab_lan_device ];then
-  cp -p /etc/sysconfig/iiab_lan_device  /tmp/$basket
-else 
-  touch /tmp/$basket/iiab_lan_device_not_set
-fi
-if [ -f /etc/sysconfig/iiab_wan_device ];then
-  cp -p /etc/sysconfig/iiab_wan_device  /tmp/$basket
-else 
-  touch /tmp/$basket/iiab_wan_device_not_set
-fi
-ls /etc/NetworkManager/system-connections > /dev/null
-if [ $? -eq 0  ]; then
-  cp -rp /etc/NetworkManager/system-connections /tmp/$basket
-fi
-cp /etc/sysconfig/network-scripts/ifcfg-* /tmp/$basket
-if [ -f /opt/schoolserver/iiab/iiab-network.log ]; then
-  cp -p /opt/schoolserver/iiab/iiab-network.log /tmp/$basket
-fi
-
-mkdir -p /etc/iiab/diagnose/
-if [ ! -z $diagnose_name ];then
-  pushd /tmp > /dev/null
-  tar czf /etc/iiab/diagnose/$basket.tgz $basket/*
-  popd > /dev/null
-  rm -rf /tmp/$basket
-  exit 0
-else
-  pushd /tmp > /dev/null
-  tar czf  /etc/iiab/diagnose/$basket.tgz $basket/*
-  popd > /dev/null
-  rm -rf /tmp/$basket
-fi
-
-# clear out all the memory variables and let auto-configure start from scratch
-rm -rf /etc/sysconfig/iiab_domain_name
-rm -rf /etc/sysconfig/iiab_lan_device
-rm -rf /etc/sysconfig/iiab_wan_device
-rm -rf /etc/NetworkManager/system-connestions/*
-if [ -f /etc/sysconfig/network-scripts/ifcfg-WAN ];then
-  mv /etc/sysconfig/network-scripts/ifcfg-WAN /root
-  echo -e "\n\nWAN setup file moved to /root for safekeeping.\n\n"
-fi
-
-ls -1 /etc/sysconfig/network-scripts/ifcfg-*|grep -v -e ifcfg-lo
-if [ $? -eq 0 ]; then
-  ls -1 /etc/sysconfig/network-scripts/ifcfg-*|grep -v -e ifcfg-lo|xargs rm
-fi
-
-echo -e "\n\nAll Network variables erased. Now run 'iiab-network' to set up the new network configuration.\n\nPlease see /opt/schoolserver/iiab/docs/GETTING_HELP.rst for ways to get help or \nprovide the feedback which will improve XSCE\n\n"
--- a/roles/all-vars/tasks/main.yml
+++ b/roles/all-vars/tasks/main.yml
@ -0,0 +1,14 @@
+- name: Print some debug information 
+  vars: 
+    msg: |
+        Module Variables ("vars"):
+        --------------------------------
+        {{ vars | to_nice_json }} 
+
+        Environment Variables ("environment"):
+        --------------------------------
+        {{ environment | to_nice_json }} 
+
+  debug: 
+    msg: "{{ msg.split('\n') }}"       
+  tags: debug_info
--- a/run-one-role.yml
+++ b/run-one-role.yml
@ -4,7 +4,7 @@

  vars_files:
  - vars/default_vars.yml
-  - vars/{{ ansible_local.local_facts.os_ver }}.yml
+  - "vars/{{ ansible_local.local_facts.os_ver }}.yml"
  - /etc/iiab/local_vars.yml
  - /etc/iiab/config_vars.yml

--- a/7
+++ b/7
@ -4,8 +4,11 @@ INVENTORY="ansible_hosts"
 PLAYBOOK="run-one-role.yml"
 #PLAYBOOK="iiab-stages.yml"
 CWD=`pwd`
-
-export ANSIBLE_LOG_PATH="$CWD/iiab-debug.log"
+if [ $# -eq 2 ]; then
+   export ANSIBLE_LOG_PATH="$2"
+else
+   export ANSIBLE_LOG_PATH="$CWD/iiab-debug.log"
+fi

 if [ ! -f $PLAYBOOK ]; then
    echo "Exiting: IIAB Playbook not found."
--- a/scripts/iiab-diagnostics
+++ b/scripts/iiab-diagnostics
@ -0,0 +1,178 @@
+#!/bin/bash
+
+# Collect IIAB diagnostic info into 1 file for easy online/offline circulation!
+# PLEASE SEE iiab-diagnostics.README.md
+
+IIAB_RELEASE=`cat /etc/iiab/iiab.env | grep IIAB_RELEASE | cut -d'=' -f2`
+OS_VER=`cat /etc/iiab/iiab.env | grep OS_VER | cut -d'=' -f2`
+#HASH=`cd /opt/iiab/iiab; git log --pretty=format:'%h' -n 1`
+HASH1=`cd /opt/iiab/iiab; git log --pretty=format:'%H' -n 1`
+HASH2=`cd /opt/iiab/iiab-admin-console; git log --pretty=format:'%H' -n 1`
+YMDT=$(date +%F_%T_%Z)
+
+echo -e "\nAccelerate troubleshooting by collecting your IIAB diagnostics into 1 file:"
+echo -e "\n   sudo iiab-diagnostics"
+echo -e "   sudo iiab-diagnostics PATH/FILE1 PATH/FILE2 ..."
+echo -ne "\nCan you provide a \e[1mshort public nickname:\e[0m (no spaces!) "
+read nickname < /dev/tty
+if [ -z "$nickname" ]; then
+    nickname="NONAME"
+fi
+
+# Build up a meaningful shared filename for DEV / IMPLEM / LEARNING team(s)
+outfile=/etc/iiab/diag/${IIAB_RELEASE}_${OS_VER}_${YMDT}_$nickname
+mkdir -p /etc/iiab/diag
+
+function cat_file_raw() {    # $1 = path/filename; $2 = # of lines, for tail
+    if [ -f $1 ]; then
+        ls -l $1 >> $outfile
+        if [ ! -s $1 ]; then
+            echo >> $outfile
+            echo "FILE EXISTS BUT IS EMPTY!" >> $outfile
+        elif [ $# -eq 1 ]; then
+            echo >> $outfile
+            cat $1 | iconv -t UTF-8//IGNORE >> $outfile
+        else    # e.g. last 100 lines, maximum
+            echo "                        ...ITS LAST $2 LINES FOLLOW..." >> $outfile
+            echo >> $outfile
+            tail -$2 $1 | iconv -t UTF-8//IGNORE >> $outfile
+        fi
+        echo >> $outfile
+    elif [ -h $1 ]; then
+        ls -l $1 >> $outfile
+        echo >> $outfile
+        echo "SYMLINK DOES NOT LEAD TO A REGULAR FILE!" >> $outfile
+        echo >> $outfile
+    elif [ -d $1 ]; then
+        ls -ld $1 >> $outfile
+        echo >> $outfile
+        echo "THIS IS A DIRECTORY NOT A FILE!" >> $outfile
+        echo >> $outfile
+    else
+        echo "FILE DOES NOT EXIST: $1" >> $outfile
+    fi
+}
+
+function cat_file() {
+    echo "     $1"
+    echo "=IIAB==========================================================================" >> $outfile
+    cat_file_raw $1
+}
+
+function cat_dir() {
+    echo "     $1"
+    echo "=IIAB==========================================================================" >> $outfile
+    if [ -d "$1" ]; then
+        echo "DIRECTORY $1 FILES WILL FOLLOW...IF THEY EXIST" >> $outfile
+        for f in $(ls $1); do
+            echo "-IIAB--------------------------------------------------------------------------" >> $outfile
+            cat_file_raw $1/$f 100
+        done
+    else
+        echo "DIRECTORY DOES NOT EXIST: $1" >> $outfile
+    fi
+}
+
+function cat_cmd() {                        # $1 = command + params, $2 = explanation
+    echo "     $1  # $2"
+    echo "=IIAB==========================================================================" >> $outfile
+    cmd=$(echo $1 | sed 's/\s.*$//')        # Keep command on left; Drop params on right
+    pth=$(which $cmd | sed 's/[^/]*$//')    # Keep only path on left; Drop command on right
+    echo "COMMAND: $pth$1    # $2" >> $outfile
+    echo >> $outfile
+    $(echo $1) >> $outfile
+    echo >> $outfile
+}
+
+function cat_tail() {     # $1 = path/filename; $2 = # of lines, for tail
+    echo "     $1"
+    echo "=IIAB==========================================================================" >> $outfile
+    cat_file_raw $1 $2    # e.g. last 100 lines, maximum
+}
+
+# START BUILDING UP THE FILE THAT'LL CONTAIN THE DIAGNOSTICS!
+echo -e "\nCompiling diagnostics..."
+
+echo -e "\n  0. Header/Hashes/OS"
+echo "This is: $outfile" >> $outfile
+echo >> $outfile
+echo "iiab commit:               $HASH1" >> $outfile
+echo "iiab-admin-console commit: $HASH2" >> $outfile
+echo >> $outfile
+echo -n "Checking /etc/rpi-issue for Raspbian OS version... " >> $outfile
+if [ -f /etc/rpi-issue ]; then
+    echo >> $outfile
+    cat /etc/rpi-issue >> $outfile
+    echo >> $outfile
+    echo "stage2 = lite; stage5 = desktop SEE https://github.com/RPi-Distro/pi-gen#stage-anatomy" >> $outfile
+else
+    echo "Not a Raspberry Pi!" >> $outfile
+fi
+echo >> $outfile
+
+echo -e '\n  1. Files specially requested: (for "iiab-diagnostics PATH/FILE1 PATH/FILE2")\n'
+for f in "$@"; do
+    cat_file $f
+done
+
+if [ $# -eq 0 ]; then
+    echo -e "  2. Regular Files:\n"
+else
+    echo -e "\n  2. Regular Files:\n"
+fi
+#cat_file /dev/sda                    # Device "file" test
+#cat_file /nonsense                   # Non-existence test
+#cat_file /opt/iiab/iiab              # Directory test
+#cat_file /tmp/empty-file             # Empty file test
+#cat_file /usr/bin/iiab-support-on    # Symlink test
+cat_file /etc/iiab/openvpn_handle
+cat_file /.iiab-image
+cat_file /etc/iiab/iiab.env
+cat_file /etc/iiab/iiab.ini
+cat_file /etc/iiab/local_vars.yml
+cat_file /etc/iiab/config_vars.yml
+cat_file /etc/resolv.conf
+cat_file /etc/network/interfaces
+cat_file /usr/bin/iiab-gen-iptables
+
+# Record all Ansible variables: SLOW! OUTPUT TOO LARGE?
+#pushd /opt/iiab/iiab > /dev/null
+#./runrole all-vars /tmp/all-ansible-vars
+#popd > /dev/null
+#cat_file /tmp/all-ansible-var
+
+echo -e "\n  3. Content of Directories, 1-level deep:\n"
+cat_dir /etc/network/interfaces.d
+cat_dir /etc/systemd/network
+cat_dir /etc/NetworkManager/system-connections
+cat_dir /etc/netplan
+#cat_dir /etc/sysconfig/network-scripts/if-cfg*    # No longer common.
+#cat_dir /etc/network    # Above file /etc/network/interfaces suffices
+
+echo -e "\n  4. Output of Commands:\n"
+cat_cmd 'uname -a' 'Linux kernel'
+cat_cmd 'free' 'RAM memory'
+cat_cmd 'lscpu' 'CPU details'
+cat_cmd 'df -h' 'Disk usage'
+cat_cmd 'lsblk' 'Partition mount points'
+cat_cmd 'blkid' 'Mount point details'
+cat_cmd 'ip addr' 'Network interfaces'
+cat_cmd 'ifconfig' 'Network interfaces (old view)'
+cat_cmd 'brctl show' 'Bridge for LAN side'
+cat_cmd 'netstat -rn' 'Routing table'
+cat_cmd 'netstat -natp' 'Ports/Services in use'
+cat_cmd 'iptables-save' 'Firewall rules'
+cat_cmd 'systemctl status dnsmasq' 'Is dnsmasq Ok?'
+cat_cmd 'journalctl -u dnsmasq' 'dnsmasq log'
+#cat_cmd 'ansible localhost -m setup 2>/dev/null' 'All Ansible facts'
+
+echo -e "\n  5. Log Files: (last 100 lines of each)\n"
+cat_tail /opt/iiab/iiab/iiab-install.log 100
+cat_tail /opt/iiab/iiab/iiab-network.log 100
+cat_tail /opt/iiab/iiab/iiab-debug.log 100
+cat_tail /opt/iiab/iiab-admin-console/admin-install.log 100
+
+linecount=$(wc -l $outfile | sed 's/\s.*$//')
+sizecount=$(du -h $outfile | sed 's/\s.*$//')
+echo -e "\n\e[32mCOMPLETE! To share this on the web ($sizecount, $linecount lines) run:\e[0m"
+echo -e "\n\e[1mpastebinit < $outfile\e[0m\n"
--- a/scripts/iiab-diagnostics.README.md
+++ b/scripts/iiab-diagnostics.README.md
@ -0,0 +1,51 @@
+## Objective
+
+To streamline troubleshooting of remote Internet-in-a-Box (IIAB) installations, we bundle up common machine/software diagnostics, all together in 1 human-readable small file, that can be easily circulated online AND offline.
+
+The ``pastebinit`` command can then be used to upload this file, creating a short URL that makes it easier to pass around.
+
+But first off, the file is compiled by harvesting 5 main kinds of things:
+
+1. Files specially requested (if you run ``sudo iiab-diagnostics PATH/FILE1 PATH/FILE2``)
+
+2. Regular Files
+
+3. Content of Directories, 1-level deep
+
+4. Output of Commands
+
+5. Log Files: (last 100 lines of each)
+
+## Usage 
+
+1. Run it as follows:
+
+   ```
+   sudo iiab-diagnostics
+   ```
+
+   To bundle in more files, run:
+
+   ```
+   sudo iiab-diagnostics PATH/FILE1 PATH/FILE2 ...
+   ```
+
+   ( All diagnostics will be bundled up into a single human-readable file, placed in: /etc/iiab/diag/ )
+
+2. Make sure you're online, and upload the new file using the ``pastebinit`` command:
+
+   ```
+   pastebinit < /etc/iiab/diag/NEW-FILE-NAME
+   ```
+   
+   This will generate an actual web link (URL).
+
+3. Post this link (URL) to a "New issue" at https://github.com/iiab/iiab/issues
+
+   Include a description of the symptoms, and how to reproduce the problem.
+
+4. If you don't understand Step 3, email everything to bugs@iiab.io instead.
+
+## Source Code
+
+Please look over the bottom of [iiab-diagnostics](iiab-diagnostics) to learn more about which common IIAB files and commands make this rapid troubleshooting possible.