feat: initial upload

Initial public release

scripts/00_prereq_check.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+set -e
+
+# Source our common vars
+scripts_path="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+. ${scripts_path}/vars.sh
+
+debug_msg "Starting 00_prereq_check.sh"
+
+# Check for required utils
+for bin in losetup docker wget sudo unsquashfs; do
+    if ! which ${bin} > /dev/null; then
+        error_msg "${bin} is missing! Exiting..."
+        exit 1
+    fi
+done
+
+# Make sure loop module is loaded
+if [ ! -d /sys/module/loop ]; then
+    error_msg "Loop module isn't loaded into the kernel! This is REQUIRED! Exiting..."
+    exit 1
+fi
+
+# Validate FW is downloaded
+if ! [ -f "${root_path}/unifi-firmware/${firmware_filename}" ]; then
+    echo "Error: File ${firmware_filename} does not exist in ./unifi-firmware! Exiting..."
+    exit 1
+fi
+
+# Does the checksum match?
+file_md5=$(md5sum ${root_path}/unifi-firmware/${firmware_filename} | awk '{print $1}')
+if [ "$file_md5" != "${firmware_md5}" ]; then
+    echo "Error: File ${firmware_filename} does not have the expected checksum! This is either the wrong file, or it's corrupted. Exiting..."
+    exit 1
+fi
+
+debug_msg "Finished 00_prereq_check.sh"

scripts/01_pre_docker.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+set -e
+
+# Source our common vars
+scripts_path="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+. ${scripts_path}/vars.sh
+
+debug_msg "Starting 01_pre_docker.sh"
+
+# Make sure our BuildEnv dir exists
+if [ -d ${build_path} ]; then
+    error_msg "BuildEnv already exists, this isn't a clean build! Things might fail, but we're going to try!"
+else
+    mkdir ${build_path}
+fi
+
+# Extract the goodies
+${scripts_path}/ubnt-fw-parse.py "${root_path}/unifi-firmware/${firmware_filename}" "${build_path}/fw-extract"
+
+# Extract the squashfs rootfs
+echo "Extracting unifi rootfs as root..."
+sudo unsquashfs -f -d "${build_path}/fw-extract/rootfs" "${build_path}/fw-extract/rootfs.bin"
+
+# Always build to pickup changes/updates/improvements
+debug_msg "Building ${docker_tag}"
+docker build -t ${docker_tag} ${root_path}
+
+debug_msg "Finished 01_pre_docker.sh"

scripts/03_docker.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+set -e
+
+# Source our common vars
+scripts_path="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+. ${scripts_path}/vars.sh
+
+debug_msg "Starting 03_docker.sh"
+
+debug_msg "Doing safety checks... please enter your password for sudo if prompted..."
+# Before we do anything, make our dirs, and validate they are not mounted atm. If they are, exit!
+if mountpoint -q ${build_path}/rootfs/boot; then
+    error_msg "ERROR: ${build_path}/rootfs/boot is mounted before it should be! Cleaning up..."
+    sudo umount ${build_path}/rootfs/boot
+fi
+if mountpoint -q ${build_path}/rootfs; then
+    error_msg "ERROR: ${build_path}/rootfs is mounted before it should be! Cleaning up..."
+    sudo umount ${build_path}/rootfs
+fi
+
+# Validate we don't have image files yet, because if we do, they may be mounted, and
+# that would be REAL BAD if we overwrite em
+if [ -f ${build_path}/boot.ext4 ]; then
+    error_msg "ERROR: ${build_path}/boot.ext4 exists already! Cleaning up..."
+    rm -f ${build_path}/boot.ext4
+fi
+if [ -f ${build_path}/rootfs.ext4 ]; then
+    error_msg "ERROR: ${build_path}/rootfs.ext4 exists already! Cleaning up..."
+    rm -f ${build_path}/rootfs.ext4
+fi
+
+debug_msg "Docker: Generating rootfs and boot partitions..."
+docker run --ulimit nofile=1024 --rm -v "${root_path}:/repo:Z" -it ${docker_tag} /repo/scripts/docker/run_mkimage_initial.sh
+
+debug_msg "Note: You might be asked for your password for losetup and mounting of said loopback devices since sudo is used..."
+
+debug_msg "Mounting generated block files for use with docker..."
+# Mount our loopbacks
+boot_loop_dev=$(sudo losetup -f --show ${build_path}/boot.ext4)
+rootfs_loop_dev=$(sudo losetup -f --show ${build_path}/rootfs.ext4)
+
+# And now mount them to the dirs :)
+mkdir -p ${build_path}/rootfs
+sudo mount -t ext4 ${rootfs_loop_dev} ${build_path}/rootfs
+sudo mkdir -p ${build_path}/rootfs/boot
+sudo mount -t ext4 ${boot_loop_dev} ${build_path}/rootfs/boot
+
+# Remove stupid placeholder files -_-
+sudo rm -f ${build_path}/rootfs/placeholder ${build_path}/rootfs/boot/placeholder
+
+# SAFETY NET - trap it, even tho we have makefile with set -e
+debug_msg "Docker: debootstraping..."
+trap "sudo umount ${build_path}/rootfs/boot; sudo umount ${build_path}/rootfs; sudo losetup -d ${boot_loop_dev}; sudo losetup -d ${rootfs_loop_dev}" SIGINT SIGTERM
+docker run --ulimit nofile=1024 --rm --privileged --cap-add=ALL -v /dev:/dev -v "${root_path}:/repo:Z" -it ${docker_tag} /repo/scripts/docker/run_debootstrap.sh
+
+debug_msg "Note: You might be asked for your password for losetup and umount since we are cleaning up mounts..."
+debug_msg "Cleaning up..."
+sudo umount ${build_path}/rootfs/boot
+sudo umount ${build_path}/rootfs
+sudo losetup -d ${boot_loop_dev}
+sudo losetup -d ${rootfs_loop_dev}
+rm -rf ${build_path}/rootfs
+
+debug_msg "Finished 03_docker.sh"

scripts/04_post_docker.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+set -e
+
+# Source our common vars
+scripts_path="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+. ${scripts_path}/vars.sh
+
+debug_msg "Starting 04_post_docker.sh"
+
+if [ -d ${build_path}/final ]; then
+    debug_msg "WARNING: final builddir already exists! Cleaning up..."
+    rm -rf ${build_path}/final
+fi
+mkdir -p ${build_path}/final
+
+# Kick off the docker to do the magics for us, since we need genimage
+docker run --rm -v "${root_path}:/repo:Z" -it ${docker_tag} /repo/scripts/docker/run_mkimage_final.sh
+
+# Just create our final dir and move bits over
+TIMESTAMP=`date +%Y%m%d-%H%M`
+mkdir -p ${root_path}/output/${TIMESTAMP}
+mv ${build_path}/final/debian*.img.gz ${root_path}/output/${TIMESTAMP}/
+sudo rm -rf ${build_path} # Be gone, we done buildin! :) 
+
+debug_msg "Finished 04_post_docker.sh"

scripts/docker/bootstrap/001-bootstrap

@@ -0,0 +1,62 @@
+#!/bin/bash
+set -e
+
+echo "Starting 001-bootstrap within chroot!"
+
+export DEBIAN_FRONTEND=noninteractive
+export APT_LISTCHANGES_FRONTEND=none
+
+# Conf debconf
+debconf-set-selections /debconf.set
+rm -f /debconf.set
+
+# Initial package install
+apt-get clean
+apt-get update
+apt-mark hold linux-image-* # We do not want these, as we run our own kernel!
+
+# Setup ulcmd
+systemctl enable ulcmd
+systemctl enable mock-ubnt-api
+
+# Now that we have our wanted kernel in place, do the rest of our installs
+apt-get -o Dpkg::Options::="--force-confold" -y --allow-downgrades \
+    --allow-remove-essential --allow-change-held-packages install cloud-init \
+    bsdextrautils git binutils ca-certificates e2fsprogs haveged parted curl \
+    locales console-common openssh-server less vim net-tools wireguard-tools \
+    ntpsec u-boot-tools wget u-boot-menu initramfs-tools python3-flask gnupg
+
+# Locale gen
+locale-gen
+
+# Setup OMV repo
+wget --quiet --output-document=- https://packages.openmediavault.org/public/archive.key | gpg --dearmor --yes --output "/usr/share/keyrings/openmediavault-archive-keyring.gpg"
+cat <<EOF >> /etc/apt/sources.list.d/openmediavault.list
+deb [signed-by=/usr/share/keyrings/openmediavault-archive-keyring.gpg] https://packages.openmediavault.org/public sandworm main
+# deb [signed-by=/usr/share/keyrings/openmediavault-archive-keyring.gpg] https://downloads.sourceforge.net/project/openmediavault/packages sandworm main
+## This software is not part of OpenMediaVault, but is offered by third-party
+## developers as a service to OpenMediaVault users.
+deb [signed-by=/usr/share/keyrings/openmediavault-archive-keyring.gpg] https://packages.openmediavault.org/public sandworm partner
+# deb [signed-by=/usr/share/keyrings/openmediavault-archive-keyring.gpg] https://downloads.sourceforge.net/project/openmediavault/packages sandworm partner
+EOF
+
+# Install OMV
+apt-get update
+apt-get --yes --auto-remove --show-upgraded \
+    --allow-downgrades --allow-change-held-packages \
+    --no-install-recommends \
+    --option DPkg::Options::="--force-confdef" \
+    --option DPkg::Options::="--force-confold" \
+    install openmediavault openmediavault-md || true # We "fail" all apt cmds from here on til we boot on HW
+
+# Disable systemd-networkd
+systemctl disable systemd-networkd
+systemctl disable systemd-networkd-wait-online
+systemctl mask systemd-networkd
+systemctl mask systemd-networkd-wait-online
+
+# Cleanup stuff we don't want floating around
+apt-get autoclean || true
+apt-get --purge -y autoremove || true
+rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /etc/resolv.conf
+rm -rf /var/lib/dbus/machine-id /etc/machine-id # Nuke machine IDs

scripts/docker/run_debootstrap.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+set -e
+
+docker_scripts_path="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+scripts_path="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )"
+. ${scripts_path}/vars.sh
+
+# Exports
+export PATH=${build_path}/toolchain/${toolchain_bin_path}:${PATH}
+export GCC_COLORS=auto
+export CROSS_COMPILE=${toolchain_cross_compile}
+export ARCH=arm64
+export DEBIAN_FRONTEND=noninteractive
+export DEBCONF_NONINTERACTIVE_SEEN=true
+
+# CD into our rootfs mount, and starts the fun!
+cd ${build_path}/rootfs
+debootstrap --no-check-gpg --foreign --arch=${deb_arch} --include=apt-transport-https ${deb_release} ${build_path}/rootfs ${deb_mirror}
+cp /usr/bin/qemu-aarch64-static usr/bin/
+chroot ${build_path}/rootfs /debootstrap/debootstrap --second-stage
+
+# Copy over our kernel modules and kernel
+mv -f "${build_path}/fw-extract/rootfs/lib/modules" ${build_path}/rootfs/lib
+cp "${build_path}/fw-extract/kernel.bin" "${build_path}/rootfs/boot/uImage"
+
+# Copy over our overlay if we have one
+if [[ -d ${root_path}/overlay/${fs_overlay_dir}/ ]]; then
+	echo "Applying ${fs_overlay_dir} overlay"
+	cp -R ${root_path}/overlay/${fs_overlay_dir}/* ./
+fi
+
+# Apply our part UUIDs to fstab
+sed -i "s|BOOTUUIDPLACEHOLDER|$(blkid -o value -s UUID ${build_path}/boot.ext4)|g" ${build_path}/rootfs/etc/fstab
+sed -i "s|ROOTUUIDPLACEHOLDER|$(blkid -o value -s UUID ${build_path}/rootfs.ext4)|g" ${build_path}/rootfs/etc/fstab
+
+# Hostname
+echo "${distrib_name}" > ${build_path}/rootfs/etc/hostname
+echo "127.0.1.1	${distrib_name}" >> ${build_path}/rootfs/etc/hosts
+
+# Console settings
+echo "console-common	console-data/keymap/policy	select	Select keymap from full list
+console-common	console-data/keymap/full	select	us
+" > ${build_path}/rootfs/debconf.set
+
+# Copy over stuff for ulcmd, this is hacky, but that's this ENTIRE repo for you
+mv "${build_path}/fw-extract/rootfs/lib/systemd/system/ulcmd.service" "${build_path}/rootfs/lib/systemd/system/ulcmd.service"
+mv "${build_path}/fw-extract/rootfs/usr/bin/ulcmd" "${build_path}/rootfs/usr/bin/ulcmd"
+mkdir -p "${build_path}/rootfs/usr/lib/ubnt-fw/"
+for file in libgrpc++.so.1 libgrpc.so.10 libprotobuf.so.23 \
+	libssl.so.1.1 libcrypto.so.1.1 libabsl*.so.20200923 libatomic.so.1; do
+	mv ${build_path}/fw-extract/rootfs/usr/lib/aarch64-linux-gnu/${file} "${build_path}/rootfs/usr/lib/ubnt-fw/"
+done
+sed -i 's|Type=simple|Type=simple\nEnvironment="LD_LIBRARY_PATH=/usr/lib/ubnt-fw"|g' "${build_path}/rootfs/lib/systemd/system/ulcmd.service"
+
+# Kick off bash setup script within chroot
+cp ${docker_scripts_path}/bootstrap/001-bootstrap ${build_path}/rootfs/bootstrap
+chroot ${build_path}/rootfs /bootstrap
+rm ${build_path}/rootfs/bootstrap
+
+# Final cleanup
+rm ${build_path}/rootfs/usr/bin/qemu-aarch64-static

scripts/docker/run_mkimage_final.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+set -e
+
+scripts_path="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )"
+. ${scripts_path}/vars.sh
+
+# Tempdir for root
+GENIMAGE_ROOT=$(mktemp -d)
+
+# setup and move bits
+mkdir -p ${build_path}/final
+cp ${build_path}/boot.ext4 ${build_path}/final/
+cp ${build_path}/rootfs.ext4 ${build_path}/final/
+cp ${root_path}/genimage_final.cfg ${build_path}/genimage.cfg
+
+# We get to gen an image per board, YAY!
+echo "Generating disk image"
+genimage                         \
+	--rootpath "${GENIMAGE_ROOT}"     \
+	--tmppath "/tmp/genimage-initial-tmppath"    \
+	--inputpath "${build_path}/final"  \
+	--outputpath "${build_path}/final" \
+	--config "${build_path}/genimage.cfg"
+mv ${build_path}/final/emmc.img ${build_path}/final/debian-UNVRPRO.img
+gzip ${build_path}/final/debian-UNVRPRO.img
+rm -rf /tmp/genimage-initial-tmppath # Cleanup
+
+# Cleanup
+rm ${build_path}/final/boot.ext4 ${build_path}/final/rootfs.ext4 ${build_path}/genimage.cfg

scripts/docker/run_mkimage_initial.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+set -e
+
+scripts_path="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )"
+. ${scripts_path}/vars.sh
+
+# Tempdir for root
+GENIMAGE_ROOT=$(mktemp -d)
+
+# "fake" file so generation is happy
+touch ${GENIMAGE_ROOT}/placeholder
+
+# Generate our boot and rootfs disk images
+genimage                         \
+	--rootpath "${GENIMAGE_ROOT}"     \
+	--tmppath "/tmp/genimage-initial-tmppath"    \
+	--inputpath "${build_path}"  \
+	--outputpath "${build_path}" \
+	--config "${root_path}/genimage_initial.cfg"

scripts/docker/setup_mkimage.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+set -e
+
+# setup_mkimage is special, as it's copied to the dockerfile and ran
+# same with vars.sh which we put in place
+. /vars.sh
+
+cd /usr/src
+echo "Downloading genimage..."
+wget ${genimage_src} -O /usr/src/${genimage_filename}
+
+echo "Extracting genimage..."
+tar -xJf /usr/src/${genimage_filename}
+
+echo "Building genimage..."
+cd ./${genimage_repopath}
+./configure
+make
+
+echo "Installing genimage..."
+make install
+
+echo "Cleaning up..."
+rm -rf /vars.sh # We wanna use it n burn it
+rm -rf /usr/src/${genimage_repopath}*
+
+exit 0

scripts/ubnt-fw-parse.py

@@ -0,0 +1,92 @@
+#!/usr/bin/python3
+import argparse
+import binascii
+import mmap
+import sys
+import zlib
+
+from io import BytesIO
+
+from pathlib import Path
+
+def main(fwfile: str, savedir: str):
+    # Does the file exist?
+    if not Path(fwfile).is_file():
+        print(f"Error: {fwfile} is not a file! Exiting...")
+        sys.exit(1)
+
+    # Does our save dir exist?
+    if not Path(savedir).is_dir():
+        print(f"Warning: {savedir} is not a directory, attempting to create it...")
+        Path(savedir).mkdir(parents=True)
+
+    # Start parsing the OTA file
+    with open(fwfile, 'rb+') as f:
+        # Read from disk, not memory
+        mm = mmap.mmap(f.fileno(), 0)
+
+        ubntheader = mm.read(0x104) # Read header in
+        ubntheadercrc = int.from_bytes(mm.read(0x4)) # Read header CRC, convert to int
+
+        # Do we have the UBNT header?
+        if ubntheader[0:4].decode("utf-8") != "UBNT":
+            print(f"Error: {fwfile} is missing the UBNT header! Is this the right firmware file?")
+            sys.exit(1)
+
+        # Is the header CRC valid?
+        if zlib.crc32(ubntheader) != ubntheadercrc:
+            print(f"Error: {fwfile} has in incorrect CRC for it's header! Please re-download the file!")
+            sys.exit(1)
+
+        # If we are here, that's a great sign :) 
+        ubnt_fw_ver_string = ubntheader[4:].decode("utf-8")
+        print(f"Loaded in firmware file {ubnt_fw_ver_string}")
+
+        # Start parsing out all of the files in the OTA
+        #files = []
+        fcount = 1
+        while True:
+            file_header_offset = mm.find(b'\x46\x49\x4C\x45') # FILE in hex bye string
+            # Are we done scanning the file?
+            if file_header_offset == -1:
+                break
+
+            # We found one, seek to it
+            mm.seek(file_header_offset)
+            file_header = mm.read(0x38) # Entire header
+            file_position = file_header[39] # Increments with files read, can be used to validate this is a file for us
+            file_location = file_header_offset+0x38 # header location - header = data :)
+
+            # Is this a VALID file?!
+            if fcount == file_position:
+                file_name = file_header[4:33].decode("utf-8").rstrip('\x00') # Name/type of FILE
+                file_length = int(file_header[48:52].hex(), 16)
+                print(f"{file_name} is at offset {"0x%0.2X" % file_location}, {file_length} bytes")
+                # print(int(file_header[52:56].hex(), 16)) # Maybe reserved memory or partition size? We don't use this tho
+                #files.extend([(file_position, file_name, file_location, file_length)])
+                fcount = fcount+1 # Increment on find!
+
+                fcontents = mm.read(file_length) # Read into memory
+                file_footer_crc32 = mm.read(0x8)[0:4] # Read in tailing 8 bytes (crc32) footer, but we only want the first 4
+
+                # Does our calculated crc32 match the unifi footer in the img?
+                if hex(zlib.crc32(file_header + fcontents)).lstrip('0x') != file_footer_crc32.hex().lstrip('0'):
+                    print(f"Error: Contents of {file_name} does not match the Unifi CRC! Please re-download the file!")
+                    sys.exit(1)                    
+
+                # Write file out since our mmap position is now AT the data, and we parsed the length
+                with open(f"{savedir}/{file_name}.bin", "wb") as wf:
+                    wf.write(fcontents) # Write out file
+                print(f"{file_name} has been written to {savedir}/{file_name}.bin")
+
+                del fcontents # Cleanup memory for next run
+
+        print(f"Finished extracting the contents of {fwfile}, enjoy!")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser("ubnt-fw-parse for Dream Machines")
+    parser.add_argument("file", help="The Ubiquiti firmware file to parse", type=str)
+    parser.add_argument("savedir", help="The directory to save the parsed files to", type=str)
+    args = parser.parse_args()
+    main(args.file, args.savedir)

scripts/vars.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+root_path="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )"
+build_path="${root_path}/BuildEnv"
+
+# Docker image name
+docker_tag=unvr-nas:builder
+
+# Expected UNVR Firmware and hash
+firmware_filename="f449-UNVRPRO-4.0.3-fdec2c4f-1855-4eb6-8711-e22f8f904922.bin"
+firmware_md5="5dcdc03bdec1524767007fcd12e81777"
+
+# Genimage
+genimage_src="https://github.com/pengutronix/genimage/releases/download/v16/genimage-16.tar.xz"
+genimage_filename="$(basename ${genimage_src})"
+genimage_repopath="${genimage_filename%.tar.xz}"
+
+# Distro
+distrib_name="debian"
+#deb_mirror="http://ftp.us.debian.org/debian"
+deb_mirror="http://debian.uchicago.edu/debian"
+deb_release="bookworm"
+deb_arch="arm64"
+fs_overlay_dir="filesystem"
+
+debug_msg () {
+    BLU='\033[0;32m'
+    NC='\033[0m'
+    printf "${BLU}${@}${NC}\n"
+}
+
+error_msg () {
+    BLU='\033[0;31m'
+    NC='\033[0m'
+    printf "${BLU}${@}${NC}\n"
+}