Backups Rework (#698)

* wip: Backup al

* wip: Backup

* backup code complete

* wip

* wip

* update types

* wip

* fix errors

* Backups wizard (#699)

* backup adjustments

* fix endpoint arg

* Update prod-key-modal.page.ts

Co-authored-by: Drew Ansbacher <drew.ansbacher@spiredigital.com>
Co-authored-by: Aiden McClelland <me@drbonez.dev>

* build errs addressed

* working

* update backup command input, nix, and apk add

* add ecryptfs-utils

* fix build

* wip

* fixes for macos

* more mac magic

* fix typo

* working

* fixes after rebase

* chore: remove unused imports

Co-authored-by: Justin Miller <dragondef@gmail.com>
Co-authored-by: Drew Ansbacher <drew.ansbacher@gmail.com>
Co-authored-by: Drew Ansbacher <drew.ansbacher@spiredigital.com>
Co-authored-by: Lucy Cifferello <12953208+elvece@users.noreply.github.com>

build/initialization.sh

@@ -2,6 +2,9 @@
 
 # Update repositories, install dependencies, do some initial configurations, set hostname, enable embassy-init, and config Tor
 set -e
+
+! test -f /etc/docker/daemon.json || rm /etc/docker/daemon.json
+
 apt update
 apt install -y \
 	docker.io \
@@ -16,7 +19,8 @@ apt install -y \
 	sqlite3 \
 	wireless-tools \
 	net-tools \
-	ifupdown
+	ifupdown \
+	ecryptfs-utils
 sed -i 's/"1"/"0"/g' /etc/apt/apt.conf.d/20auto-upgrades
 sed -i 's/Restart=on-failure/Restart=always/g' /lib/systemd/system/tor@default.service
 sed -i '/}/i \ \ \ \ application\/wasm \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ wasm;' /etc/nginx/mime.types
@@ -28,7 +32,7 @@ echo "iface wlan0 inet dhcp" >> /etc/network/interfaces
 mkdir -p /etc/nginx/ssl
 
 # fix to suppress docker warning, fixed in 21.xx release of docker cli: https://github.com/docker/cli/pull/2934
-mkdir /root/.docker
+mkdir -p /root/.docker
 touch /root/.docker/config.json
 
 docker run --privileged --rm tonistiigi/binfmt --install all

build/make-image.sh

@@ -16,4 +16,11 @@ export OUTPUT_DEVICE=$(sudo losetup --show -fP eos.img)
 export LOOPDEV=$(sudo losetup --show -fP ubuntu.img)
 ./build/partitioning.sh
 ./build/write-image.sh
-sudo losetup -d $OUTPUT_DEVICE
+sudo e2fsck -f ${OUTPUT_DEVICE}p3
+sudo resize2fs -M ${OUTPUT_DEVICE}p3
+BLOCK_INFO=$(sudo dumpe2fs ${OUTPUT_DEVICE}p3)
+BLOCK_COUNT=$(echo "$BLOCK_INFO" | grep "Block count:" | sed 's/Block count:\s\+//g')
+BLOCK_SIZE=$(echo "$BLOCK_INFO" | grep "Block size:" | sed 's/Block size:\s\+//g')
+echo "YOUR GREEN FILESYSTEM is '$[$BLOCK_COUNT*$BLOCK_SIZE]' BYTES"
+echo "IF YOU ARE QUICK-FLASHING FROM MAC-OS, NOTE THIS NUMBER FOR LATER"
+sudo losetup -d $OUTPUT_DEVICE

build/quick-flash.sh

@@ -2,14 +2,20 @@
 
 set -e
 
+function mktmpfifo () {
+	TMP_PATH=$(mktemp)
+	rm $TMP_PATH
+	mkfifo $TMP_PATH
+	echo $TMP_PATH
+}
+
 echo 'This script will only work on a card that has previously had a full image written to it.'
 echo 'It will *only* flash the ext4 portion (`green` partition) of the img file onto the card.'
 echo 'The product key, disk guid, and kernel data will *not* be affected.'
 read -p "Continue? [y/N]" -n 1 -r
+echo
 if ! [[ "$REPLY" =~ ^[Yy]$ ]]; then
         exit 1
-else
-	echo
 fi
 
 if ! which pv > /dev/null; then
@@ -26,48 +32,101 @@ if ! which pv > /dev/null; then
 		if [[ "$REPLY" =~ ^[Yy]$ ]]; then
 			sudo pacman -S pv
 		fi
+	elif which brew > /dev/null; then
+		read -p "Install? [y/N]" -n 1 -r
+		echo
+		if [[ "$REPLY" =~ ^[Yy]$ ]]; then
+			brew install pv
+		fi
 	else
 		>&2 echo 'This script does not recognize what package manager you have available on your system.'
 		>&2 echo 'Please go install the utility manually if you want progress reporting.'
 	fi
 fi
 
-if ! test -e /dev/disk/by-label/green; then
-	>&2 echo '`green` partition not found'
-	exit 1
+if [[ "$(uname)" == "Darwin" ]]; then
+	export TARGET_PARTITION="disk$(diskutil list | grep EMBASSY | head -1 | rev | cut -b 3)s3"
+	if ! test -e $TARGET_PARTITION; then
+		>&2 echo '`green` partition not found'
+                exit 1
+	fi
+	export SOURCE_DEVICE="$(hdiutil attach -nomount eos.img | head -n1 | sed -E 's/([^ ]+).*$/\1/g')"
+	export SOURCE_PARTITION="${SOURCE_DEVICE}s3"
+	function detach () {
+		hdiutil detach $SOURCE_DEVICE
+	}
+else
+	if ! test -e /dev/disk/by-label/green; then
+		>&2 echo '`green` partition not found'
+		exit 1
+	fi
+	export TARGET_PARTITION=$(readlink -f /dev/disk/by-label/green)
+	export SOURCE_DEVICE="$(sudo losetup --show -fP eos.img)"
+	export SOURCE_PARTITION="${SOURCE_DEVICE}p3"
+	function detach () {
+		sudo losetup -d ${SOURCE_DEVICE}
+	}
 fi
-export TARGET_PARTITION=$(readlink -f /dev/disk/by-label/green)
 
 if [[ "$TARGET_PARTITION" =~ ^/dev/loop ]]; then
 	>&2 echo 'You are currently flashing onto a loop device.'
 	>&2 echo 'This is probably a mistake, and usually means you failed to detach a .img file.'
 	read -p "Continue anyway? [y/N]" -n 1 -r
+	echo
 	if ! [[ "$REPLY" =~ ^[Yy]$ ]]; then
-        exit 1
-	else
-		echo
+        	exit 1
 	fi
 fi
 
-export SOURCE_DEVICE=$(sudo losetup --show -fP eos.img)
-sudo e2fsck -f ${SOURCE_DEVICE}p3
-sudo resize2fs -M ${SOURCE_DEVICE}p3
-export BLOCK_INFO=$(sudo dumpe2fs ${SOURCE_DEVICE}p3)
-export BLOCK_COUNT=$(echo "$BLOCK_INFO" | grep "Block count:" | sed 's/Block count:\s\+//g')
-export BLOCK_SIZE=$(echo "$BLOCK_INFO" | grep "Block size:" | sed 's/Block size:\s\+//g')
-export FS_SIZE=$[$BLOCK_COUNT*$BLOCK_SIZE]
-echo "Flashing $FS_SIZE bytes to $TARGET_PARTITION"
-if which pv > /dev/null; then
-	sudo cat ${SOURCE_DEVICE}p3 | head -c $FS_SIZE | pv -s $FS_SIZE | sudo dd of=${TARGET_PARTITION} bs=1M iflag=fullblock oflag=direct 2>/dev/null
+if [[ "$(uname)" == "Darwin" ]]; then
+	if test -z "$FS_SIZE"; then
+		read -p "Enter FS Size (shown during make of eos.img)" -r
+		export FS_SIZE=$REPLY
+	fi
 else
-	sudo cat ${SOURCE_DEVICE}p3 | head -c $FS_SIZE | sudo dd of=${TARGET_PARTITION} bs=1M iflag=fullblock oflag=direct
+	sudo e2fsck -f ${SOURCE_PARTITION}
+	sudo resize2fs -M ${SOURCE_PARTITION}
+	export BLOCK_INFO=$(sudo dumpe2fs ${SOURCE_PARTITION})
+	export BLOCK_COUNT=$(echo "$BLOCK_INFO" | grep "Block count:" | sed 's/Block count:\s\+//g')
+	export BLOCK_SIZE=$(echo "$BLOCK_INFO" | grep "Block size:" | sed 's/Block size:\s\+//g')
+	export FS_SIZE=$[$BLOCK_COUNT*$BLOCK_SIZE]
+fi
+echo "Flashing $FS_SIZE bytes to $TARGET_PARTITION"
+if [[ "$(uname)" == "Darwin" ]]; then
+	if which pv > /dev/null; then
+		sudo cat ${SOURCE_PARTITION} | head -c $FS_SIZE | pv -s $FS_SIZE | sudo dd of=${TARGET_PARTITION} bs=1m 2>/dev/null
+	else
+		sudo cat ${SOURCE_PARTITION} | head -c $FS_SIZE | sudo dd of=${TARGET_PARTITION} bs=1M iflag=fullblock oflag=direct
+	fi
+else
+	if which pv > /dev/null; then
+		sudo cat ${SOURCE_PARTITION} | head -c $FS_SIZE | pv -s $FS_SIZE | sudo dd of=${TARGET_PARTITION} bs=1M iflag=fullblock oflag=direct 2>/dev/null
+	else
+		sudo cat ${SOURCE_PARTITION} | head -c $FS_SIZE | sudo dd of=${TARGET_PARTITION} bs=1M iflag=fullblock oflag=direct
+	fi
 fi
 echo Verifying...
-export INPUT_HASH=$(sudo cat ${SOURCE_DEVICE}p3 | head -c $FS_SIZE | sha256sum)
-export OUTPUT_HASH=$(sudo cat ${TARGET_PARTITION} | head -c $FS_SIZE | sha256sum)
-sudo losetup -d ${SOURCE_DEVICE}
-if ! [[ "$INPUT_HASH" == "$OUTPUT_HASH" ]]; then
+export INPUT_HASH=$(mktemp)
+export OUTPUT_HASH=$(mktemp)
+if which pv > /dev/null; then
+	export PV_IN=$(mktmpfifo)
+fi
+sudo cat ${SOURCE_PARTITION} | head -c $FS_SIZE | tee -a $PV_IN | sha256sum > $INPUT_HASH &
+export INPUT_CHILD=$!
+sudo cat ${TARGET_PARTITION} | head -c $FS_SIZE | tee -a $PV_IN | sha256sum > $OUTPUT_HASH &
+export OUTPUT_CHILD=$!
+if which pv > /dev/null; then
+	pv -s $[$FS_SIZE*2] < $PV_IN > /dev/null &
+fi
+wait $INPUT_CHILD $OUTPUT_CHILD
+if which pv > /dev/null; then
+	rm $PV_IN
+fi
+detach
+if ! [[ "$(cat $INPUT_HASH)" == "$(cat $OUTPUT_HASH)" ]]; then
+	rm $INPUT_HASH $OUTPUT_HASH
 	>&2 echo Verification Failed
 	exit 1
 fi
+rm $INPUT_HASH $OUTPUT_HASH
 echo "Verification Succeeded"

build/write-image.sh

@@ -21,6 +21,7 @@ sudo e2label ${OUTPUT_DEVICE}p4 blue
 mkdir -p /tmp/eos-mnt
 sudo mount ${OUTPUT_DEVICE}p1 /tmp/eos-mnt
 
+sudo sed -i 's/^/usb-storage.quirks=152d:0562:u /g' /tmp/eos-mnt/cmdline.txt
 sudo sed -i 's/LABEL=writable/LABEL=green/g' /tmp/eos-mnt/cmdline.txt
 cat /tmp/eos-mnt/config.txt | grep -v "dtoverlay=" | sudo tee /tmp/eos-mnt/config.txt.tmp
 echo "dtoverlay=pwm-2chan" | sudo tee -a /tmp/eos-mnt/config.txt.tmp