continuous deployment (#2485)

* continuous deployment

* fix

* escape braces in format string

* Update upload-ota.sh

* curl fail on http error

.github/workflows/startos-iso.yaml

@@ -31,6 +31,13 @@ on:
           - aarch64
           - aarch64-nonfree
           - raspberrypi
+      deploy:
+        type: choice
+        description: Deploy
+        options:
+          - NONE
+          - alpha
+          - beta
   push:
     branches:
       - master
@@ -191,3 +198,40 @@ jobs:
           name: ${{ matrix.platform }}.img
           path: results/*.img
         if: ${{ matrix.platform == 'raspberrypi' }}
+
+      - name: Upload OTA to registry
+        run: >-
+          PLATFORM=${{ matrix.platform }} make upload-ota TARGET="${{
+            fromJson('{
+              "alpha": "alpha-registry-x.start9.com",
+              "beta": "beta-registry.start9.com",
+            }')[github.event.inputs.deploy]
+          }}" KEY="${{
+            fromJson(
+              format('{{
+                "alpha": "{0}",
+                "beta": "{1}",
+              }}', secrets.ALPHA_INDEX_KEY, secrets.BETA_INDEX_KEY)
+            )[github.event.inputs.deploy]
+          }}"
+        if: ${{ github.event.inputs.deploy != '' && github.event.inputs.deploy != 'NONE' }}
+
+  index:
+    if: ${{ github.event.inputs.deploy != '' && github.event.inputs.deploy != 'NONE' }}
+    needs: [image]
+    runs-on: ubuntu-22.04
+    steps:
+      - run: >-
+          curl "https://${{
+            fromJson('{
+              "alpha": "alpha-registry-x.start9.com",
+              "beta": "beta-registry.start9.com",
+            }')[github.event.inputs.deploy]
+          }}:8443/resync.cgi?key=${{
+            fromJson(
+              format('{{
+                "alpha": "{0}",
+                "beta": "{1}",
+              }}', secrets.ALPHA_INDEX_KEY, secrets.BETA_INDEX_KEY)
+            )[github.event.inputs.deploy]
+          }}"

Makefile

@@ -158,6 +158,9 @@ emulate-reflash: $(ALL_TARGETS)
 	$(MAKE) install REMOTE=$(REMOTE) SSHPASS=$(SSHPASS) DESTDIR=/media/embassy/next PLATFORM=$(PLATFORM)
 	$(call ssh,"sudo touch /media/embassy/config/upgrade && sudo rm -f /media/embassy/config/disk.guid && sudo sync && sudo reboot")
 
+upload-ota: results/$(BASENAME).squashfs
+	TARGET=$(TARGET) KEY=$(KEY) ./upload-ota.sh
+
 build/lib/depends build/lib/conflicts: build/dpkg-deps/*
 	build/dpkg-deps/generate.sh
 

basename.sh

@@ -2,7 +2,7 @@
 
 cd "$(dirname "${BASH_SOURCE[0]}")"
 
-PLATFORM=$(if [ -f ./PLATFORM.txt ]; then cat ./PLATFORM.txt; else echo unknown; fi)
+PLATFORM="$(if [ -f ./PLATFORM.txt ]; then cat ./PLATFORM.txt; else echo unknown; fi)"
 VERSION="$(cat ./VERSION.txt)"
 GIT_HASH="$(cat ./GIT_HASH.txt)"
 if [[ "$GIT_HASH" =~ ^@ ]]; then

build/registry/downloadIndexActionResult

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+set -e
+
+RUN_ID=$1
+
+if [ -z "$RUN_ID" ]; then
+        >&2 echo usage: $0 '<run-id>'
+        exit 1
+fi
+
+TMP_DIR=/var/tmp/action-run-results/$RUN_ID
+
+rm -rf $TMP_DIR
+mkdir -p $TMP_DIR
+
+cd $TMP_DIR
+
+for arch in x86_64 x86_64-nonfree aarch64 aarch64-nonfree raspberrypi; do
+        gh run download -R Start9Labs/start-os $RUN_ID -n $arch.squashfs
+done
+
+VERSION=
+HASH=
+for file in $(ls *.squashfs); do
+        if [[ $file =~ ^startos-([0-9]+\.[0-9]+\.[0-9]+(\.[0-9]+)?)-([a-f0-9]{7}(~[a-z-]+)?|unknown)_([a-z0-9_-]+).squashfs$ ]]; then
+                if [ -n "$VERSION" ] && [ "$VERSION" != "${BASH_REMATCH[1]}" ]; then
+                        >&2 echo "VERSION MISMATCH: expected $VERSION got ${BASH_REMATCH[1]}"
+                        exit 2
+                fi
+                if [ -n "$HASH" ] && [ "$HASH" != "${BASH_REMATCH[3]}" ]; then
+                        >&2 echo "HASH MISMATCH: expected $HASH got ${BASH_REMATCH[3]}"
+                        exit 3
+                fi
+                VERSION="${BASH_REMATCH[1]}"
+                HASH="${BASH_REMATCH[3]}"
+        fi
+done
+
+mkdir -p /root/resources/eos/$VERSION
+rm -rf /root/resources/eos/$VERSION/$HASH
+mv $TMP_DIR /root/resources/eos/$VERSION/$HASH
+
+cd /root/resources/eos/$VERSION
+setOsCommitHash $HASH

build/registry/resync.cgi

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+declare -A params
+while IFS='=' read -r -d '&' key value && [[ -n "$key" ]]; do
+    params["$key"]=$value
+done <<<"${QUERY_STRING}&"
+
+index_key="${params['key']}"
+if [ -z "$index_key" ] || [ "$index_key" != "$(cat /var/www/index_key.txt)" ]; then
+    echo "HTTP/1.1 401 UNAUTHORIZED"
+    echo "Content-Type: text/html"
+    echo
+    echo "UNAUTHORIZED"
+    exit
+fi
+
+touch /tmp/resync
+
+echo "HTTP/1.1 200 OK"
+echo "Content-Type: text/html"
+echo
+echo "OK: Upload successful"

build/registry/resyncRsyncRegistry

@@ -6,12 +6,14 @@
 # Then we are going to make sure that each of these files is then put on the rsyncd server
 # so the embassies can pull them down
 
+date >> /var/log/resyncRsyncRegistry.runlog
+
 
 cat > /etc/rsyncd.conf << RD
 uid = root
 gid = root
 use chroot = yes
-max connections = 50
+max connections = 4
 pid file = /var/run/rsyncd.pid
 exclude = lost+found/
 timeout = 900
@@ -27,7 +29,7 @@ do
     filename=${dir##*/}
     version=$(echo $directory | sed -r 's/.*\///')
     version_dir="/srv/rsync/$version"
-    type=$(echo "$filename" | sed -r "s/^.*?\.(\w+)\.squashfs$/\1/")
+    type=$(echo "$filename" | sed -r "s/^.*?\.([a-z0-9_-]+)\.squashfs$/\1/")
     new_dir="$version_dir/$type"
 
 
@@ -51,4 +53,4 @@ INSERTING
 done 
 
 echo "Created rsyncd.conf file, restarting service"
-systemctl restart rsync
+systemctl restart rsync

build/registry/setOsCommitHash

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# Get the current directory
+PWD=$(pwd)
+HASH=$1
+
+if [ -z "$HASH" ]; then
+  >&2 echo "usage: setOsCommitHash <hash>"
+  exit 1
+fi
+
+# Define the expected pattern for the directory
+pattern="/root/resources/eos/"
+
+# Check if the current directory matches the pattern
+if [[ $PWD =~ ^$pattern([0-9.]+)$ ]]; then
+  # Extract the version number from the directory path
+  version="${BASH_REMATCH[1]}"
+else
+  >&2 echo "MUST BE IN OS VERSION DIRECTORY"
+  exit 1
+fi
+
+if ! [ -d "$HASH" ]; then
+  >&2 echo "$HASH: No such directory"
+  exit 1
+fi
+
+for file in $(ls $HASH/startos-$version-${HASH}_*.squashfs); do
+  if [[ $file =~ ^$HASH/startos-$version-${HASH}_([a-z0-9_-]+).squashfs$ ]]; then
+    arch="${BASH_REMATCH[1]}"
+    echo "Found arch $arch"
+    umount /srv/rsync/$version/$arch
+    rm eos.$arch.squashfs
+    ln -s $file eos.$arch.squashfs
+  fi
+done
+
+resyncRsyncRegistry

build/registry/upload.cgi

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+declare -A params
+while IFS='=' read -r -d '&' key value && [[ -n "$key" ]]; do
+    params["$key"]=$value
+done <<<"${QUERY_STRING}&"
+
+index_key="${params['key']}"
+if [ -z "$index_key" ] || [ "$index_key" != "$(cat /var/www/index_key.txt)" ]; then
+    echo "HTTP/1.1 401 UNAUTHORIZED"
+    echo "Content-Type: text/html"
+    echo
+    echo "UNAUTHORIZED"
+    exit
+fi
+
+git_hash="${params['gitHash']}"
+version="${params['version']}"
+platform="${params['platform']}"
+shasum="${params['shasum']}"
+if [ -z "$git_hash" ] || [ -z "$version" ] || [ -z "$platform" ] || [ -z "$shasum" ]; then
+    echo "HTTP/1.1 400 BAD REQUEST"
+    echo "Content-Type: text/html"
+    echo
+    echo "BAD REQUEST: missing param"
+    exit
+fi
+
+tmp_file=$(mktemp /var/tmp/tmp.XXXXXXXXXX.squashfs)
+cat > $tmp_file
+
+if ! sha256sum $tmp_file | grep "$shasum"; then
+    rm $tmp_file
+    echo "HTTP/1.1 400 BAD REQUEST"
+    echo "Content-Type: text/html"
+    echo
+    echo "BAD REQUEST: shasum mismatch"
+fi
+
+mkdir -p /var/www/resources/eos/${version}/${git_hash}
+mv $tmp_file /var/www/resources/eos/${version}/${git_hash}/startos-${version}-${git_hash}_${platform}.squashfs
+rm /var/www/resources/eos/${version}/eos.${platform}.squashfs
+ln -rs /var/www/resources/eos/${version}/${git_hash}/startos-${version}-${git_hash}_${platform}.squashfs /var/www/resources/eos/${version}/eos.${platform}.squashfs
+
+echo "HTTP/1.1 200 OK"
+echo "Content-Type: text/html"
+echo
+echo "OK: Upload successful"

upload-ota.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+set -e
+
+cd "$(dirname "${BASH_SOURCE[0]}")"
+
+if [ -z "$TARGET" ]; then
+    >&2 echo "TARGET is required"
+    exit 1
+fi
+
+if [ -z "$KEY" ]; then
+    >&2 echo "KEY is required"
+    exit 1
+fi
+
+PLATFORM="$(cat ./PLATFORM.txt)"
+VERSION="$(cat ./VERSION.txt)"
+GIT_HASH="$(cat ./GIT_HASH.txt)"
+if [[ "$GIT_HASH" =~ ^@ ]]; then
+  GIT_HASH=unknown
+else
+  GIT_HASH="$(echo -n "$GIT_HASH" |  head -c 7)"
+fi
+STARTOS_ENV="$(cat ./ENVIRONMENT.txt)"
+if [ -n "$STARTOS_ENV" ]; then
+  GIT_HASH="$GIT_HASH~${STARTOS_ENV}"
+fi
+
+BASENAME="startos-${VERSION}-${GIT_HASH}_${PLATFORM}"
+
+SHASUM=$(sha256sum results/$BASENAME.squashfs | awk '{print $1}')
+
+curl --fail-with-body -T results/${BASENAME}.squashfs "https://${TARGET}:8443/upload.cgi?key=${KEY}&gitHash=${GIT_HASH}&version=${VERSION}&platform=${PLATFORM}&shasum=${SHASUM}"