diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md new file mode 100644 index 000000000..cb9385f41 --- /dev/null +++ b/DEVELOPMENT.md @@ -0,0 +1,133 @@ +# Setting up your development environment on Debian/Ubuntu + +A step-by-step guide + +> This is the only officially supported build environment. +> MacOS has limited build capabilities and Windows requires [WSL2](https://learn.microsoft.com/en-us/windows/wsl/install) + +## Installing dependencies + +Run the following commands one at a time + +```sh +sudo apt update +sudo apt install -y ca-certificates curl gpg build-essential +curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg +echo "deb [arch=$(dpkg-architecture -q DEB_HOST_ARCH) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian bookworm stable" | sudo tee /etc/apt/sources.list.d/docker.list +sudo apt update +sudo apt install -y sed grep gawk jq gzip brotli containerd.io docker-ce docker-ce-cli docker-compose-plugin qemu-user-static binfmt-support squashfs-tools git debspawn rsync b3sum +sudo mkdir -p /etc/debspawn/ +echo "AllowUnsafePermissions=true" | sudo tee /etc/debspawn/global.toml +sudo usermod -aG docker $USER +sudo su $USER +docker run --privileged --rm tonistiigi/binfmt --install all +docker buildx create --use +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh # proceed with default installation +curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/master/install.sh | bash +source ~/.bashrc +nvm install 20 +nvm use 20 +``` + +## Cloning the repository + +```sh +git clone --recursive https://github.com/Start9Labs/start-os.git --branch next/minor +cd start-os +``` + +## Building an ISO + +```sh +PLATFORM=$(uname -m) ENVIRONMENT=dev make iso +``` + +This will build an ISO for your current architecture. If you are building to run on an architecture other than the one you are currently on, replace `$(uname -m)` with the correct platform for the device (one of `aarch64`, `aarch64-nonfree`, `x86_64`, `x86_64-nonfree`, `raspberrypi`) + +## Creating a VM + +### Install virt-manager + +```sh +sudo apt update +sudo apt install -y virt-manager +sudo usermod -aG libvirt $USER +sudo su $USER +``` + +### Launch virt-manager + +```sh +virt-manager +``` + +### Create new virtual machine + +![Select "Create a new virtual machine"](assets/create-vm/step-1.png) +![Click "Forward"](assets/create-vm/step-2.png) +![Click "Browse"](assets/create-vm/step-3.png) +![Click "+"](assets/create-vm/step-4.png) + +#### make sure to set "Target Path" to the path to your results directory in start-os + +![Create storage pool](assets/create-vm/step-5.png) +![Select storage pool](assets/create-vm/step-6.png) +![Select ISO](assets/create-vm/step-7.png) +![Select "Generic or unknown OS" and click "Forward"](assets/create-vm/step-8.png) +![Set Memory and CPUs](assets/create-vm/step-9.png) +![Create disk](assets/create-vm/step-10.png) +![Name VM](assets/create-vm/step-11.png) +![Create network](assets/create-vm/step-12.png) + +## Updating a VM + +The fastest way to update a VM to your latest code depends on what you changed: + +### UI or startd: + +```sh +PLATFORM=$(uname -m) ENVIRONMENT=dev make update-startbox REMOTE=start9@ +``` + +### Container runtime or debian dependencies: + +```sh +PLATFORM=$(uname -m) ENVIRONMENT=dev make update-deb REMOTE=start9@ +``` + +### Image recipe: + +```sh +PLATFORM=$(uname -m) ENVIRONMENT=dev make update-squashfs REMOTE=start9@ +``` + +--- + +If the device you are building for is not available via ssh, it is also possible to use `magic-wormhole` to send the relevant files. + +### Prerequisites: + +```sh +sudo apt update +sudo apt install -y magic-wormhole +``` + +As before, the fastest way to update a VM to your latest code depends on what you changed. Each of the following commands will return a command to paste into the shell of the device you would like to upgrade. + +### UI or startd: + +```sh +PLATFORM=$(uname -m) ENVIRONMENT=dev make wormhole +``` + +### Container runtime or debian dependencies: + +```sh +PLATFORM=$(uname -m) ENVIRONMENT=dev make wormhole-deb +``` + +### Image recipe: + +```sh +PLATFORM=$(uname -m) ENVIRONMENT=dev make wormhole-squashfs +``` diff --git a/Makefile b/Makefile index 5a0440196..5e038a08d 100644 --- a/Makefile +++ b/Makefile @@ -152,16 +152,21 @@ update-overlay: $(ALL_TARGETS) $(call ssh,"sudo systemctl start startd") wormhole: core/target/$(ARCH)-unknown-linux-musl/release/startbox - @echo "Paste the following command into the shell of your start-os server:" + @echo "Paste the following command into the shell of your StartOS server:" + @echo @wormhole send core/target/$(ARCH)-unknown-linux-musl/release/startbox 2>&1 | awk -Winteractive '/wormhole receive/ { printf "sudo /usr/lib/startos/scripts/chroot-and-upgrade \"cd /usr/bin && rm startbox && wormhole receive --accept-file %s && chmod +x startbox\"\n", $$3 }' wormhole-deb: results/$(BASENAME).deb - @echo "Paste the following command into the shell of your start-os server:" + @echo "Paste the following command into the shell of your StartOS server:" + @echo @wormhole send results/$(BASENAME).deb 2>&1 | awk -Winteractive '/wormhole receive/ { printf "sudo /usr/lib/startos/scripts/chroot-and-upgrade '"'"'cd $$(mktemp -d) && wormhole receive --accept-file %s && apt-get install -y --reinstall ./$(BASENAME).deb'"'"'\n", $$3 }' -wormhole-cli: core/target/$(ARCH)-unknown-linux-musl/release/start-cli - @echo "Paste the following command into the shell of your start-os server:" - @wormhole send results/$(BASENAME).deb 2>&1 | awk -Winteractive '/wormhole receive/ { printf "sudo /usr/lib/startos/scripts/chroot-and-upgrade '"'"'cd $$(mktemp -d) && wormhole receive --accept-file %s && apt-get install -y --reinstall ./$(BASENAME).deb'"'"'\n", $$3 }' +wormhole-squashfs: results/$(BASENAME).squashfs + $(eval SQFS_SUM := $(shell b3sum results/$(BASENAME).squashfs | head -c 32)) + $(eval SQFS_SIZE := $(shell du -s --bytes results/$(BASENAME).squashfs | awk '{print $$1}')) + @echo "Paste the following command into the shell of your StartOS server:" + @echo + @wormhole send results/$(BASENAME).squashfs 2>&1 | awk -Winteractive '/wormhole receive/ { printf "sudo sh -c '"'"'/usr/lib/startos/scripts/prune-images $(SQFS_SIZE) && cd /media/startos/images && wormhole receive --accept-file %s && mv $(BASENAME).squashfs $(SQFS_SUM).rootfs && ln -rsf ./$(SQFS_SUM).rootfs ../config/current.rootfs && sync && reboot'"'"'\n", $$3 }' update: $(ALL_TARGETS) @if [ -z "$(REMOTE)" ]; then >&2 echo "Must specify REMOTE" && false; fi @@ -169,6 +174,28 @@ update: $(ALL_TARGETS) $(MAKE) install REMOTE=$(REMOTE) SSHPASS=$(SSHPASS) DESTDIR=/media/startos/next PLATFORM=$(PLATFORM) $(call ssh,'sudo /media/startos/next/usr/lib/startos/scripts/chroot-and-upgrade --no-sync "apt-get install -y $(shell cat ./build/lib/depends)"') +update-startbox: core/target/$(ARCH)-unknown-linux-musl/release/startbox # only update binary (faster than full update) + @if [ -z "$(REMOTE)" ]; then >&2 echo "Must specify REMOTE" && false; fi + $(call ssh,'sudo /usr/lib/startos/scripts/chroot-and-upgrade --create') + $(call cp,core/target/$(ARCH)-unknown-linux-musl/release/startbox,/media/startos/next/usr/bin/startbox) + $(call ssh,'sudo /media/startos/next/usr/lib/startos/scripts/chroot-and-upgrade --no-sync true') + +update-deb: results/$(BASENAME).deb # better than update, but only available from debian + @if [ -z "$(REMOTE)" ]; then >&2 echo "Must specify REMOTE" && false; fi + $(call ssh,'sudo /usr/lib/startos/scripts/chroot-and-upgrade --create') + $(call mkdir,/media/startos/next/tmp/startos-deb) + $(call cp,results/$(BASENAME).deb,/media/startos/next/tmp/startos-deb/$(BASENAME).deb) + $(call ssh,'sudo /media/startos/next/usr/lib/startos/scripts/chroot-and-upgrade --no-sync "apt-get install -y --reinstall /tmp/startos-deb/$(BASENAME).deb"') + +update-squashfs: results/$(BASENAME).squashfs + @if [ -z "$(REMOTE)" ]; then >&2 echo "Must specify REMOTE" && false; fi + $(eval SQFS_SUM := $(shell b3sum results/$(BASENAME).squashfs)) + $(eval SQFS_SIZE := $(shell du -s --bytes results/$(BASENAME).squashfs | awk '{print $$1}')) + $(call ssh,'/usr/lib/startos/scripts/prune-images $(SQFS_SIZE)') + $(call cp,results/$(BASENAME).squashfs,/media/startos/images/$(SQFS_SUM).rootfs) + $(call ssh,'sudo ln -rsf /media/startos/images/$(SQFS_SUM).rootfs /media/startos/config/current.rootfs') + $(call ssh,'sudo reboot') + emulate-reflash: $(ALL_TARGETS) @if [ -z "$(REMOTE)" ]; then >&2 echo "Must specify REMOTE" && false; fi $(call ssh,'sudo /usr/lib/startos/scripts/chroot-and-upgrade --create') diff --git a/assets/create-vm/step-1.png b/assets/create-vm/step-1.png new file mode 100644 index 000000000..2dfafc25f Binary files /dev/null and b/assets/create-vm/step-1.png differ diff --git a/assets/create-vm/step-10.png b/assets/create-vm/step-10.png new file mode 100644 index 000000000..bc1985394 Binary files /dev/null and b/assets/create-vm/step-10.png differ diff --git a/assets/create-vm/step-11.png b/assets/create-vm/step-11.png new file mode 100644 index 000000000..322dd5394 Binary files /dev/null and b/assets/create-vm/step-11.png differ diff --git a/assets/create-vm/step-12.png b/assets/create-vm/step-12.png new file mode 100644 index 000000000..52f14f56e Binary files /dev/null and b/assets/create-vm/step-12.png differ diff --git a/assets/create-vm/step-2.png b/assets/create-vm/step-2.png new file mode 100644 index 000000000..020f3d7d1 Binary files /dev/null and b/assets/create-vm/step-2.png differ diff --git a/assets/create-vm/step-3.png b/assets/create-vm/step-3.png new file mode 100644 index 000000000..0295ba6bc Binary files /dev/null and b/assets/create-vm/step-3.png differ diff --git a/assets/create-vm/step-4.png b/assets/create-vm/step-4.png new file mode 100644 index 000000000..85832d260 Binary files /dev/null and b/assets/create-vm/step-4.png differ diff --git a/assets/create-vm/step-5.png b/assets/create-vm/step-5.png new file mode 100644 index 000000000..d34cb16a7 Binary files /dev/null and b/assets/create-vm/step-5.png differ diff --git a/assets/create-vm/step-6.png b/assets/create-vm/step-6.png new file mode 100644 index 000000000..92f2c2f1f Binary files /dev/null and b/assets/create-vm/step-6.png differ diff --git a/assets/create-vm/step-7.png b/assets/create-vm/step-7.png new file mode 100644 index 000000000..ad8eb9b81 Binary files /dev/null and b/assets/create-vm/step-7.png differ diff --git a/assets/create-vm/step-8.png b/assets/create-vm/step-8.png new file mode 100644 index 000000000..ce5443c76 Binary files /dev/null and b/assets/create-vm/step-8.png differ diff --git a/assets/create-vm/step-9.png b/assets/create-vm/step-9.png new file mode 100644 index 000000000..042735490 Binary files /dev/null and b/assets/create-vm/step-9.png differ diff --git a/build/lib/scripts/chroot-and-upgrade b/build/lib/scripts/chroot-and-upgrade index 7adaaaccb..59581fbc6 100755 --- a/build/lib/scripts/chroot-and-upgrade +++ b/build/lib/scripts/chroot-and-upgrade @@ -1,5 +1,7 @@ #!/bin/bash +SOURCE_DIR="$(dirname "${BASH_SOURCE[0]}")" + if [ "$UID" -ne 0 ]; then >&2 echo 'Must be run as root' exit 1 @@ -77,20 +79,7 @@ umount /media/startos/next/boot if [ "$CHROOT_RES" -eq 0 ]; then if [ -h /media/startos/config/current.rootfs ] && [ -e /media/startos/config/current.rootfs ]; then - echo 'Pruning...' - current="$(readlink -f /media/startos/config/current.rootfs)" - needed=$(du -s --bytes /media/startos/next | awk '{print $1}') - while [[ "$(df -B1 --output=avail --sync /media/startos/images | tail -n1)" -lt "$needed" ]]; do - to_prune="$(ls -t1 /media/startos/images/*.rootfs | grep -v "$current" | tail -n1)" - if [ -e "$to_prune" ]; then - echo " Pruning $to_prune" - rm -rf "$to_prune" - else - >&2 echo "Not enough space and nothing to prune!" - exit 1 - fi - done - echo 'done.' + ${SOURCE_DIR}/prune-images $(du -s --bytes /media/startos/next | awk '{print $1}') fi echo 'Upgrading...' diff --git a/build/lib/scripts/prune-images b/build/lib/scripts/prune-images new file mode 100755 index 000000000..21861467f --- /dev/null +++ b/build/lib/scripts/prune-images @@ -0,0 +1,49 @@ +#!/bin/bash + +if [ "$UID" -ne 0 ]; then + >&2 echo 'Must be run as root' + exit 1 +fi + +POSITIONAL_ARGS=() + +while [[ $# -gt 0 ]]; do + case $1 in + -*|--*) + echo "Unknown option $1" + exit 1 + ;; + *) + POSITIONAL_ARGS+=("$1") # save positional arg + shift # past argument + ;; + esac +done + +set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters + +needed=$1 + +if [ -z "$needed" ]; then + >&2 echo "usage: $0 " + exit 1 +fi + +if [ -h /media/startos/config/current.rootfs ] && [ -e /media/startos/config/current.rootfs ]; then + echo 'Pruning...' + current="$(readlink -f /media/startos/config/current.rootfs)" + while [[ "$(df -B1 --output=avail --sync /media/startos/images | tail -n1)" -lt "$needed" ]]; do + to_prune="$(ls -t1 /media/startos/images/*.rootfs | grep -v "$current" | tail -n1)" + if [ -e "$to_prune" ]; then + echo " Pruning $to_prune" + rm -rf "$to_prune" + else + >&2 echo "Not enough space and nothing to prune!" + exit 1 + fi + done + echo 'done.' +else + >&2 echo 'No current.rootfs, not safe to prune' + exit 1 +fi \ No newline at end of file