make it faster (#2328)

* make it faster

* better pipelining

* remove unnecessary test

* use tmpfs for debspawn

* don't download intermediate artifacts

* fix upload dir path

* switch to buildjet

* use buildjet cache on buildjet runner

* native builds when fast

* remove quotes

* always use buildjet cache

* remove newlines

* delete data after done with it

* skip aarch64 for fast dev builds

* don't tmpfs for arm

* don't try to remove debspawn tmpdir

.github/workflows/README.md

@@ -1,29 +0,0 @@
-# This folder contains GitHub Actions workflows for building the project
-
-## backend
-Runs: manually (on: workflow_dispatch) or called by product-pipeline (on: workflow_call)
-
-This workflow uses the actions and docker/setup-buildx-action@v1 to prepare the environment for aarch64 cross complilation using docker buildx.
-When execution of aarch64 containers is required the action docker/setup-qemu-action@v1 is added.
-A matrix-strategy has been used to build for both x86_64 and aarch64 platforms in parallel.
-
-### Running unittests
-
-Unittests are run using [cargo-nextest]( https://nexte.st/). First the sources are (cross-)compiled and archived. The archive is then run on the correct platform.
-
-## frontend
-Runs: manually (on: workflow_dispatch) or called by product-pipeline (on: workflow_call)
-
-This workflow builds the frontends.
-
-## product
-Runs: when a pull request targets the master or next branch and when a change to the master or next branch is made
-
-This workflow builds everything, re-using the backend and frontend workflows.
-The download and extraction order of artifacts is relevant to `make`, as it checks the file timestamps to decide which targets need to be executed.
-
-Result: eos.img
-
-## a note on uploading artifacts
-
-Artifacts are used to share data between jobs. File permissions are not maintained during artifact upload. Where file permissions are relevant, the workaround using tar has been used. See (here)[https://github.com/actions/upload-artifact#maintaining-file-permissions-and-case-sensitive-files].

.github/workflows/backend.yaml

@@ -1,233 +0,0 @@
-name: Backend
-
-on:
-  workflow_call:
-  workflow_dispatch:
-
-env:
-  RUST_VERSION: "1.67.1"
-  ENVIRONMENT: "dev"
-
-jobs:
-  build_libs:
-    name: Build libs
-    strategy:
-      fail-fast: false
-      matrix:
-        target: [x86_64, aarch64]
-        include:
-          - target: x86_64
-            snapshot_command: ./build-v8-snapshot.sh
-            artifact_name: js_snapshot
-            artifact_path: libs/js_engine/src/artifacts/JS_SNAPSHOT.bin
-          - target: aarch64
-            snapshot_command: ./build-arm-v8-snapshot.sh
-            artifact_name: arm_js_snapshot
-            artifact_path: libs/js_engine/src/artifacts/ARM_JS_SNAPSHOT.bin
-    runs-on: ubuntu-latest
-    timeout-minutes: 60
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        submodules: recursive
-        
-    - name: Set up QEMU
-      uses: docker/setup-qemu-action@v2
-      if: ${{ matrix.target == 'aarch64' }}
-      
-    - name: Set up Docker Buildx        
-      uses: docker/setup-buildx-action@v2
-      if: ${{ matrix.target == 'aarch64' }}
-
-    - name: "Install Rust"
-      run: |
-        rustup toolchain install ${{ env.RUST_VERSION }} --profile minimal --no-self-update
-        rustup default ${{ inputs.rust }}
-      shell: bash
-      if: ${{ matrix.target == 'x86_64' }}
-    
-    - uses: actions/cache@v3
-      with:
-        path: |
-          ~/.cargo/bin/
-          ~/.cargo/registry/index/
-          ~/.cargo/registry/cache/
-          ~/.cargo/git/db/
-          libs/target/
-        key: ${{ runner.os }}-cargo-libs-${{ matrix.target }}-${{ hashFiles('libs/Cargo.lock') }}
-
-    - name: Build v8 snapshot
-      run: ${{ matrix.snapshot_command }}
-      working-directory: libs
-
-    - uses: actions/upload-artifact@v3
-      with:
-        name: ${{ matrix.artifact_name }}
-        path: ${{ matrix.artifact_path }}
-
-  build_backend:
-    name: Build backend
-    strategy:
-      fail-fast: false
-      matrix:
-        target: [x86_64, aarch64]
-        include:
-          - target: x86_64
-            snapshot_download: js_snapshot
-          - target: aarch64
-            snapshot_download: arm_js_snapshot
-    runs-on: ubuntu-latest
-    timeout-minutes: 120
-    needs: build_libs
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        submodules: recursive
-    
-    - name: Download ${{ matrix.snapshot_download }} artifact
-      uses: actions/download-artifact@v3
-      with:
-        name: ${{ matrix.snapshot_download }}
-        path: libs/js_engine/src/artifacts/
-        
-    - name: "Install Rust"
-      run: |
-        rustup toolchain install ${{ env.RUST_VERSION }} --profile minimal --no-self-update
-        rustup default ${{ inputs.rust }}
-      shell: bash
-      if: ${{ matrix.target == 'x86_64' }}
-    
-    - uses: actions/cache@v3
-      with:
-        path: |
-          ~/.cargo/bin/
-          ~/.cargo/registry/index/
-          ~/.cargo/registry/cache/
-          ~/.cargo/git/db/
-          backend/target/
-        key: ${{ runner.os }}-cargo-backend-${{ matrix.target }}-${{ hashFiles('backend/Cargo.lock') }}
-
-    - name: Install dependencies
-      run: |
-        sudo apt-get update
-        sudo apt-get install libavahi-client-dev
-      if: ${{ matrix.target == 'x86_64' }}
-
-    - name: Check Git Hash
-      run: ./check-git-hash.sh
-
-    - name: Check Environment
-      run: ./check-environment.sh
-
-    - name: Build backend
-      run: make ARCH=${{ matrix.target }} backend
-
-    - name: 'Tar files to preserve file permissions'
-      run: make ARCH=${{ matrix.target }} backend-${{ matrix.target }}.tar
-
-    - uses: actions/upload-artifact@v3
-      with:
-        name: backend-${{ matrix.target }}
-        path: backend-${{ matrix.target }}.tar
-    
-    - name: Install nextest
-      uses: taiki-e/install-action@nextest
-
-    - name: Build and archive tests
-      run: cargo nextest archive --archive-file nextest-archive-${{ matrix.target }}.tar.zst --target ${{ matrix.target }}-unknown-linux-gnu
-      working-directory: backend
-      if: ${{ matrix.target == 'x86_64' }}
-
-    - name: Build and archive tests
-      run: |
-        docker run --rm \
-        -v "$HOME/.cargo/registry":/root/.cargo/registry \
-        -v "$(pwd)":/home/rust/src \
-        -P start9/rust-arm-cross:aarch64 \
-        sh -c 'cd /home/rust/src/backend &&
-          rustup install ${{ env.RUST_VERSION }} &&
-          rustup override set ${{ env.RUST_VERSION }} &&
-          rustup target add aarch64-unknown-linux-gnu &&
-          curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin &&
-          cargo nextest archive --archive-file nextest-archive-${{ matrix.target }}.tar.zst --target ${{ matrix.target }}-unknown-linux-gnu'
-      if: ${{ matrix.target == 'aarch64' }}
-
-    - name: Reset permissions
-      run: sudo chown -R $USER target
-      working-directory: backend
-      if: ${{ matrix.target == 'aarch64' }}
-
-    - name: Upload archive to workflow
-      uses: actions/upload-artifact@v3
-      with:
-        name: nextest-archive-${{ matrix.target }}
-        path: backend/nextest-archive-${{ matrix.target }}.tar.zst
-
-  run_tests_backend:
-    name: Test backend
-    strategy:
-      fail-fast: false
-      matrix:
-        target: [x86_64, aarch64]
-        include:
-          - target: x86_64
-          - target: aarch64
-    runs-on: ubuntu-latest
-    timeout-minutes: 60
-    needs: build_backend
-    env:
-      CARGO_TERM_COLOR: always
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        submodules: recursive
-
-    - name: Set up QEMU
-      uses: docker/setup-qemu-action@v2
-      if: ${{ matrix.target == 'aarch64' }}
-
-    - name: Set up Docker Buildx        
-      uses: docker/setup-buildx-action@v2
-      if: ${{ matrix.target == 'aarch64' }}
-
-    - run: mkdir -p ~/.cargo/bin
-      if: ${{ matrix.target == 'x86_64' }}
-
-    - name: Install nextest
-      uses: taiki-e/install-action@v2
-      with:
-        tool: nextest@0.9.47
-      if: ${{ matrix.target == 'x86_64' }}
-
-    - name: Download archive
-      uses: actions/download-artifact@v3
-      with:
-        name: nextest-archive-${{ matrix.target }}
-
-    - name: Download nextest (aarch64)
-      run: wget -O nextest-aarch64.tar.gz https://get.nexte.st/0.9.47/linux-arm
-      if: ${{ matrix.target == 'aarch64' }}
-
-    - name: Run tests
-      run: |
-        ${CARGO_HOME:-~/.cargo}/bin/cargo-nextest nextest run --no-fail-fast --archive-file nextest-archive-${{ matrix.target }}.tar.zst \
-        --filter-expr 'not (test(system::test_get_temp) | test(net::tor::test) | test(system::test_get_disk_usage) | test(net::ssl::certificate_details_persist) | test(net::ssl::ca_details_persist))'
-      if: ${{ matrix.target == 'x86_64' }}
-
-    - name: Run tests
-      run: |
-        docker run --rm --platform linux/arm64/v8 \
-        -v "/home/runner/.cargo/registry":/usr/local/cargo/registry \
-        -v "$(pwd)":/home/rust/src \
-        -e CARGO_TERM_COLOR=${{ env.CARGO_TERM_COLOR }} \
-        -P ubuntu:20.04 \
-        sh -c '
-          apt-get update &&
-          apt-get install -y ca-certificates &&
-          apt-get install -y rsync &&
-          cd /home/rust/src &&
-          mkdir -p ~/.cargo/bin &&
-          tar -zxvf nextest-aarch64.tar.gz -C ${CARGO_HOME:-~/.cargo}/bin &&
-          ${CARGO_HOME:-~/.cargo}/bin/cargo-nextest nextest run --archive-file nextest-archive-${{ matrix.target }}.tar.zst \
-          --filter-expr "not (test(system::test_get_temp) | test(net::tor::test) | test(system::test_get_disk_usage) | test(net::ssl::certificate_details_persist) | test(net::ssl::ca_details_persist))"'
-      if: ${{ matrix.target == 'aarch64' }}

.github/workflows/frontend.yaml

@@ -1,46 +0,0 @@
-name: Frontend
-
-on:
-  workflow_call:
-  workflow_dispatch:
-
-env:
-  NODEJS_VERSION: '18.15.0'
-  ENVIRONMENT: "dev"
-
-jobs:
-  frontend:
-    name: Build frontend
-    runs-on: ubuntu-latest
-    timeout-minutes: 60
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        submodules: recursive
-
-    - uses: actions/setup-node@v3
-      with:
-        node-version: ${{ env.NODEJS_VERSION }}
-    
-    - name: Get npm cache directory
-      id: npm-cache-dir
-      run: |
-        echo "dir=$(npm config get cache)" >> $GITHUB_OUTPUT
-    - uses: actions/cache@v3
-      id: npm-cache
-      with:
-        path: ${{ steps.npm-cache-dir.outputs.dir }}
-        key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
-        restore-keys: |
-          ${{ runner.os }}-node-
-
-    - name: Build frontends
-      run: make frontends
-
-    - name: 'Tar files to preserve file permissions'
-      run: tar -cvf frontend.tar ENVIRONMENT.txt GIT_HASH.txt VERSION.txt frontend/dist frontend/config.json
-
-    - uses: actions/upload-artifact@v3
-      with:
-        name: frontend
-        path: frontend.tar

.github/workflows/reusable-workflow.yaml

@@ -1,37 +0,0 @@
-name: Reusable Workflow
-
-on:
-  workflow_call:
-    inputs:
-      build_command:
-        required: true
-        type: string
-      artifact_name:
-        required: true
-        type: string
-      artifact_path:
-        required: true
-        type: string
-
-env:
-  ENVIRONMENT: "dev"
-
-jobs:
-  generic_build_job:
-    runs-on: ubuntu-latest
-    timeout-minutes: 60
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        submodules: recursive
-
-    - name: Set up Docker Buildx        
-      uses: docker/setup-buildx-action@v2
-
-    - name: Build image
-      run: ${{ inputs.build_command }}
-
-    - uses: actions/upload-artifact@v3
-      with:
-        name: ${{ inputs.artifact_name }}
-        path: ${{ inputs.artifact_path }}

.github/workflows/startos-iso.yaml

@@ -12,6 +12,12 @@ on:
           - dev
           - unstable
           - dev-unstable
+      runner:
+        type: choice
+        description: Runner
+        options:
+          - standard
+          - fast
   push:
     branches:
       - master
@@ -26,28 +32,49 @@ env:
   ENVIRONMENT: '${{ fromJson(format(''["{0}", ""]'', github.event.inputs.environment || ''dev''))[github.event.inputs.environment == ''<NONE>''] }}'
 
 jobs:
-  dpkg:
-    name: Build dpkg
+  all:
+    name: Build
     strategy:
       fail-fast: false
       matrix:
         platform:
           [x86_64, x86_64-nonfree, aarch64, aarch64-nonfree, raspberrypi]
-    runs-on: '${{ fromJson(''["ubuntu-22.04", "fatboi"]'')[github.event.inputs.environment == ''<NONE>''] }}'
+    runs-on: >-
+      ${{
+        fromJson(
+          format(
+            '["ubuntu-22.04", "{0}"]',
+            fromJson('{
+              "x86_64": "buildjet-32vcpu-ubuntu-2204",
+              "x86_64-nonfree": "buildjet-32vcpu-ubuntu-2204",
+              "aarch64": "buildjet-8vcpu-ubuntu-2204-arm",
+              "aarch64-nonfree": "buildjet-8vcpu-ubuntu-2204-arm",
+              "raspberrypi": "buildjet-16vcpu-ubuntu-2204-arm",
+            }')[matrix.platform]
+          )
+        )[github.event.inputs.runner == 'fast']
+      }}
     steps:
+      - run: |
+          sudo mount -t tmpfs tmpfs .
+        if: ${{ github.event.inputs.runner == 'fast' && (matrix.platform == 'x86_64' || matrix.platform == 'x86_64-nonfree') }}
+
       - uses: actions/checkout@v3
         with:
           repository: Start9Labs/embassy-os-deb
+          path: embassy-os-deb
 
       - uses: actions/checkout@v3
         with:
           submodules: recursive
-          path: embassyos-0.3.x
+          path: embassy-os-deb/embassyos-0.3.x
+
       - run: |
           cp -r debian embassyos-0.3.x/
           VERSION=0.3.x ./control.sh
           cp embassyos-0.3.x/backend/embassyd.service embassyos-0.3.x/debian/embassyos.embassyd.service
           cp embassyos-0.3.x/backend/embassy-init.service embassyos-0.3.x/debian/embassyos.embassy-init.service
+        working-directory: embassy-os-deb
 
       - uses: actions/setup-node@v3
         with:
@@ -57,7 +84,7 @@ jobs:
         id: npm-cache-dir
         run: |
           echo "dir=$(npm config get cache)" >> $GITHUB_OUTPUT
-      - uses: actions/cache@v3
+      - uses: buildjet/cache@v3
         id: npm-cache
         with:
           path: ${{ steps.npm-cache-dir.outputs.dir }}
@@ -79,29 +106,16 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
 
-      - name: Run build
+      - name: Run dpkg build
+        working-directory: embassy-os-deb
         run: "make VERSION=0.3.x TAG=${{ github.ref_name }}"
         env:
           OS_ARCH: ${{ matrix.platform }}
 
-      - uses: actions/upload-artifact@v3
-        with:
-          name: ${{ matrix.platform }}.deb
-          path: embassyos_0.3.x-1_*.deb
-
-  iso:
-    name: Build iso
-    strategy:
-      fail-fast: false
-      matrix:
-        platform:
-          [x86_64, x86_64-nonfree, aarch64, aarch64-nonfree, raspberrypi]
-    runs-on: '${{ fromJson(''["ubuntu-22.04", "fatboi"]'')[github.event.inputs.environment == ''<NONE>''] }}'
-    needs: [dpkg]
-    steps:
       - uses: actions/checkout@v3
         with:
           repository: Start9Labs/startos-image-recipes
+          path: startos-image-recipes
 
       - name: Install dependencies
         run: |
@@ -115,56 +129,57 @@ jobs:
         run: |
           sudo mkdir -p /etc/debspawn/
           echo "AllowUnsafePermissions=true" | sudo tee /etc/debspawn/global.toml
+          sudo mkdir -p /var/tmp/debspawn
 
-      - uses: actions/cache@v3
+      - run: sudo mount -t tmpfs tmpfs /var/tmp/debspawn
+        if: ${{ github.event.inputs.runner == 'fast' && (matrix.platform == 'x86_64' || matrix.platform == 'x86_64-nonfree') }}
+
+      - uses: buildjet/cache@v3
         with:
           path: /var/lib/debspawn
-          key: ${{ runner.os }}-debspawn-init
+          key: ${{ runner.os }}-${{ matrix.platform }}-debspawn-init
 
-      - run: "mkdir -p overlays/deb"
+      - run: "mkdir -p startos-image-recipes/overlays/deb"
 
-      - name: Download dpkg
-        uses: actions/download-artifact@v3
-        with:
-          name: ${{ matrix.platform }}.deb
-          path: overlays/deb
+      - run: "mv embassy-os-deb/embassyos_0.3.x-1_*.deb startos-image-recipes/overlays/deb/"
 
-      - name: Run build
+      - run: "rm -rf embassy-os-deb"
+
+      - name: Run iso build
+        working-directory: startos-image-recipes
         run: |
           ./run-local-build.sh ${{ matrix.platform }}
 
       - uses: actions/upload-artifact@v3
         with:
           name: ${{ matrix.platform }}.squashfs
-          path: results/*.squashfs
+          path: startos-image-recipes/results/*.squashfs
 
       - uses: actions/upload-artifact@v3
         with:
           name: ${{ matrix.platform }}.iso
-          path: results/*.iso
+          path: startos-image-recipes/results/*.iso
         if: ${{ matrix.platform != 'raspberrypi' }}
 
-  image:
-    name: Build image
-    runs-on: '${{ fromJson(''["ubuntu-22.04", "fatboi"]'')[github.event.inputs.environment == ''<NONE>''] }}'
-    timeout-minutes: 60
-    needs: [iso]
-    steps:
       - uses: actions/checkout@v3
         with:
           submodules: recursive
+          path: start-os
+        if: ${{ matrix.platform == 'raspberrypi' }}
 
-      - name: Download raspberrypi.squashfs artifact
-        uses: actions/download-artifact@v3
-        with:
-          name: raspberrypi.squashfs
+      - run: "mv startos-image-recipes/results/startos-*_raspberrypi.squashfs start-os/startos.raspberrypi.squashfs"
+        if: ${{ matrix.platform == 'raspberrypi' }}
 
-      - run: mv startos-*_raspberrypi.squashfs startos.raspberrypi.squashfs
+      - run: rm -rf startos-image-recipes
+        if: ${{ matrix.platform == 'raspberrypi' }}
 
       - name: Build image
+        working-directory: start-os
         run: make startos_raspberrypi.img
+        if: ${{ matrix.platform == 'raspberrypi' }}
 
       - uses: actions/upload-artifact@v3
         with:
           name: raspberrypi.img
-          path: startos-*_raspberrypi.img
+          path: start-os/startos-*_raspberrypi.img
+        if: ${{ matrix.platform == 'raspberrypi' }}

Makefile

@@ -192,7 +192,7 @@ ui: frontend/dist/ui
 backend: $(EMBASSY_BINS)
 
 cargo-deps/aarch64-unknown-linux-gnu/release/nc-broadcast:
-	./build-cargo-dep.sh nc-broadcast
+	ARCH=$(ARCH) ./build-cargo-dep.sh nc-broadcast
 
 cargo-deps/aarch64-unknown-linux-gnu/release/pi-beep:
-	./build-cargo-dep.sh pi-beep
+	ARCH=$(ARCH) ./build-cargo-dep.sh pi-beep

backend/build-prod.sh

@@ -22,7 +22,7 @@ if tty -s; then
 	USE_TTY="-it"
 fi
 
-alias 'rust-gnu-builder'='docker run $USE_TTY --rm -e "OS_ARCH=$OS_ARCH" -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$(pwd)":/home/rust/src -P start9/rust-arm-cross:aarch64'
+alias 'rust-gnu-builder'='docker run $USE_TTY --rm -e "OS_ARCH=$OS_ARCH" -v "$HOME/.cargo/registry":/usr/local/cargo/registry -v "$(pwd)":/home/rust/src -w /home/rust/src -P start9/rust-arm-cross:aarch64'
 alias 'rust-musl-builder'='docker run $USE_TTY --rm -e "OS_ARCH=$OS_ARCH" -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$(pwd)":/home/rust/src -P messense/rust-musl-cross:$ARCH-musl'
 
 cd ..
@@ -37,26 +37,26 @@ fi
 set +e
 fail=
 if [[ "$FLAGS" = "" ]]; then
-	rust-gnu-builder sh -c "(git config --global --add safe.directory '*'; cd backend && cargo build --release --locked  --target=$ARCH-unknown-linux-gnu)"
+	rust-gnu-builder sh -c "(cd backend && cargo build --release --locked  --target=$ARCH-unknown-linux-gnu)"
 	if test $? -ne 0; then 
 		fail=true
 	fi
 	for ARCH in x86_64 aarch64
 	do
-		rust-musl-builder sh -c "(git config --global --add safe.directory '*'; cd libs && cargo build --release --locked --bin embassy_container_init )"
+		rust-musl-builder sh -c "(cd libs && cargo build --release --locked --bin embassy_container_init )"
 		if test $? -ne 0; then 
 			fail=true
 		fi
 	done
 else
 	echo "FLAGS=$FLAGS"
-	rust-gnu-builder sh -c "(git config --global --add safe.directory '*'; cd backend && cargo build --release --features $FLAGS --locked --target=$ARCH-unknown-linux-gnu)"
+	rust-gnu-builder sh -c "(cd backend && cargo build --release --features $FLAGS --locked --target=$ARCH-unknown-linux-gnu)"
 	if test $? -ne 0; then 
 		fail=true
 	fi
 	for ARCH in x86_64 aarch64
 	do
-		rust-musl-builder sh -c "(git config --global --add safe.directory '*'; cd libs && cargo build --release --features $FLAGS --locked --bin embassy_container_init)"
+		rust-musl-builder sh -c "(cd libs && cargo build --release --features $FLAGS --locked --bin embassy_container_init)"
 		if test $? -ne 0; then 
 			fail=true
 		fi

build-cargo-dep.sh

@@ -13,9 +13,13 @@ if tty -s; then
 	USE_TTY="-it"
 fi
 
-mkdir -p cargo-deps
-alias 'rust-arm64-builder'='docker run $USE_TTY --rm -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$(pwd)"/cargo-deps:/home/rust/src -P start9/rust-arm-cross:aarch64'
+if [ -z "$ARCH" ]; then
+	ARCH=$(uname -m)
+fi
 
-rust-arm64-builder cargo install "$1" --target-dir /home/rust/src
+mkdir -p cargo-deps
+alias 'rust-arm64-builder'='docker run $USE_TTY --rm -v "$HOME/.cargo/registry":/usr/local/cargo/registry -v "$(pwd)"/cargo-deps:/home/rust/src -P start9/rust-arm-cross:aarch64'
+
+rust-arm64-builder cargo install "$1" --target-dir /home/rust/src --target=$ARCH-unknown-linux-gnu
 sudo chown -R $USER cargo-deps
 sudo chown -R $USER ~/.cargo