From d3316ff6ff4244b4c4fa87dfa72fb72e997a508d Mon Sep 17 00:00:00 2001 From: Aiden McClelland <3732071+dr-bonez@users.noreply.github.com> Date: Wed, 28 Jun 2023 13:37:26 -0600 Subject: [PATCH] make it faster (#2328) * make it faster * better pipelining * remove unnecessary test * use tmpfs for debspawn * don't download intermediate artifacts * fix upload dir path * switch to buildjet * use buildjet cache on buildjet runner * native builds when fast * remove quotes * always use buildjet cache * remove newlines * delete data after done with it * skip aarch64 for fast dev builds * don't tmpfs for arm * don't try to remove debspawn tmpdir --- .github/workflows/README.md | 29 --- .github/workflows/backend.yaml | 233 ----------------------- .github/workflows/frontend.yaml | 46 ----- .github/workflows/reusable-workflow.yaml | 37 ---- .github/workflows/startos-iso.yaml | 103 +++++----- Makefile | 4 +- backend/build-prod.sh | 10 +- build-cargo-dep.sh | 10 +- 8 files changed, 73 insertions(+), 399 deletions(-) delete mode 100644 .github/workflows/README.md delete mode 100644 .github/workflows/backend.yaml delete mode 100644 .github/workflows/frontend.yaml delete mode 100644 .github/workflows/reusable-workflow.yaml diff --git a/.github/workflows/README.md b/.github/workflows/README.md deleted file mode 100644 index 4235613eb..000000000 --- a/.github/workflows/README.md +++ /dev/null @@ -1,29 +0,0 @@ -# This folder contains GitHub Actions workflows for building the project - -## backend -Runs: manually (on: workflow_dispatch) or called by product-pipeline (on: workflow_call) - -This workflow uses the actions and docker/setup-buildx-action@v1 to prepare the environment for aarch64 cross complilation using docker buildx. -When execution of aarch64 containers is required the action docker/setup-qemu-action@v1 is added. -A matrix-strategy has been used to build for both x86_64 and aarch64 platforms in parallel. - -### Running unittests - -Unittests are run using [cargo-nextest]( https://nexte.st/). First the sources are (cross-)compiled and archived. The archive is then run on the correct platform. - -## frontend -Runs: manually (on: workflow_dispatch) or called by product-pipeline (on: workflow_call) - -This workflow builds the frontends. - -## product -Runs: when a pull request targets the master or next branch and when a change to the master or next branch is made - -This workflow builds everything, re-using the backend and frontend workflows. -The download and extraction order of artifacts is relevant to `make`, as it checks the file timestamps to decide which targets need to be executed. - -Result: eos.img - -## a note on uploading artifacts - -Artifacts are used to share data between jobs. File permissions are not maintained during artifact upload. Where file permissions are relevant, the workaround using tar has been used. See (here)[https://github.com/actions/upload-artifact#maintaining-file-permissions-and-case-sensitive-files]. \ No newline at end of file diff --git a/.github/workflows/backend.yaml b/.github/workflows/backend.yaml deleted file mode 100644 index dd0255a3c..000000000 --- a/.github/workflows/backend.yaml +++ /dev/null @@ -1,233 +0,0 @@ -name: Backend - -on: - workflow_call: - workflow_dispatch: - -env: - RUST_VERSION: "1.67.1" - ENVIRONMENT: "dev" - -jobs: - build_libs: - name: Build libs - strategy: - fail-fast: false - matrix: - target: [x86_64, aarch64] - include: - - target: x86_64 - snapshot_command: ./build-v8-snapshot.sh - artifact_name: js_snapshot - artifact_path: libs/js_engine/src/artifacts/JS_SNAPSHOT.bin - - target: aarch64 - snapshot_command: ./build-arm-v8-snapshot.sh - artifact_name: arm_js_snapshot - artifact_path: libs/js_engine/src/artifacts/ARM_JS_SNAPSHOT.bin - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - uses: actions/checkout@v3 - with: - submodules: recursive - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - if: ${{ matrix.target == 'aarch64' }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - if: ${{ matrix.target == 'aarch64' }} - - - name: "Install Rust" - run: | - rustup toolchain install ${{ env.RUST_VERSION }} --profile minimal --no-self-update - rustup default ${{ inputs.rust }} - shell: bash - if: ${{ matrix.target == 'x86_64' }} - - - uses: actions/cache@v3 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - libs/target/ - key: ${{ runner.os }}-cargo-libs-${{ matrix.target }}-${{ hashFiles('libs/Cargo.lock') }} - - - name: Build v8 snapshot - run: ${{ matrix.snapshot_command }} - working-directory: libs - - - uses: actions/upload-artifact@v3 - with: - name: ${{ matrix.artifact_name }} - path: ${{ matrix.artifact_path }} - - build_backend: - name: Build backend - strategy: - fail-fast: false - matrix: - target: [x86_64, aarch64] - include: - - target: x86_64 - snapshot_download: js_snapshot - - target: aarch64 - snapshot_download: arm_js_snapshot - runs-on: ubuntu-latest - timeout-minutes: 120 - needs: build_libs - steps: - - uses: actions/checkout@v3 - with: - submodules: recursive - - - name: Download ${{ matrix.snapshot_download }} artifact - uses: actions/download-artifact@v3 - with: - name: ${{ matrix.snapshot_download }} - path: libs/js_engine/src/artifacts/ - - - name: "Install Rust" - run: | - rustup toolchain install ${{ env.RUST_VERSION }} --profile minimal --no-self-update - rustup default ${{ inputs.rust }} - shell: bash - if: ${{ matrix.target == 'x86_64' }} - - - uses: actions/cache@v3 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - backend/target/ - key: ${{ runner.os }}-cargo-backend-${{ matrix.target }}-${{ hashFiles('backend/Cargo.lock') }} - - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install libavahi-client-dev - if: ${{ matrix.target == 'x86_64' }} - - - name: Check Git Hash - run: ./check-git-hash.sh - - - name: Check Environment - run: ./check-environment.sh - - - name: Build backend - run: make ARCH=${{ matrix.target }} backend - - - name: 'Tar files to preserve file permissions' - run: make ARCH=${{ matrix.target }} backend-${{ matrix.target }}.tar - - - uses: actions/upload-artifact@v3 - with: - name: backend-${{ matrix.target }} - path: backend-${{ matrix.target }}.tar - - - name: Install nextest - uses: taiki-e/install-action@nextest - - - name: Build and archive tests - run: cargo nextest archive --archive-file nextest-archive-${{ matrix.target }}.tar.zst --target ${{ matrix.target }}-unknown-linux-gnu - working-directory: backend - if: ${{ matrix.target == 'x86_64' }} - - - name: Build and archive tests - run: | - docker run --rm \ - -v "$HOME/.cargo/registry":/root/.cargo/registry \ - -v "$(pwd)":/home/rust/src \ - -P start9/rust-arm-cross:aarch64 \ - sh -c 'cd /home/rust/src/backend && - rustup install ${{ env.RUST_VERSION }} && - rustup override set ${{ env.RUST_VERSION }} && - rustup target add aarch64-unknown-linux-gnu && - curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin && - cargo nextest archive --archive-file nextest-archive-${{ matrix.target }}.tar.zst --target ${{ matrix.target }}-unknown-linux-gnu' - if: ${{ matrix.target == 'aarch64' }} - - - name: Reset permissions - run: sudo chown -R $USER target - working-directory: backend - if: ${{ matrix.target == 'aarch64' }} - - - name: Upload archive to workflow - uses: actions/upload-artifact@v3 - with: - name: nextest-archive-${{ matrix.target }} - path: backend/nextest-archive-${{ matrix.target }}.tar.zst - - run_tests_backend: - name: Test backend - strategy: - fail-fast: false - matrix: - target: [x86_64, aarch64] - include: - - target: x86_64 - - target: aarch64 - runs-on: ubuntu-latest - timeout-minutes: 60 - needs: build_backend - env: - CARGO_TERM_COLOR: always - steps: - - uses: actions/checkout@v3 - with: - submodules: recursive - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - if: ${{ matrix.target == 'aarch64' }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - if: ${{ matrix.target == 'aarch64' }} - - - run: mkdir -p ~/.cargo/bin - if: ${{ matrix.target == 'x86_64' }} - - - name: Install nextest - uses: taiki-e/install-action@v2 - with: - tool: nextest@0.9.47 - if: ${{ matrix.target == 'x86_64' }} - - - name: Download archive - uses: actions/download-artifact@v3 - with: - name: nextest-archive-${{ matrix.target }} - - - name: Download nextest (aarch64) - run: wget -O nextest-aarch64.tar.gz https://get.nexte.st/0.9.47/linux-arm - if: ${{ matrix.target == 'aarch64' }} - - - name: Run tests - run: | - ${CARGO_HOME:-~/.cargo}/bin/cargo-nextest nextest run --no-fail-fast --archive-file nextest-archive-${{ matrix.target }}.tar.zst \ - --filter-expr 'not (test(system::test_get_temp) | test(net::tor::test) | test(system::test_get_disk_usage) | test(net::ssl::certificate_details_persist) | test(net::ssl::ca_details_persist))' - if: ${{ matrix.target == 'x86_64' }} - - - name: Run tests - run: | - docker run --rm --platform linux/arm64/v8 \ - -v "/home/runner/.cargo/registry":/usr/local/cargo/registry \ - -v "$(pwd)":/home/rust/src \ - -e CARGO_TERM_COLOR=${{ env.CARGO_TERM_COLOR }} \ - -P ubuntu:20.04 \ - sh -c ' - apt-get update && - apt-get install -y ca-certificates && - apt-get install -y rsync && - cd /home/rust/src && - mkdir -p ~/.cargo/bin && - tar -zxvf nextest-aarch64.tar.gz -C ${CARGO_HOME:-~/.cargo}/bin && - ${CARGO_HOME:-~/.cargo}/bin/cargo-nextest nextest run --archive-file nextest-archive-${{ matrix.target }}.tar.zst \ - --filter-expr "not (test(system::test_get_temp) | test(net::tor::test) | test(system::test_get_disk_usage) | test(net::ssl::certificate_details_persist) | test(net::ssl::ca_details_persist))"' - if: ${{ matrix.target == 'aarch64' }} diff --git a/.github/workflows/frontend.yaml b/.github/workflows/frontend.yaml deleted file mode 100644 index 239bf0a5a..000000000 --- a/.github/workflows/frontend.yaml +++ /dev/null @@ -1,46 +0,0 @@ -name: Frontend - -on: - workflow_call: - workflow_dispatch: - -env: - NODEJS_VERSION: '18.15.0' - ENVIRONMENT: "dev" - -jobs: - frontend: - name: Build frontend - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - uses: actions/checkout@v3 - with: - submodules: recursive - - - uses: actions/setup-node@v3 - with: - node-version: ${{ env.NODEJS_VERSION }} - - - name: Get npm cache directory - id: npm-cache-dir - run: | - echo "dir=$(npm config get cache)" >> $GITHUB_OUTPUT - - uses: actions/cache@v3 - id: npm-cache - with: - path: ${{ steps.npm-cache-dir.outputs.dir }} - key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} - restore-keys: | - ${{ runner.os }}-node- - - - name: Build frontends - run: make frontends - - - name: 'Tar files to preserve file permissions' - run: tar -cvf frontend.tar ENVIRONMENT.txt GIT_HASH.txt VERSION.txt frontend/dist frontend/config.json - - - uses: actions/upload-artifact@v3 - with: - name: frontend - path: frontend.tar diff --git a/.github/workflows/reusable-workflow.yaml b/.github/workflows/reusable-workflow.yaml deleted file mode 100644 index c4f6e0d15..000000000 --- a/.github/workflows/reusable-workflow.yaml +++ /dev/null @@ -1,37 +0,0 @@ -name: Reusable Workflow - -on: - workflow_call: - inputs: - build_command: - required: true - type: string - artifact_name: - required: true - type: string - artifact_path: - required: true - type: string - -env: - ENVIRONMENT: "dev" - -jobs: - generic_build_job: - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - uses: actions/checkout@v3 - with: - submodules: recursive - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Build image - run: ${{ inputs.build_command }} - - - uses: actions/upload-artifact@v3 - with: - name: ${{ inputs.artifact_name }} - path: ${{ inputs.artifact_path }} diff --git a/.github/workflows/startos-iso.yaml b/.github/workflows/startos-iso.yaml index 2298a2032..079f28caa 100644 --- a/.github/workflows/startos-iso.yaml +++ b/.github/workflows/startos-iso.yaml @@ -12,6 +12,12 @@ on: - dev - unstable - dev-unstable + runner: + type: choice + description: Runner + options: + - standard + - fast push: branches: - master @@ -26,28 +32,49 @@ env: ENVIRONMENT: '${{ fromJson(format(''["{0}", ""]'', github.event.inputs.environment || ''dev''))[github.event.inputs.environment == ''''] }}' jobs: - dpkg: - name: Build dpkg + all: + name: Build strategy: fail-fast: false matrix: platform: [x86_64, x86_64-nonfree, aarch64, aarch64-nonfree, raspberrypi] - runs-on: '${{ fromJson(''["ubuntu-22.04", "fatboi"]'')[github.event.inputs.environment == ''''] }}' + runs-on: >- + ${{ + fromJson( + format( + '["ubuntu-22.04", "{0}"]', + fromJson('{ + "x86_64": "buildjet-32vcpu-ubuntu-2204", + "x86_64-nonfree": "buildjet-32vcpu-ubuntu-2204", + "aarch64": "buildjet-8vcpu-ubuntu-2204-arm", + "aarch64-nonfree": "buildjet-8vcpu-ubuntu-2204-arm", + "raspberrypi": "buildjet-16vcpu-ubuntu-2204-arm", + }')[matrix.platform] + ) + )[github.event.inputs.runner == 'fast'] + }} steps: + - run: | + sudo mount -t tmpfs tmpfs . + if: ${{ github.event.inputs.runner == 'fast' && (matrix.platform == 'x86_64' || matrix.platform == 'x86_64-nonfree') }} + - uses: actions/checkout@v3 with: repository: Start9Labs/embassy-os-deb + path: embassy-os-deb - uses: actions/checkout@v3 with: submodules: recursive - path: embassyos-0.3.x + path: embassy-os-deb/embassyos-0.3.x + - run: | cp -r debian embassyos-0.3.x/ VERSION=0.3.x ./control.sh cp embassyos-0.3.x/backend/embassyd.service embassyos-0.3.x/debian/embassyos.embassyd.service cp embassyos-0.3.x/backend/embassy-init.service embassyos-0.3.x/debian/embassyos.embassy-init.service + working-directory: embassy-os-deb - uses: actions/setup-node@v3 with: @@ -57,7 +84,7 @@ jobs: id: npm-cache-dir run: | echo "dir=$(npm config get cache)" >> $GITHUB_OUTPUT - - uses: actions/cache@v3 + - uses: buildjet/cache@v3 id: npm-cache with: path: ${{ steps.npm-cache-dir.outputs.dir }} @@ -79,29 +106,16 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - name: Run build + - name: Run dpkg build + working-directory: embassy-os-deb run: "make VERSION=0.3.x TAG=${{ github.ref_name }}" env: OS_ARCH: ${{ matrix.platform }} - - uses: actions/upload-artifact@v3 - with: - name: ${{ matrix.platform }}.deb - path: embassyos_0.3.x-1_*.deb - - iso: - name: Build iso - strategy: - fail-fast: false - matrix: - platform: - [x86_64, x86_64-nonfree, aarch64, aarch64-nonfree, raspberrypi] - runs-on: '${{ fromJson(''["ubuntu-22.04", "fatboi"]'')[github.event.inputs.environment == ''''] }}' - needs: [dpkg] - steps: - uses: actions/checkout@v3 with: repository: Start9Labs/startos-image-recipes + path: startos-image-recipes - name: Install dependencies run: | @@ -115,56 +129,57 @@ jobs: run: | sudo mkdir -p /etc/debspawn/ echo "AllowUnsafePermissions=true" | sudo tee /etc/debspawn/global.toml + sudo mkdir -p /var/tmp/debspawn - - uses: actions/cache@v3 + - run: sudo mount -t tmpfs tmpfs /var/tmp/debspawn + if: ${{ github.event.inputs.runner == 'fast' && (matrix.platform == 'x86_64' || matrix.platform == 'x86_64-nonfree') }} + + - uses: buildjet/cache@v3 with: path: /var/lib/debspawn - key: ${{ runner.os }}-debspawn-init + key: ${{ runner.os }}-${{ matrix.platform }}-debspawn-init - - run: "mkdir -p overlays/deb" + - run: "mkdir -p startos-image-recipes/overlays/deb" - - name: Download dpkg - uses: actions/download-artifact@v3 - with: - name: ${{ matrix.platform }}.deb - path: overlays/deb + - run: "mv embassy-os-deb/embassyos_0.3.x-1_*.deb startos-image-recipes/overlays/deb/" - - name: Run build + - run: "rm -rf embassy-os-deb" + + - name: Run iso build + working-directory: startos-image-recipes run: | ./run-local-build.sh ${{ matrix.platform }} - uses: actions/upload-artifact@v3 with: name: ${{ matrix.platform }}.squashfs - path: results/*.squashfs + path: startos-image-recipes/results/*.squashfs - uses: actions/upload-artifact@v3 with: name: ${{ matrix.platform }}.iso - path: results/*.iso + path: startos-image-recipes/results/*.iso if: ${{ matrix.platform != 'raspberrypi' }} - image: - name: Build image - runs-on: '${{ fromJson(''["ubuntu-22.04", "fatboi"]'')[github.event.inputs.environment == ''''] }}' - timeout-minutes: 60 - needs: [iso] - steps: - uses: actions/checkout@v3 with: submodules: recursive + path: start-os + if: ${{ matrix.platform == 'raspberrypi' }} - - name: Download raspberrypi.squashfs artifact - uses: actions/download-artifact@v3 - with: - name: raspberrypi.squashfs + - run: "mv startos-image-recipes/results/startos-*_raspberrypi.squashfs start-os/startos.raspberrypi.squashfs" + if: ${{ matrix.platform == 'raspberrypi' }} - - run: mv startos-*_raspberrypi.squashfs startos.raspberrypi.squashfs + - run: rm -rf startos-image-recipes + if: ${{ matrix.platform == 'raspberrypi' }} - name: Build image + working-directory: start-os run: make startos_raspberrypi.img + if: ${{ matrix.platform == 'raspberrypi' }} - uses: actions/upload-artifact@v3 with: name: raspberrypi.img - path: startos-*_raspberrypi.img + path: start-os/startos-*_raspberrypi.img + if: ${{ matrix.platform == 'raspberrypi' }} diff --git a/Makefile b/Makefile index 2aed23cdf..23abc2a30 100644 --- a/Makefile +++ b/Makefile @@ -192,7 +192,7 @@ ui: frontend/dist/ui backend: $(EMBASSY_BINS) cargo-deps/aarch64-unknown-linux-gnu/release/nc-broadcast: - ./build-cargo-dep.sh nc-broadcast + ARCH=$(ARCH) ./build-cargo-dep.sh nc-broadcast cargo-deps/aarch64-unknown-linux-gnu/release/pi-beep: - ./build-cargo-dep.sh pi-beep \ No newline at end of file + ARCH=$(ARCH) ./build-cargo-dep.sh pi-beep \ No newline at end of file diff --git a/backend/build-prod.sh b/backend/build-prod.sh index 075360e8c..4f16ee558 100755 --- a/backend/build-prod.sh +++ b/backend/build-prod.sh @@ -22,7 +22,7 @@ if tty -s; then USE_TTY="-it" fi -alias 'rust-gnu-builder'='docker run $USE_TTY --rm -e "OS_ARCH=$OS_ARCH" -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$(pwd)":/home/rust/src -P start9/rust-arm-cross:aarch64' +alias 'rust-gnu-builder'='docker run $USE_TTY --rm -e "OS_ARCH=$OS_ARCH" -v "$HOME/.cargo/registry":/usr/local/cargo/registry -v "$(pwd)":/home/rust/src -w /home/rust/src -P start9/rust-arm-cross:aarch64' alias 'rust-musl-builder'='docker run $USE_TTY --rm -e "OS_ARCH=$OS_ARCH" -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$(pwd)":/home/rust/src -P messense/rust-musl-cross:$ARCH-musl' cd .. @@ -37,26 +37,26 @@ fi set +e fail= if [[ "$FLAGS" = "" ]]; then - rust-gnu-builder sh -c "(git config --global --add safe.directory '*'; cd backend && cargo build --release --locked --target=$ARCH-unknown-linux-gnu)" + rust-gnu-builder sh -c "(cd backend && cargo build --release --locked --target=$ARCH-unknown-linux-gnu)" if test $? -ne 0; then fail=true fi for ARCH in x86_64 aarch64 do - rust-musl-builder sh -c "(git config --global --add safe.directory '*'; cd libs && cargo build --release --locked --bin embassy_container_init )" + rust-musl-builder sh -c "(cd libs && cargo build --release --locked --bin embassy_container_init )" if test $? -ne 0; then fail=true fi done else echo "FLAGS=$FLAGS" - rust-gnu-builder sh -c "(git config --global --add safe.directory '*'; cd backend && cargo build --release --features $FLAGS --locked --target=$ARCH-unknown-linux-gnu)" + rust-gnu-builder sh -c "(cd backend && cargo build --release --features $FLAGS --locked --target=$ARCH-unknown-linux-gnu)" if test $? -ne 0; then fail=true fi for ARCH in x86_64 aarch64 do - rust-musl-builder sh -c "(git config --global --add safe.directory '*'; cd libs && cargo build --release --features $FLAGS --locked --bin embassy_container_init)" + rust-musl-builder sh -c "(cd libs && cargo build --release --features $FLAGS --locked --bin embassy_container_init)" if test $? -ne 0; then fail=true fi diff --git a/build-cargo-dep.sh b/build-cargo-dep.sh index 08fad4799..f4ad4c951 100755 --- a/build-cargo-dep.sh +++ b/build-cargo-dep.sh @@ -13,9 +13,13 @@ if tty -s; then USE_TTY="-it" fi -mkdir -p cargo-deps -alias 'rust-arm64-builder'='docker run $USE_TTY --rm -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$(pwd)"/cargo-deps:/home/rust/src -P start9/rust-arm-cross:aarch64' +if [ -z "$ARCH" ]; then + ARCH=$(uname -m) +fi -rust-arm64-builder cargo install "$1" --target-dir /home/rust/src +mkdir -p cargo-deps +alias 'rust-arm64-builder'='docker run $USE_TTY --rm -v "$HOME/.cargo/registry":/usr/local/cargo/registry -v "$(pwd)"/cargo-deps:/home/rust/src -P start9/rust-arm-cross:aarch64' + +rust-arm64-builder cargo install "$1" --target-dir /home/rust/src --target=$ARCH-unknown-linux-gnu sudo chown -R $USER cargo-deps sudo chown -R $USER ~/.cargo \ No newline at end of file