Add basic GitHub workflows builds (#1578)

* add easy target for backend build

* add reusable backend build workflow

* add reusable frontend build workflow

* add full build workflow

* add some comments

.github/workflows/README.md

@@ -0,0 +1,24 @@
+# This folder contains GitHub Actions workflows for building the project
+
+## backend-pr
+Runs: when a pull request targets the master branch and changes the libs/ and/or backend/ folders
+
+This workflow uses the actions docker/setup-qemu-action@v1 and docker/setup-buildx-action@v1 to prepare the environment for aarch64 cross complilation using docker buildx.
+A matrix-strategy has been used for building the v8 snapshot instead of the makefile to allow parallel job execution.
+
+## frontend-pr
+Runs: when a pull request targets the master branch and changes the frontend/ folder
+
+This workflow builds the frontends.
+
+## product
+Runs: when a change to the master branch is made
+
+This workflow builds everything, re-using the backend-pr and frontend-pr workflows.
+The download and extraction order of artifacts is relevant to `make`, as it checks the file timestamps to decide which targets need to be executed.
+
+Result: eos.img
+
+## a note on uploading artifacts
+
+Artifacts are used to share data between jobs. File permissions are not maintained during artifact upload. Where file permissions are relevant, the workaround using tar has been used. See (here)[https://github.com/actions/upload-artifact#maintaining-file-permissions-and-case-sensitive-files].

.github/workflows/backend-pr.yaml

@@ -0,0 +1,104 @@
+name: Backend PR
+
+on:
+  workflow_call:
+  workflow_dispatch:
+  pull_request:
+    branches:
+    - master
+    paths:
+    - backend
+    - libs
+
+jobs:
+  libs:
+    name: Build libs
+    strategy:
+      matrix:
+        target: [amd64, arm64]
+        include:
+          - target: amd64
+            snapshot_command: ./build-v8-snapshot.sh
+            artifact_name: js_snapshot
+            artifact_path: libs/js_engine/src/artifacts/JS_SNAPSHOT.bin
+          - target: arm64
+            snapshot_command: ./build-arm-v8-snapshot.sh
+            artifact_name: arm_js_snapshot
+            artifact_path: libs/js_engine/src/artifacts/ARM_JS_SNAPSHOT.bin
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: recursive
+        
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v1
+      
+    - name: Set up Docker Buildx        
+      uses: docker/setup-buildx-action@v1
+    
+    - uses: actions/cache@v3
+      with:
+        path: |
+          ~/.cargo/bin/
+          ~/.cargo/registry/index/
+          ~/.cargo/registry/cache/
+          ~/.cargo/git/db/
+          target/
+        key: ${{ runner.os }}-cargo-libs-${{ matrix.target }}-${{ hashFiles('libs/Cargo.lock') }}
+
+    - name: Build v8 snapshot
+      run: ${{ matrix.snapshot_command }}
+      working-directory: libs
+
+    - uses: actions/upload-artifact@v3
+      with:
+        name: ${{ matrix.artifact_name }}
+        path: ${{ matrix.artifact_path }}
+
+  backend:
+    name: Build backend
+    runs-on: ubuntu-latest
+    needs: libs
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: recursive
+    
+    - name: Download arm_js_snapshot artifact
+      uses: actions/download-artifact@v3
+      with:
+        name: arm_js_snapshot
+        path: libs/js_engine/src/artifacts/
+        
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v1
+      
+    - name: Set up Docker Buildx        
+      uses: docker/setup-buildx-action@v1
+
+    - uses: actions-rs/toolchain@v1
+      with:
+        toolchain: stable
+        override: true
+    
+    - uses: actions/cache@v3
+      with:
+        path: |
+          ~/.cargo/bin/
+          ~/.cargo/registry/index/
+          ~/.cargo/registry/cache/
+          ~/.cargo/git/db/
+          target/
+        key: ${{ runner.os }}-cargo-backend-${{ hashFiles('backend/Cargo.lock') }}
+
+    - name: Build backend
+      run: make backend
+
+    - name: 'Tar files to preserve file permissions'
+      run: tar -cvf backend.tar backend/target/aarch64-unknown-linux-gnu/release/embassy*
+
+    - uses: actions/upload-artifact@v3
+      with:
+        name: backend
+        path: backend.tar

.github/workflows/frontend-pr.yaml

@@ -0,0 +1,46 @@
+name: Frontend PR
+
+on:
+  workflow_call:
+  workflow_dispatch:
+  pull_request:
+    branches:
+    - master
+    paths:
+    - frontend
+
+jobs:
+  frontend:
+    name: Build frontend
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: recursive
+
+    - uses: actions/setup-node@v3
+      with:
+        node-version: 16
+    
+    - name: Get npm cache directory
+      id: npm-cache-dir
+      run: |
+        echo "::set-output name=dir::$(npm config get cache)"
+    - uses: actions/cache@v3
+      id: npm-cache
+      with:
+        path: ${{ steps.npm-cache-dir.outputs.dir }}
+        key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+        restore-keys: |
+          ${{ runner.os }}-node-
+
+    - name: Build frontend
+      run: make frontend
+
+    - name: 'Tar files to preserve file permissions'
+      run: tar -cvf frontend.tar frontend/dist frontend/config.json
+
+    - uses: actions/upload-artifact@v3
+      with:
+        name: frontend
+        path: frontend.tar

.github/workflows/product.yaml

@@ -0,0 +1,142 @@
+name: Build Pipeline
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+    - master
+
+jobs:
+  compat:
+    name: Build compat.tar
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: recursive
+        
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v1
+      
+    - name: Set up Docker Buildx        
+      uses: docker/setup-buildx-action@v1
+
+    - uses: actions-rs/toolchain@v1
+      with:
+        toolchain: stable
+        override: true
+
+    - uses: actions/cache@v3
+      with:
+        path: |
+          ~/.cargo/bin/
+          ~/.cargo/registry/index/
+          ~/.cargo/registry/cache/
+          ~/.cargo/git/db/
+          target/
+        key: ${{ runner.os }}-cargo-compat-${{ hashFiles('**/system-images/compat/Cargo.lock') }}
+
+    - name: Build image
+      run: make system-images/compat/compat.tar
+
+    - uses: actions/upload-artifact@v3
+      with:
+        name: compat.tar
+        path: system-images/compat/compat.tar
+
+  utils:
+    name: Build utils.tar
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: recursive
+        
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v1
+      
+    - name: Set up Docker Buildx        
+      uses: docker/setup-buildx-action@v1
+
+    - name: Build image
+      run: make system-images/utils/utils.tar
+
+    - uses: actions/upload-artifact@v3
+      with:
+        name: utils.tar
+        path: system-images/utils/utils.tar
+
+  backend:
+    uses: ./.github/workflows/backend-pr.yaml
+  
+  frontend:
+    uses: ./.github/workflows/frontend-pr.yaml
+    
+  image:
+    name: Build image
+    runs-on: ubuntu-latest
+    needs: [compat,utils,backend,frontend]
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: recursive
+
+    - name: Download compat.tar artifact
+      uses: actions/download-artifact@v3
+      with:
+        name: compat.tar
+        path: system-images/compat
+    
+    - name: Download utils.tar artifact
+      uses: actions/download-artifact@v3
+      with:
+        name: utils.tar
+        path: system-images/utils
+
+    - name: Download js_snapshot artifact
+      uses: actions/download-artifact@v3
+      with:
+        name: js_snapshot
+        path: libs/js_engine/src/artifacts/
+  
+    - name: Download arm_js_snapshot artifact
+      uses: actions/download-artifact@v3
+      with:
+        name: arm_js_snapshot
+        path: libs/js_engine/src/artifacts/
+
+    - name: Download backend artifact
+      uses: actions/download-artifact@v3
+      with:
+        name: backend
+
+    - name: 'Extract backend'
+      run: 
+        tar -mxvf backend.tar
+
+    - name: Download frontend artifact
+      uses: actions/download-artifact@v3
+      with:
+        name: frontend
+
+    - name: Skip frontend build
+      run: |
+        mkdir frontend/node_modules
+        mkdir frontend/dist
+        mkdir patch-db/client/node_modules
+        mkdir patch-db/client/dist
+
+    - name: 'Extract frontend'
+      run: |
+        tar -mxvf frontend.tar frontend/config.json
+        tar -mxvf frontend.tar frontend/dist
+
+    - name: Cache raspiOS
+      id: cache-raspios
+      uses: actions/cache@v3
+      with:
+        path: raspios.img
+        key: cache-raspios
+    
+    - name: Build image
+      run: "make V=1 eos.img --debug"

Makefile

@@ -82,3 +82,6 @@ frontends: frontend/node_modules frontend/config.json $(EMBASSY_UIS)
 
 # this is a convenience step to build the UI
 ui: frontend/node_modules frontend/config.json frontend/dist/ui
+
+# this is a convenience step to build the backend
+backend: $(EMBASSY_BINS)