be resilient to bad lshw output (#2390)

* update readme layout and assets

* Update README.md

---------

Co-authored-by: Matt Hill <MattDHill@users.noreply.github.com>

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -1,6 +1,6 @@
 name: 🐛 Bug Report
-description: Create a report to help us improve embassyOS
-title: '[bug]: '
+description: Create a report to help us improve StartOS
+title: "[bug]: "
 labels: [Bug, Needs Triage]
 assignees:
   - MattDHill
@@ -10,27 +10,25 @@ body:
       label: Prerequisites
       description: Please confirm you have completed the following.
       options:
-        - label: I have searched for [existing issues](https://github.com/start9labs/embassy-os/issues) that already report this problem.
+        - label: I have searched for [existing issues](https://github.com/start9labs/start-os/issues) that already report this problem.
           required: true
   - type: input
     attributes:
-      label: embassyOS Version
-      description: What version of embassyOS are you running?
-      placeholder: e.g. 0.3.0
+      label: Server Hardware
+      description: On what hardware are you running StartOS? Please be as detailed as possible!
+      placeholder: Pi (8GB) w/ 32GB microSD & Samsung T7 SSD
+    validations:
+      required: true
+  - type: input
+    attributes:
+      label: StartOS Version
+      description: What version of StartOS are you running?
+      placeholder: e.g. 0.3.4.3
     validations:
       required: true
   - type: dropdown
     attributes:
-      label: Device
-      description: What device are you using to connect to Embassy?
-      options:
-        - Phone/tablet
-        - Laptop/Desktop
-    validations:
-      required: true
-  - type: dropdown
-    attributes:
-      label: Device OS
+      label: Client OS
       description: What operating system is your device running?
       options:
         - MacOS
@@ -45,14 +43,14 @@ body:
       required: true
   - type: input
     attributes:
-      label: Device OS Version
+      label: Client OS Version
       description: What version is your device OS?
     validations:
       required: true
   - type: dropdown
     attributes:
       label: Browser
-      description: What browser are you using to connect to Embassy?
+      description: What browser are you using to connect to your server?
       options:
         - Firefox
         - Brave

.github/ISSUE_TEMPLATE/feature-request.yml

@@ -1,6 +1,6 @@
 name: 💡 Feature Request
-description: Suggest an idea for embassyOS
-title: '[feat]: '
+description: Suggest an idea for StartOS
+title: "[feat]: "
 labels: [Enhancement]
 assignees:
   - MattDHill
@@ -10,7 +10,7 @@ body:
       label: Prerequisites
       description: Please confirm you have completed the following.
       options:
-        - label: I have searched for [existing issues](https://github.com/start9labs/embassy-os/issues) that already suggest this feature.
+        - label: I have searched for [existing issues](https://github.com/start9labs/start-os/issues) that already suggest this feature.
           required: true
   - type: textarea
     attributes:
@@ -27,7 +27,7 @@ body:
   - type: textarea
     attributes:
       label: Describe Preferred Solution
-      description: How you want this feature added to embassyOS?
+      description: How you want this feature added to StartOS?
   - type: textarea
     attributes:
       label: Describe Alternatives

.github/workflows/README.md

@@ -1,29 +0,0 @@
-# This folder contains GitHub Actions workflows for building the project
-
-## backend
-Runs: manually (on: workflow_dispatch) or called by product-pipeline (on: workflow_call)
-
-This workflow uses the actions and docker/setup-buildx-action@v1 to prepare the environment for aarch64 cross complilation using docker buildx.
-When execution of aarch64 containers is required the action docker/setup-qemu-action@v1 is added.
-A matrix-strategy has been used to build for both x86_64 and aarch64 platforms in parallel.
-
-### Running unittests
-
-Unittests are run using [cargo-nextest]( https://nexte.st/). First the sources are (cross-)compiled and archived. The archive is then run on the correct platform.
-
-## frontend
-Runs: manually (on: workflow_dispatch) or called by product-pipeline (on: workflow_call)
-
-This workflow builds the frontends.
-
-## product
-Runs: when a pull request targets the master or next branch and when a change to the master or next branch is made
-
-This workflow builds everything, re-using the backend and frontend workflows.
-The download and extraction order of artifacts is relevant to `make`, as it checks the file timestamps to decide which targets need to be executed.
-
-Result: eos.img
-
-## a note on uploading artifacts
-
-Artifacts are used to share data between jobs. File permissions are not maintained during artifact upload. Where file permissions are relevant, the workaround using tar has been used. See (here)[https://github.com/actions/upload-artifact#maintaining-file-permissions-and-case-sensitive-files].

.github/workflows/backend.yaml

@@ -1,233 +0,0 @@
-name: Backend
-
-on:
-  workflow_call:
-  workflow_dispatch:
-
-env:
-  RUST_VERSION: "1.67.1"
-  ENVIRONMENT: "dev"
-
-jobs:
-  build_libs:
-    name: Build libs
-    strategy:
-      fail-fast: false
-      matrix:
-        target: [x86_64, aarch64]
-        include:
-          - target: x86_64
-            snapshot_command: ./build-v8-snapshot.sh
-            artifact_name: js_snapshot
-            artifact_path: libs/js_engine/src/artifacts/JS_SNAPSHOT.bin
-          - target: aarch64
-            snapshot_command: ./build-arm-v8-snapshot.sh
-            artifact_name: arm_js_snapshot
-            artifact_path: libs/js_engine/src/artifacts/ARM_JS_SNAPSHOT.bin
-    runs-on: ubuntu-latest
-    timeout-minutes: 60
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        submodules: recursive
-        
-    - name: Set up QEMU
-      uses: docker/setup-qemu-action@v2
-      if: ${{ matrix.target == 'aarch64' }}
-      
-    - name: Set up Docker Buildx        
-      uses: docker/setup-buildx-action@v2
-      if: ${{ matrix.target == 'aarch64' }}
-
-    - name: "Install Rust"
-      run: |
-        rustup toolchain install ${{ env.RUST_VERSION }} --profile minimal --no-self-update
-        rustup default ${{ inputs.rust }}
-      shell: bash
-      if: ${{ matrix.target == 'x86_64' }}
-    
-    - uses: actions/cache@v3
-      with:
-        path: |
-          ~/.cargo/bin/
-          ~/.cargo/registry/index/
-          ~/.cargo/registry/cache/
-          ~/.cargo/git/db/
-          libs/target/
-        key: ${{ runner.os }}-cargo-libs-${{ matrix.target }}-${{ hashFiles('libs/Cargo.lock') }}
-
-    - name: Build v8 snapshot
-      run: ${{ matrix.snapshot_command }}
-      working-directory: libs
-
-    - uses: actions/upload-artifact@v3
-      with:
-        name: ${{ matrix.artifact_name }}
-        path: ${{ matrix.artifact_path }}
-
-  build_backend:
-    name: Build backend
-    strategy:
-      fail-fast: false
-      matrix:
-        target: [x86_64, aarch64]
-        include:
-          - target: x86_64
-            snapshot_download: js_snapshot
-          - target: aarch64
-            snapshot_download: arm_js_snapshot
-    runs-on: ubuntu-latest
-    timeout-minutes: 120
-    needs: build_libs
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        submodules: recursive
-    
-    - name: Download ${{ matrix.snapshot_download }} artifact
-      uses: actions/download-artifact@v3
-      with:
-        name: ${{ matrix.snapshot_download }}
-        path: libs/js_engine/src/artifacts/
-        
-    - name: "Install Rust"
-      run: |
-        rustup toolchain install ${{ env.RUST_VERSION }} --profile minimal --no-self-update
-        rustup default ${{ inputs.rust }}
-      shell: bash
-      if: ${{ matrix.target == 'x86_64' }}
-    
-    - uses: actions/cache@v3
-      with:
-        path: |
-          ~/.cargo/bin/
-          ~/.cargo/registry/index/
-          ~/.cargo/registry/cache/
-          ~/.cargo/git/db/
-          backend/target/
-        key: ${{ runner.os }}-cargo-backend-${{ matrix.target }}-${{ hashFiles('backend/Cargo.lock') }}
-
-    - name: Install dependencies
-      run: |
-        sudo apt-get update
-        sudo apt-get install libavahi-client-dev
-      if: ${{ matrix.target == 'x86_64' }}
-
-    - name: Check Git Hash
-      run: ./check-git-hash.sh
-
-    - name: Check Environment
-      run: ./check-environment.sh
-
-    - name: Build backend
-      run: make ARCH=${{ matrix.target }} backend
-
-    - name: 'Tar files to preserve file permissions'
-      run: make ARCH=${{ matrix.target }} backend-${{ matrix.target }}.tar
-
-    - uses: actions/upload-artifact@v3
-      with:
-        name: backend-${{ matrix.target }}
-        path: backend-${{ matrix.target }}.tar
-    
-    - name: Install nextest
-      uses: taiki-e/install-action@nextest
-
-    - name: Build and archive tests
-      run: cargo nextest archive --archive-file nextest-archive-${{ matrix.target }}.tar.zst --target ${{ matrix.target }}-unknown-linux-gnu
-      working-directory: backend
-      if: ${{ matrix.target == 'x86_64' }}
-
-    - name: Build and archive tests
-      run: |
-        docker run --rm \
-        -v "$HOME/.cargo/registry":/root/.cargo/registry \
-        -v "$(pwd)":/home/rust/src \
-        -P start9/rust-arm-cross:aarch64 \
-        sh -c 'cd /home/rust/src/backend &&
-          rustup install ${{ env.RUST_VERSION }} &&
-          rustup override set ${{ env.RUST_VERSION }} &&
-          rustup target add aarch64-unknown-linux-gnu &&
-          curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin &&
-          cargo nextest archive --archive-file nextest-archive-${{ matrix.target }}.tar.zst --target ${{ matrix.target }}-unknown-linux-gnu'
-      if: ${{ matrix.target == 'aarch64' }}
-
-    - name: Reset permissions
-      run: sudo chown -R $USER target
-      working-directory: backend
-      if: ${{ matrix.target == 'aarch64' }}
-
-    - name: Upload archive to workflow
-      uses: actions/upload-artifact@v3
-      with:
-        name: nextest-archive-${{ matrix.target }}
-        path: backend/nextest-archive-${{ matrix.target }}.tar.zst
-
-  run_tests_backend:
-    name: Test backend
-    strategy:
-      fail-fast: false
-      matrix:
-        target: [x86_64, aarch64]
-        include:
-          - target: x86_64
-          - target: aarch64
-    runs-on: ubuntu-latest
-    timeout-minutes: 60
-    needs: build_backend
-    env:
-      CARGO_TERM_COLOR: always
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        submodules: recursive
-
-    - name: Set up QEMU
-      uses: docker/setup-qemu-action@v2
-      if: ${{ matrix.target == 'aarch64' }}
-
-    - name: Set up Docker Buildx        
-      uses: docker/setup-buildx-action@v2
-      if: ${{ matrix.target == 'aarch64' }}
-
-    - run: mkdir -p ~/.cargo/bin
-      if: ${{ matrix.target == 'x86_64' }}
-
-    - name: Install nextest
-      uses: taiki-e/install-action@v2
-      with:
-        tool: nextest@0.9.47
-      if: ${{ matrix.target == 'x86_64' }}
-
-    - name: Download archive
-      uses: actions/download-artifact@v3
-      with:
-        name: nextest-archive-${{ matrix.target }}
-
-    - name: Download nextest (aarch64)
-      run: wget -O nextest-aarch64.tar.gz https://get.nexte.st/0.9.47/linux-arm
-      if: ${{ matrix.target == 'aarch64' }}
-
-    - name: Run tests
-      run: |
-        ${CARGO_HOME:-~/.cargo}/bin/cargo-nextest nextest run --no-fail-fast --archive-file nextest-archive-${{ matrix.target }}.tar.zst \
-        --filter-expr 'not (test(system::test_get_temp) | test(net::tor::test) | test(system::test_get_disk_usage) | test(net::ssl::certificate_details_persist) | test(net::ssl::ca_details_persist))'
-      if: ${{ matrix.target == 'x86_64' }}
-
-    - name: Run tests
-      run: |
-        docker run --rm --platform linux/arm64/v8 \
-        -v "/home/runner/.cargo/registry":/usr/local/cargo/registry \
-        -v "$(pwd)":/home/rust/src \
-        -e CARGO_TERM_COLOR=${{ env.CARGO_TERM_COLOR }} \
-        -P ubuntu:20.04 \
-        sh -c '
-          apt-get update &&
-          apt-get install -y ca-certificates &&
-          apt-get install -y rsync &&
-          cd /home/rust/src &&
-          mkdir -p ~/.cargo/bin &&
-          tar -zxvf nextest-aarch64.tar.gz -C ${CARGO_HOME:-~/.cargo}/bin &&
-          ${CARGO_HOME:-~/.cargo}/bin/cargo-nextest nextest run --archive-file nextest-archive-${{ matrix.target }}.tar.zst \
-          --filter-expr "not (test(system::test_get_temp) | test(net::tor::test) | test(system::test_get_disk_usage) | test(net::ssl::certificate_details_persist) | test(net::ssl::ca_details_persist))"'
-      if: ${{ matrix.target == 'aarch64' }}

.github/workflows/debian.yaml

@@ -1,63 +0,0 @@
-name: Debian Package
-
-on:
-  workflow_call:
-  workflow_dispatch:
-
-env:
-  NODEJS_VERSION: '16.11.0'
-  ENVIRONMENT: "dev"
-
-jobs:
-  dpkg:
-    name: Build dpkg
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        repository: Start9Labs/embassy-os-deb
-
-    - uses: actions/checkout@v3
-      with:
-        submodules: recursive
-        path: embassyos-0.3.x
-    - run: |
-        cp -r debian embassyos-0.3.x/
-        VERSION=0.3.x ./control.sh
-        cp embassyos-0.3.x/backend/embassyd.service embassyos-0.3.x/debian/embassyos.embassyd.service
-        cp embassyos-0.3.x/backend/embassy-init.service embassyos-0.3.x/debian/embassyos.embassy-init.service
-    
-    - uses: actions/setup-node@v3
-      with:
-        node-version: ${{ env.NODEJS_VERSION }}
-    
-    - name: Get npm cache directory
-      id: npm-cache-dir
-      run: |
-        echo "dir=$(npm config get cache)" >> $GITHUB_OUTPUT
-    - uses: actions/cache@v3
-      id: npm-cache
-      with:
-        path: ${{ steps.npm-cache-dir.outputs.dir }}
-        key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
-        restore-keys: |
-          ${{ runner.os }}-node-
-
-    - name: Install dependencies
-      run: |
-        sudo apt-get update
-        sudo apt-get install debmake debhelper-compat
-
-    - name: Set up QEMU
-      uses: docker/setup-qemu-action@v2
-      
-    - name: Set up Docker Buildx        
-      uses: docker/setup-buildx-action@v2
-
-    - name: Run build
-      run: "make VERSION=0.3.x TAG=${{ github.ref_name }}"
-
-    - uses: actions/upload-artifact@v3
-      with:
-        name: deb
-        path: embassyos_0.3.x-1_amd64.deb

.github/workflows/frontend.yaml

@@ -1,46 +0,0 @@
-name: Frontend
-
-on:
-  workflow_call:
-  workflow_dispatch:
-
-env:
-  NODEJS_VERSION: '16.11.0'
-  ENVIRONMENT: "dev"
-
-jobs:
-  frontend:
-    name: Build frontend
-    runs-on: ubuntu-latest
-    timeout-minutes: 60
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        submodules: recursive
-
-    - uses: actions/setup-node@v3
-      with:
-        node-version: ${{ env.NODEJS_VERSION }}
-    
-    - name: Get npm cache directory
-      id: npm-cache-dir
-      run: |
-        echo "dir=$(npm config get cache)" >> $GITHUB_OUTPUT
-    - uses: actions/cache@v3
-      id: npm-cache
-      with:
-        path: ${{ steps.npm-cache-dir.outputs.dir }}
-        key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
-        restore-keys: |
-          ${{ runner.os }}-node-
-
-    - name: Build frontends
-      run: make frontends
-
-    - name: 'Tar files to preserve file permissions'
-      run: tar -cvf frontend.tar ENVIRONMENT.txt GIT_HASH.txt VERSION.txt frontend/dist frontend/config.json
-
-    - uses: actions/upload-artifact@v3
-      with:
-        name: frontend
-        path: frontend.tar

.github/workflows/product.yaml

@@ -1,129 +0,0 @@
-name: Build Pipeline
-
-on:
-  workflow_dispatch:
-  push:
-    branches:
-    - master
-    - next
-  pull_request:
-    branches:
-    - master
-    - next
-
-env:
-  ENVIRONMENT: "dev"
-
-jobs:
-  compat:
-    uses: ./.github/workflows/reusable-workflow.yaml
-    with:
-      build_command: make system-images/compat/docker-images/aarch64.tar
-      artifact_name: compat.tar
-      artifact_path: system-images/compat/docker-images/aarch64.tar
-
-  utils:
-    uses: ./.github/workflows/reusable-workflow.yaml
-    with:
-      build_command: make system-images/utils/docker-images/aarch64.tar
-      artifact_name: utils.tar
-      artifact_path: system-images/utils/docker-images/aarch64.tar
-
-  binfmt:
-    uses: ./.github/workflows/reusable-workflow.yaml
-    with:
-      build_command: make system-images/binfmt/docker-images/aarch64.tar
-      artifact_name: binfmt.tar
-      artifact_path: system-images/binfmt/docker-images/aarch64.tar
-
-  backend:
-    uses: ./.github/workflows/backend.yaml
-
-  frontend:
-    uses: ./.github/workflows/frontend.yaml
-    
-  image:
-    name: Build image
-    runs-on: ubuntu-latest
-    timeout-minutes: 60
-    needs: [compat,utils,binfmt,backend,frontend]
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        submodules: recursive
-
-    - name: Download compat.tar artifact
-      uses: actions/download-artifact@v3
-      with:
-        name: compat.tar
-        path: system-images/compat/docker-images/
-
-    - name: Download utils.tar artifact
-      uses: actions/download-artifact@v3
-      with:
-        name: utils.tar
-        path: system-images/utils/docker-images/
-    
-    - name: Download binfmt.tar artifact
-      uses: actions/download-artifact@v3
-      with:
-        name: binfmt.tar
-        path: system-images/binfmt/docker-images/
-
-    - name: Download js_snapshot artifact
-      uses: actions/download-artifact@v3
-      with:
-        name: js_snapshot
-        path: libs/js_engine/src/artifacts/
-  
-    - name: Download arm_js_snapshot artifact
-      uses: actions/download-artifact@v3
-      with:
-        name: arm_js_snapshot
-        path: libs/js_engine/src/artifacts/
-
-    - name: Download backend artifact
-      uses: actions/download-artifact@v3
-      with:
-        name: backend-aarch64
-
-    - name: 'Extract backend'
-      run: 
-        tar -mxvf backend-aarch64.tar
-
-    - name: Download frontend artifact
-      uses: actions/download-artifact@v3
-      with:
-        name: frontend
-
-    - name: Skip frontend build
-      run: |
-        mkdir frontend/node_modules
-        mkdir frontend/dist
-        mkdir patch-db/client/node_modules
-        mkdir patch-db/client/dist
-
-    - name: 'Extract frontend'
-      run: |
-        tar -mxvf frontend.tar frontend/config.json
-        tar -mxvf frontend.tar frontend/dist
-        tar -xvf frontend.tar GIT_HASH.txt
-        tar -xvf frontend.tar ENVIRONMENT.txt
-        tar -xvf frontend.tar VERSION.txt
-        rm frontend.tar
-
-    - name: Cache raspiOS
-      id: cache-raspios
-      uses: actions/cache@v3
-      with:
-        path: raspios.img
-        key: cache-raspios
-    
-    - name: Build image
-      run: |
-        make V=1 eos_raspberrypi-uninit.img --debug
-      
-    - uses: actions/upload-artifact@v3
-      with:
-        name: image
-        path: eos_raspberrypi-uninit.img

.github/workflows/pureos-iso.yaml

@@ -1,70 +0,0 @@
-name: PureOS Based ISO
-
-on:
-  workflow_call:
-  workflow_dispatch:
-  push:
-    branches:
-    - master
-    - next
-  pull_request:
-    branches:
-    - master
-    - next
-
-env:
-  ENVIRONMENT: "dev"
-
-jobs:
-  dpkg:
-    uses: ./.github/workflows/debian.yaml
-
-  iso:
-    name: Build iso
-    runs-on: ubuntu-22.04
-    needs: [dpkg]
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        repository: Start9Labs/eos-image-recipes
-
-    - name: Install dependencies
-      run: |
-        sudo apt update
-        wget http://ftp.us.debian.org/debian/pool/main/d/debspawn/debspawn_0.6.1-1_all.deb
-        sha256sum ./debspawn_0.6.1-1_all.deb | grep fb8a3f588438ff9ef51e713ec1d83306db893f0aa97447565e28bbba9c6e90c6
-        sudo apt-get install -y ./debspawn_0.6.1-1_all.deb
-        wget https://repo.pureos.net/pureos/pool/main/d/debootstrap/debootstrap_1.0.125pureos1_all.deb
-        sudo apt-get install -y --allow-downgrades ./debootstrap_1.0.125pureos1_all.deb
-        wget https://repo.pureos.net/pureos/pool/main/p/pureos-archive-keyring/pureos-archive-keyring_2021.11.0_all.deb
-        sudo apt-get install -y ./pureos-archive-keyring_2021.11.0_all.deb
-
-    - name: Configure debspawn
-      run: |
-        sudo mkdir -p /etc/debspawn/
-        echo "AllowUnsafePermissions=true" | sudo tee /etc/debspawn/global.toml
-
-    - uses: actions/cache@v3
-      with:
-        path: /var/lib/debspawn
-        key: ${{ runner.os }}-debspawn-init-byzantium
-
-    - name: Make build container
-      run: "debspawn list | grep byzantium || debspawn create --with-init byzantium"
-
-    - run: "mkdir -p overlays/vendor/root"
-
-    - name: Download dpkg
-      uses: actions/download-artifact@v3
-      with:
-        name: deb
-        path: overlays/vendor/root
-
-    - name: Run build
-      run: |
-        ./run-local-build.sh --no-fakemachine byzantium none custom "" true
-
-    - uses: actions/upload-artifact@v3
-      with:
-        name: iso
-        path: results/*.iso

.github/workflows/reusable-workflow.yaml

@@ -1,37 +0,0 @@
-name: Reusable Workflow
-
-on:
-  workflow_call:
-    inputs:
-      build_command:
-        required: true
-        type: string
-      artifact_name:
-        required: true
-        type: string
-      artifact_path:
-        required: true
-        type: string
-
-env:
-  ENVIRONMENT: "dev"
-
-jobs:
-  generic_build_job:
-    runs-on: ubuntu-latest
-    timeout-minutes: 60
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        submodules: recursive
-
-    - name: Set up Docker Buildx        
-      uses: docker/setup-buildx-action@v2
-
-    - name: Build image
-      run: ${{ inputs.build_command }}
-
-    - uses: actions/upload-artifact@v3
-      with:
-        name: ${{ inputs.artifact_name }}
-        path: ${{ inputs.artifact_path }}

.github/workflows/startos-iso.yaml

@@ -0,0 +1,208 @@
+name: Debian-based ISO and SquashFS
+
+on:
+  workflow_call:
+  workflow_dispatch:
+    inputs:
+      environment:
+        type: choice
+        description: Environment
+        options:
+          - NONE
+          - dev
+          - unstable
+          - dev-unstable
+      runner:
+        type: choice
+        description: Runner
+        options:
+          - standard
+          - fast
+      platform:
+        type: choice
+        description: Platform
+        options:
+          - ALL
+          - x86_64
+          - x86_64-nonfree
+          - aarch64
+          - aarch64-nonfree
+          - raspberrypi
+  push:
+    branches:
+      - master
+      - next
+  pull_request:
+    branches:
+      - master
+      - next
+
+env:
+  NODEJS_VERSION: "18.15.0"
+  ENVIRONMENT: '${{ fromJson(format(''["{0}", ""]'', github.event.inputs.environment || ''dev''))[github.event.inputs.environment == ''NONE''] }}'
+
+jobs:
+  all:
+    name: Build
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: >-
+          ${{
+            fromJson(
+              format(
+                '[
+                  ["{0}"],
+                  ["x86_64", "x86_64-nonfree", "aarch64", "aarch64-nonfree", "raspberrypi"]
+                ]',
+                github.event.inputs.platform || 'ALL'
+              )
+            )[(github.event.inputs.platform || 'ALL') == 'ALL']
+          }}
+    runs-on: >-
+      ${{
+        fromJson(
+          format(
+            '["ubuntu-22.04", "{0}"]',
+            fromJson('{
+              "x86_64": ["buildjet-32vcpu-ubuntu-2204", "buildjet-32vcpu-ubuntu-2204"],
+              "x86_64-nonfree": ["buildjet-32vcpu-ubuntu-2204", "buildjet-32vcpu-ubuntu-2204"],
+              "aarch64": ["buildjet-16vcpu-ubuntu-2204-arm", "buildjet-32vcpu-ubuntu-2204-arm"],
+              "aarch64-nonfree": ["buildjet-16vcpu-ubuntu-2204-arm", "buildjet-32vcpu-ubuntu-2204-arm"],
+              "raspberrypi": ["buildjet-16vcpu-ubuntu-2204-arm", "buildjet-32vcpu-ubuntu-2204-arm"],
+            }')[matrix.platform][github.event.inputs.platform == matrix.platform]
+          )
+        )[github.event.inputs.runner == 'fast']
+      }}
+    steps:
+      - name: Free space
+        run: df -h && rm -rf /opt/hostedtoolcache* && df -h
+        if: ${{ github.event.inputs.runner != 'fast' }}
+
+      - run: |
+          sudo mount -t tmpfs tmpfs .
+        if: ${{ github.event.inputs.runner == 'fast' && (matrix.platform == 'x86_64' || matrix.platform == 'x86_64-nonfree' || github.event.inputs.platform == matrix.platform) }}
+
+      - uses: actions/checkout@v3
+        with:
+          repository: Start9Labs/embassy-os-deb
+          path: embassy-os-deb
+
+      - uses: actions/checkout@v3
+        with:
+          submodules: recursive
+          path: embassy-os-deb/embassyos-0.3.x
+
+      - run: |
+          cp -r debian embassyos-0.3.x/
+          VERSION=0.3.x ./control.sh
+          cp embassyos-0.3.x/backend/startd.service embassyos-0.3.x/debian/embassyos.startd.service
+        working-directory: embassy-os-deb
+
+      - uses: actions/setup-node@v3
+        with:
+          node-version: ${{ env.NODEJS_VERSION }}
+
+      - name: Get npm cache directory
+        id: npm-cache-dir
+        run: |
+          echo "dir=$(npm config get cache)" >> $GITHUB_OUTPUT
+      - uses: actions/cache@v3
+        id: npm-cache
+        with:
+          path: ${{ steps.npm-cache-dir.outputs.dir }}
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install \
+            debmake \
+            debhelper-compat \
+            crossbuild-essential-arm64
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Run dpkg build
+        working-directory: embassy-os-deb
+        run: "make VERSION=0.3.x TAG=${{ github.ref_name }}"
+        env:
+          OS_ARCH: ${{ matrix.platform }}
+
+      - uses: actions/checkout@v3
+        with:
+          repository: Start9Labs/startos-image-recipes
+          path: startos-image-recipes
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y qemu-user-static
+          wget https://deb.debian.org/debian/pool/main/d/debspawn/debspawn_0.6.2-1_all.deb
+          sha256sum ./debspawn_0.6.2-1_all.deb | grep 37ef27458cb1e35e8bce4d4f639b06b4b3866fc0b9191ec6b9bd157afd06a817
+          sudo apt-get install -y ./debspawn_0.6.2-1_all.deb
+
+      - name: Configure debspawn
+        run: |
+          sudo mkdir -p /etc/debspawn/
+          echo "AllowUnsafePermissions=true" | sudo tee /etc/debspawn/global.toml
+          sudo mkdir -p /var/tmp/debspawn
+
+      - run: sudo mount -t tmpfs tmpfs /var/tmp/debspawn
+        if: ${{ github.event.inputs.runner == 'fast' && (matrix.platform == 'x86_64' || matrix.platform == 'x86_64-nonfree') }}
+
+      - uses: actions/cache@v3
+        with:
+          path: /var/lib/debspawn
+          key: ${{ runner.os }}-${{ matrix.platform }}-debspawn-init
+
+      - run: "mkdir -p startos-image-recipes/overlays/deb"
+
+      - run: "mv embassy-os-deb/embassyos_0.3.x-1_*.deb startos-image-recipes/overlays/deb/"
+
+      - run: "rm -rf embassy-os-deb ${{ steps.npm-cache-dir.outputs.dir }} $HOME/.cargo"
+
+      - name: Run iso build
+        working-directory: startos-image-recipes
+        run: |
+          ./run-local-build.sh ${{ matrix.platform }}
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: ${{ matrix.platform }}.squashfs
+          path: startos-image-recipes/results/*.squashfs
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: ${{ matrix.platform }}.iso
+          path: startos-image-recipes/results/*.iso
+        if: ${{ matrix.platform != 'raspberrypi' }}
+
+      - uses: actions/checkout@v3
+        with:
+          submodules: recursive
+          path: start-os
+        if: ${{ matrix.platform == 'raspberrypi' }}
+
+      - run: "mv startos-image-recipes/results/startos-*_raspberrypi.squashfs start-os/startos.raspberrypi.squashfs"
+        if: ${{ matrix.platform == 'raspberrypi' }}
+
+      - run: rm -rf startos-image-recipes
+        if: ${{ matrix.platform == 'raspberrypi' }}
+
+      - name: Build image
+        working-directory: start-os
+        run: make startos_raspberrypi.img
+        if: ${{ matrix.platform == 'raspberrypi' }}
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: raspberrypi.img
+          path: start-os/startos-*_raspberrypi.img
+        if: ${{ matrix.platform == 'raspberrypi' }}

CONTRIBUTING.md

@@ -1,6 +1,6 @@
 <!-- omit in toc -->
 
-# Contributing to Embassy OS
+# Contributing to StartOS
 
 First off, thanks for taking the time to contribute! ❤️
 
@@ -19,7 +19,7 @@ forward to your contributions. 🎉
 > - Tweet about it
 > - Refer this project in your project's readme
 > - Mention the project at local meetups and tell your friends/colleagues
-> - Buy an [Embassy](https://start9labs.com)
+> - Buy a [Start9 server](https://start9.com)
 
 <!-- omit in toc -->
 
@@ -49,7 +49,7 @@ forward to your contributions. 🎉
 > [Documentation](https://docs.start9labs.com).
 
 Before you ask a question, it is best to search for existing
-[Issues](https://github.com/Start9Labs/embassy-os/issues) that might help you.
+[Issues](https://github.com/Start9Labs/start-os/issues) that might help you.
 In case you have found a suitable issue and still need clarification, you can
 write your question in this issue. It is also advisable to search the internet
 for answers first.
@@ -57,7 +57,7 @@ for answers first.
 If you then still feel the need to ask a question and need clarification, we
 recommend the following:
 
-- Open an [Issue](https://github.com/Start9Labs/embassy-os/issues/new).
+- Open an [Issue](https://github.com/Start9Labs/start-os/issues/new).
 - Provide as much context as you can about what you're running into.
 - Provide project and platform versions, depending on what seems relevant.
 
@@ -105,7 +105,7 @@ steps in advance to help us fix any potential bug as fast as possible.
 - To see if other users have experienced (and potentially already solved) the
   same issue you are having, check if there is not already a bug report existing
   for your bug or error in the
-  [bug tracker](https://github.com/Start9Labs/embassy-os/issues?q=label%3Abug).
+  [bug tracker](https://github.com/Start9Labs/start-os/issues?q=label%3Abug).
 - Also make sure to search the internet (including Stack Overflow) to see if
   users outside of the GitHub community have discussed the issue.
 - Collect information about the bug:
@@ -131,7 +131,7 @@ steps in advance to help us fix any potential bug as fast as possible.
 We use GitHub issues to track bugs and errors. If you run into an issue with the
 project:
 
-- Open an [Issue](https://github.com/Start9Labs/embassy-os/issues/new/choose)
+- Open an [Issue](https://github.com/Start9Labs/start-os/issues/new/choose)
   selecting the appropriate type.
 - Explain the behavior you would expect and the actual behavior.
 - Please provide as much context as possible and describe the _reproduction
@@ -155,8 +155,7 @@ Once it's filed:
 
 ### Suggesting Enhancements
 
-This section guides you through submitting an enhancement suggestion for Embassy
-OS, **including completely new features and minor improvements to existing
+This section guides you through submitting an enhancement suggestion for StartOS, **including completely new features and minor improvements to existing
 functionality**. Following these guidelines will help maintainers and the
 community to understand your suggestion and find related suggestions.
 
@@ -168,7 +167,7 @@ community to understand your suggestion and find related suggestions.
 - Read the [documentation](https://start9.com/latest/user-manual) carefully and
   find out if the functionality is already covered, maybe by an individual
   configuration.
-- Perform a [search](https://github.com/Start9Labs/embassy-os/issues) to see if
+- Perform a [search](https://github.com/Start9Labs/start-os/issues) to see if
   the enhancement has already been suggested. If it has, add a comment to the
   existing issue instead of opening a new one.
 - Find out whether your idea fits with the scope and aims of the project. It's
@@ -182,7 +181,7 @@ community to understand your suggestion and find related suggestions.
 #### How Do I Submit a Good Enhancement Suggestion?
 
 Enhancement suggestions are tracked as
-[GitHub issues](https://github.com/Start9Labs/embassy-os/issues).
+[GitHub issues](https://github.com/Start9Labs/start-os/issues).
 
 - Use a **clear and descriptive title** for the issue to identify the
   suggestion.
@@ -197,7 +196,7 @@ Enhancement suggestions are tracked as
   macOS and Windows, and [this tool](https://github.com/colinkeenan/silentcast)
   or [this tool](https://github.com/GNOME/byzanz) on Linux.
   <!-- this should only be included if the project has a GUI -->
-- **Explain why this enhancement would be useful** to most Embassy OS users. You
+- **Explain why this enhancement would be useful** to most StartOS users. You
   may also want to point out the other projects that solved it better and which
   could serve as inspiration.
 
@@ -205,24 +204,24 @@ Enhancement suggestions are tracked as
 
 ### Project Structure
 
-embassyOS is composed of the following components. Please visit the README for
+StartOS is composed of the following components. Please visit the README for
 each component to understand the dependency requirements and installation
 instructions.
 
 - [`backend`](backend/README.md) (Rust) is a command line utility, daemon, and
   software development kit that sets up and manages services and their
   environments, provides the interface for the ui, manages system state, and
-  provides utilities for packaging services for embassyOS.
+  provides utilities for packaging services for StartOS.
 - [`build`](build/README.md) contains scripts and necessary for deploying
-  embassyOS to a debian/raspbian system.
+  StartOS to a debian/raspbian system.
 - [`frontend`](frontend/README.md) (Typescript Ionic Angular) is the code that
-  is deployed to the browser to provide the user interface for embassyOS.
-  - `projects/ui` - Code for the user interface that is displayed when embassyOS
+  is deployed to the browser to provide the user interface for StartOS.
+  - `projects/ui` - Code for the user interface that is displayed when StartOS
     is running normally.
   - `projects/setup-wizard`(frontend/README.md) - Code for the user interface
-    that is displayed during the setup and recovery process for embassyOS.
+    that is displayed during the setup and recovery process for StartOS.
   - `projects/diagnostic-ui` - Code for the user interface that is displayed
-    when something has gone wrong with starting up embassyOS, which provides
+    when something has gone wrong with starting up StartOS, which provides
     helpful debugging tools.
 - `libs` (Rust) is a set of standalone crates that were separated out of
   `backend` for the purpose of portability
@@ -232,18 +231,18 @@ instructions.
     [client](https://github.com/Start9Labs/patch-db/tree/master/client) with its
     own dependency and installation requirements.
 - `system-images` - (Docker, Rust) A suite of utility Docker images that are
-  preloaded with embassyOS to assist with functions relating to services (eg.
+  preloaded with StartOS to assist with functions relating to services (eg.
   configuration, backups, health checks).
 
 ### Your First Code Contribution
 
 #### Setting Up Your Development Environment
 
-First, clone the embassyOS repository and from the project root, pull in the
+First, clone the StartOS repository and from the project root, pull in the
 submodules for dependent libraries.
 
 ```sh
-git clone https://github.com/Start9Labs/embassy-os.git
+git clone https://github.com/Start9Labs/start-os.git
 git submodule update --init --recursive
 ```
 
@@ -254,7 +253,7 @@ to, follow the installation requirements listed in that component's README
 #### Building The Raspberry Pi Image
 
 This step is for setting up an environment in which to test your code changes if
-you do not yet have a embassyOS.
+you do not yet have a StartOS.
 
 - Requirements
   - `ext4fs` (available if running on the Linux kernel)
@@ -262,7 +261,7 @@ you do not yet have a embassyOS.
   - GNU Make
 - Building
   - see setup instructions [here](build/README.md)
-  - run `make embassyos-raspi.img ARCH=aarch64` from the project root
+  - run `make startos-raspi.img ARCH=aarch64` from the project root
 
 ### Improving The Documentation
 
@@ -286,7 +285,7 @@ seamless and intuitive experience.
 
 ### Formatting
 
-Each component of embassyOS contains its own style guide. Code must be formatted
+Each component of StartOS contains its own style guide. Code must be formatted
 with the formatter designated for each component. These are outlined within each
 component folder's README.
 
@@ -306,7 +305,7 @@ component. i.e. `backend: update to tokio v0.3`.
 
 The body of a pull request should contain sufficient description of what the
 changes do, as well as a justification. You should include references to any
-relevant [issues](https://github.com/Start9Labs/embassy-os/issues).
+relevant [issues](https://github.com/Start9Labs/start-os/issues).
 
 ### Rebasing Changes
 

Makefile

@@ -1,33 +1,34 @@
-RASPI_TARGETS := eos_raspberrypi-uninit.img eos_raspberrypi-uninit.tar.gz
-OS_ARCH := $(shell if echo $(RASPI_TARGETS) | grep -qw "$(MAKECMDGOALS)"; then echo raspberrypi; else uname -m; fi)
-ARCH := $(shell if [ "$(OS_ARCH)" = "raspberrypi" ]; then echo aarch64; else echo $(OS_ARCH); fi)
+OS_ARCH := $(shell echo "${OS_ARCH}")
+ARCH := $(shell if [ "$(OS_ARCH)" = "raspberrypi" ]; then echo aarch64; else echo $(OS_ARCH) | sed 's/-nonfree$$//g'; fi)
 ENVIRONMENT_FILE = $(shell ./check-environment.sh)
 GIT_HASH_FILE = $(shell ./check-git-hash.sh)
 VERSION_FILE = $(shell ./check-version.sh)
-EMBASSY_BINS := backend/target/$(ARCH)-unknown-linux-gnu/release/embassyd backend/target/$(ARCH)-unknown-linux-gnu/release/embassy-init backend/target/$(ARCH)-unknown-linux-gnu/release/embassy-cli backend/target/$(ARCH)-unknown-linux-gnu/release/embassy-sdk backend/target/$(ARCH)-unknown-linux-gnu/release/avahi-alias libs/target/aarch64-unknown-linux-musl/release/embassy_container_init libs/target/x86_64-unknown-linux-musl/release/embassy_container_init
-EMBASSY_UIS := frontend/dist/ui frontend/dist/setup-wizard frontend/dist/diagnostic-ui frontend/dist/install-wizard
+EMBASSY_BINS := backend/target/$(ARCH)-unknown-linux-gnu/release/startbox libs/target/aarch64-unknown-linux-musl/release/embassy_container_init libs/target/x86_64-unknown-linux-musl/release/embassy_container_init
+EMBASSY_UIS := frontend/dist/raw/ui frontend/dist/raw/setup-wizard frontend/dist/raw/diagnostic-ui frontend/dist/raw/install-wizard
 BUILD_SRC := $(shell find build)
-EMBASSY_SRC := backend/embassyd.service backend/embassy-init.service $(EMBASSY_UIS) $(BUILD_SRC)
+EMBASSY_SRC := backend/startd.service $(BUILD_SRC)
 COMPAT_SRC := $(shell find system-images/compat/ -not -path 'system-images/compat/target/*' -and -not -name *.tar -and -not -name target)
 UTILS_SRC := $(shell find system-images/utils/ -not -name *.tar)
 BINFMT_SRC := $(shell find system-images/binfmt/ -not -name *.tar)
-BACKEND_SRC := $(shell find backend/src) $(shell find backend/migrations) $(shell find patch-db/*/src) $(shell find libs/*/src) libs/*/Cargo.toml backend/Cargo.toml backend/Cargo.lock
+BACKEND_SRC := $(shell find backend/src) $(shell find backend/migrations) $(shell find patch-db/*/src) $(shell find libs/*/src) libs/*/Cargo.toml backend/Cargo.toml backend/Cargo.lock frontend/dist/static
 FRONTEND_SHARED_SRC := $(shell find frontend/projects/shared) $(shell ls -p frontend/ | grep -v / | sed 's/^/frontend\//g') frontend/package.json frontend/node_modules frontend/config.json patch-db/client/dist frontend/patchdb-ui-seed.json
 FRONTEND_UI_SRC := $(shell find frontend/projects/ui)
 FRONTEND_SETUP_WIZARD_SRC := $(shell find frontend/projects/setup-wizard)
 FRONTEND_DIAGNOSTIC_UI_SRC := $(shell find frontend/projects/diagnostic-ui)
 FRONTEND_INSTALL_WIZARD_SRC := $(shell find frontend/projects/install-wizard)
-PATCH_DB_CLIENT_SRC := $(shell find patch-db/client -not -path patch-db/client/dist)
+PATCH_DB_CLIENT_SRC := $(shell find patch-db/client -not -path patch-db/client/dist -and -not -path patch-db/client/node_modules)
 GZIP_BIN := $(shell which pigz || which gzip)
-ALL_TARGETS := $(EMBASSY_BINS) system-images/compat/docker-images/$(ARCH).tar system-images/utils/docker-images/$(ARCH).tar system-images/binfmt/docker-images/$(ARCH).tar $(EMBASSY_SRC) $(ENVIRONMENT_FILE) $(GIT_HASH_FILE) $(VERSION_FILE)
+ALL_TARGETS := $(EMBASSY_BINS) system-images/compat/docker-images/$(ARCH).tar system-images/utils/docker-images/$(ARCH).tar system-images/binfmt/docker-images/$(ARCH).tar $(EMBASSY_SRC) $(shell if [ "$(OS_ARCH)" = "raspberrypi" ]; then echo cargo-deps/aarch64-unknown-linux-gnu/release/pi-beep; fi) $(ENVIRONMENT_FILE) $(GIT_HASH_FILE) $(VERSION_FILE)
 
 ifeq ($(REMOTE),)
 	mkdir = mkdir -p $1
 	rm = rm -rf $1
 	cp = cp -r $1 $2
+	ln = ln -sf $1 $2
 else
 	mkdir = ssh $(REMOTE) 'mkdir -p $1'
 	rm  = ssh $(REMOTE) 'sudo rm -rf $1'
+	ln = ssh $(REMOTE) 'sudo ln -sf $1 $2'
 define cp
 	tar --transform "s|^$1|x|" -czv -f- $1 | ssh $(REMOTE) "sudo tar --transform 's|^x|$2|' -xzv -f- -C /"
 endef
@@ -35,7 +36,7 @@ endif
 
 .DELETE_ON_ERROR:
 
-.PHONY: all gzip install clean format sdk snapshots frontends ui backend reflash eos_raspberrypi.img sudo
+.PHONY: all gzip install clean format sdk snapshots frontends ui backend reflash startos_raspberrypi.img sudo
 
 all: $(ALL_TARGETS)
 
@@ -43,12 +44,6 @@ sudo:
 	sudo true
 
 clean:
-	rm -f 2022-01-28-raspios-bullseye-arm64-lite.zip
-	rm -f raspios.img
-	rm -f eos_raspberrypi-uninit.img
-	rm -f eos_raspberrypi-uninit.tar.gz
-	rm -f ubuntu.img
-	rm -f product_key.txt
 	rm -f system-images/**/*.tar
 	rm -rf system-images/compat/target
 	rm -rf backend/target
@@ -72,25 +67,19 @@ format:
 sdk:
 	cd backend/ && ./install-sdk.sh
 
-eos_raspberrypi-uninit.img: $(ALL_TARGETS) raspios.img cargo-deps/aarch64-unknown-linux-gnu/release/nc-broadcast cargo-deps/aarch64-unknown-linux-gnu/release/pi-beep | sudo
-	! test -f eos_raspberrypi-uninit.img || rm eos_raspberrypi-uninit.img
-	./build/raspberry-pi/make-image.sh
-
-lite-upgrade.img: raspios.img cargo-deps/aarch64-unknown-linux-gnu/release/nc-broadcast cargo-deps/aarch64-unknown-linux-gnu/release/pi-beep $(BUILD_SRC) eos.raspberrypi.squashfs
-	! test -f lite-upgrade.img || rm lite-upgrade.img
-	./build/raspberry-pi/make-upgrade-image.sh
-
-eos_raspberrypi.img: raspios.img $(BUILD_SRC) eos.raspberrypi.squashfs $(VERSION_FILE) $(ENVIRONMENT_FILE) $(GIT_HASH_FILE) | sudo
-	! test -f eos_raspberrypi.img || rm eos_raspberrypi.img
-	./build/raspberry-pi/make-initialized-image.sh
+startos_raspberrypi.img: $(BUILD_SRC) startos.raspberrypi.squashfs $(VERSION_FILE) $(ENVIRONMENT_FILE) $(GIT_HASH_FILE) cargo-deps/aarch64-unknown-linux-gnu/release/pi-beep | sudo
+	./build/raspberrypi/make-image.sh
 
 # For creating os images. DO NOT USE
 install: $(ALL_TARGETS)
 	$(call mkdir,$(DESTDIR)/usr/bin)
-	$(call cp,backend/target/$(ARCH)-unknown-linux-gnu/release/embassy-init,$(DESTDIR)/usr/bin/embassy-init)
-	$(call cp,backend/target/$(ARCH)-unknown-linux-gnu/release/embassyd,$(DESTDIR)/usr/bin/embassyd)
-	$(call cp,backend/target/$(ARCH)-unknown-linux-gnu/release/embassy-cli,$(DESTDIR)/usr/bin/embassy-cli)
-	$(call cp,backend/target/$(ARCH)-unknown-linux-gnu/release/avahi-alias,$(DESTDIR)/usr/bin/avahi-alias)
+	$(call cp,backend/target/$(ARCH)-unknown-linux-gnu/release/startbox,$(DESTDIR)/usr/bin/startbox)
+	$(call ln,/usr/bin/startbox,$(DESTDIR)/usr/bin/startd)
+	$(call ln,/usr/bin/startbox,$(DESTDIR)/usr/bin/start-cli)
+	$(call ln,/usr/bin/startbox,$(DESTDIR)/usr/bin/start-sdk)
+	$(call ln,/usr/bin/startbox,$(DESTDIR)/usr/bin/avahi-alias)
+	$(call ln,/usr/bin/startbox,$(DESTDIR)/usr/bin/embassy-cli)
+	if [ "$(OS_ARCH)" = "raspberrypi" ]; then $(call cp,cargo-deps/aarch64-unknown-linux-gnu/release/pi-beep,$(DESTDIR)/usr/bin/pi-beep); fi
 	
 	$(call mkdir,$(DESTDIR)/usr/lib)
 	$(call rm,$(DESTDIR)/usr/lib/embassy)
@@ -109,22 +98,14 @@ install: $(ALL_TARGETS)
 	$(call cp,system-images/utils/docker-images/$(ARCH).tar,$(DESTDIR)/usr/lib/embassy/system-images/utils.tar)
 	$(call cp,system-images/binfmt/docker-images/$(ARCH).tar,$(DESTDIR)/usr/lib/embassy/system-images/binfmt.tar)
 
-	$(call mkdir,$(DESTDIR)/var/www/html)
-	$(call cp,frontend/dist/diagnostic-ui,$(DESTDIR)/var/www/html/diagnostic)
-	$(call cp,frontend/dist/setup-wizard,$(DESTDIR)/var/www/html/setup)
-	$(call cp,frontend/dist/install-wizard,$(DESTDIR)/var/www/html/install)
-	$(call cp,frontend/dist/ui,$(DESTDIR)/var/www/html/main)
-	$(call cp,index.html,$(DESTDIR)/var/www/html/index.html)
-
 update-overlay:
 	@echo "\033[33m!!! THIS WILL ONLY REFLASH YOUR DEVICE IN MEMORY !!!\033[0m"
 	@echo "\033[33mALL CHANGES WILL BE REVERTED IF YOU RESTART THE DEVICE\033[0m"
 	@if [ -z "$(REMOTE)" ]; then >&2 echo "Must specify REMOTE" && false; fi
-	@if [ "`ssh $(REMOTE) 'cat /usr/lib/embassy/VERSION.txt'`" != "`cat ./VERSION.txt`" ]; then >&2 echo "Embassy requires migrations: update-overlay is unavailable." && false; fi
-	@if ssh $(REMOTE) "pidof embassy-init"; then >&2 echo "Embassy in INIT: update-overlay is unavailable." && false; fi
-	ssh $(REMOTE) "sudo systemctl stop embassyd"
+	@if [ "`ssh $(REMOTE) 'cat /usr/lib/embassy/VERSION.txt'`" != "`cat ./VERSION.txt`" ]; then >&2 echo "StartOS requires migrations: update-overlay is unavailable." && false; fi
+	ssh $(REMOTE) "sudo systemctl stop startd"
 	$(MAKE) install REMOTE=$(REMOTE) OS_ARCH=$(OS_ARCH)
-	ssh $(REMOTE) "sudo systemctl start embassyd"
+	ssh $(REMOTE) "sudo systemctl start startd"
 
 update:
 	@if [ -z "$(REMOTE)" ]; then >&2 echo "Must specify REMOTE" && false; fi
@@ -147,11 +128,6 @@ system-images/utils/docker-images/aarch64.tar system-images/utils/docker-images/
 system-images/binfmt/docker-images/aarch64.tar system-images/binfmt/docker-images/x86_64.tar: $(BINFMT_SRC)
 	cd system-images/binfmt && make
 
-raspios.img:
-	wget --continue https://downloads.raspberrypi.org/raspios_lite_arm64/images/raspios_lite_arm64-2022-01-28/2022-01-28-raspios-bullseye-arm64-lite.zip
-	unzip 2022-01-28-raspios-bullseye-arm64-lite.zip
-	mv 2022-01-28-raspios-bullseye-arm64-lite.img raspios.img
-
 snapshots: libs/snapshot_creator/Cargo.toml
 	cd libs/  && ./build-v8-snapshot.sh
 	cd libs/  && ./build-arm-v8-snapshot.sh
@@ -163,18 +139,21 @@ $(EMBASSY_BINS): $(BACKEND_SRC) $(ENVIRONMENT_FILE) $(GIT_HASH_FILE) frontend/pa
 frontend/node_modules: frontend/package.json
 	npm --prefix frontend ci
 
-frontend/dist/ui: $(FRONTEND_UI_SRC) $(FRONTEND_SHARED_SRC) $(ENVIRONMENT_FILE)
+frontend/dist/raw/ui: $(FRONTEND_UI_SRC) $(FRONTEND_SHARED_SRC) $(ENVIRONMENT_FILE)
 	npm --prefix frontend run build:ui
 
-frontend/dist/setup-wizard: $(FRONTEND_SETUP_WIZARD_SRC) $(FRONTEND_SHARED_SRC) $(ENVIRONMENT_FILE)
+frontend/dist/raw/setup-wizard: $(FRONTEND_SETUP_WIZARD_SRC) $(FRONTEND_SHARED_SRC) $(ENVIRONMENT_FILE)
 	npm --prefix frontend run build:setup
 
-frontend/dist/diagnostic-ui: $(FRONTEND_DIAGNOSTIC_UI_SRC) $(FRONTEND_SHARED_SRC) $(ENVIRONMENT_FILE)
+frontend/dist/raw/diagnostic-ui: $(FRONTEND_DIAGNOSTIC_UI_SRC) $(FRONTEND_SHARED_SRC) $(ENVIRONMENT_FILE)
 	npm --prefix frontend run build:dui
 
-frontend/dist/install-wizard: $(FRONTEND_INSTALL_WIZARD_SRC) $(FRONTEND_SHARED_SRC) $(ENVIRONMENT_FILE)
+frontend/dist/raw/install-wizard: $(FRONTEND_INSTALL_WIZARD_SRC) $(FRONTEND_SHARED_SRC) $(ENVIRONMENT_FILE)
 	npm --prefix frontend run build:install-wiz
 
+frontend/dist/static: $(EMBASSY_UIS)
+	./compress-uis.sh
+
 frontend/config.json: $(GIT_HASH_FILE) frontend/config-sample.json
 	jq '.useMocks = false' frontend/config-sample.json > frontend/config.json
 	jq '.packageArch = "$(ARCH)"' frontend/config.json > frontend/config.json.tmp
@@ -183,7 +162,7 @@ frontend/config.json: $(GIT_HASH_FILE) frontend/config-sample.json
 	npm --prefix frontend run-script build-config
 
 frontend/patchdb-ui-seed.json: frontend/package.json
-	jq '."ack-welcome" = "$(shell yq '.version' frontend/package.json)"' frontend/patchdb-ui-seed.json > ui-seed.tmp
+	jq '."ack-welcome" = $(shell yq '.version' frontend/package.json)' frontend/patchdb-ui-seed.json > ui-seed.tmp
 	mv ui-seed.tmp frontend/patchdb-ui-seed.json
 
 patch-db/client/node_modules: patch-db/client/package.json
@@ -201,13 +180,10 @@ backend-$(ARCH).tar: $(EMBASSY_BINS)
 frontends: $(EMBASSY_UIS) 
 
 # this is a convenience step to build the UI
-ui: frontend/dist/ui
+ui: frontend/dist/raw/ui
 
 # used by github actions
 backend: $(EMBASSY_BINS)
 
-cargo-deps/aarch64-unknown-linux-gnu/release/nc-broadcast:
-	./build-cargo-dep.sh nc-broadcast
-
 cargo-deps/aarch64-unknown-linux-gnu/release/pi-beep:
-	./build-cargo-dep.sh pi-beep
+	ARCH=aarch64 ./build-cargo-dep.sh pi-beep

README.md

@@ -1,51 +1,81 @@
-# embassyOS
-[![version](https://img.shields.io/github/v/tag/Start9Labs/embassy-os?color=success)](https://github.com/Start9Labs/embassy-os/releases)
-[![build](https://github.com/Start9Labs/embassy-os/actions/workflows/product.yaml/badge.svg)](https://github.com/Start9Labs/embassy-os/actions/workflows/product.yaml)
-[![community](https://img.shields.io/badge/community-matrix-yellow)](https://matrix.to/#/#community:matrix.start9labs.com)
-[![community](https://img.shields.io/badge/community-telegram-informational)](https://t.me/start9_labs)
-[![support](https://img.shields.io/badge/support-docs-important)](https://docs.start9.com)
-[![developer](https://img.shields.io/badge/developer-matrix-blueviolet)](https://matrix.to/#/#community-dev:matrix.start9labs.com)
-[![website](https://img.shields.io/website?down_color=lightgrey&down_message=offline&up_color=green&up_message=online&url=https%3A%2F%2Fstart9.com)](https://start9.com)
+<div align="center">
+  <img src="frontend/projects/shared/assets/img/icon_pwa.png" alt="StartOS Logo" width="16%" />
+  <h1 style="margin-top: 0;">StartOS</h1>
+  <a href="https://github.com/Start9Labs/start-os/releases">
+    <img src="https://img.shields.io/github/v/tag/Start9Labs/start-os?color=success" />
+  </a>
+  <a href="https://github.com/Start9Labs/start-os/actions/workflows/startos-iso.yaml">
+    <img src="https://github.com/Start9Labs/start-os/actions/workflows/startos-iso.yaml/badge.svg">
+  </a>
+  <a href="https://twitter.com/start9labs">
+    <img src="https://img.shields.io/twitter/follow/start9labs?label=Follow">
+  </a>
+  <a href="http://mastodon.start9labs.com">
+    <img src="https://img.shields.io/mastodon/follow/000000001?domain=https%3A%2F%2Fmastodon.start9labs.com&label=Follow&style=social">
+  </a>
+  <a href="https://matrix.to/#/#community:matrix.start9labs.com">
+    <img src="https://img.shields.io/badge/community-matrix-yellow">
+  </a>
+  <a href="https://t.me/start9_labs">
+    <img src="https://img.shields.io/badge/community-telegram-informational">
+  </a>
+  <a href="https://docs.start9.com">
+    <img src="https://img.shields.io/badge/support-docs-important">
+  </a>
+  <a href="https://matrix.to/#/#community-dev:matrix.start9labs.com">
+    <img src="https://img.shields.io/badge/developer-matrix-blueviolet">
+  </a>
+  <a href="https://start9.com">
+    <img src="https://img.shields.io/website?down_color=lightgrey&down_message=offline&up_color=green&up_message=online&url=https%3A%2F%2Fstart9.com">
+  </a>
+</div>
+<br />
+<div align="center">
+  <h3>
+    Welcome to the era of Sovereign Computing
+  </h3>
+  <p>
+    StartOS is a Debian-based Linux distro optimized for running a personal server. It facilitates the discovery,       installation, network configuration, service configuration, data backup, dependency management, and health monitoring of self-hosted software services.
+  </p>
+</div>
+<br />
+<p align="center">
+<img src="assets/StartOS.png" alt="StartOS" width="85%">
+</p>
+<br />
 
-[![mastodon](https://img.shields.io/mastodon/follow/000000001?domain=https%3A%2F%2Fmastodon.start9labs.com&label=Follow&style=social)](http://mastodon.start9labs.com)
-[![twitter](https://img.shields.io/twitter/follow/start9labs?label=Follow)](https://twitter.com/start9labs)
+## Running StartOS
+There are multiple ways to get started with StartOS:
 
-### _Welcome to the era of Sovereign Computing_ ###
+### 💰 Buy a Start9 server
+This is the most convenient option. Simply [buy a server](https://store.start9.com) from Start9 and plug it in.
 
-embassyOS is a browser-based, graphical operating system for a personal server. embassyOS facilitates the discovery, installation, network configuration, service configuration, data backup, dependency management, and health monitoring of self-hosted software services. It is the most advanced, secure, reliable, and user friendly personal server OS in the world.
-
-## Running embassyOS
-There are multiple ways to get your hands on embassyOS.
-
-### :moneybag: Buy an Embassy
-This is the most convenient option. Simply [buy an Embassy](https://start9.com) from Start9 and plug it in. Depending on where you live, shipping costs and import duties will vary.
-
-### :construction_worker: Build your own Embassy
-While not as convenient as buying an Embassy, this option is easier than you might imagine, and there are 4 reasons why you might prefer it:
-1. You already have your own hardware.
-1. You want to save on shipping costs.
-1. You prefer not to divulge your physical address.
-1. You just like building things.
+### 👷 Build your own server
+This option is easier than you might imagine, and there are 4 reasons why you might prefer it:
+1. You already have hardware
+1. You want to save on shipping costs
+1. You prefer not to divulge your physical address
+1. You just like building things
 
 To pursue this option, follow one of our [DIY guides](https://start9.com/latest/diy).
 
-### :hammer_and_wrench: Build embassyOS from Source
+## ❤️ Contributing
+There are multiple ways to contribute: work directly on StartOS, package a service for the marketplace, or help with documentation and guides. To learn more about contributing, see [here](https://start9.com/contribute/).
 
-embassyOS can be built from source, for personal use, for free.
-A detailed guide for doing so can be found [here](https://github.com/Start9Labs/embassy-os/blob/master/build/README.md).
+To report security issues, please email our security team - security@start9.com.
 
-## :heart: Contributing
-There are multiple ways to contribute: work directly on embassyOS, package a service for the marketplace, or help with documentation and guides. To learn more about contributing, see [here](https://docs.start9.com/latest/contribute/) or [here](https://github.com/Start9Labs/embassy-os/blob/master/CONTRIBUTING.md).
+## 🌎 Marketplace
+There are dozens of service available for StartOS, and new ones are being added all the time. Check out the full list of available services [here](https://marketplace.start9.com/marketplace). To read more about the Marketplace ecosystem, check out this [blog post](https://blog.start9.com/start9-marketplace-strategy/)
+
+## 🖥️ User Interface Screenshots
 
-## UI Screenshots
 <p align="center">
-<img src="assets/embassyOS.png" alt="embassyOS" width="85%">
-</p>
-<p align="center">
-<img src="assets/eOS-preferences.png" alt="Embassy Preferences" width="49%">
-<img src="assets/eOS-ghost.png" alt="Embassy Ghost Service" width="49%">
-</p>
-<p align="center">
-<img src="assets/eOS-synapse-health-check.png" alt="Embassy Synapse Health Checks" width="49%">
-<img src="assets/eOS-sideload.png" alt="Embassy Sideload Service" width="49%">
+<img src="assets/registry.png" alt="StartOS Marketplace" width="49%">
+<img src="assets/community.png" alt="StartOS Community Registry" width="49%">
+<img src="assets/c-lightning.png" alt="StartOS NextCloud Service" width="49%">
+<img src="assets/btcpay.png" alt="StartOS BTCPay Service" width="49%">
+<img src="assets/nextcloud.png" alt="StartOS System Settings" width="49%">
+<img src="assets/system.png" alt="StartOS System Settings" width="49%">
+<img src="assets/welcome.png" alt="StartOS System Settings" width="49%">
+<img src="assets/logs.png" alt="StartOS System Settings" width="49%">
 </p>

backend/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 authors = ["Aiden McClelland <me@drbonez.dev>"]
-description = "The core of the Start9 Embassy Operating System"
-documentation = "https://docs.rs/embassy-os"
+description = "The core of StartOS"
+documentation = "https://docs.rs/start-os"
 edition = "2021"
 keywords = [
   "self-hosted",
@@ -11,40 +11,28 @@ keywords = [
   "full-node",
   "lightning",
 ]
-name = "embassy-os"
+name = "start-os"
 readme = "README.md"
-repository = "https://github.com/Start9Labs/embassy-os"
-version = "0.3.4"
+repository = "https://github.com/Start9Labs/start-os"
+version = "0.3.4-rev.4"
 
 [lib]
-name = "embassy"
+name = "startos"
 path = "src/lib.rs"
 
 [[bin]]
-name = "embassyd"
-path = "src/bin/embassyd.rs"
-
-[[bin]]
-name = "embassy-init"
-path = "src/bin/embassy-init.rs"
-
-[[bin]]
-name = "embassy-sdk"
-path = "src/bin/embassy-sdk.rs"
-
-[[bin]]
-name = "embassy-cli"
-path = "src/bin/embassy-cli.rs"
-
-[[bin]]
-name = "avahi-alias"
-path = "src/bin/avahi-alias.rs"
+name = "startbox"
+path = "src/main.rs"
 
 [features]
 avahi = ["avahi-sys"]
-default = ["avahi", "js_engine"]
+default = ["avahi-alias", "cli", "sdk", "daemon", "js_engine"]
 dev = []
 unstable = ["patch-db/unstable"]
+avahi-alias = ["avahi"]
+cli = []
+sdk = []
+daemon = []
 
 [dependencies]
 aes = { version = "0.7.5", features = ["ctr"] }
@@ -90,11 +78,14 @@ http = "0.2.8"
 hyper = { version = "0.14.20", features = ["full"] }
 hyper-ws-listener = "0.2.0"
 imbl = "2.0.0"
+include_dir = "0.7.3"
 indexmap = { version = "1.9.1", features = ["serde"] }
 ipnet = { version = "2.7.1", features = ["serde"] }
 iprange = { version = "0.6.7", features = ["serde"] }
 isocountry = "0.3.2"
 itertools = "0.10.3"
+jaq-core = "0.10.0"
+jaq-std = "0.10.0"
 josekit = "0.8.1"
 js_engine = { path = '../libs/js_engine', optional = true }
 jsonpath_lib = "0.3.0"
@@ -103,6 +94,7 @@ libc = "0.2.126"
 log = "0.4.17"
 mbrman = "0.5.0"
 models = { version = "*", path = "../libs/models" }
+new_mime_guess = "4"
 nix = "0.25.0"
 nom = "7.1.1"
 num = "0.4.0"
@@ -152,6 +144,7 @@ tokio-stream = { version = "0.1.11", features = ["io-util", "sync", "net"] }
 tokio-tar = { git = "https://github.com/dr-bonez/tokio-tar.git" }
 tokio-tungstenite = { version = "0.17.1", features = ["native-tls"] }
 tokio-rustls = "0.23.4"
+tokio-socks = "0.5.1"
 tokio-util = { version = "0.7.3", features = ["io"] }
 torut = "0.2.1"
 tracing = "0.1.35"
@@ -161,6 +154,7 @@ tracing-subscriber = { version = "0.3.14", features = ["env-filter"] }
 trust-dns-server = "0.22.0"
 typed-builder = "0.10.0"
 url = { version = "2.2.2", features = ["serde"] }
+urlencoding = "2.1.2"
 uuid = { version = "1.1.2", features = ["v4"] }
 zeroize = "1.5.7"
 

backend/README.md

@@ -1,36 +1,35 @@
-# embassyOS Backend
+# StartOS Backend
 
 - Requirements:
   - [Install Rust](https://rustup.rs)
   - Recommended: [rust-analyzer](https://rust-analyzer.github.io/)
   - [Docker](https://docs.docker.com/get-docker/)
   - [Rust ARM64 Build Container](https://github.com/Start9Labs/rust-arm-builder)
-- Scripts (run withing the `./backend` directory)
+- Scripts (run within the `./backend` directory)
   - `build-prod.sh` - compiles a release build of the artifacts for running on
     ARM64
 - A Linux computer or VM
 
 ## Structure
 
-The embassyOS backend is broken up into 4 different binaries:
+The StartOS backend is packed into a single binary `startbox` that is symlinked under 
+several different names for different behaviour:
 
-- embassyd: This is the main workhorse of embassyOS - any new functionality you
+- startd: This is the main workhorse of StartOS - any new functionality you
   want will likely go here
-- embassy-init: This is the component responsible for allowing you to set up
-  your device, and handles system initialization on startup
-- embassy-cli: This is a CLI tool that will allow you to issue commands to
-  embassyd and control it similarly to the UI
-- embassy-sdk: This is a CLI tool that aids in building and packaging services
-  you wish to deploy to the Embassy
+- start-cli: This is a CLI tool that will allow you to issue commands to
+  startd and control it similarly to the UI
+- start-sdk: This is a CLI tool that aids in building and packaging services
+  you wish to deploy to StartOS
 
-Finally there is a library `embassy` that supports all four of these tools.
+Finally there is a library `startos` that supports all of these tools.
 
 See [here](/backend/Cargo.toml) for details.
 
 ## Building
 
 You can build the entire operating system image using `make` from the root of
-the embassyOS project. This will subsequently invoke the build scripts above to
+the StartOS project. This will subsequently invoke the build scripts above to
 actually create the requisite binaries and put them onto the final operating
 system image.
 

backend/build-dev.sh

@@ -1,24 +0,0 @@
-#!/bin/bash
-
-set -e
-shopt -s expand_aliases
-
-if [ "$0" != "./build-dev.sh" ]; then
-	>&2 echo "Must be run from backend directory"
-	exit 1
-fi
-
-USE_TTY=
-if tty -s; then
-	USE_TTY="-it"
-fi
-
-alias 'rust-arm64-builder'='docker run $USE_TTY --rm -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$(pwd)":/home/rust/src start9/rust-arm-cross:aarch64'
-
-cd ..
-rust-arm64-builder sh -c "(cd backend && cargo build --locked)"
-cd backend
-
-sudo chown -R $USER target
-sudo chown -R $USER ~/.cargo
-#rust-arm64-builder aarch64-linux-gnu-strip target/aarch64-unknown-linux-gnu/release/embassyd

backend/build-portable-dev.sh

@@ -1,23 +0,0 @@
-#!/bin/bash
-
-set -e
-shopt -s expand_aliases
-
-if [ "$0" != "./build-portable-dev.sh" ]; then
-	>&2 echo "Must be run from backend directory"
-	exit 1
-fi
-
-USE_TTY=
-if tty -s; then
-	USE_TTY="-it"
-fi
-
-alias 'rust-musl-builder'='docker run $USE_TTY --rm -v "$HOME"/.cargo/registry:/root/.cargo/registry -v "$(pwd)":/home/rust/src start9/rust-musl-cross:x86_64-musl'
-
-cd ..
-rust-musl-builder sh -c "(cd backend && cargo +beta build --target=x86_64-unknown-linux-musl --no-default-features --locked)"
-cd backend
-
-sudo chown -R $USER target
-sudo chown -R $USER ~/.cargo

backend/build-prod.sh

@@ -3,6 +3,11 @@
 set -e
 shopt -s expand_aliases
 
+if [ -z "$OS_ARCH" ]; then
+	>&2 echo '$OS_ARCH is required'
+	exit 1
+fi
+
 if [ -z "$ARCH" ]; then
 	ARCH=$(uname -m)
 fi
@@ -17,8 +22,8 @@ if tty -s; then
 	USE_TTY="-it"
 fi
 
-alias 'rust-gnu-builder'='docker run $USE_TTY --rm -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$(pwd)":/home/rust/src -P start9/rust-arm-cross:aarch64'
-alias 'rust-musl-builder'='docker run $USE_TTY --rm -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$(pwd)":/home/rust/src -P messense/rust-musl-cross:$ARCH-musl'
+alias 'rust-gnu-builder'='docker run $USE_TTY --rm -e "OS_ARCH=$OS_ARCH" -v "$HOME/.cargo/registry":/usr/local/cargo/registry -v "$(pwd)":/home/rust/src -w /home/rust/src -P start9/rust-arm-cross:aarch64'
+alias 'rust-musl-builder'='docker run $USE_TTY --rm -e "OS_ARCH=$OS_ARCH" -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$(pwd)":/home/rust/src -P messense/rust-musl-cross:$ARCH-musl'
 
 cd ..
 FLAGS=""
@@ -32,26 +37,26 @@ fi
 set +e
 fail=
 if [[ "$FLAGS" = "" ]]; then
-	rust-gnu-builder sh -c "(git config --global --add safe.directory '*'; cd backend && cargo build --release --locked  --target=$ARCH-unknown-linux-gnu)"
+	rust-gnu-builder sh -c "(cd backend && cargo build --release --locked  --target=$ARCH-unknown-linux-gnu)"
 	if test $? -ne 0; then 
 		fail=true
 	fi
 	for ARCH in x86_64 aarch64
 	do
-		rust-musl-builder sh -c "(git config --global --add safe.directory '*'; cd libs && cargo build --release --locked --bin embassy_container_init )"
+		rust-musl-builder sh -c "(cd libs && cargo build --release --locked --bin embassy_container_init )"
 		if test $? -ne 0; then 
 			fail=true
 		fi
 	done
 else
 	echo "FLAGS=$FLAGS"
-	rust-gnu-builder sh -c "(git config --global --add safe.directory '*'; cd backend && cargo build --release --features $FLAGS --locked --target=$ARCH-unknown-linux-gnu)"
+	rust-gnu-builder sh -c "(cd backend && cargo build --release --features $FLAGS --locked --target=$ARCH-unknown-linux-gnu)"
 	if test $? -ne 0; then 
 		fail=true
 	fi
 	for ARCH in x86_64 aarch64
 	do
-		rust-musl-builder sh -c "(git config --global --add safe.directory '*'; cd libs && cargo build --release --features $FLAGS --locked --bin embassy_container_init)"
+		rust-musl-builder sh -c "(cd libs && cargo build --release --features $FLAGS --locked --bin embassy_container_init)"
 		if test $? -ne 0; then 
 			fail=true
 		fi
@@ -67,5 +72,3 @@ sudo chown -R $USER ../libs/target
 if [ -n "$fail" ]; then
 	exit 1
 fi
-
-#rust-arm64-builder aarch64-linux-gnu-strip target/aarch64-unknown-linux-gnu/release/embassyd

backend/embassy-init.service

@@ -1,15 +0,0 @@
-[Unit]
-Description=Embassy Init
-After=network-online.target
-Requires=network-online.target
-Wants=avahi-daemon.service
-
-[Service]
-Type=oneshot
-Environment=RUST_LOG=embassy_init=debug,embassy=debug,js_engine=debug,patch_db=warn
-ExecStart=/usr/bin/embassy-init
-RemainAfterExit=true
-StandardOutput=append:/var/log/embassy-init.log
-
-[Install]
-WantedBy=embassyd.service

backend/embassyd.service

@@ -1,17 +0,0 @@
-[Unit]
-Description=Embassy Daemon
-After=embassy-init.service
-Requires=embassy-init.service
-
-[Service]
-Type=simple
-Environment=RUST_LOG=embassyd=debug,embassy=debug,js_engine=debug,patch_db=warn
-ExecStart=/usr/bin/embassyd
-Restart=always
-RestartSec=3
-ManagedOOMPreference=avoid
-CPUAccounting=true
-CPUWeight=1000
-
-[Install]
-WantedBy=multi-user.target

backend/install-sdk.sh

@@ -8,4 +8,11 @@ if [ "$0" != "./install-sdk.sh" ]; then
 	exit 1
 fi
 
-cargo install --bin=embassy-sdk --bin=embassy-cli --path=. --no-default-features --features=js_engine --locked
+if [ -z "$OS_ARCH" ]; then
+  export OS_ARCH=$(uname -m)
+fi
+
+cargo install --path=. --no-default-features --features=js_engine,sdk,cli --locked
+startbox_loc=$(which startbox)
+ln -sf $startbox_loc $(dirname $startbox_loc)/start-cli
+ln -sf $startbox_loc $(dirname $startbox_loc)/start-sdk

backend/src/backup/backup_bulk.rs

@@ -1,5 +1,5 @@
 use std::collections::{BTreeMap, BTreeSet};
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 
 use chrono::Utc;
 use clap::ArgMatches;
@@ -8,6 +8,7 @@ use helpers::AtomicFile;
 use patch_db::{DbHandle, LockType, PatchDbHandle};
 use rpc_toolkit::command;
 use tokio::io::AsyncWriteExt;
+use tokio::process::Command;
 use tracing::instrument;
 
 use super::target::BackupTargetId;
@@ -23,8 +24,9 @@ use crate::disk::mount::guard::TmpMountGuard;
 use crate::notifications::NotificationLevel;
 use crate::s9pk::manifest::PackageId;
 use crate::status::MainStatus;
-use crate::util::display_none;
+use crate::util::io::dir_copy;
 use crate::util::serde::IoFormat;
+use crate::util::{display_none, Invoke};
 use crate::version::VersionT;
 use crate::{Error, ErrorKind, ResultExt};
 
@@ -358,6 +360,19 @@ async fn perform_backup<Db: DbHandle>(
         .await
         .with_kind(ErrorKind::Filesystem)?;
 
+    let luks_folder_old = backup_guard.as_ref().join("luks.old");
+    if tokio::fs::metadata(&luks_folder_old).await.is_ok() {
+        tokio::fs::remove_dir_all(&luks_folder_old).await?;
+    }
+    let luks_folder_bak = backup_guard.as_ref().join("luks");
+    if tokio::fs::metadata(&luks_folder_bak).await.is_ok() {
+        tokio::fs::rename(&luks_folder_bak, &luks_folder_old).await?;
+    }
+    let luks_folder = Path::new("/media/embassy/config/luks");
+    if tokio::fs::metadata(&luks_folder).await.is_ok() {
+        dir_copy(&luks_folder, &luks_folder_bak, None).await?;
+    }
+
     let timestamp = Some(Utc::now());
 
     backup_guard.unencrypted_metadata.version = crate::version::Current::new().semver().into();

backend/src/backup/os.rs

@@ -1,12 +1,13 @@
+use openssl::pkey::PKey;
+use openssl::x509::X509;
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+
 use crate::account::AccountInfo;
 use crate::hostname::{generate_hostname, generate_id, Hostname};
 use crate::net::keys::Key;
 use crate::util::serde::Base64;
 use crate::Error;
-use openssl::pkey::PKey;
-use openssl::x509::X509;
-use serde::{Deserialize, Serialize};
-use serde_json::Value;
 
 pub struct OsBackup {
     pub account: AccountInfo,

backend/src/backup/restore.rs

@@ -6,8 +6,8 @@ use std::time::Duration;
 
 use clap::ArgMatches;
 use color_eyre::eyre::eyre;
-use futures::{future::BoxFuture, stream};
-use futures::{FutureExt, StreamExt};
+use futures::future::BoxFuture;
+use futures::{stream, FutureExt, StreamExt};
 use openssl::x509::X509;
 use patch_db::{DbHandle, PatchDbHandle};
 use rpc_toolkit::command;
@@ -109,7 +109,7 @@ async fn approximate_progress(
         if tokio::fs::metadata(&dir).await.is_err() {
             *size = 0;
         } else {
-            *size = dir_size(&dir).await?;
+            *size = dir_size(&dir, None).await?;
         }
     }
     Ok(())
@@ -285,7 +285,7 @@ async fn restore_packages(
         progress_info.package_installs.insert(id.clone(), progress);
         progress_info
             .src_volume_size
-            .insert(id.clone(), dir_size(backup_dir(&id)).await?);
+            .insert(id.clone(), dir_size(backup_dir(&id), None).await?);
         progress_info.target_volume_size.insert(id.clone(), 0);
         let package_id = id.clone();
         tasks.push(
@@ -443,7 +443,7 @@ async fn restore_package<'a>(
     Ok((
         progress.clone(),
         async move {
-            download_install_s9pk(&ctx, &manifest, None, progress, file).await?;
+            download_install_s9pk(&ctx, &manifest, None, progress, file, None).await?;
 
             guard.unmount().await?;
 

backend/src/backup/target/mod.rs

@@ -7,10 +7,12 @@ use clap::ArgMatches;
 use color_eyre::eyre::eyre;
 use digest::generic_array::GenericArray;
 use digest::OutputSizeUser;
+use lazy_static::lazy_static;
 use rpc_toolkit::command;
 use serde::{Deserialize, Serialize};
 use sha2::Sha256;
 use sqlx::{Executor, Postgres};
+use tokio::sync::Mutex;
 use tracing::instrument;
 
 use self::cifs::CifsBackupTarget;
@@ -23,7 +25,7 @@ use crate::disk::mount::guard::TmpMountGuard;
 use crate::disk::util::PartitionInfo;
 use crate::s9pk::manifest::PackageId;
 use crate::util::serde::{deserialize_from_str, display_serializable, serialize_display};
-use crate::util::Version;
+use crate::util::{display_none, Version};
 use crate::Error;
 
 pub mod cifs;
@@ -42,7 +44,7 @@ pub enum BackupTarget {
     Cifs(CifsBackupTarget),
 }
 
-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)]
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
 pub enum BackupTargetId {
     Disk { logicalname: PathBuf },
     Cifs { id: i32 },
@@ -129,7 +131,7 @@ impl FileSystem for BackupTargetFS {
     }
 }
 
-#[command(subcommands(cifs::cifs, list, info))]
+#[command(subcommands(cifs::cifs, list, info, mount, umount))]
 pub fn target() -> Result<(), Error> {
     Ok(())
 }
@@ -247,3 +249,61 @@ pub async fn info(
 
     Ok(res)
 }
+
+lazy_static! {
+    static ref USER_MOUNTS: Mutex<BTreeMap<BackupTargetId, BackupMountGuard<TmpMountGuard>>> =
+        Mutex::new(BTreeMap::new());
+}
+
+#[command]
+#[instrument(skip_all)]
+pub async fn mount(
+    #[context] ctx: RpcContext,
+    #[arg(rename = "target-id")] target_id: BackupTargetId,
+    #[arg] password: String,
+) -> Result<String, Error> {
+    let mut mounts = USER_MOUNTS.lock().await;
+
+    if let Some(existing) = mounts.get(&target_id) {
+        return Ok(existing.as_ref().display().to_string());
+    }
+
+    let guard = BackupMountGuard::mount(
+        TmpMountGuard::mount(
+            &target_id
+                .clone()
+                .load(&mut ctx.secret_store.acquire().await?)
+                .await?,
+            ReadWrite,
+        )
+        .await?,
+        &password,
+    )
+    .await?;
+
+    let res = guard.as_ref().display().to_string();
+
+    mounts.insert(target_id, guard);
+
+    Ok(res)
+}
+
+#[command(display(display_none))]
+#[instrument(skip_all)]
+pub async fn umount(
+    #[context] ctx: RpcContext,
+    #[arg(rename = "target-id")] target_id: Option<BackupTargetId>,
+) -> Result<(), Error> {
+    let mut mounts = USER_MOUNTS.lock().await;
+    if let Some(target_id) = target_id {
+        if let Some(existing) = mounts.remove(&target_id) {
+            existing.unmount().await?;
+        }
+    } else {
+        for (_, existing) in std::mem::take(&mut *mounts) {
+            existing.unmount().await?;
+        }
+    }
+
+    Ok(())
+}

backend/src/bins/avahi_alias.rs

@@ -14,7 +14,7 @@ fn log_str_error(action: &str, e: i32) {
     }
 }
 
-fn main() {
+pub fn main() {
     let aliases: Vec<_> = std::env::args().skip(1).collect();
     unsafe {
         let simple_poll = avahi_sys::avahi_simple_poll_new();

backend/src/bins/deprecated.rs

@@ -0,0 +1,9 @@
+pub fn renamed(old: &str, new: &str) -> ! {
+    eprintln!("{old} has been renamed to {new}");
+    std::process::exit(1)
+}
+
+pub fn removed(name: &str) -> ! {
+    eprintln!("{name} has been removed");
+    std::process::exit(1)
+}

backend/src/bins/mod.rs

@@ -0,0 +1,55 @@
+use std::path::Path;
+
+#[cfg(feature = "avahi-alias")]
+pub mod avahi_alias;
+pub mod deprecated;
+#[cfg(feature = "cli")]
+pub mod start_cli;
+#[cfg(feature = "daemon")]
+pub mod start_init;
+#[cfg(feature = "sdk")]
+pub mod start_sdk;
+#[cfg(feature = "daemon")]
+pub mod startd;
+
+fn select_executable(name: &str) -> Option<fn()> {
+    match name {
+        #[cfg(feature = "avahi-alias")]
+        "avahi-alias" => Some(avahi_alias::main),
+        #[cfg(feature = "cli")]
+        "start-cli" => Some(start_cli::main),
+        #[cfg(feature = "sdk")]
+        "start-sdk" => Some(start_sdk::main),
+        #[cfg(feature = "daemon")]
+        "startd" => Some(startd::main),
+        "embassy-cli" => Some(|| deprecated::renamed("embassy-cli", "start-cli")),
+        "embassy-sdk" => Some(|| deprecated::renamed("embassy-sdk", "start-sdk")),
+        "embassyd" => Some(|| deprecated::renamed("embassyd", "startd")),
+        "embassy-init" => Some(|| deprecated::removed("embassy-init")),
+        _ => None,
+    }
+}
+
+pub fn startbox() {
+    let args = std::env::args().take(2).collect::<Vec<_>>();
+    if let Some(x) = args
+        .get(0)
+        .and_then(|s| Path::new(&*s).file_name())
+        .and_then(|s| s.to_str())
+        .and_then(|s| select_executable(&s))
+    {
+        x()
+    } else if let Some(x) = args.get(1).and_then(|s| select_executable(&s)) {
+        x()
+    } else {
+        eprintln!(
+            "unknown executable: {}",
+            args.get(0)
+                .filter(|x| &**x != "startbox")
+                .or_else(|| args.get(1))
+                .map(|s| s.as_str())
+                .unwrap_or("N/A")
+        );
+        std::process::exit(1);
+    }
+}

backend/src/bins/start_cli.rs

@@ -1,21 +1,22 @@
 use clap::Arg;
-use embassy::context::CliContext;
-use embassy::util::logger::EmbassyLogger;
-use embassy::version::{Current, VersionT};
-use embassy::Error;
 use rpc_toolkit::run_cli;
 use rpc_toolkit::yajrc::RpcError;
 use serde_json::Value;
 
+use crate::context::CliContext;
+use crate::util::logger::EmbassyLogger;
+use crate::version::{Current, VersionT};
+use crate::Error;
+
 lazy_static::lazy_static! {
     static ref VERSION_STRING: String = Current::new().semver().to_string();
 }
 
 fn inner_main() -> Result<(), Error> {
     run_cli!({
-        command: embassy::main_api,
+        command: crate::main_api,
         app: app => app
-            .name("Embassy CLI")
+            .name("StartOS CLI")
             .version(&**VERSION_STRING)
             .arg(
                 clap::Arg::with_name("config")
@@ -48,7 +49,7 @@ fn inner_main() -> Result<(), Error> {
     Ok(())
 }
 
-fn main() {
+pub fn main() {
     match inner_main() {
         Ok(_) => (),
         Err(e) => {

backend/src/bins/start_init.rs

@@ -1,28 +1,71 @@
+use std::net::{Ipv6Addr, SocketAddr};
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
 use std::time::Duration;
 
-use embassy::context::rpc::RpcContextConfig;
-use embassy::context::{DiagnosticContext, InstallContext, SetupContext};
-use embassy::disk::fsck::RepairStrategy;
-use embassy::disk::main::DEFAULT_PASSWORD;
-use embassy::disk::REPAIR_DISK_PATH;
-use embassy::init::STANDBY_MODE_PATH;
-use embassy::net::web_server::WebServer;
-use embassy::shutdown::Shutdown;
-use embassy::sound::CHIME;
-use embassy::util::logger::EmbassyLogger;
-use embassy::util::Invoke;
-use embassy::{Error, ErrorKind, ResultExt, IS_RASPBERRY_PI};
 use tokio::process::Command;
 use tracing::instrument;
 
+use crate::context::rpc::RpcContextConfig;
+use crate::context::{DiagnosticContext, InstallContext, SetupContext};
+use crate::disk::fsck::RepairStrategy;
+use crate::disk::main::DEFAULT_PASSWORD;
+use crate::disk::REPAIR_DISK_PATH;
+use crate::init::STANDBY_MODE_PATH;
+use crate::net::web_server::WebServer;
+use crate::shutdown::Shutdown;
+use crate::sound::CHIME;
+use crate::util::logger::EmbassyLogger;
+use crate::util::Invoke;
+use crate::{Error, ErrorKind, ResultExt, OS_ARCH};
+
 #[instrument(skip_all)]
 async fn setup_or_init(cfg_path: Option<PathBuf>) -> Result<(), Error> {
-    if tokio::fs::metadata("/cdrom").await.is_ok() {
+    Command::new("ln")
+        .arg("-sf")
+        .arg("/usr/lib/embassy/scripts/fake-apt")
+        .arg("/usr/local/bin/apt")
+        .invoke(crate::ErrorKind::OpenSsh)
+        .await?;
+    Command::new("ln")
+        .arg("-sf")
+        .arg("/usr/lib/embassy/scripts/fake-apt")
+        .arg("/usr/local/bin/apt-get")
+        .invoke(crate::ErrorKind::OpenSsh)
+        .await?;
+    Command::new("ln")
+        .arg("-sf")
+        .arg("/usr/lib/embassy/scripts/fake-apt")
+        .arg("/usr/local/bin/aptitude")
+        .invoke(crate::ErrorKind::OpenSsh)
+        .await?;
+
+    Command::new("make-ssl-cert")
+        .arg("generate-default-snakeoil")
+        .arg("--force-overwrite")
+        .invoke(crate::ErrorKind::OpenSsl)
+        .await?;
+
+    if tokio::fs::metadata("/run/live/medium").await.is_ok() {
+        Command::new("sed")
+            .arg("-i")
+            .arg("s/PasswordAuthentication no/PasswordAuthentication yes/g")
+            .arg("/etc/ssh/sshd_config")
+            .invoke(crate::ErrorKind::Filesystem)
+            .await?;
+        Command::new("systemctl")
+            .arg("reload")
+            .arg("ssh")
+            .invoke(crate::ErrorKind::OpenSsh)
+            .await?;
+
         let ctx = InstallContext::init(cfg_path).await?;
 
-        let server = WebServer::install(([0, 0, 0, 0], 80).into(), ctx.clone()).await?;
+        let server = WebServer::install(
+            SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 80),
+            ctx.clone(),
+        )
+        .await?;
 
         tokio::time::sleep(Duration::from_secs(1)).await; // let the record state that I hate this
         CHIME.play().await?;
@@ -36,7 +79,7 @@ async fn setup_or_init(cfg_path: Option<PathBuf>) -> Result<(), Error> {
         server.shutdown().await;
 
         Command::new("reboot")
-            .invoke(embassy::ErrorKind::Unknown)
+            .invoke(crate::ErrorKind::Unknown)
             .await?;
     } else if tokio::fs::metadata("/media/embassy/config/disk.guid")
         .await
@@ -44,7 +87,11 @@ async fn setup_or_init(cfg_path: Option<PathBuf>) -> Result<(), Error> {
     {
         let ctx = SetupContext::init(cfg_path).await?;
 
-        let server = WebServer::setup(([0, 0, 0, 0], 80).into(), ctx.clone()).await?;
+        let server = WebServer::setup(
+            SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 80),
+            ctx.clone(),
+        )
+        .await?;
 
         tokio::time::sleep(Duration::from_secs(1)).await; // let the record state that I hate this
         CHIME.play().await?;
@@ -70,7 +117,7 @@ async fn setup_or_init(cfg_path: Option<PathBuf>) -> Result<(), Error> {
         let guid_string = tokio::fs::read_to_string("/media/embassy/config/disk.guid") // unique identifier for volume group - keeps track of the disk that goes with your embassy
             .await?;
         let guid = guid_string.trim();
-        let requires_reboot = embassy::disk::main::import(
+        let requires_reboot = crate::disk::main::import(
             guid,
             cfg.datadir(),
             if tokio::fs::metadata(REPAIR_DISK_PATH).await.is_ok() {
@@ -78,22 +125,26 @@ async fn setup_or_init(cfg_path: Option<PathBuf>) -> Result<(), Error> {
             } else {
                 RepairStrategy::Preen
             },
-            DEFAULT_PASSWORD,
+            if guid.ends_with("_UNENC") {
+                None
+            } else {
+                Some(DEFAULT_PASSWORD)
+            },
         )
         .await?;
         if tokio::fs::metadata(REPAIR_DISK_PATH).await.is_ok() {
             tokio::fs::remove_file(REPAIR_DISK_PATH)
                 .await
-                .with_ctx(|_| (embassy::ErrorKind::Filesystem, REPAIR_DISK_PATH))?;
+                .with_ctx(|_| (crate::ErrorKind::Filesystem, REPAIR_DISK_PATH))?;
         }
         if requires_reboot.0 {
-            embassy::disk::main::export(guid, cfg.datadir()).await?;
+            crate::disk::main::export(guid, cfg.datadir()).await?;
             Command::new("reboot")
-                .invoke(embassy::ErrorKind::Unknown)
+                .invoke(crate::ErrorKind::Unknown)
                 .await?;
         }
         tracing::info!("Loaded Disk");
-        embassy::init::init(&cfg).await?;
+        crate::init::init(&cfg).await?;
     }
 
     Ok(())
@@ -119,14 +170,14 @@ async fn run_script_if_exists<P: AsRef<Path>>(path: P) {
 
 #[instrument(skip_all)]
 async fn inner_main(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Error> {
-    if *IS_RASPBERRY_PI && tokio::fs::metadata(STANDBY_MODE_PATH).await.is_ok() {
+    if OS_ARCH == "raspberrypi" && tokio::fs::metadata(STANDBY_MODE_PATH).await.is_ok() {
         tokio::fs::remove_file(STANDBY_MODE_PATH).await?;
         Command::new("sync").invoke(ErrorKind::Filesystem).await?;
-        embassy::sound::SHUTDOWN.play().await?;
+        crate::sound::SHUTDOWN.play().await?;
         futures::future::pending::<()>().await;
     }
 
-    embassy::sound::BEP.play().await?;
+    crate::sound::BEP.play().await?;
 
     run_script_if_exists("/media/embassy/config/preinit.sh").await;
 
@@ -134,7 +185,7 @@ async fn inner_main(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Error
         async move {
             tracing::error!("{}", e.source);
             tracing::debug!("{}", e.source);
-            embassy::sound::BEETHOVEN.play().await?;
+            crate::sound::BEETHOVEN.play().await?;
 
             let ctx = DiagnosticContext::init(
                 cfg_path,
@@ -155,7 +206,11 @@ async fn inner_main(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Error
             )
             .await?;
 
-            let server = WebServer::diagnostic(([0, 0, 0, 0], 80).into(), ctx.clone()).await?;
+            let server = WebServer::diagnostic(
+                SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 80),
+                ctx.clone(),
+            )
+            .await?;
 
             let shutdown = ctx.shutdown.subscribe().recv().await.unwrap();
 
@@ -173,8 +228,8 @@ async fn inner_main(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Error
     res
 }
 
-fn main() {
-    let matches = clap::App::new("embassy-init")
+pub fn main() {
+    let matches = clap::App::new("start-init")
         .arg(
             clap::Arg::with_name("config")
                 .short('c')
@@ -183,8 +238,6 @@ fn main() {
         )
         .get_matches();
 
-    EmbassyLogger::init();
-
     let cfg_path = matches.value_of("config").map(|p| Path::new(p).to_owned());
     let res = {
         let rt = tokio::runtime::Builder::new_multi_thread()

backend/src/bins/start_sdk.rs

@@ -1,20 +1,21 @@
-use embassy::context::SdkContext;
-use embassy::util::logger::EmbassyLogger;
-use embassy::version::{Current, VersionT};
-use embassy::Error;
 use rpc_toolkit::run_cli;
 use rpc_toolkit::yajrc::RpcError;
 use serde_json::Value;
 
+use crate::context::SdkContext;
+use crate::util::logger::EmbassyLogger;
+use crate::version::{Current, VersionT};
+use crate::Error;
+
 lazy_static::lazy_static! {
     static ref VERSION_STRING: String = Current::new().semver().to_string();
 }
 
 fn inner_main() -> Result<(), Error> {
     run_cli!({
-        command: embassy::portable_api,
+        command: crate::portable_api,
         app: app => app
-            .name("Embassy SDK")
+            .name("StartOS SDK")
             .version(&**VERSION_STRING)
             .arg(
                 clap::Arg::with_name("config")
@@ -47,7 +48,7 @@ fn inner_main() -> Result<(), Error> {
     Ok(())
 }
 
-fn main() {
+pub fn main() {
     match inner_main() {
         Ok(_) => (),
         Err(e) => {

backend/src/bins/startd.rs

@@ -1,17 +1,19 @@
+use std::net::{Ipv6Addr, SocketAddr};
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
 
 use color_eyre::eyre::eyre;
-use embassy::context::{DiagnosticContext, RpcContext};
-use embassy::net::web_server::WebServer;
-use embassy::shutdown::Shutdown;
-use embassy::system::launch_metrics_task;
-use embassy::util::logger::EmbassyLogger;
-use embassy::{Error, ErrorKind, ResultExt};
 use futures::{FutureExt, TryFutureExt};
 use tokio::signal::unix::signal;
 use tracing::instrument;
 
+use crate::context::{DiagnosticContext, RpcContext};
+use crate::net::web_server::WebServer;
+use crate::shutdown::Shutdown;
+use crate::system::launch_metrics_task;
+use crate::util::logger::EmbassyLogger;
+use crate::{Error, ErrorKind, ResultExt};
+
 #[instrument(skip_all)]
 async fn inner_main(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Error> {
     let (rpc_ctx, server, shutdown) = {
@@ -25,8 +27,12 @@ async fn inner_main(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Error
             ),
         )
         .await?;
-        embassy::hostname::sync_hostname(&*rpc_ctx.account.read().await).await?;
-        let server = WebServer::main(([0, 0, 0, 0], 80).into(), rpc_ctx.clone()).await?;
+        crate::hostname::sync_hostname(&rpc_ctx.account.read().await.hostname).await?;
+        let server = WebServer::main(
+            SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 80),
+            rpc_ctx.clone(),
+        )
+        .await?;
 
         let mut shutdown_recv = rpc_ctx.shutdown.subscribe();
 
@@ -66,7 +72,7 @@ async fn inner_main(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Error
             .await
         });
 
-        embassy::sound::CHIME.play().await?;
+        crate::sound::CHIME.play().await?;
 
         metrics_task
             .map_err(|e| {
@@ -95,8 +101,15 @@ async fn inner_main(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Error
     Ok(shutdown)
 }
 
-fn main() {
-    let matches = clap::App::new("embassyd")
+pub fn main() {
+    EmbassyLogger::init();
+
+    if !Path::new("/run/embassy/initialized").exists() {
+        super::start_init::main();
+        std::fs::write("/run/embassy/initialized", "").unwrap();
+    }
+
+    let matches = clap::App::new("startd")
         .arg(
             clap::Arg::with_name("config")
                 .short('c')
@@ -105,8 +118,6 @@ fn main() {
         )
         .get_matches();
 
-    EmbassyLogger::init();
-
     let cfg_path = matches.value_of("config").map(|p| Path::new(p).to_owned());
 
     let res = {
@@ -121,7 +132,7 @@ fn main() {
                     async {
                         tracing::error!("{}", e.source);
                         tracing::debug!("{:?}", e.source);
-                        embassy::sound::BEETHOVEN.play().await?;
+                        crate::sound::BEETHOVEN.play().await?;
                         let ctx = DiagnosticContext::init(
                             cfg_path,
                             if tokio::fs::metadata("/media/embassy/config/disk.guid")
@@ -141,8 +152,11 @@ fn main() {
                         )
                         .await?;
 
-                        let server =
-                            WebServer::diagnostic(([0, 0, 0, 0], 80).into(), ctx.clone()).await?;
+                        let server = WebServer::diagnostic(
+                            SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 80),
+                            ctx.clone(),
+                        )
+                        .await?;
 
                         let mut shutdown = ctx.shutdown.subscribe();
 

backend/src/config/mod.rs

@@ -503,19 +503,27 @@ pub fn configure_rec<'a, Db: DbHandle>(
             .config_actions
             .get(db, id)
             .await?
-            .ok_or_else(not_found)?;
+            .ok_or_else(|| not_found!(id))?;
         let dependencies = receipts
             .dependencies
             .get(db, id)
             .await?
-            .ok_or_else(not_found)?;
-        let volumes = receipts.volumes.get(db, id).await?.ok_or_else(not_found)?;
+            .ok_or_else(|| not_found!(id))?;
+        let volumes = receipts
+            .volumes
+            .get(db, id)
+            .await?
+            .ok_or_else(|| not_found!(id))?;
         let is_needs_config = !receipts
             .configured
             .get(db, id)
             .await?
-            .ok_or_else(not_found)?;
-        let version = receipts.version.get(db, id).await?.ok_or_else(not_found)?;
+            .ok_or_else(|| not_found!(id))?;
+        let version = receipts
+            .version
+            .get(db, id)
+            .await?
+            .ok_or_else(|| not_found!(id))?;
 
         // get current config and current spec
         let ConfigRes {
@@ -530,7 +538,11 @@ pub fn configure_rec<'a, Db: DbHandle>(
             spec.gen(&mut rand::rngs::StdRng::from_entropy(), timeout)?
         };
 
-        let manifest = receipts.manifest.get(db, id).await?.ok_or_else(not_found)?;
+        let manifest = receipts
+            .manifest
+            .get(db, id)
+            .await?
+            .ok_or_else(|| not_found!(id))?;
 
         spec.validate(&manifest)?;
         spec.matches(&config)?; // check that new config matches spec
@@ -549,7 +561,7 @@ pub fn configure_rec<'a, Db: DbHandle>(
             .system_pointers
             .get(db, &id)
             .await?
-            .ok_or_else(not_found)?;
+            .ok_or_else(|| not_found!(id))?;
         sys.truncate(0);
         let mut current_dependencies: CurrentDependencies = CurrentDependencies(
             dependencies
@@ -655,7 +667,7 @@ pub fn configure_rec<'a, Db: DbHandle>(
             .dependency_errors
             .get(db, &id)
             .await?
-            .ok_or_else(not_found)?;
+            .ok_or_else(|| not_found!(id))?;
         tracing::warn!("Dependency Errors: {:?}", errs);
         let errs = DependencyErrors::init(
             ctx,
@@ -675,7 +687,7 @@ pub fn configure_rec<'a, Db: DbHandle>(
             .current_dependents
             .get(db, id)
             .await?
-            .ok_or_else(not_found)?;
+            .ok_or_else(|| not_found!(id))?;
         let prev = if is_needs_config { None } else { old_config }
             .map(Value::Object)
             .unwrap_or_default();
@@ -693,7 +705,7 @@ pub fn configure_rec<'a, Db: DbHandle>(
                     .manifest
                     .get(db, &dependent)
                     .await?
-                    .ok_or_else(not_found)?;
+                    .ok_or_else(|| not_found!(id))?;
                 if let Err(error) = cfg
                     .check(
                         ctx,
@@ -771,10 +783,16 @@ pub fn configure_rec<'a, Db: DbHandle>(
     }
     .boxed()
 }
-#[instrument(skip_all)]
-pub fn not_found() -> Error {
-    Error::new(eyre!("Could not find"), crate::ErrorKind::Incoherent)
+
+macro_rules! not_found {
+    ($x:expr) => {
+        crate::Error::new(
+            color_eyre::eyre::eyre!("Could not find {} at {}:{}", $x, module_path!(), line!()),
+            crate::ErrorKind::Incoherent,
+        )
+    };
 }
+pub(crate) use not_found;
 
 /// We want to have a double check that the paths are what we expect them to be.
 /// Found that earlier the paths where not what we expected them to be.

backend/src/context/cli.rs

@@ -17,12 +17,11 @@ use rpc_toolkit::Context;
 use serde::Deserialize;
 use tracing::instrument;
 
+use super::setup::CURRENT_SECRET;
 use crate::middleware::auth::LOCAL_AUTH_COOKIE_PATH;
 use crate::util::config::{load_config_from_paths, local_config_path};
 use crate::ResultExt;
 
-use super::setup::CURRENT_SECRET;
-
 #[derive(Debug, Default, Deserialize)]
 #[serde(rename_all = "kebab-case")]
 pub struct CliContextConfig {
@@ -54,7 +53,8 @@ impl Drop for CliContextSeed {
             true,
         )
         .unwrap();
-        let store = self.cookie_store.lock().unwrap();
+        let mut store = self.cookie_store.lock().unwrap();
+        store.remove("localhost", "", "local");
         store.save_json(&mut *writer).unwrap();
         writer.sync_all().unwrap();
         std::fs::rename(tmp, &self.cookie_path).unwrap();
@@ -101,19 +101,22 @@ impl CliContext {
                 .unwrap_or(Path::new("/"))
                 .join(".cookies.json")
         });
-        let cookie_store = Arc::new(CookieStoreMutex::new(if cookie_path.exists() {
-            let mut store = CookieStore::load_json(BufReader::new(File::open(&cookie_path)?))
-                .map_err(|e| eyre!("{}", e))
-                .with_kind(crate::ErrorKind::Deserialization)?;
+        let cookie_store = Arc::new(CookieStoreMutex::new({
+            let mut store = if cookie_path.exists() {
+                CookieStore::load_json(BufReader::new(File::open(&cookie_path)?))
+                    .map_err(|e| eyre!("{}", e))
+                    .with_kind(crate::ErrorKind::Deserialization)?
+            } else {
+                CookieStore::default()
+            };
             if let Ok(local) = std::fs::read_to_string(LOCAL_AUTH_COOKIE_PATH) {
                 store
                     .insert_raw(&Cookie::new("local", local), &"http://localhost".parse()?)
                     .with_kind(crate::ErrorKind::Network)?;
             }
             store
-        } else {
-            CookieStore::default()
         }));
+
         Ok(CliContext(Arc::new(CliContextSeed {
             base_url: url.clone(),
             rpc_url: {

backend/src/context/rpc.rs

@@ -11,7 +11,7 @@ use helpers::to_tmp_path;
 use josekit::jwk::Jwk;
 use patch_db::json_ptr::JsonPointer;
 use patch_db::{DbHandle, LockReceipt, LockType, PatchDb};
-use reqwest::Url;
+use reqwest::{Client, Proxy, Url};
 use rpc_toolkit::Context;
 use serde::Deserialize;
 use sqlx::postgres::PgConnectOptions;
@@ -19,11 +19,12 @@ use sqlx::PgPool;
 use tokio::sync::{broadcast, oneshot, Mutex, RwLock};
 use tracing::instrument;
 
+use super::setup::CURRENT_SECRET;
 use crate::account::AccountInfo;
 use crate::core::rpc_continuations::{RequestGuid, RestHandler, RpcContinuation};
-use crate::db::model::{Database, InstalledPackageDataEntry, PackageDataEntry};
+use crate::db::model::{CurrentDependents, Database, InstalledPackageDataEntry, PackageDataEntry};
 use crate::disk::OsPartitionInfo;
-use crate::init::{init_postgres, pgloader};
+use crate::init::init_postgres;
 use crate::install::cleanup::{cleanup_failed, uninstall, CleanupFailedReceipts};
 use crate::manager::ManagerMap;
 use crate::middleware::auth::HashSessionToken;
@@ -33,11 +34,11 @@ use crate::net::wifi::WpaCli;
 use crate::notifications::NotificationManager;
 use crate::shutdown::Shutdown;
 use crate::status::{MainStatus, Status};
+use crate::system::get_mem_info;
 use crate::util::config::load_config_from_paths;
+use crate::util::lshw::{lshw, LshwDevice};
 use crate::{Error, ErrorKind, ResultExt};
 
-use super::setup::CURRENT_SECRET;
-
 #[derive(Debug, Default, Deserialize)]
 #[serde(rename_all = "kebab-case")]
 pub struct RpcContextConfig {
@@ -96,15 +97,6 @@ impl RpcContextConfig {
             .run(&secret_store)
             .await
             .with_kind(crate::ErrorKind::Database)?;
-        let old_db_path = self.datadir().join("main/secrets.db");
-        if tokio::fs::metadata(&old_db_path).await.is_ok() {
-            pgloader(
-                &old_db_path,
-                self.migration_batch_rows.unwrap_or(25000),
-                self.migration_prefetch_rows.unwrap_or(100_000),
-            )
-            .await?;
-        }
         Ok(secret_store)
     }
 }
@@ -130,6 +122,13 @@ pub struct RpcContextSeed {
     pub rpc_stream_continuations: Mutex<BTreeMap<RequestGuid, RpcContinuation>>,
     pub wifi_manager: Option<Arc<RwLock<WpaCli>>>,
     pub current_secret: Arc<Jwk>,
+    pub client: Client,
+    pub hardware: Hardware,
+}
+
+pub struct Hardware {
+    pub devices: Vec<LshwDevice>,
+    pub ram: u64,
 }
 
 pub struct RpcCleanReceipts {
@@ -197,6 +196,7 @@ impl RpcContext {
             NetController::init(
                 base.tor_control
                     .unwrap_or(SocketAddr::from(([127, 0, 0, 1], 9051))),
+                tor_proxy,
                 base.dns_bind
                     .as_ref()
                     .map(|v| v.as_slice())
@@ -212,6 +212,9 @@ impl RpcContext {
         let metrics_cache = RwLock::new(None);
         let notification_manager = NotificationManager::new(secret_store.clone());
         tracing::info!("Initialized Notification Manager");
+        let tor_proxy_url = format!("socks5h://{tor_proxy}");
+        let devices = lshw().await?;
+        let ram = get_mem_info().await?.total.0 as u64 * 1024 * 1024;
         let seed = Arc::new(RpcContextSeed {
             is_closed: AtomicBool::new(false),
             datadir: base.datadir().to_path_buf(),
@@ -244,6 +247,17 @@ impl RpcContext {
                     )
                 })?,
             ),
+            client: Client::builder()
+                .proxy(Proxy::custom(move |url| {
+                    if url.host_str().map_or(false, |h| h.ends_with(".onion")) {
+                        Some(tor_proxy_url.clone())
+                    } else {
+                        None
+                    }
+                }))
+                .build()
+                .with_kind(crate::ErrorKind::ParseUrl)?,
+            hardware: Hardware { devices, ram },
         });
 
         let res = Self(seed);
@@ -274,6 +288,45 @@ impl RpcContext {
     pub async fn cleanup(&self) -> Result<(), Error> {
         let mut db = self.db.handle();
         let receipts = RpcCleanReceipts::new(&mut db).await?;
+        let packages = receipts.packages.get(&mut db).await?.0;
+        let mut current_dependents = packages
+            .keys()
+            .map(|k| (k.clone(), BTreeMap::new()))
+            .collect::<BTreeMap<_, _>>();
+        for (package_id, package) in packages {
+            for (k, v) in package
+                .into_installed()
+                .into_iter()
+                .flat_map(|i| i.current_dependencies.0)
+            {
+                let mut entry: BTreeMap<_, _> = current_dependents.remove(&k).unwrap_or_default();
+                entry.insert(package_id.clone(), v);
+                current_dependents.insert(k, entry);
+            }
+        }
+        for (package_id, current_dependents) in current_dependents {
+            if let Some(deps) = crate::db::DatabaseModel::new()
+                .package_data()
+                .idx_model(&package_id)
+                .and_then(|pde| pde.installed())
+                .map::<_, CurrentDependents>(|i| i.current_dependents())
+                .check(&mut db)
+                .await?
+            {
+                deps.put(&mut db, &CurrentDependents(current_dependents))
+                    .await?;
+            } else if let Some(deps) = crate::db::DatabaseModel::new()
+                .package_data()
+                .idx_model(&package_id)
+                .and_then(|pde| pde.removing())
+                .map::<_, CurrentDependents>(|i| i.current_dependents())
+                .check(&mut db)
+                .await?
+            {
+                deps.put(&mut db, &CurrentDependents(current_dependents))
+                    .await?;
+            }
+        }
         for (package_id, package) in receipts.packages.get(&mut db).await?.0 {
             if let Err(e) = async {
                 match package {

backend/src/context/setup.rs

@@ -17,7 +17,7 @@ use tracing::instrument;
 use crate::account::AccountInfo;
 use crate::db::model::Database;
 use crate::disk::OsPartitionInfo;
-use crate::init::{init_postgres, pgloader};
+use crate::init::init_postgres;
 use crate::setup::SetupStatus;
 use crate::util::config::load_config_from_paths;
 use crate::{Error, ResultExt};
@@ -45,6 +45,8 @@ pub struct SetupContextConfig {
     pub migration_batch_rows: Option<usize>,
     pub migration_prefetch_rows: Option<usize>,
     pub datadir: Option<PathBuf>,
+    #[serde(default)]
+    pub disable_encryption: bool,
 }
 impl SetupContextConfig {
     #[instrument(skip_all)]
@@ -75,6 +77,7 @@ pub struct SetupContextSeed {
     pub config_path: Option<PathBuf>,
     pub migration_batch_rows: usize,
     pub migration_prefetch_rows: usize,
+    pub disable_encryption: bool,
     pub shutdown: Sender<()>,
     pub datadir: PathBuf,
     pub selected_v2_drive: RwLock<Option<PathBuf>>,
@@ -102,6 +105,7 @@ impl SetupContext {
             config_path: path.as_ref().map(|p| p.as_ref().to_owned()),
             migration_batch_rows: cfg.migration_batch_rows.unwrap_or(25000),
             migration_prefetch_rows: cfg.migration_prefetch_rows.unwrap_or(100_000),
+            disable_encryption: cfg.disable_encryption,
             shutdown,
             datadir,
             selected_v2_drive: RwLock::new(None),
@@ -132,15 +136,6 @@ impl SetupContext {
             .run(&secret_store)
             .await
             .with_kind(crate::ErrorKind::Database)?;
-        let old_db_path = self.datadir.join("main/secrets.db");
-        if tokio::fs::metadata(&old_db_path).await.is_ok() {
-            pgloader(
-                &old_db_path,
-                self.migration_batch_rows,
-                self.migration_prefetch_rows,
-            )
-            .await?;
-        }
         Ok(secret_store)
     }
 }

backend/src/db/mod.rs

@@ -4,9 +4,10 @@ pub mod package;
 use std::future::Future;
 use std::sync::Arc;
 
+use color_eyre::eyre::eyre;
 use futures::{FutureExt, SinkExt, StreamExt};
 use patch_db::json_ptr::JsonPointer;
-use patch_db::{Dump, Revision};
+use patch_db::{DbHandle, Dump, LockType, Revision};
 use rpc_toolkit::command;
 use rpc_toolkit::hyper::upgrade::Upgraded;
 use rpc_toolkit::hyper::{Body, Error as HyperError, Request, Response};
@@ -24,6 +25,7 @@ use tracing::instrument;
 pub use self::model::DatabaseModel;
 use crate::context::RpcContext;
 use crate::middleware::auth::{HasValidSession, HashSessionToken};
+use crate::util::display_none;
 use crate::util::serde::{display_serializable, IoFormat};
 use crate::{Error, ResultExt};
 
@@ -163,7 +165,7 @@ pub async fn subscribe(ctx: RpcContext, req: Request<Body>) -> Result<Response<B
     Ok(res)
 }
 
-#[command(subcommands(revisions, dump, put))]
+#[command(subcommands(revisions, dump, put, apply))]
 pub fn db() -> Result<(), RpcError> {
     Ok(())
 }
@@ -199,6 +201,85 @@ pub async fn dump(
     Ok(ctx.db.dump().await?)
 }
 
+fn apply_expr(input: jaq_core::Val, expr: &str) -> Result<jaq_core::Val, Error> {
+    let (expr, errs) = jaq_core::parse::parse(expr, jaq_core::parse::main());
+
+    let Some(expr) = expr else {
+        return Err(Error::new(
+            eyre!("Failed to parse expression: {:?}", errs),
+            crate::ErrorKind::InvalidRequest,
+        ));
+    };
+
+    let mut errs = Vec::new();
+
+    let mut defs = jaq_core::Definitions::core();
+    for def in jaq_std::std() {
+        defs.insert(def, &mut errs);
+    }
+
+    let filter = defs.finish(expr, Vec::new(), &mut errs);
+
+    if !errs.is_empty() {
+        return Err(Error::new(
+            eyre!("Failed to compile expression: {:?}", errs),
+            crate::ErrorKind::InvalidRequest,
+        ));
+    };
+
+    let inputs = jaq_core::RcIter::new(std::iter::empty());
+    let mut res_iter = filter.run(jaq_core::Ctx::new([], &inputs), input);
+
+    let Some(res) = res_iter
+        .next()
+        .transpose()
+        .map_err(|e| eyre!("{e}"))
+        .with_kind(crate::ErrorKind::Deserialization)?
+    else {
+        return Err(Error::new(
+            eyre!("expr returned no results"),
+            crate::ErrorKind::InvalidRequest,
+        ));
+    };
+
+    if res_iter.next().is_some() {
+        return Err(Error::new(
+            eyre!("expr returned too many results"),
+            crate::ErrorKind::InvalidRequest,
+        ));
+    }
+
+    Ok(res)
+}
+
+#[command(display(display_none))]
+pub async fn apply(#[context] ctx: RpcContext, #[arg] expr: String) -> Result<(), Error> {
+    let mut db = ctx.db.handle();
+
+    DatabaseModel::new().lock(&mut db, LockType::Write).await?;
+
+    let root_ptr = JsonPointer::<String>::default();
+
+    let input = db.get_value(&root_ptr, None).await?;
+
+    let res = (|| {
+        let res = apply_expr(input.into(), &expr)?;
+
+        serde_json::from_value::<model::Database>(res.clone().into()).with_ctx(|_| {
+            (
+                crate::ErrorKind::Deserialization,
+                "result does not match database model",
+            )
+        })?;
+
+        Ok::<serde_json::Value, Error>(res.into())
+    })()?;
+
+    db.put_value(&root_ptr, &res).await?;
+
+    Ok(())
+}
+
 #[command(subcommands(ui))]
 pub fn put() -> Result<(), RpcError> {
     Ok(())

backend/src/db/model.rs

@@ -49,7 +49,7 @@ impl Database {
                 last_wifi_region: None,
                 eos_version_compat: Current::new().compat().clone(),
                 lan_address,
-                tor_address: format!("http://{}", account.key.tor_address())
+                tor_address: format!("https://{}", account.key.tor_address())
                     .parse()
                     .unwrap(),
                 ip_info: BTreeMap::new(),
@@ -80,6 +80,7 @@ impl Database {
                     .map(|x| format!("{x:X}"))
                     .join(":"),
                 system_start_time: Utc::now().to_rfc3339(),
+                zram: false,
             },
             package_data: AllPackageData::default(),
             ui: serde_json::from_str(include_str!("../../../frontend/patchdb-ui-seed.json"))
@@ -106,6 +107,7 @@ pub struct ServerInfo {
     pub lan_address: Url,
     pub tor_address: Url,
     #[model]
+    #[serde(default)]
     pub ip_info: BTreeMap<String, IpInfo>,
     #[model]
     #[serde(default)]
@@ -117,6 +119,8 @@ pub struct ServerInfo {
     pub pubkey: String,
     pub ca_fingerprint: String,
     pub system_start_time: String,
+    #[serde(default)]
+    pub zram: bool,
 }
 
 #[derive(Debug, Deserialize, Serialize, HasModel)]

backend/src/dependencies.rs

@@ -237,13 +237,16 @@ impl DependencyError {
                     }
                 }
                 DependencyError::ConfigUnsatisfied { .. } => {
-                    let dependent_manifest =
-                        receipts.manifest.get(db, id).await?.ok_or_else(not_found)?;
+                    let dependent_manifest = receipts
+                        .manifest
+                        .get(db, id)
+                        .await?
+                        .ok_or_else(|| not_found!(id))?;
                     let dependency_manifest = receipts
                         .manifest
                         .get(db, dependency)
                         .await?
-                        .ok_or_else(not_found)?;
+                        .ok_or_else(|| not_found!(dependency))?;
 
                     let dependency_config = if let Some(cfg) = dependency_config.take() {
                         cfg
@@ -294,7 +297,7 @@ impl DependencyError {
                         .status
                         .get(db, dependency)
                         .await?
-                        .ok_or_else(not_found)?;
+                        .ok_or_else(|| not_found!(dependency))?;
                     if status.main.running() {
                         DependencyError::HealthChecksFailed {
                             failures: BTreeMap::new(),
@@ -310,7 +313,7 @@ impl DependencyError {
                         .status
                         .get(db, dependency)
                         .await?
-                        .ok_or_else(not_found)?;
+                        .ok_or_else(|| not_found!(dependency))?;
                     match status.main {
                         MainStatus::BackingUp {
                             started: Some(_),
@@ -324,7 +327,7 @@ impl DependencyError {
                                         .current_dependencies
                                         .get(db, id)
                                         .await?
-                                        .ok_or_else(not_found)?
+                                        .ok_or_else(|| not_found!(id))?
                                         .get(dependency)
                                         .map(|x| x.health_checks.contains(&check))
                                         .unwrap_or(false)
@@ -934,7 +937,7 @@ pub fn break_transitive<'a, Db: DbHandle>(
             .dependency_errors
             .get(&mut tx, id)
             .await?
-            .ok_or_else(not_found)?;
+            .ok_or_else(|| not_found!(id))?;
 
         let old = dependency_errors.0.remove(dependency);
         let newly_broken = if let Some(e) = &old {
@@ -997,7 +1000,7 @@ pub async fn heal_all_dependents_transitive<'a, Db: DbHandle>(
         .current_dependents
         .get(db, id)
         .await?
-        .ok_or_else(not_found)?;
+        .ok_or_else(|| not_found!(id))?;
     for dependent in dependents.0.keys().filter(|dependent| id != *dependent) {
         heal_transitive(ctx, db, dependent, id, locks).await?;
     }
@@ -1013,7 +1016,11 @@ pub fn heal_transitive<'a, Db: DbHandle>(
     receipts: &'a DependencyReceipt,
 ) -> BoxFuture<'a, Result<(), Error>> {
     async move {
-        let mut status = receipts.status.get(db, id).await?.ok_or_else(not_found)?;
+        let mut status = receipts
+            .status
+            .get(db, id)
+            .await?
+            .ok_or_else(|| not_found!(id))?;
 
         let old = status.dependency_errors.0.remove(dependency);
 
@@ -1022,7 +1029,7 @@ pub fn heal_transitive<'a, Db: DbHandle>(
                 .dependency
                 .get(db, (id, dependency))
                 .await?
-                .ok_or_else(not_found)?;
+                .ok_or_else(|| not_found!(format!("{id}'s dependency: {dependency}")))?;
             if let Some(new) = old
                 .try_heal(ctx, db, id, dependency, None, &info, &receipts.try_heal)
                 .await?

backend/src/diagnostic.rs

@@ -9,11 +9,10 @@ use crate::disk::repair;
 use crate::init::SYSTEM_REBUILD_PATH;
 use crate::logs::{fetch_logs, LogResponse, LogSource};
 use crate::shutdown::Shutdown;
+use crate::system::SYSTEMD_UNIT;
 use crate::util::display_none;
 use crate::Error;
 
-pub const SYSTEMD_UNIT: &'static str = "embassy-init";
-
 #[command(subcommands(error, logs, exit, restart, forget_disk, disk, rebuild))]
 pub fn diagnostic() -> Result<(), Error> {
     Ok(())

backend/src/disk/fsck/btrfs.rs

@@ -0,0 +1,31 @@
+use std::path::Path;
+
+use tokio::process::Command;
+use tracing::instrument;
+
+use crate::disk::fsck::RequiresReboot;
+use crate::util::Invoke;
+use crate::Error;
+
+#[instrument(skip_all)]
+pub async fn btrfs_check_readonly(logicalname: impl AsRef<Path>) -> Result<RequiresReboot, Error> {
+    Command::new("btrfs")
+        .arg("check")
+        .arg("--readonly")
+        .arg(logicalname.as_ref())
+        .invoke(crate::ErrorKind::DiskManagement)
+        .await?;
+
+    Ok(RequiresReboot(false))
+}
+
+pub async fn btrfs_check_repair(logicalname: impl AsRef<Path>) -> Result<RequiresReboot, Error> {
+    Command::new("btrfs")
+        .arg("check")
+        .arg("--repair")
+        .arg(logicalname.as_ref())
+        .invoke(crate::ErrorKind::DiskManagement)
+        .await?;
+
+    Ok(RequiresReboot(false))
+}

backend/src/disk/fsck/ext4.rs

@@ -7,34 +7,9 @@ use futures::FutureExt;
 use tokio::process::Command;
 use tracing::instrument;
 
+use crate::disk::fsck::RequiresReboot;
 use crate::Error;
 
-#[derive(Debug, Clone, Copy)]
-#[must_use]
-pub struct RequiresReboot(pub bool);
-impl std::ops::BitOrAssign for RequiresReboot {
-    fn bitor_assign(&mut self, rhs: Self) {
-        self.0 |= rhs.0
-    }
-}
-
-#[derive(Debug, Clone, Copy)]
-pub enum RepairStrategy {
-    Preen,
-    Aggressive,
-}
-impl RepairStrategy {
-    pub async fn e2fsck(
-        &self,
-        logicalname: impl AsRef<Path> + std::fmt::Debug,
-    ) -> Result<RequiresReboot, Error> {
-        match self {
-            RepairStrategy::Preen => e2fsck_preen(logicalname).await,
-            RepairStrategy::Aggressive => e2fsck_aggressive(logicalname).await,
-        }
-    }
-}
-
 #[instrument(skip_all)]
 pub async fn e2fsck_preen(
     logicalname: impl AsRef<Path> + std::fmt::Debug,

backend/src/disk/fsck/mod.rs

@@ -0,0 +1,70 @@
+use std::path::Path;
+
+use color_eyre::eyre::eyre;
+use tokio::process::Command;
+
+use crate::disk::fsck::btrfs::{btrfs_check_readonly, btrfs_check_repair};
+use crate::disk::fsck::ext4::{e2fsck_aggressive, e2fsck_preen};
+use crate::util::Invoke;
+use crate::Error;
+
+pub mod btrfs;
+pub mod ext4;
+
+#[derive(Debug, Clone, Copy)]
+#[must_use]
+pub struct RequiresReboot(pub bool);
+impl std::ops::BitOrAssign for RequiresReboot {
+    fn bitor_assign(&mut self, rhs: Self) {
+        self.0 |= rhs.0
+    }
+}
+
+#[derive(Debug, Clone, Copy)]
+pub enum RepairStrategy {
+    Preen,
+    Aggressive,
+}
+impl RepairStrategy {
+    pub async fn fsck(
+        &self,
+        logicalname: impl AsRef<Path> + std::fmt::Debug,
+    ) -> Result<RequiresReboot, Error> {
+        match &*String::from_utf8(
+            Command::new("grub-probe")
+                .arg("-d")
+                .arg(logicalname.as_ref())
+                .invoke(crate::ErrorKind::DiskManagement)
+                .await?,
+        )?
+        .trim()
+        {
+            "ext2" => self.e2fsck(logicalname).await,
+            "btrfs" => self.btrfs_check(logicalname).await,
+            fs => {
+                return Err(Error::new(
+                    eyre!("Unknown filesystem {fs}"),
+                    crate::ErrorKind::DiskManagement,
+                ))
+            }
+        }
+    }
+    pub async fn e2fsck(
+        &self,
+        logicalname: impl AsRef<Path> + std::fmt::Debug,
+    ) -> Result<RequiresReboot, Error> {
+        match self {
+            RepairStrategy::Preen => e2fsck_preen(logicalname).await,
+            RepairStrategy::Aggressive => e2fsck_aggressive(logicalname).await,
+        }
+    }
+    pub async fn btrfs_check(
+        &self,
+        logicalname: impl AsRef<Path> + std::fmt::Debug,
+    ) -> Result<RequiresReboot, Error> {
+        match self {
+            RepairStrategy::Preen => btrfs_check_readonly(logicalname).await,
+            RepairStrategy::Aggressive => btrfs_check_repair(logicalname).await,
+        }
+    }
+}

backend/src/disk/main.rs

@@ -13,7 +13,7 @@ use crate::disk::mount::util::unmount;
 use crate::util::Invoke;
 use crate::{Error, ErrorKind, ResultExt};
 
-pub const PASSWORD_PATH: &'static str = "/etc/embassy/password";
+pub const PASSWORD_PATH: &'static str = "/run/embassy/password";
 pub const DEFAULT_PASSWORD: &'static str = "password";
 pub const MAIN_FS_SIZE: FsSize = FsSize::Gigabytes(8);
 
@@ -22,13 +22,13 @@ pub async fn create<I, P>(
     disks: &I,
     pvscan: &BTreeMap<PathBuf, Option<String>>,
     datadir: impl AsRef<Path>,
-    password: &str,
+    password: Option<&str>,
 ) -> Result<String, Error>
 where
     for<'a> &'a I: IntoIterator<Item = &'a P>,
     P: AsRef<Path>,
 {
-    let guid = create_pool(disks, pvscan).await?;
+    let guid = create_pool(disks, pvscan, password.is_some()).await?;
     create_all_fs(&guid, &datadir, password).await?;
     export(&guid, datadir).await?;
     Ok(guid)
@@ -38,6 +38,7 @@ where
 pub async fn create_pool<I, P>(
     disks: &I,
     pvscan: &BTreeMap<PathBuf, Option<String>>,
+    encrypted: bool,
 ) -> Result<String, Error>
 where
     for<'a> &'a I: IntoIterator<Item = &'a P>,
@@ -62,13 +63,16 @@ where
             .invoke(crate::ErrorKind::DiskManagement)
             .await?;
     }
-    let guid = format!(
+    let mut guid = format!(
         "EMBASSY_{}",
         base32::encode(
             base32::Alphabet::RFC4648 { padding: false },
             &rand::random::<[u8; 32]>(),
         )
     );
+    if !encrypted {
+        guid += "_UNENC";
+    }
     let mut cmd = Command::new("vgcreate");
     cmd.arg("-y").arg(&guid);
     for disk in disks {
@@ -90,11 +94,8 @@ pub async fn create_fs<P: AsRef<Path>>(
     datadir: P,
     name: &str,
     size: FsSize,
-    password: &str,
+    password: Option<&str>,
 ) -> Result<(), Error> {
-    tokio::fs::write(PASSWORD_PATH, password)
-        .await
-        .with_ctx(|_| (crate::ErrorKind::Filesystem, PASSWORD_PATH))?;
     let mut cmd = Command::new("lvcreate");
     match size {
         FsSize::Gigabytes(a) => cmd.arg("-L").arg(format!("{}G", a)),
@@ -106,36 +107,41 @@ pub async fn create_fs<P: AsRef<Path>>(
         .arg(guid)
         .invoke(crate::ErrorKind::DiskManagement)
         .await?;
-    Command::new("cryptsetup")
-        .arg("-q")
-        .arg("luksFormat")
-        .arg(format!("--key-file={}", PASSWORD_PATH))
-        .arg(format!("--keyfile-size={}", password.len()))
-        .arg(Path::new("/dev").join(guid).join(name))
+    let mut blockdev_path = Path::new("/dev").join(guid).join(name);
+    if let Some(password) = password {
+        if let Some(parent) = Path::new(PASSWORD_PATH).parent() {
+            tokio::fs::create_dir_all(parent).await?;
+        }
+        tokio::fs::write(PASSWORD_PATH, password)
+            .await
+            .with_ctx(|_| (crate::ErrorKind::Filesystem, PASSWORD_PATH))?;
+        Command::new("cryptsetup")
+            .arg("-q")
+            .arg("luksFormat")
+            .arg(format!("--key-file={}", PASSWORD_PATH))
+            .arg(format!("--keyfile-size={}", password.len()))
+            .arg(&blockdev_path)
+            .invoke(crate::ErrorKind::DiskManagement)
+            .await?;
+        Command::new("cryptsetup")
+            .arg("-q")
+            .arg("luksOpen")
+            .arg(format!("--key-file={}", PASSWORD_PATH))
+            .arg(format!("--keyfile-size={}", password.len()))
+            .arg(&blockdev_path)
+            .arg(format!("{}_{}", guid, name))
+            .invoke(crate::ErrorKind::DiskManagement)
+            .await?;
+        tokio::fs::remove_file(PASSWORD_PATH)
+            .await
+            .with_ctx(|_| (crate::ErrorKind::Filesystem, PASSWORD_PATH))?;
+        blockdev_path = Path::new("/dev/mapper").join(format!("{}_{}", guid, name));
+    }
+    Command::new("mkfs.btrfs")
+        .arg(&blockdev_path)
         .invoke(crate::ErrorKind::DiskManagement)
         .await?;
-    Command::new("cryptsetup")
-        .arg("-q")
-        .arg("luksOpen")
-        .arg(format!("--key-file={}", PASSWORD_PATH))
-        .arg(format!("--keyfile-size={}", password.len()))
-        .arg(Path::new("/dev").join(guid).join(name))
-        .arg(format!("{}_{}", guid, name))
-        .invoke(crate::ErrorKind::DiskManagement)
-        .await?;
-    Command::new("mkfs.ext4")
-        .arg(Path::new("/dev/mapper").join(format!("{}_{}", guid, name)))
-        .invoke(crate::ErrorKind::DiskManagement)
-        .await?;
-    mount(
-        Path::new("/dev/mapper").join(format!("{}_{}", guid, name)),
-        datadir.as_ref().join(name),
-        ReadWrite,
-    )
-    .await?;
-    tokio::fs::remove_file(PASSWORD_PATH)
-        .await
-        .with_ctx(|_| (crate::ErrorKind::Filesystem, PASSWORD_PATH))?;
+    mount(&blockdev_path, datadir.as_ref().join(name), ReadWrite).await?;
     Ok(())
 }
 
@@ -143,7 +149,7 @@ pub async fn create_fs<P: AsRef<Path>>(
 pub async fn create_all_fs<P: AsRef<Path>>(
     guid: &str,
     datadir: P,
-    password: &str,
+    password: Option<&str>,
 ) -> Result<(), Error> {
     create_fs(guid, &datadir, "main", MAIN_FS_SIZE, password).await?;
     create_fs(
@@ -160,12 +166,14 @@ pub async fn create_all_fs<P: AsRef<Path>>(
 #[instrument(skip_all)]
 pub async fn unmount_fs<P: AsRef<Path>>(guid: &str, datadir: P, name: &str) -> Result<(), Error> {
     unmount(datadir.as_ref().join(name)).await?;
-    Command::new("cryptsetup")
-        .arg("-q")
-        .arg("luksClose")
-        .arg(format!("{}_{}", guid, name))
-        .invoke(crate::ErrorKind::DiskManagement)
-        .await?;
+    if !guid.ends_with("_UNENC") {
+        Command::new("cryptsetup")
+            .arg("-q")
+            .arg("luksClose")
+            .arg(format!("{}_{}", guid, name))
+            .invoke(crate::ErrorKind::DiskManagement)
+            .await?;
+    }
 
     Ok(())
 }
@@ -202,7 +210,7 @@ pub async fn import<P: AsRef<Path>>(
     guid: &str,
     datadir: P,
     repair: RepairStrategy,
-    password: &str,
+    password: Option<&str>,
 ) -> Result<RequiresReboot, Error> {
     let scan = pvscan().await?;
     if scan
@@ -213,7 +221,7 @@ pub async fn import<P: AsRef<Path>>(
         .is_none()
     {
         return Err(Error::new(
-            eyre!("Embassy disk not found."),
+            eyre!("StartOS disk not found."),
             crate::ErrorKind::DiskNotAvailable,
         ));
     }
@@ -223,7 +231,7 @@ pub async fn import<P: AsRef<Path>>(
         .any(|id| id == guid)
     {
         return Err(Error::new(
-            eyre!("An Embassy disk was found, but it is not the correct disk for this device."),
+            eyre!("A StartOS disk was found, but it is not the correct disk for this device."),
             crate::ErrorKind::IncorrectDisk,
         ));
     }
@@ -260,27 +268,56 @@ pub async fn mount_fs<P: AsRef<Path>>(
     datadir: P,
     name: &str,
     repair: RepairStrategy,
-    password: &str,
+    password: Option<&str>,
 ) -> Result<RequiresReboot, Error> {
-    tokio::fs::write(PASSWORD_PATH, password)
-        .await
-        .with_ctx(|_| (crate::ErrorKind::Filesystem, PASSWORD_PATH))?;
-    Command::new("cryptsetup")
-        .arg("-q")
-        .arg("luksOpen")
-        .arg(format!("--key-file={}", PASSWORD_PATH))
-        .arg(format!("--keyfile-size={}", password.len()))
-        .arg(Path::new("/dev").join(guid).join(name))
-        .arg(format!("{}_{}", guid, name))
-        .invoke(crate::ErrorKind::DiskManagement)
-        .await?;
-    let mapper_path = Path::new("/dev/mapper").join(format!("{}_{}", guid, name));
-    let reboot = repair.e2fsck(&mapper_path).await?;
-    mount(&mapper_path, datadir.as_ref().join(name), ReadWrite).await?;
+    let orig_path = Path::new("/dev").join(guid).join(name);
+    let mut blockdev_path = orig_path.clone();
+    let full_name = format!("{}_{}", guid, name);
+    if !guid.ends_with("_UNENC") {
+        let password = password.unwrap_or(DEFAULT_PASSWORD);
+        if let Some(parent) = Path::new(PASSWORD_PATH).parent() {
+            tokio::fs::create_dir_all(parent).await?;
+        }
+        tokio::fs::write(PASSWORD_PATH, password)
+            .await
+            .with_ctx(|_| (crate::ErrorKind::Filesystem, PASSWORD_PATH))?;
+        Command::new("cryptsetup")
+            .arg("-q")
+            .arg("luksOpen")
+            .arg(format!("--key-file={}", PASSWORD_PATH))
+            .arg(format!("--keyfile-size={}", password.len()))
+            .arg(&blockdev_path)
+            .arg(&full_name)
+            .invoke(crate::ErrorKind::DiskManagement)
+            .await?;
+        tokio::fs::remove_file(PASSWORD_PATH)
+            .await
+            .with_ctx(|_| (crate::ErrorKind::Filesystem, PASSWORD_PATH))?;
+        blockdev_path = Path::new("/dev/mapper").join(&full_name);
+    }
+    let reboot = repair.fsck(&blockdev_path).await?;
 
-    tokio::fs::remove_file(PASSWORD_PATH)
-        .await
-        .with_ctx(|_| (crate::ErrorKind::Filesystem, PASSWORD_PATH))?;
+    if !guid.ends_with("_UNENC") {
+        // Backup LUKS header if e2fsck succeeded
+        let luks_folder = Path::new("/media/embassy/config/luks");
+        tokio::fs::create_dir_all(luks_folder).await?;
+        let tmp_luks_bak = luks_folder.join(format!(".{full_name}.luks.bak.tmp"));
+        if tokio::fs::metadata(&tmp_luks_bak).await.is_ok() {
+            tokio::fs::remove_file(&tmp_luks_bak).await?;
+        }
+        let luks_bak = luks_folder.join(format!("{full_name}.luks.bak"));
+        Command::new("cryptsetup")
+            .arg("-q")
+            .arg("luksHeaderBackup")
+            .arg("--header-backup-file")
+            .arg(&tmp_luks_bak)
+            .arg(&orig_path)
+            .invoke(crate::ErrorKind::DiskManagement)
+            .await?;
+        tokio::fs::rename(&tmp_luks_bak, &luks_bak).await?;
+    }
+
+    mount(&blockdev_path, datadir.as_ref().join(name), ReadWrite).await?;
 
     Ok(reboot)
 }
@@ -290,7 +327,7 @@ pub async fn mount_all_fs<P: AsRef<Path>>(
     guid: &str,
     datadir: P,
     repair: RepairStrategy,
-    password: &str,
+    password: Option<&str>,
 ) -> Result<RequiresReboot, Error> {
     let mut reboot = RequiresReboot(false);
     reboot |= mount_fs(guid, &datadir, "main", repair, password).await?;

backend/src/disk/mod.rs

@@ -22,6 +22,7 @@ pub const REPAIR_DISK_PATH: &str = "/media/embassy/config/repair-disk";
 #[serde(rename_all = "kebab-case")]
 pub struct OsPartitionInfo {
     pub efi: Option<PathBuf>,
+    pub bios: Option<PathBuf>,
     pub boot: PathBuf,
     pub root: PathBuf,
 }
@@ -31,6 +32,11 @@ impl OsPartitionInfo {
             .as_ref()
             .map(|p| p == logicalname.as_ref())
             .unwrap_or(false)
+            || self
+                .bios
+                .as_ref()
+                .map(|p| p == logicalname.as_ref())
+                .unwrap_or(false)
             || &*self.boot == logicalname.as_ref()
             || &*self.root == logicalname.as_ref()
     }

backend/src/disk/mount/filesystem/cifs.rs

@@ -16,6 +16,9 @@ use crate::util::Invoke;
 use crate::Error;
 
 async fn resolve_hostname(hostname: &str) -> Result<IpAddr, Error> {
+    if let Ok(addr) = hostname.parse() {
+        return Ok(addr);
+    }
     #[cfg(feature = "avahi")]
     if hostname.ends_with(".local") {
         return Ok(IpAddr::V4(crate::net::mdns::resolve_mdns(hostname).await?));

backend/src/disk/mount/filesystem/efivarfs.rs

@@ -1,4 +1,3 @@
-use std::os::unix::ffi::OsStrExt;
 use std::path::Path;
 
 use async_trait::async_trait;

backend/src/disk/util.rs

@@ -1,4 +1,4 @@
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, BTreeSet};
 use std::path::{Path, PathBuf};
 
 use color_eyre::eyre::{self, eyre};
@@ -251,7 +251,7 @@ pub async fn recovery_info(
 #[instrument(skip_all)]
 pub async fn list(os: &OsPartitionInfo) -> Result<Vec<DiskInfo>, Error> {
     struct DiskIndex {
-        parts: IndexSet<PathBuf>,
+        parts: BTreeSet<PathBuf>,
         internal: bool,
     }
     let disk_guids = pvscan().await?;
@@ -301,7 +301,7 @@ pub async fn list(os: &OsPartitionInfo) -> Result<Vec<DiskInfo>, Error> {
                     disks.insert(
                         disk.clone(),
                         DiskIndex {
-                            parts: IndexSet::new(),
+                            parts: BTreeSet::new(),
                             internal: false,
                         },
                     );
@@ -324,11 +324,13 @@ pub async fn list(os: &OsPartitionInfo) -> Result<Vec<DiskInfo>, Error> {
         if index.internal {
             for part in index.parts {
                 let mut disk_info = disk_info(disk.clone()).await;
-                disk_info.logicalname = part;
+                let part_info = part_info(part).await;
+                disk_info.logicalname = part_info.logicalname.clone();
+                disk_info.capacity = part_info.capacity;
                 if let Some(g) = disk_guids.get(&disk_info.logicalname) {
                     disk_info.guid = g.clone();
                 } else {
-                    disk_info.partitions = vec![part_info(disk_info.logicalname.clone()).await];
+                    disk_info.partitions = vec![part_info];
                 }
                 res.push(disk_info);
             }

backend/src/hostname.rs

@@ -56,7 +56,8 @@ pub async fn get_current_hostname() -> Result<Hostname, Error> {
 #[instrument(skip_all)]
 pub async fn set_hostname(hostname: &Hostname) -> Result<(), Error> {
     let hostname: &String = &hostname.0;
-    let _out = Command::new("hostnamectl")
+    Command::new("hostnamectl")
+        .arg("--static")
         .arg("set-hostname")
         .arg(hostname)
         .invoke(ErrorKind::ParseSysInfo)
@@ -65,8 +66,8 @@ pub async fn set_hostname(hostname: &Hostname) -> Result<(), Error> {
 }
 
 #[instrument(skip_all)]
-pub async fn sync_hostname(account: &AccountInfo) -> Result<(), Error> {
-    set_hostname(&account.hostname).await?;
+pub async fn sync_hostname(hostname: &Hostname) -> Result<(), Error> {
+    set_hostname(hostname).await?;
     Command::new("systemctl")
         .arg("restart")
         .arg("avahi-daemon")

backend/src/init.rs

@@ -1,8 +1,7 @@
-use std::collections::{BTreeMap, HashMap};
+use std::collections::HashMap;
 use std::fs::Permissions;
 use std::os::unix::fs::PermissionsExt;
 use std::path::Path;
-use std::process::Stdio;
 use std::time::Duration;
 
 use color_eyre::eyre::eyre;
@@ -15,7 +14,8 @@ use tokio::process::Command;
 
 use crate::account::AccountInfo;
 use crate::context::rpc::RpcContextConfig;
-use crate::db::model::{IpInfo, ServerStatus};
+use crate::db::model::{ServerInfo, ServerStatus};
+use crate::disk::mount::util::unmount;
 use crate::install::PKG_ARCHIVE_DIR;
 use crate::middleware::auth::LOCAL_AUTH_COOKIE_PATH;
 use crate::sound::BEP;
@@ -40,17 +40,18 @@ pub async fn check_time_is_synchronized() -> Result<bool, Error> {
 }
 
 pub struct InitReceipts {
+    pub server_info: LockReceipt<ServerInfo, ()>,
     pub server_version: LockReceipt<crate::util::Version, ()>,
     pub version_range: LockReceipt<emver::VersionRange, ()>,
-    pub last_wifi_region: LockReceipt<Option<isocountry::CountryCode>, ()>,
-    pub status_info: LockReceipt<ServerStatus, ()>,
-    pub ip_info: LockReceipt<BTreeMap<String, IpInfo>, ()>,
-    pub system_start_time: LockReceipt<String, ()>,
 }
 impl InitReceipts {
     pub async fn new(db: &mut impl DbHandle) -> Result<Self, Error> {
         let mut locks = Vec::new();
 
+        let server_info = crate::db::DatabaseModel::new()
+            .server_info()
+            .make_locker(LockType::Write)
+            .add_to_keys(&mut locks);
         let server_version = crate::db::DatabaseModel::new()
             .server_info()
             .version()
@@ -61,112 +62,29 @@ impl InitReceipts {
             .eos_version_compat()
             .make_locker(LockType::Write)
             .add_to_keys(&mut locks);
-        let last_wifi_region = crate::db::DatabaseModel::new()
-            .server_info()
-            .last_wifi_region()
-            .make_locker(LockType::Write)
-            .add_to_keys(&mut locks);
-        let ip_info = crate::db::DatabaseModel::new()
-            .server_info()
-            .ip_info()
-            .make_locker(LockType::Write)
-            .add_to_keys(&mut locks);
-        let status_info = crate::db::DatabaseModel::new()
-            .server_info()
-            .status_info()
-            .into_model()
-            .make_locker(LockType::Write)
-            .add_to_keys(&mut locks);
-        let system_start_time = crate::db::DatabaseModel::new()
-            .server_info()
-            .system_start_time()
-            .make_locker(LockType::Write)
-            .add_to_keys(&mut locks);
 
         let skeleton_key = db.lock_all(locks).await?;
         Ok(Self {
+            server_info: server_info.verify(&skeleton_key)?,
             server_version: server_version.verify(&skeleton_key)?,
             version_range: version_range.verify(&skeleton_key)?,
-            ip_info: ip_info.verify(&skeleton_key)?,
-            status_info: status_info.verify(&skeleton_key)?,
-            last_wifi_region: last_wifi_region.verify(&skeleton_key)?,
-            system_start_time: system_start_time.verify(&skeleton_key)?,
         })
     }
 }
 
-pub async fn pgloader(
-    old_db_path: impl AsRef<Path>,
-    batch_rows: usize,
-    prefetch_rows: usize,
-) -> Result<(), Error> {
-    tokio::fs::write(
-        "/etc/embassy/migrate.load",
-        format!(
-            include_str!("migrate.load"),
-            sqlite_path = old_db_path.as_ref().display(),
-            batch_rows = batch_rows,
-            prefetch_rows = prefetch_rows
-        ),
-    )
-    .await?;
-    match tokio::fs::remove_dir_all("/tmp/pgloader").await {
-        Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
-        a => a,
-    }?;
-    tracing::info!("Running pgloader");
-    let out = Command::new("pgloader")
-        .arg("-v")
-        .arg("/etc/embassy/migrate.load")
-        .stdout(Stdio::piped())
-        .stderr(Stdio::piped())
-        .output()
-        .await?;
-    let stdout = String::from_utf8(out.stdout)?;
-    for line in stdout.lines() {
-        tracing::debug!("pgloader: {}", line);
-    }
-    let stderr = String::from_utf8(out.stderr)?;
-    for line in stderr.lines() {
-        tracing::debug!("pgloader err: {}", line);
-    }
-    tracing::debug!("pgloader exited with code {:?}", out.status);
-    if let Some(err) = stdout.lines().chain(stderr.lines()).find_map(|l| {
-        if l.split_ascii_whitespace()
-            .any(|word| word == "ERROR" || word == "FATAL")
-        {
-            Some(l)
-        } else {
-            None
-        }
-    }) {
-        return Err(Error::new(
-            eyre!("pgloader error: {}", err),
-            crate::ErrorKind::Database,
-        ));
-    }
-    tokio::fs::rename(
-        old_db_path.as_ref(),
-        old_db_path.as_ref().with_extension("bak"),
-    )
-    .await?;
-    Ok(())
-}
-
 // must be idempotent
 pub async fn init_postgres(datadir: impl AsRef<Path>) -> Result<(), Error> {
     let db_dir = datadir.as_ref().join("main/postgresql");
-    let is_mountpoint = || async {
-        Ok::<_, Error>(
-            tokio::process::Command::new("mountpoint")
-                .arg("/var/lib/postgresql")
-                .stdout(std::process::Stdio::null())
-                .stderr(std::process::Stdio::null())
-                .status()
-                .await?
-                .success(),
-        )
-    };
+    if tokio::process::Command::new("mountpoint")
+        .arg("/var/lib/postgresql")
+        .stdout(std::process::Stdio::null())
+        .stderr(std::process::Stdio::null())
+        .status()
+        .await?
+        .success()
+    {
+        unmount("/var/lib/postgresql").await?;
+    }
     let exists = tokio::fs::metadata(&db_dir).await.is_ok();
     if !exists {
         Command::new("cp")
@@ -176,18 +94,73 @@ pub async fn init_postgres(datadir: impl AsRef<Path>) -> Result<(), Error> {
             .invoke(crate::ErrorKind::Filesystem)
             .await?;
     }
-    if !is_mountpoint().await? {
-        crate::disk::mount::util::bind(&db_dir, "/var/lib/postgresql", false).await?;
-    }
     Command::new("chown")
         .arg("-R")
-        .arg("postgres")
-        .arg("/var/lib/postgresql")
+        .arg("postgres:postgres")
+        .arg(&db_dir)
         .invoke(crate::ErrorKind::Database)
         .await?;
+
+    let mut pg_paths = tokio::fs::read_dir("/usr/lib/postgresql").await?;
+    let mut pg_version = None;
+    while let Some(pg_path) = pg_paths.next_entry().await? {
+        let pg_path_version = pg_path
+            .file_name()
+            .to_str()
+            .map(|v| v.parse())
+            .transpose()?
+            .unwrap_or(0);
+        if pg_path_version > pg_version.unwrap_or(0) {
+            pg_version = Some(pg_path_version)
+        }
+    }
+    let pg_version = pg_version.ok_or_else(|| {
+        Error::new(
+            eyre!("could not determine postgresql version"),
+            crate::ErrorKind::Database,
+        )
+    })?;
+
+    crate::disk::mount::util::bind(&db_dir, "/var/lib/postgresql", false).await?;
+
+    let pg_version_string = pg_version.to_string();
+    let pg_version_path = db_dir.join(&pg_version_string);
+    if tokio::fs::metadata(&pg_version_path).await.is_err() {
+        let conf_dir = Path::new("/etc/postgresql").join(pg_version.to_string());
+        let conf_dir_tmp = {
+            let mut tmp = conf_dir.clone();
+            tmp.set_extension("tmp");
+            tmp
+        };
+        if tokio::fs::metadata(&conf_dir).await.is_ok() {
+            tokio::fs::rename(&conf_dir, &conf_dir_tmp).await?;
+        }
+        let mut old_version = pg_version;
+        while old_version > 13
+        /* oldest pg version included in startos */
+        {
+            old_version -= 1;
+            let old_datadir = db_dir.join(old_version.to_string());
+            if tokio::fs::metadata(&old_datadir).await.is_ok() {
+                Command::new("pg_upgradecluster")
+                    .arg(old_version.to_string())
+                    .arg("main")
+                    .invoke(crate::ErrorKind::Database)
+                    .await?;
+                break;
+            }
+        }
+        if tokio::fs::metadata(&conf_dir).await.is_ok() {
+            if tokio::fs::metadata(&conf_dir).await.is_ok() {
+                tokio::fs::remove_dir_all(&conf_dir).await?;
+            }
+            tokio::fs::rename(&conf_dir_tmp, &conf_dir).await?;
+        }
+    }
+
     Command::new("systemctl")
         .arg("start")
-        .arg("postgresql")
+        .arg(format!("postgresql@{pg_version}-main.service"))
         .invoke(crate::ErrorKind::Database)
         .await?;
     if !exists {
@@ -208,6 +181,7 @@ pub async fn init_postgres(datadir: impl AsRef<Path>) -> Result<(), Error> {
             .invoke(crate::ErrorKind::Database)
             .await?;
     }
+
     Ok(())
 }
 
@@ -250,15 +224,15 @@ pub async fn init(cfg: &RpcContextConfig) -> Result<InitResult, Error> {
     let db = cfg.db(&account).await?;
     tracing::info!("Opened PatchDB");
     let mut handle = db.handle();
-    crate::db::DatabaseModel::new()
+    let mut server_info = crate::db::DatabaseModel::new()
         .server_info()
-        .lock(&mut handle, LockType::Write)
+        .get_mut(&mut handle)
         .await?;
     let receipts = InitReceipts::new(&mut handle).await?;
 
     // write to ca cert store
     tokio::fs::write(
-        "/usr/local/share/ca-certificates/embassy-root-ca.crt",
+        "/usr/local/share/ca-certificates/startos-root-ca.crt",
         account.root_ca_cert.to_pem()?,
     )
     .await?;
@@ -270,17 +244,15 @@ pub async fn init(cfg: &RpcContextConfig) -> Result<InitResult, Error> {
         crate::net::wifi::synchronize_wpa_supplicant_conf(
             &cfg.datadir().join("main"),
             wifi_interface,
-            &receipts.last_wifi_region.get(&mut handle).await?,
+            &server_info.last_wifi_region,
         )
         .await?;
         tracing::info!("Synchronized WiFi");
     }
 
     let should_rebuild = tokio::fs::metadata(SYSTEM_REBUILD_PATH).await.is_ok()
-        || &*receipts.server_version.get(&mut handle).await? < &emver::Version::new(0, 3, 2, 0)
-        || (*ARCH == "x86_64"
-            && &*receipts.server_version.get(&mut handle).await?
-                < &emver::Version::new(0, 3, 4, 0));
+        || &*server_info.version < &emver::Version::new(0, 3, 2, 0)
+        || (*ARCH == "x86_64" && &*server_info.version < &emver::Version::new(0, 3, 4, 0));
 
     let song = if should_rebuild {
         Some(NonDetachingJoinHandle::from(tokio::spawn(async {
@@ -398,32 +370,19 @@ pub async fn init(cfg: &RpcContextConfig) -> Result<InitResult, Error> {
         tracing::info!("Syncronized system clock");
     }
 
-    Command::new("systemctl")
-        .arg("start")
-        .arg("tor")
-        .invoke(crate::ErrorKind::Tor)
-        .await?;
+    if server_info.zram {
+        crate::system::enable_zram().await?
+    }
+    server_info.ip_info = crate::net::dhcp::init_ips().await?;
+    server_info.status_info = ServerStatus {
+        updated: false,
+        update_progress: None,
+        backup_progress: None,
+    };
 
-    receipts
-        .ip_info
-        .set(&mut handle, crate::net::dhcp::init_ips().await?)
-        .await?;
-    receipts
-        .status_info
-        .set(
-            &mut handle,
-            ServerStatus {
-                updated: false,
-                update_progress: None,
-                backup_progress: None,
-            },
-        )
-        .await?;
+    server_info.system_start_time = time().await?;
 
-    receipts
-        .system_start_time
-        .set(&mut handle, time().await?)
-        .await?;
+    server_info.save(&mut handle).await?;
 
     crate::version::init(&mut handle, &secret_store, &receipts).await?;
 

backend/src/install/cleanup.rs

@@ -82,7 +82,7 @@ pub async fn update_dependency_errors_of_dependents<'a, Db: DbHandle>(
                     .dependency_errors
                     .get(db, dep)
                     .await?
-                    .ok_or_else(not_found)?;
+                    .ok_or_else(|| not_found!(dep))?;
                 errs.0.insert(id.clone(), e);
                 receipts.dependency_errors.set(db, errs, dep).await?
             } else {
@@ -90,7 +90,7 @@ pub async fn update_dependency_errors_of_dependents<'a, Db: DbHandle>(
                     .dependency_errors
                     .get(db, dep)
                     .await?
-                    .ok_or_else(not_found)?;
+                    .ok_or_else(|| not_found!(dep))?;
                 errs.0.remove(id);
                 receipts.dependency_errors.set(db, errs, dep).await?
             }
@@ -215,7 +215,7 @@ pub async fn cleanup_failed<Db: DbHandle>(
         .package_data_entry
         .get(db, id)
         .await?
-        .ok_or_else(not_found)?;
+        .ok_or_else(|| not_found!(id))?;
     if let Some(manifest) = match &pde {
         PackageDataEntry::Installing { manifest, .. }
         | PackageDataEntry::Restoring { manifest, .. } => Some(manifest),

backend/src/install/mod.rs

@@ -21,6 +21,7 @@ use rpc_toolkit::yajrc::RpcError;
 use tokio::fs::{File, OpenOptions};
 use tokio::io::{AsyncRead, AsyncSeek, AsyncSeekExt};
 use tokio::process::Command;
+use tokio::sync::oneshot;
 use tokio_stream::wrappers::ReadDirStream;
 use tracing::instrument;
 
@@ -39,13 +40,14 @@ use crate::dependencies::{
 };
 use crate::install::cleanup::{cleanup, update_dependency_errors_of_dependents};
 use crate::install::progress::{InstallProgress, InstallProgressTracker};
+use crate::marketplace::with_query_params;
 use crate::notifications::NotificationLevel;
 use crate::s9pk::manifest::{Manifest, PackageId};
 use crate::s9pk::reader::S9pkReader;
 use crate::status::{MainStatus, Status};
 use crate::util::io::{copy_and_shutdown, response_to_reader};
 use crate::util::serde::{display_serializable, Port};
-use crate::util::{assure_send, display_none, AsyncFileExt, Version};
+use crate::util::{display_none, AsyncFileExt, Version};
 use crate::version::{Current, VersionT};
 use crate::volume::{asset_dir, script_dir};
 use crate::{Error, ErrorKind, ResultExt};
@@ -135,35 +137,39 @@ pub async fn install(
     let marketplace_url =
         marketplace_url.unwrap_or_else(|| crate::DEFAULT_MARKETPLACE.parse().unwrap());
     let version_priority = version_priority.unwrap_or_default();
-    let man: Manifest = reqwest::get(format!(
-        "{}/package/v0/manifest/{}?spec={}&version-priority={}&eos-version-compat={}&arch={}",
-        marketplace_url,
-        id,
-        version,
-        version_priority,
-        Current::new().compat(),
-        &*crate::ARCH,
-    ))
-    .await
-    .with_kind(crate::ErrorKind::Registry)?
-    .error_for_status()
-    .with_kind(crate::ErrorKind::Registry)?
-    .json()
-    .await
-    .with_kind(crate::ErrorKind::Registry)?;
-    let s9pk = reqwest::get(format!(
-        "{}/package/v0/{}.s9pk?spec=={}&version-priority={}&eos-version-compat={}&arch={}",
-        marketplace_url,
-        id,
-        man.version,
-        version_priority,
-        Current::new().compat(),
-        &*crate::ARCH,
-    ))
-    .await
-    .with_kind(crate::ErrorKind::Registry)?
-    .error_for_status()
-    .with_kind(crate::ErrorKind::Registry)?;
+    let man: Manifest = ctx
+        .client
+        .get(with_query_params(
+            &ctx,
+            format!(
+                "{}/package/v0/manifest/{}?spec={}&version-priority={}",
+                marketplace_url, id, version, version_priority,
+            )
+            .parse()?,
+        ))
+        .send()
+        .await
+        .with_kind(crate::ErrorKind::Registry)?
+        .error_for_status()
+        .with_kind(crate::ErrorKind::Registry)?
+        .json()
+        .await
+        .with_kind(crate::ErrorKind::Registry)?;
+    let s9pk = ctx
+        .client
+        .get(with_query_params(
+            &ctx,
+            format!(
+                "{}/package/v0/{}.s9pk?spec=={}&version-priority={}",
+                marketplace_url, id, man.version, version_priority,
+            )
+            .parse()?,
+        ))
+        .send()
+        .await
+        .with_kind(crate::ErrorKind::Registry)?
+        .error_for_status()
+        .with_kind(crate::ErrorKind::Registry)?;
 
     if man.id.as_str() != id || !man.version.satisfies(&version) {
         return Err(Error::new(
@@ -184,16 +190,18 @@ pub async fn install(
         async {
             tokio::io::copy(
                 &mut response_to_reader(
-                    reqwest::get(format!(
-                        "{}/package/v0/license/{}?spec=={}&eos-version-compat={}&arch={}",
-                        marketplace_url,
-                        id,
-                        man.version,
-                        Current::new().compat(),
-                        &*crate::ARCH,
-                    ))
-                    .await?
-                    .error_for_status()?,
+                    ctx.client
+                        .get(with_query_params(
+                            &ctx,
+                            format!(
+                                "{}/package/v0/license/{}?spec=={}",
+                                marketplace_url, id, man.version,
+                            )
+                            .parse()?,
+                        ))
+                        .send()
+                        .await?
+                        .error_for_status()?,
                 ),
                 &mut File::create(public_dir_path.join("LICENSE.md")).await?,
             )
@@ -203,16 +211,18 @@ pub async fn install(
         async {
             tokio::io::copy(
                 &mut response_to_reader(
-                    reqwest::get(format!(
-                        "{}/package/v0/instructions/{}?spec=={}&eos-version-compat={}&arch={}",
-                        marketplace_url,
-                        id,
-                        man.version,
-                        Current::new().compat(),
-                        &*crate::ARCH,
-                    ))
-                    .await?
-                    .error_for_status()?,
+                    ctx.client
+                        .get(with_query_params(
+                            &ctx,
+                            format!(
+                                "{}/package/v0/instructions/{}?spec=={}",
+                                marketplace_url, id, man.version,
+                            )
+                            .parse()?,
+                        ))
+                        .send()
+                        .await?
+                        .error_for_status()?,
                 ),
                 &mut File::create(public_dir_path.join("INSTRUCTIONS.md")).await?,
             )
@@ -222,16 +232,18 @@ pub async fn install(
         async {
             tokio::io::copy(
                 &mut response_to_reader(
-                    reqwest::get(format!(
-                        "{}/package/v0/icon/{}?spec=={}&eos-version-compat={}&arch={}",
-                        marketplace_url,
-                        id,
-                        man.version,
-                        Current::new().compat(),
-                        &*crate::ARCH,
-                    ))
-                    .await?
-                    .error_for_status()?,
+                    ctx.client
+                        .get(with_query_params(
+                            &ctx,
+                            format!(
+                                "{}/package/v0/icon/{}?spec=={}",
+                                marketplace_url, id, man.version,
+                            )
+                            .parse()?,
+                        ))
+                        .send()
+                        .await?
+                        .error_for_status()?,
                 ),
                 &mut File::create(public_dir_path.join(format!("icon.{}", icon_type))).await?,
             )
@@ -297,6 +309,7 @@ pub async fn install(
             Some(marketplace_url),
             InstallProgress::new(s9pk.content_length()),
             response_to_reader(s9pk),
+            None,
         )
         .await
         {
@@ -425,52 +438,64 @@ pub async fn sideload(
             pde.save(&mut tx).await?;
             tx.commit().await?;
 
-            if let Err(e) = download_install_s9pk(
-                &new_ctx,
-                &manifest,
-                None,
-                progress,
-                tokio_util::io::StreamReader::new(req.into_body().map_err(|e| {
-                    std::io::Error::new(
-                        match &e {
-                            e if e.is_connect() => std::io::ErrorKind::ConnectionRefused,
-                            e if e.is_timeout() => std::io::ErrorKind::TimedOut,
-                            _ => std::io::ErrorKind::Other,
-                        },
-                        e,
-                    )
-                })),
-            )
-            .await
-            {
-                let err_str = format!(
-                    "Install of {}@{} Failed: {}",
-                    manifest.id, manifest.version, e
-                );
-                tracing::error!("{}", err_str);
-                tracing::debug!("{:?}", e);
-                if let Err(e) = new_ctx
-                    .notification_manager
-                    .notify(
-                        &mut hdl,
-                        Some(manifest.id),
-                        NotificationLevel::Error,
-                        String::from("Install Failed"),
-                        err_str,
-                        (),
-                        None,
-                    )
-                    .await
-                {
-                    tracing::error!("Failed to issue Notification: {}", e);
-                    tracing::debug!("{:?}", e);
-                }
-            }
+            let (send, recv) = oneshot::channel();
 
-            Response::builder()
-                .status(StatusCode::OK)
-                .body(Body::empty())
-                .with_kind(ErrorKind::Network)
+            tokio::spawn(async move {
+                if let Err(e) = download_install_s9pk(
+                    &new_ctx,
+                    &manifest,
+                    None,
+                    progress,
+                    tokio_util::io::StreamReader::new(req.into_body().map_err(|e| {
+                        std::io::Error::new(
+                            match &e {
+                                e if e.is_connect() => std::io::ErrorKind::ConnectionRefused,
+                                e if e.is_timeout() => std::io::ErrorKind::TimedOut,
+                                _ => std::io::ErrorKind::Other,
+                            },
+                            e,
+                        )
+                    })),
+                    Some(send),
+                )
+                .await
+                {
+                    let err_str = format!(
+                        "Install of {}@{} Failed: {}",
+                        manifest.id, manifest.version, e
+                    );
+                    tracing::error!("{}", err_str);
+                    tracing::debug!("{:?}", e);
+                    if let Err(e) = new_ctx
+                        .notification_manager
+                        .notify(
+                            &mut hdl,
+                            Some(manifest.id),
+                            NotificationLevel::Error,
+                            String::from("Install Failed"),
+                            err_str,
+                            (),
+                            None,
+                        )
+                        .await
+                    {
+                        tracing::error!("Failed to issue Notification: {}", e);
+                        tracing::debug!("{:?}", e);
+                    }
+                }
+            });
+
+            if let Ok(_) = recv.await {
+                Response::builder()
+                    .status(StatusCode::OK)
+                    .body(Body::empty())
+                    .with_kind(ErrorKind::Network)
+            } else {
+                Response::builder()
+                    .status(StatusCode::INTERNAL_SERVER_ERROR)
+                    .body(Body::from("installation aborted before upload completed"))
+                    .with_kind(ErrorKind::Network)
+            }
         }
         .boxed()
     });
@@ -707,6 +732,7 @@ pub async fn download_install_s9pk(
     marketplace_url: Option<Url>,
     progress: Arc<InstallProgress>,
     mut s9pk: impl AsyncRead + Unpin,
+    download_complete: Option<oneshot::Sender<()>>,
 ) -> Result<(), Error> {
     let pkg_id = &temp_manifest.id;
     let version = &temp_manifest.version;
@@ -799,6 +825,9 @@ pub async fn download_install_s9pk(
                 let mut progress_writer = InstallProgressTracker::new(&mut dst, progress.clone());
                 tokio::io::copy(&mut s9pk, &mut progress_writer).await?;
                 progress.download_complete();
+                if let Some(complete) = download_complete {
+                    complete.send(()).unwrap_or_default();
+                }
                 Ok(())
             })
             .await?;
@@ -910,17 +939,20 @@ pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin + Send + Sync>(
         {
             Some(local_man)
         } else if let Some(marketplace_url) = &marketplace_url {
-            match reqwest::get(format!(
-                "{}/package/v0/manifest/{}?spec={}&eos-version-compat={}&arch={}",
-                marketplace_url,
-                dep,
-                info.version,
-                Current::new().compat(),
-                &*crate::ARCH,
-            ))
-            .await
-            .with_kind(crate::ErrorKind::Registry)?
-            .error_for_status()
+            match ctx
+                .client
+                .get(with_query_params(
+                    ctx,
+                    format!(
+                        "{}/package/v0/manifest/{}?spec={}",
+                        marketplace_url, dep, info.version,
+                    )
+                    .parse()?,
+                ))
+                .send()
+                .await
+                .with_kind(crate::ErrorKind::Registry)?
+                .error_for_status()
             {
                 Ok(a) => Ok(Some(
                     a.json()
@@ -945,16 +977,19 @@ pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin + Send + Sync>(
                 let icon_path = dir.join(format!("icon.{}", manifest.assets.icon_type()));
                 if tokio::fs::metadata(&icon_path).await.is_err() {
                     tokio::fs::create_dir_all(&dir).await?;
-                    let icon = reqwest::get(format!(
-                        "{}/package/v0/icon/{}?spec={}&eos-version-compat={}&arch={}",
-                        marketplace_url,
-                        dep,
-                        info.version,
-                        Current::new().compat(),
-                        &*crate::ARCH,
-                    ))
-                    .await
-                    .with_kind(crate::ErrorKind::Registry)?;
+                    let icon = ctx
+                        .client
+                        .get(with_query_params(
+                            ctx,
+                            format!(
+                                "{}/package/v0/icon/{}?spec={}",
+                                marketplace_url, dep, info.version,
+                            )
+                            .parse()?,
+                        ))
+                        .send()
+                        .await
+                        .with_kind(crate::ErrorKind::Registry)?;
                     let mut dst = File::create(&icon_path).await?;
                     tokio::io::copy(&mut response_to_reader(icon), &mut dst).await?;
                     dst.sync_all().await?;
@@ -1279,6 +1314,14 @@ pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin + Send + Sync>(
             migration.or(prev_migration)
         };
 
+        remove_from_current_dependents_lists(
+            &mut tx,
+            pkg_id,
+            &prev.current_dependencies,
+            &receipts.config.current_dependents,
+        )
+        .await?; // remove previous
+
         let configured = if let Some(f) = viable_migration {
             f.await?.configured && prev_is_configured
         } else {
@@ -1298,13 +1341,6 @@ pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin + Send + Sync>(
             )
             .await?;
         } else {
-            remove_from_current_dependents_lists(
-                &mut tx,
-                pkg_id,
-                &prev.current_dependencies,
-                &receipts.config.current_dependents,
-            )
-            .await?; // remove previous
             add_dependent_to_current_dependents_lists(
                 &mut tx,
                 pkg_id,
@@ -1435,16 +1471,23 @@ pub fn load_images<'a, P: AsRef<Path> + 'a + Send + Sync>(
                                     copy_and_shutdown(&mut File::open(&path).await?, load_in)
                                         .await?
                                 }
-                                Some("s9pk") => {
-                                    copy_and_shutdown(
-                                        &mut S9pkReader::open(&path, false)
-                                            .await?
-                                            .docker_images()
-                                            .await?,
-                                        load_in,
-                                    )
-                                    .await?
+                                Some("s9pk") => match async {
+                                    let mut reader = S9pkReader::open(&path, true).await?;
+                                    copy_and_shutdown(&mut reader.docker_images().await?, load_in)
+                                        .await?;
+                                    Ok::<_, Error>(())
                                 }
+                                .await
+                                {
+                                    Ok(()) => (),
+                                    Err(e) => {
+                                        tracing::error!(
+                                            "Error loading docker images from s9pk: {e}"
+                                        );
+                                        tracing::debug!("{e:?}");
+                                        return Ok(());
+                                    }
+                                },
                                 _ => unreachable!(),
                             };
 

backend/src/install/update.rs

@@ -76,7 +76,7 @@ pub async fn dry(
         .current_dependents
         .get(&mut tx, &id)
         .await?
-        .ok_or_else(not_found)?
+        .ok_or_else(|| not_found!(id))?
         .0
         .keys()
         .into_iter()

backend/src/lib.rs

@@ -5,20 +5,19 @@ pub const DEFAULT_MARKETPLACE: &str = "https://registry.start9.com";
 pub const BUFFER_SIZE: usize = 1024;
 pub const HOST_IP: [u8; 4] = [172, 18, 0, 1];
 pub const TARGET: &str = current_platform::CURRENT_PLATFORM;
+pub const OS_ARCH: &str = env!("OS_ARCH");
 lazy_static::lazy_static! {
     pub static ref ARCH: &'static str = {
         let (arch, _) = TARGET.split_once("-").unwrap();
         arch
     };
-    pub static ref IS_RASPBERRY_PI: bool = {
-        *ARCH == "aarch64"
-    };
 }
 
 pub mod account;
 pub mod action;
 pub mod auth;
 pub mod backup;
+pub mod bins;
 pub mod config;
 pub mod context;
 pub mod control;
@@ -87,6 +86,7 @@ pub fn main_api() -> Result<(), RpcError> {
 
 #[command(subcommands(
     system::time,
+    system::experimental,
     system::logs,
     system::kernel_logs,
     system::metrics,

backend/src/logs.rs

@@ -33,7 +33,7 @@ use crate::util::serde::Reversible;
 use crate::{Error, ErrorKind};
 
 #[pin_project::pin_project]
-struct LogStream {
+pub struct LogStream {
     _child: Child,
     #[pin]
     entries: BoxStream<'static, Result<JournalctlEntry, Error>>,
@@ -141,14 +141,14 @@ impl std::fmt::Display for LogEntry {
 }
 
 #[derive(Serialize, Deserialize, Debug)]
-struct JournalctlEntry {
+pub struct JournalctlEntry {
     #[serde(rename = "__REALTIME_TIMESTAMP")]
-    timestamp: String,
+    pub timestamp: String,
     #[serde(rename = "MESSAGE")]
     #[serde(deserialize_with = "deserialize_string_or_utf8_array")]
-    message: String,
+    pub message: String,
     #[serde(rename = "__CURSOR")]
-    cursor: String,
+    pub cursor: String,
 }
 impl JournalctlEntry {
     fn log_entry(self) -> Result<(String, LogEntry), Error> {
@@ -344,7 +344,7 @@ pub async fn cli_logs_generic_follow(
     Ok(())
 }
 
-async fn journalctl(
+pub async fn journalctl(
     id: LogSource,
     limit: usize,
     cursor: Option<&str>,

backend/src/main.rs

@@ -0,0 +1,3 @@
+fn main() {
+    startos::bins::startbox()
+}

backend/src/manager/health.rs

@@ -1,6 +1,7 @@
 use std::collections::BTreeMap;
 use std::sync::atomic::{AtomicBool, Ordering};
 
+use itertools::Itertools;
 use patch_db::{DbHandle, LockReceipt, LockType};
 use tracing::instrument;
 
@@ -111,6 +112,7 @@ pub async fn check<Db: DbHandle>(
     };
 
     let health_results = if let Some(started) = started {
+        tracing::debug!("Checking health of {}", id);
         manifest
             .health_checks
             .check_all(
@@ -129,6 +131,24 @@ pub async fn check<Db: DbHandle>(
     if !should_commit.load(Ordering::SeqCst) {
         return Ok(());
     }
+
+    if !health_results
+        .iter()
+        .any(|(_, res)| matches!(res, HealthCheckResult::Failure { .. }))
+    {
+        tracing::debug!("All health checks succeeded for {}", id);
+    } else {
+        tracing::debug!(
+            "Some health checks failed for {}: {}",
+            id,
+            health_results
+                .iter()
+                .filter(|(_, res)| matches!(res, HealthCheckResult::Failure { .. }))
+                .map(|(id, _)| &*id)
+                .join(", ")
+        );
+    }
+
     let current_dependents = {
         let mut checkpoint = tx.begin().await?;
         let receipts = HealthCheckStatusReceipt::new(&mut checkpoint, id).await?;
@@ -153,9 +173,7 @@ pub async fn check<Db: DbHandle>(
         current_dependents
     };
 
-    tracing::debug!("Checking health of {}", id);
     let receipts = crate::dependencies::BreakTransitiveReceipts::new(&mut tx).await?;
-    tracing::debug!("Got receipts {}", id);
 
     for (dependent, info) in (current_dependents).0.iter() {
         let failures: BTreeMap<HealthCheckId, HealthCheckResult> = health_results

backend/src/manager/mod.rs

@@ -21,6 +21,7 @@ use tracing::instrument;
 use crate::context::RpcContext;
 use crate::manager::sync::synchronizer;
 use crate::net::net_controller::NetService;
+use crate::net::vhost::AlpnInfo;
 use crate::procedure::docker::{DockerContainer, DockerProcedure, LongRunning};
 #[cfg(feature = "js_engine")]
 use crate::procedure::js_scripts::JsProcedure;
@@ -573,8 +574,14 @@ async fn add_network_for_main(
     let mut tx = secrets.begin().await?;
     for (id, interface) in &seed.manifest.interfaces.0 {
         for (external, internal) in interface.lan_config.iter().flatten() {
-            svc.add_lan(&mut tx, id.clone(), external.0, internal.internal, false)
-                .await?;
+            svc.add_lan(
+                &mut tx,
+                id.clone(),
+                external.0,
+                internal.internal,
+                Err(AlpnInfo::Specified(vec![])),
+            )
+            .await?;
         }
         for (external, internal) in interface.tor_config.iter().flat_map(|t| &t.port_mapping) {
             svc.add_tor(&mut tx, id.clone(), external.0, internal.0)

backend/src/marketplace.rs

@@ -3,6 +3,8 @@ use reqwest::{StatusCode, Url};
 use rpc_toolkit::command;
 use serde_json::Value;
 
+use crate::context::RpcContext;
+use crate::version::VersionT;
 use crate::{Error, ResultExt};
 
 #[command(subcommands(get))]
@@ -10,9 +12,34 @@ pub fn marketplace() -> Result<(), Error> {
     Ok(())
 }
 
+pub fn with_query_params(ctx: &RpcContext, mut url: Url) -> Url {
+    url.query_pairs_mut()
+        .append_pair(
+            "os.version",
+            &crate::version::Current::new().semver().to_string(),
+        )
+        .append_pair(
+            "os.compat",
+            &crate::version::Current::new().compat().to_string(),
+        )
+        .append_pair("os.arch", crate::OS_ARCH)
+        .append_pair("hardware.arch", &*crate::ARCH)
+        .append_pair("hardware.ram", &ctx.hardware.ram.to_string());
+
+    for hw in &ctx.hardware.devices {
+        url.query_pairs_mut()
+            .append_pair(&format!("hardware.device.{}", hw.class()), hw.product());
+    }
+
+    url
+}
+
 #[command]
-pub async fn get(#[arg] url: Url) -> Result<Value, Error> {
-    let mut response = reqwest::get(url)
+pub async fn get(#[context] ctx: RpcContext, #[arg] url: Url) -> Result<Value, Error> {
+    let mut response = ctx
+        .client
+        .get(with_query_params(&ctx, url))
+        .send()
         .await
         .with_kind(crate::ErrorKind::Network)?;
     let status = response.status();

backend/src/net/cert-local.csr.conf.template

@@ -7,4 +7,4 @@ prompt = no
 [req_distinguished_name]
 CN = {hostname}.local
 O = Start9 Labs
-OU = Embassy
+OU = StartOS

backend/src/net/dns.rs

@@ -11,6 +11,7 @@ use models::PackageId;
 use tokio::net::{TcpListener, UdpSocket};
 use tokio::process::Command;
 use tokio::sync::RwLock;
+use tracing::instrument;
 use trust_dns_server::authority::MessageResponseBuilder;
 use trust_dns_server::client::op::{Header, ResponseCode};
 use trust_dns_server::client::rr::{Name, Record, RecordType};
@@ -147,6 +148,7 @@ impl RequestHandler for Resolver {
 }
 
 impl DnsController {
+    #[instrument(skip_all)]
     pub async fn init(bind: &[SocketAddr]) -> Result<Self, Error> {
         let services = Arc::new(RwLock::new(BTreeMap::new()));
 
@@ -161,10 +163,16 @@ impl DnsController {
         );
         server.register_socket(UdpSocket::bind(bind).await.with_kind(ErrorKind::Network)?);
 
-        Command::new("systemd-resolve")
-            .arg("--set-dns=127.0.0.1")
-            .arg("--interface=br-start9")
-            .arg("--set-domain=embassy")
+        Command::new("resolvectl")
+            .arg("dns")
+            .arg("br-start9")
+            .arg("127.0.0.1")
+            .invoke(ErrorKind::Network)
+            .await?;
+        Command::new("resolvectl")
+            .arg("domain")
+            .arg("br-start9")
+            .arg("embassy")
             .invoke(ErrorKind::Network)
             .await?;
 

backend/src/net/mdns.rs

@@ -5,6 +5,7 @@ use std::sync::{Arc, Weak};
 use color_eyre::eyre::eyre;
 use tokio::process::{Child, Command};
 use tokio::sync::Mutex;
+use tracing::instrument;
 
 use crate::util::Invoke;
 use crate::{Error, ResultExt};
@@ -51,6 +52,7 @@ pub struct MdnsControllerInner {
 }
 
 impl MdnsControllerInner {
+    #[instrument(skip_all)]
     async fn init() -> Result<Self, Error> {
         let mut res = MdnsControllerInner {
             alias_cmd: None,
@@ -59,6 +61,7 @@ impl MdnsControllerInner {
         res.sync().await?;
         Ok(res)
     }
+    #[instrument(skip_all)]
     async fn sync(&mut self) -> Result<(), Error> {
         if let Some(mut cmd) = self.alias_cmd.take() {
             cmd.kill().await.with_kind(crate::ErrorKind::Network)?;

backend/src/net/net_controller.rs

@@ -15,7 +15,7 @@ use crate::net::keys::Key;
 use crate::net::mdns::MdnsController;
 use crate::net::ssl::{export_cert, export_key, SslManager};
 use crate::net::tor::TorController;
-use crate::net::vhost::VHostController;
+use crate::net::vhost::{AlpnInfo, VHostController};
 use crate::s9pk::manifest::PackageId;
 use crate::volume::cert_dir;
 use crate::{Error, HOST_IP};
@@ -34,6 +34,7 @@ impl NetController {
     #[instrument(skip_all)]
     pub async fn init(
         tor_control: SocketAddr,
+        tor_socks: SocketAddr,
         dns_bind: &[SocketAddr],
         ssl: SslManager,
         hostname: &Hostname,
@@ -41,7 +42,7 @@ impl NetController {
     ) -> Result<Self, Error> {
         let ssl = Arc::new(ssl);
         let mut res = Self {
-            tor: TorController::init(tor_control).await?,
+            tor: TorController::new(tor_control, tor_socks),
             #[cfg(feature = "avahi")]
             mdns: MdnsController::init().await?,
             vhost: VHostController::new(ssl.clone()),
@@ -54,6 +55,8 @@ impl NetController {
     }
 
     async fn add_os_bindings(&mut self, hostname: &Hostname, key: &Key) -> Result<(), Error> {
+        let alpn = Err(AlpnInfo::Specified(vec!["http/1.1".into(), "h2".into()]));
+
         // Internal DNS
         self.vhost
             .add(
@@ -61,7 +64,7 @@ impl NetController {
                 Some("embassy".into()),
                 443,
                 ([127, 0, 0, 1], 80).into(),
-                false,
+                alpn.clone(),
             )
             .await?;
         self.os_bindings
@@ -70,7 +73,13 @@ impl NetController {
         // LAN IP
         self.os_bindings.push(
             self.vhost
-                .add(key.clone(), None, 443, ([127, 0, 0, 1], 80).into(), false)
+                .add(
+                    key.clone(),
+                    None,
+                    443,
+                    ([127, 0, 0, 1], 80).into(),
+                    alpn.clone(),
+                )
                 .await?,
         );
 
@@ -82,7 +91,7 @@ impl NetController {
                     Some("localhost".into()),
                     443,
                     ([127, 0, 0, 1], 80).into(),
-                    false,
+                    alpn.clone(),
                 )
                 .await?,
         );
@@ -93,7 +102,7 @@ impl NetController {
                     Some(hostname.no_dot_host_name()),
                     443,
                     ([127, 0, 0, 1], 80).into(),
-                    false,
+                    alpn.clone(),
                 )
                 .await?,
         );
@@ -106,7 +115,7 @@ impl NetController {
                     Some(hostname.local_domain_name()),
                     443,
                     ([127, 0, 0, 1], 80).into(),
-                    false,
+                    alpn.clone(),
                 )
                 .await?,
         );
@@ -114,7 +123,7 @@ impl NetController {
         // Tor (http)
         self.os_bindings.push(
             self.tor
-                .add(&key.tor_key(), 80, ([127, 0, 0, 1], 80).into())
+                .add(key.tor_key(), 80, ([127, 0, 0, 1], 80).into())
                 .await?,
         );
 
@@ -126,13 +135,13 @@ impl NetController {
                     Some(key.tor_address().to_string()),
                     443,
                     ([127, 0, 0, 1], 80).into(),
-                    false,
+                    alpn.clone(),
                 )
                 .await?,
         );
         self.os_bindings.push(
             self.tor
-                .add(&key.tor_key(), 443, ([127, 0, 0, 1], 443).into())
+                .add(key.tor_key(), 443, ([127, 0, 0, 1], 443).into())
                 .await?,
         );
 
@@ -164,13 +173,13 @@ impl NetController {
         target: SocketAddr,
     ) -> Result<Vec<Arc<()>>, Error> {
         let mut rcs = Vec::with_capacity(1);
-        rcs.push(self.tor.add(&key.tor_key(), external, target).await?);
+        rcs.push(self.tor.add(key.tor_key(), external, target).await?);
         Ok(rcs)
     }
 
     async fn remove_tor(&self, key: &Key, external: u16, rcs: Vec<Arc<()>>) -> Result<(), Error> {
         drop(rcs);
-        self.tor.gc(&key.tor_key(), external).await
+        self.tor.gc(Some(key.tor_key()), Some(external)).await
     }
 
     async fn add_lan(
@@ -178,7 +187,7 @@ impl NetController {
         key: Key,
         external: u16,
         target: SocketAddr,
-        connect_ssl: bool,
+        connect_ssl: Result<(), AlpnInfo>,
     ) -> Result<Vec<Arc<()>>, Error> {
         let mut rcs = Vec::with_capacity(2);
         rcs.push(
@@ -260,7 +269,7 @@ impl NetService {
         id: InterfaceId,
         external: u16,
         internal: u16,
-        connect_ssl: bool,
+        connect_ssl: Result<(), AlpnInfo>,
     ) -> Result<(), Error>
     where
         for<'a> &'a mut Ex: PgExecutor<'a>,

backend/src/net/ssl.rs

@@ -173,7 +173,7 @@ pub fn make_root_cert(root_key: &PKey<Private>, hostname: &Hostname) -> Result<X
     let mut subject_name_builder = X509NameBuilder::new()?;
     subject_name_builder.append_entry_by_text("CN", &format!("{} Local Root CA", &*hostname.0))?;
     subject_name_builder.append_entry_by_text("O", "Start9")?;
-    subject_name_builder.append_entry_by_text("OU", "Embassy")?;
+    subject_name_builder.append_entry_by_text("OU", "StartOS")?;
     let subject_name = subject_name_builder.build();
     builder.set_subject_name(&subject_name)?;
 
@@ -225,9 +225,9 @@ pub fn make_int_cert(
     builder.set_serial_number(&*rand_serial()?)?;
 
     let mut subject_name_builder = X509NameBuilder::new()?;
-    subject_name_builder.append_entry_by_text("CN", "Embassy Local Intermediate CA")?;
+    subject_name_builder.append_entry_by_text("CN", "StartOS Local Intermediate CA")?;
     subject_name_builder.append_entry_by_text("O", "Start9")?;
-    subject_name_builder.append_entry_by_text("OU", "Embassy")?;
+    subject_name_builder.append_entry_by_text("OU", "StartOS")?;
     let subject_name = subject_name_builder.build();
     builder.set_subject_name(&subject_name)?;
 
@@ -370,7 +370,7 @@ pub fn make_leaf_cert(
             .unwrap_or("localhost"),
     )?;
     subject_name_builder.append_entry_by_text("O", "Start9")?;
-    subject_name_builder.append_entry_by_text("OU", "Embassy")?;
+    subject_name_builder.append_entry_by_text("OU", "StartOS")?;
     let subject_name = subject_name_builder.build();
     builder.set_subject_name(&subject_name)?;
 

backend/src/net/static_server.rs

@@ -1,18 +1,19 @@
+use std::borrow::Cow;
 use std::fs::Metadata;
-use std::path::Path;
+use std::path::{Path, PathBuf};
 use std::sync::Arc;
 use std::time::UNIX_EPOCH;
 
-use async_compression::tokio::bufread::BrotliEncoder;
 use async_compression::tokio::bufread::GzipEncoder;
 use color_eyre::eyre::eyre;
 use digest::Digest;
 use futures::FutureExt;
 use http::header::ACCEPT_ENCODING;
-use http::header::CONTENT_ENCODING;
 use http::request::Parts as RequestParts;
 use http::response::Builder;
 use hyper::{Body, Method, Request, Response, StatusCode};
+use include_dir::{include_dir, Dir};
+use new_mime_guess::MimeGuess;
 use openssl::hash::MessageDigest;
 use openssl::x509::X509;
 use rpc_toolkit::rpc_handler;
@@ -35,10 +36,9 @@ static NOT_FOUND: &[u8] = b"Not Found";
 static METHOD_NOT_ALLOWED: &[u8] = b"Method Not Allowed";
 static NOT_AUTHORIZED: &[u8] = b"Not Authorized";
 
-pub const MAIN_UI_WWW_DIR: &str = "/var/www/html/main";
-pub const SETUP_UI_WWW_DIR: &str = "/var/www/html/setup";
-pub const DIAG_UI_WWW_DIR: &str = "/var/www/html/diagnostic";
-pub const INSTALL_UI_WWW_DIR: &str = "/var/www/html/install";
+static EMBEDDED_UIS: Dir<'_> = include_dir!("$CARGO_MANIFEST_DIR/../frontend/dist/static");
+
+const PROXY_STRIP_HEADERS: &[&str] = &["cookie", "host", "origin", "referer", "user-agent"];
 
 fn status_fn(_: i32) -> StatusCode {
     StatusCode::OK
@@ -52,6 +52,17 @@ pub enum UiMode {
     Main,
 }
 
+impl UiMode {
+    fn path(&self, path: &str) -> PathBuf {
+        match self {
+            Self::Setup => Path::new("setup-wizard").join(path),
+            Self::Diag => Path::new("diagnostic-ui").join(path),
+            Self::Install => Path::new("install-wizard").join(path),
+            Self::Main => Path::new("ui").join(path),
+        }
+    }
+}
+
 pub async fn setup_ui_file_router(ctx: SetupContext) -> Result<HttpHandler, Error> {
     let handler: HttpHandler = Arc::new(move |req| {
         let ctx = ctx.clone();
@@ -226,65 +237,35 @@ pub async fn main_ui_server_router(ctx: RpcContext) -> Result<HttpHandler, Error
 }
 
 async fn alt_ui(req: Request<Body>, ui_mode: UiMode) -> Result<Response<Body>, Error> {
-    let selected_root_dir = match ui_mode {
-        UiMode::Setup => SETUP_UI_WWW_DIR,
-        UiMode::Diag => DIAG_UI_WWW_DIR,
-        UiMode::Install => INSTALL_UI_WWW_DIR,
-        UiMode::Main => MAIN_UI_WWW_DIR,
-    };
-
     let (request_parts, _body) = req.into_parts();
-    let accept_encoding = request_parts
-        .headers
-        .get_all(ACCEPT_ENCODING)
-        .into_iter()
-        .filter_map(|h| h.to_str().ok())
-        .flat_map(|s| s.split(","))
-        .filter_map(|s| s.split(";").next())
-        .map(|s| s.trim())
-        .collect::<Vec<_>>();
     match &request_parts.method {
         &Method::GET => {
-            let uri_path = request_parts
-                .uri
-                .path()
-                .strip_prefix('/')
-                .unwrap_or(request_parts.uri.path());
+            let uri_path = ui_mode.path(
+                request_parts
+                    .uri
+                    .path()
+                    .strip_prefix('/')
+                    .unwrap_or(request_parts.uri.path()),
+            );
 
-            let full_path = Path::new(selected_root_dir).join(uri_path);
-            file_send(
-                &request_parts,
-                if tokio::fs::metadata(&full_path)
+            let file = EMBEDDED_UIS
+                .get_file(&*uri_path)
+                .or_else(|| EMBEDDED_UIS.get_file(&*ui_mode.path("index.html")));
+
+            if let Some(file) = file {
+                FileData::from_embedded(&request_parts, file)
+                    .into_response(&request_parts)
                     .await
-                    .ok()
-                    .map(|f| f.is_file())
-                    .unwrap_or(false)
-                {
-                    full_path
-                } else {
-                    Path::new(selected_root_dir).join("index.html")
-                },
-                &accept_encoding,
-            )
-            .await
+            } else {
+                Ok(not_found())
+            }
         }
         _ => Ok(method_not_allowed()),
     }
 }
 
 async fn main_embassy_ui(req: Request<Body>, ctx: RpcContext) -> Result<Response<Body>, Error> {
-    let selected_root_dir = MAIN_UI_WWW_DIR;
-
     let (request_parts, _body) = req.into_parts();
-    let accept_encoding = request_parts
-        .headers
-        .get_all(ACCEPT_ENCODING)
-        .into_iter()
-        .filter_map(|h| h.to_str().ok())
-        .flat_map(|s| s.split(","))
-        .filter_map(|s| s.split(";").next())
-        .map(|s| s.trim())
-        .collect::<Vec<_>>();
     match (
         &request_parts.method,
         request_parts
@@ -299,11 +280,12 @@ async fn main_embassy_ui(req: Request<Body>, ctx: RpcContext) -> Result<Response
                 Ok(_) => {
                     let sub_path = Path::new(path);
                     if let Ok(rest) = sub_path.strip_prefix("package-data") {
-                        file_send(
+                        FileData::from_path(
                             &request_parts,
-                            ctx.datadir.join(PKG_PUBLIC_DIR).join(rest),
-                            &accept_encoding,
+                            &ctx.datadir.join(PKG_PUBLIC_DIR).join(rest),
                         )
+                        .await?
+                        .into_response(&request_parts)
                         .await
                     } else if let Ok(rest) = sub_path.strip_prefix("eos") {
                         match rest.to_str() {
@@ -318,6 +300,40 @@ async fn main_embassy_ui(req: Request<Body>, ctx: RpcContext) -> Result<Response
                 Err(e) => un_authorized(e, &format!("public/{path}")),
             }
         }
+        (&Method::GET, Some(("proxy", target))) => {
+            match HasValidSession::from_request_parts(&request_parts, &ctx).await {
+                Ok(_) => {
+                    let target = urlencoding::decode(target)?;
+                    let res = ctx
+                        .client
+                        .get(target.as_ref())
+                        .headers(
+                            request_parts
+                                .headers
+                                .iter()
+                                .filter(|(h, _)| {
+                                    !PROXY_STRIP_HEADERS
+                                        .iter()
+                                        .any(|bad| h.as_str().eq_ignore_ascii_case(bad))
+                                })
+                                .map(|(h, v)| (h.clone(), v.clone()))
+                                .collect(),
+                        )
+                        .send()
+                        .await
+                        .with_kind(crate::ErrorKind::Network)?;
+                    let mut hres = Response::builder().status(res.status());
+                    for (h, v) in res.headers().clone() {
+                        if let Some(h) = h {
+                            hres = hres.header(h, v);
+                        }
+                    }
+                    hres.body(Body::wrap_stream(res.bytes_stream()))
+                        .with_kind(crate::ErrorKind::Network)
+                }
+                Err(e) => un_authorized(e, &format!("proxy/{target}")),
+            }
+        }
         (&Method::GET, Some(("eos", "local.crt"))) => {
             match HasValidSession::from_request_parts(&request_parts, &ctx).await {
                 Ok(_) => cert_send(&ctx.account.read().await.root_ca_cert),
@@ -325,28 +341,25 @@ async fn main_embassy_ui(req: Request<Body>, ctx: RpcContext) -> Result<Response
             }
         }
         (&Method::GET, _) => {
-            let uri_path = request_parts
-                .uri
-                .path()
-                .strip_prefix('/')
-                .unwrap_or(request_parts.uri.path());
+            let uri_path = UiMode::Main.path(
+                request_parts
+                    .uri
+                    .path()
+                    .strip_prefix('/')
+                    .unwrap_or(request_parts.uri.path()),
+            );
 
-            let full_path = Path::new(selected_root_dir).join(uri_path);
-            file_send(
-                &request_parts,
-                if tokio::fs::metadata(&full_path)
+            let file = EMBEDDED_UIS
+                .get_file(&*uri_path)
+                .or_else(|| EMBEDDED_UIS.get_file(&*UiMode::Main.path("index.html")));
+
+            if let Some(file) = file {
+                FileData::from_embedded(&request_parts, file)
+                    .into_response(&request_parts)
                     .await
-                    .ok()
-                    .map(|f| f.is_file())
-                    .unwrap_or(false)
-                {
-                    full_path
-                } else {
-                    Path::new(selected_root_dir).join("index.html")
-                },
-                &accept_encoding,
-            )
-            .await
+            } else {
+                Ok(not_found())
+            }
         }
         _ => Ok(method_not_allowed()),
     }
@@ -409,118 +422,158 @@ fn cert_send(cert: &X509) -> Result<Response<Body>, Error> {
         .with_kind(ErrorKind::Network)
 }
 
-async fn file_send(
-    req: &RequestParts,
-    path: impl AsRef<Path>,
-    accept_encoding: &[&str],
-) -> Result<Response<Body>, Error> {
-    // Serve a file by asynchronously reading it by chunks using tokio-util crate.
+struct FileData {
+    data: Body,
+    len: Option<u64>,
+    encoding: Option<&'static str>,
+    e_tag: String,
+    mime: Option<String>,
+}
+impl FileData {
+    fn from_embedded(req: &RequestParts, file: &'static include_dir::File<'static>) -> Self {
+        let path = file.path();
+        let (encoding, data) = req
+            .headers
+            .get_all(ACCEPT_ENCODING)
+            .into_iter()
+            .filter_map(|h| h.to_str().ok())
+            .flat_map(|s| s.split(","))
+            .filter_map(|s| s.split(";").next())
+            .map(|s| s.trim())
+            .fold((None, file.contents()), |acc, e| {
+                if let Some(file) = (e == "br")
+                    .then_some(())
+                    .and_then(|_| EMBEDDED_UIS.get_file(format!("{}.br", path.display())))
+                {
+                    (Some("br"), file.contents())
+                } else if let Some(file) = (e == "gzip" && acc.0 != Some("br"))
+                    .then_some(())
+                    .and_then(|_| EMBEDDED_UIS.get_file(format!("{}.gz", path.display())))
+                {
+                    (Some("gzip"), file.contents())
+                } else {
+                    acc
+                }
+            });
 
-    let path = path.as_ref();
-
-    let file = File::open(path)
-        .await
-        .with_ctx(|_| (ErrorKind::Filesystem, path.display().to_string()))?;
-    let metadata = file
-        .metadata()
-        .await
-        .with_ctx(|_| (ErrorKind::Filesystem, path.display().to_string()))?;
-
-    let e_tag = e_tag(path, &metadata)?;
-
-    let mut builder = Response::builder();
-    builder = with_content_type(path, builder);
-    builder = builder.header(http::header::ETAG, &e_tag);
-    builder = builder.header(
-        http::header::CACHE_CONTROL,
-        "public, max-age=21000000, immutable",
-    );
-
-    if req
-        .headers
-        .get_all(http::header::CONNECTION)
-        .iter()
-        .flat_map(|s| s.to_str().ok())
-        .flat_map(|s| s.split(","))
-        .any(|s| s.trim() == "keep-alive")
-    {
-        builder = builder.header(http::header::CONNECTION, "keep-alive");
+        Self {
+            len: Some(data.len() as u64),
+            encoding,
+            data: data.into(),
+            e_tag: e_tag(path, None),
+            mime: MimeGuess::from_path(path)
+                .first()
+                .map(|m| m.essence_str().to_owned()),
+        }
     }
 
-    if req
-        .headers
-        .get("if-none-match")
-        .and_then(|h| h.to_str().ok())
-        == Some(e_tag.as_str())
-    {
-        builder = builder.status(StatusCode::NOT_MODIFIED);
-        builder.body(Body::empty())
-    } else {
-        let body = if false && accept_encoding.contains(&"br") && metadata.len() > u16::MAX as u64 {
-            builder = builder.header(CONTENT_ENCODING, "br");
-            Body::wrap_stream(ReaderStream::new(BrotliEncoder::new(BufReader::new(file))))
-        } else if accept_encoding.contains(&"gzip") && metadata.len() > u16::MAX as u64 {
-            builder = builder.header(CONTENT_ENCODING, "gzip");
-            Body::wrap_stream(ReaderStream::new(GzipEncoder::new(BufReader::new(file))))
+    async fn from_path(req: &RequestParts, path: &Path) -> Result<Self, Error> {
+        let encoding = req
+            .headers
+            .get_all(ACCEPT_ENCODING)
+            .into_iter()
+            .filter_map(|h| h.to_str().ok())
+            .flat_map(|s| s.split(","))
+            .filter_map(|s| s.split(";").next())
+            .map(|s| s.trim())
+            .any(|e| e == "gzip")
+            .then_some("gzip");
+
+        let file = File::open(path)
+            .await
+            .with_ctx(|_| (ErrorKind::Filesystem, path.display().to_string()))?;
+        let metadata = file
+            .metadata()
+            .await
+            .with_ctx(|_| (ErrorKind::Filesystem, path.display().to_string()))?;
+
+        let e_tag = e_tag(path, Some(&metadata));
+
+        let (len, data) = if encoding == Some("gzip") {
+            (
+                None,
+                Body::wrap_stream(ReaderStream::new(GzipEncoder::new(BufReader::new(file)))),
+            )
         } else {
-            builder = with_content_length(&metadata, builder);
-            Body::wrap_stream(ReaderStream::new(file))
+            (
+                Some(metadata.len()),
+                Body::wrap_stream(ReaderStream::new(file)),
+            )
         };
-        builder.body(body)
+
+        Ok(Self {
+            data,
+            len,
+            encoding,
+            e_tag,
+            mime: MimeGuess::from_path(path)
+                .first()
+                .map(|m| m.essence_str().to_owned()),
+        })
+    }
+
+    async fn into_response(self, req: &RequestParts) -> Result<Response<Body>, Error> {
+        let mut builder = Response::builder();
+        if let Some(mime) = self.mime {
+            builder = builder.header(http::header::CONTENT_TYPE, &*mime);
+        }
+        builder = builder.header(http::header::ETAG, &*self.e_tag);
+        builder = builder.header(
+            http::header::CACHE_CONTROL,
+            "public, max-age=21000000, immutable",
+        );
+
+        if req
+            .headers
+            .get_all(http::header::CONNECTION)
+            .iter()
+            .flat_map(|s| s.to_str().ok())
+            .flat_map(|s| s.split(","))
+            .any(|s| s.trim() == "keep-alive")
+        {
+            builder = builder.header(http::header::CONNECTION, "keep-alive");
+        }
+
+        if req
+            .headers
+            .get("if-none-match")
+            .and_then(|h| h.to_str().ok())
+            == Some(self.e_tag.as_ref())
+        {
+            builder = builder.status(StatusCode::NOT_MODIFIED);
+            builder.body(Body::empty())
+        } else {
+            if let Some(len) = self.len {
+                builder = builder.header(http::header::CONTENT_LENGTH, len);
+            }
+            if let Some(encoding) = self.encoding {
+                builder = builder.header(http::header::CONTENT_ENCODING, encoding);
+            }
+
+            builder.body(self.data)
+        }
+        .with_kind(ErrorKind::Network)
     }
-    .with_kind(ErrorKind::Network)
 }
 
-fn e_tag(path: &Path, metadata: &Metadata) -> Result<String, Error> {
-    let modified = metadata.modified().with_kind(ErrorKind::Filesystem)?;
+fn e_tag(path: &Path, metadata: Option<&Metadata>) -> String {
     let mut hasher = sha2::Sha256::new();
     hasher.update(format!("{:?}", path).as_bytes());
-    hasher.update(
-        format!(
-            "{}",
-            modified
-                .duration_since(UNIX_EPOCH)
-                .unwrap_or_default()
-                .as_secs()
-        )
-        .as_bytes(),
-    );
+    if let Some(modified) = metadata.and_then(|m| m.modified().ok()) {
+        hasher.update(
+            format!(
+                "{}",
+                modified
+                    .duration_since(UNIX_EPOCH)
+                    .unwrap_or_default()
+                    .as_secs()
+            )
+            .as_bytes(),
+        );
+    }
     let res = hasher.finalize();
-    Ok(format!(
+    format!(
         "\"{}\"",
         base32::encode(base32::Alphabet::RFC4648 { padding: false }, res.as_slice()).to_lowercase()
-    ))
-}
-
-///https://en.wikipedia.org/wiki/Media_type
-fn with_content_type(path: &Path, builder: Builder) -> Builder {
-    let content_type = match path.extension() {
-        Some(os_str) => match os_str.to_str() {
-            Some("apng") => "image/apng",
-            Some("avif") => "image/avif",
-            Some("flif") => "image/flif",
-            Some("gif") => "image/gif",
-            Some("jpg") | Some("jpeg") | Some("jfif") | Some("pjpeg") | Some("pjp") => "image/jpeg",
-            Some("jxl") => "image/jxl",
-            Some("png") => "image/png",
-            Some("svg") => "image/svg+xml",
-            Some("webp") => "image/webp",
-            Some("mng") | Some("x-mng") => "image/x-mng",
-            Some("css") => "text/css",
-            Some("csv") => "text/csv",
-            Some("html") => "text/html",
-            Some("php") => "text/php",
-            Some("plain") | Some("md") | Some("txt") => "text/plain",
-            Some("xml") => "text/xml",
-            Some("js") => "text/javascript",
-            Some("wasm") => "application/wasm",
-            None | Some(_) => "text/plain",
-        },
-        None => "text/plain",
-    };
-    builder.header(http::header::CONTENT_TYPE, content_type)
-}
-
-fn with_content_length(metadata: &Metadata, builder: Builder) -> Builder {
-    builder.header(http::header::CONTENT_LENGTH, metadata.len())
+    )
 }

backend/src/net/tor.rs

@@ -1,32 +1,80 @@
 use std::collections::BTreeMap;
 use std::net::SocketAddr;
+use std::sync::atomic::AtomicBool;
 use std::sync::{Arc, Weak};
+use std::time::Duration;
 
 use clap::ArgMatches;
 use color_eyre::eyre::eyre;
 use futures::future::BoxFuture;
-use futures::FutureExt;
+use futures::{FutureExt, TryStreamExt};
+use helpers::NonDetachingJoinHandle;
+use itertools::Itertools;
+use lazy_static::lazy_static;
+use regex::Regex;
 use rpc_toolkit::command;
+use rpc_toolkit::yajrc::RpcError;
 use tokio::net::TcpStream;
-use tokio::sync::Mutex;
+use tokio::process::Command;
+use tokio::sync::{mpsc, oneshot};
+use tokio::time::Instant;
 use torut::control::{AsyncEvent, AuthenticatedConn, ConnError};
 use torut::onion::{OnionAddressV3, TorSecretKeyV3};
 use tracing::instrument;
 
-use crate::context::RpcContext;
+use crate::context::{CliContext, RpcContext};
+use crate::logs::{
+    cli_logs_generic_follow, cli_logs_generic_nofollow, fetch_logs, follow_logs, journalctl,
+    LogFollowResponse, LogResponse, LogSource,
+};
 use crate::util::serde::{display_serializable, IoFormat};
+use crate::util::{display_none, Invoke};
 use crate::{Error, ErrorKind, ResultExt as _};
 
+pub const SYSTEMD_UNIT: &str = "tor@default";
+const STARTING_HEALTH_TIMEOUT: u64 = 120; // 2min
+
+enum ErrorLogSeverity {
+    Fatal { wipe_state: bool },
+    Unknown { wipe_state: bool },
+}
+
+lazy_static! {
+    static ref LOG_REGEXES: Vec<(Regex, ErrorLogSeverity)> = vec![(
+        Regex::new("This could indicate a route manipulation attack, network overload, bad local network connectivity, or a bug\\.").unwrap(),
+        ErrorLogSeverity::Unknown { wipe_state: true }
+    ),(
+        Regex::new("died due to an invalid selected path").unwrap(),
+        ErrorLogSeverity::Fatal { wipe_state: false }
+    ),(
+        Regex::new("Tor has not observed any network activity for the past").unwrap(),
+        ErrorLogSeverity::Unknown { wipe_state: false }
+    )];
+    static ref PROGRESS_REGEX: Regex = Regex::new("PROGRESS=([0-9]+)").unwrap();
+}
+
 #[test]
 fn random_key() {
     println!("x'{}'", hex::encode(rand::random::<[u8; 32]>()));
 }
 
-#[command(subcommands(list_services))]
+#[command(subcommands(list_services, logs, reset))]
 pub fn tor() -> Result<(), Error> {
     Ok(())
 }
 
+#[command(display(display_none))]
+pub async fn reset(
+    #[context] ctx: RpcContext,
+    #[arg(rename = "wipe-state", short = 'w', long = "wipe-state")] wipe_state: bool,
+    #[arg] reason: String,
+) -> Result<(), Error> {
+    ctx.net_controller
+        .tor
+        .reset(wipe_state, Error::new(eyre!("{reason}"), ErrorKind::Tor))
+        .await
+}
+
 fn display_services(services: Vec<OnionAddressV3>, matches: &ArgMatches) {
     use prettytable::*;
 
@@ -52,133 +100,227 @@ pub async fn list_services(
     ctx.net_controller.tor.list_services().await
 }
 
+#[command(
+    custom_cli(cli_logs(async, context(CliContext))),
+    subcommands(self(logs_nofollow(async)), logs_follow),
+    display(display_none)
+)]
+pub async fn logs(
+    #[arg(short = 'l', long = "limit")] limit: Option<usize>,
+    #[arg(short = 'c', long = "cursor")] cursor: Option<String>,
+    #[arg(short = 'B', long = "before", default)] before: bool,
+    #[arg(short = 'f', long = "follow", default)] follow: bool,
+) -> Result<(Option<usize>, Option<String>, bool, bool), Error> {
+    Ok((limit, cursor, before, follow))
+}
+pub async fn cli_logs(
+    ctx: CliContext,
+    (limit, cursor, before, follow): (Option<usize>, Option<String>, bool, bool),
+) -> Result<(), RpcError> {
+    if follow {
+        if cursor.is_some() {
+            return Err(RpcError::from(Error::new(
+                eyre!("The argument '--cursor <cursor>' cannot be used with '--follow'"),
+                crate::ErrorKind::InvalidRequest,
+            )));
+        }
+        if before {
+            return Err(RpcError::from(Error::new(
+                eyre!("The argument '--before' cannot be used with '--follow'"),
+                crate::ErrorKind::InvalidRequest,
+            )));
+        }
+        cli_logs_generic_follow(ctx, "net.tor.logs.follow", None, limit).await
+    } else {
+        cli_logs_generic_nofollow(ctx, "net.tor.logs", None, limit, cursor, before).await
+    }
+}
+pub async fn logs_nofollow(
+    _ctx: (),
+    (limit, cursor, before, _): (Option<usize>, Option<String>, bool, bool),
+) -> Result<LogResponse, Error> {
+    fetch_logs(LogSource::Service(SYSTEMD_UNIT), limit, cursor, before).await
+}
+
+#[command(rpc_only, rename = "follow", display(display_none))]
+pub async fn logs_follow(
+    #[context] ctx: RpcContext,
+    #[parent_data] (limit, _, _, _): (Option<usize>, Option<String>, bool, bool),
+) -> Result<LogFollowResponse, Error> {
+    follow_logs(ctx, LogSource::Service(SYSTEMD_UNIT), limit).await
+}
+
 fn event_handler(_event: AsyncEvent<'static>) -> BoxFuture<'static, Result<(), ConnError>> {
     async move { Ok(()) }.boxed()
 }
 
-pub struct TorController(Mutex<TorControllerInner>);
+pub struct TorController(TorControl);
 impl TorController {
-    pub async fn init(tor_control: SocketAddr) -> Result<Self, Error> {
-        Ok(TorController(Mutex::new(
-            TorControllerInner::init(tor_control).await?,
-        )))
+    pub fn new(tor_control: SocketAddr, tor_socks: SocketAddr) -> Self {
+        TorController(TorControl::new(tor_control, tor_socks))
     }
 
     pub async fn add(
         &self,
-        key: &TorSecretKeyV3,
+        key: TorSecretKeyV3,
         external: u16,
         target: SocketAddr,
     ) -> Result<Arc<()>, Error> {
-        self.0.lock().await.add(key, external, target).await
+        let (reply, res) = oneshot::channel();
+        self.0
+            .send
+            .send(TorCommand::AddOnion {
+                key,
+                external,
+                target,
+                reply,
+            })
+            .ok()
+            .ok_or_else(|| Error::new(eyre!("TorControl died"), ErrorKind::Tor))?;
+        res.await
+            .ok()
+            .ok_or_else(|| Error::new(eyre!("TorControl died"), ErrorKind::Tor))
     }
 
-    pub async fn gc(&self, key: &TorSecretKeyV3, external: u16) -> Result<(), Error> {
-        self.0.lock().await.gc(key, external).await
+    pub async fn gc(
+        &self,
+        key: Option<TorSecretKeyV3>,
+        external: Option<u16>,
+    ) -> Result<(), Error> {
+        self.0
+            .send
+            .send(TorCommand::GC { key, external })
+            .ok()
+            .ok_or_else(|| Error::new(eyre!("TorControl died"), ErrorKind::Tor))
+    }
+
+    pub async fn reset(&self, wipe_state: bool, context: Error) -> Result<(), Error> {
+        self.0
+            .send
+            .send(TorCommand::Reset {
+                wipe_state,
+                context,
+            })
+            .ok()
+            .ok_or_else(|| Error::new(eyre!("TorControl died"), ErrorKind::Tor))
     }
 
     pub async fn list_services(&self) -> Result<Vec<OnionAddressV3>, Error> {
-        self.0.lock().await.list_services().await
+        let (reply, res) = oneshot::channel();
+        self.0
+            .send
+            .send(TorCommand::GetInfo {
+                query: "onions/current".into(),
+                reply,
+            })
+            .ok()
+            .ok_or_else(|| Error::new(eyre!("TorControl died"), ErrorKind::Tor))?;
+        res.await
+            .ok()
+            .ok_or_else(|| Error::new(eyre!("TorControl died"), ErrorKind::Tor))??
+            .lines()
+            .map(|l| l.trim())
+            .filter(|l| !l.is_empty())
+            .map(|l| l.parse().with_kind(ErrorKind::Tor))
+            .collect()
     }
 }
 
 type AuthenticatedConnection = AuthenticatedConn<
     TcpStream,
-    fn(AsyncEvent<'static>) -> BoxFuture<'static, Result<(), ConnError>>,
+    Box<dyn Fn(AsyncEvent<'static>) -> BoxFuture<'static, Result<(), ConnError>> + Send + Sync>,
 >;
 
-pub struct TorControllerInner {
-    control_addr: SocketAddr,
-    connection: AuthenticatedConnection,
-    services: BTreeMap<String, BTreeMap<u16, BTreeMap<SocketAddr, Weak<()>>>>,
-}
-impl TorControllerInner {
-    #[instrument(skip_all)]
-    async fn add(
-        &mut self,
-        key: &TorSecretKeyV3,
+enum TorCommand {
+    AddOnion {
+        key: TorSecretKeyV3,
         external: u16,
         target: SocketAddr,
-    ) -> Result<Arc<()>, Error> {
-        let mut rm_res = Ok(());
-        let onion_base = key
-            .public()
-            .get_onion_address()
-            .get_address_without_dot_onion();
-        let mut service = if let Some(service) = self.services.remove(&onion_base) {
-            rm_res = self.connection.del_onion(&onion_base).await;
-            service
-        } else {
-            BTreeMap::new()
-        };
-        let mut binding = service.remove(&external).unwrap_or_default();
-        let rc = if let Some(rc) = Weak::upgrade(&binding.remove(&target).unwrap_or_default()) {
-            rc
-        } else {
-            Arc::new(())
-        };
-        binding.insert(target, Arc::downgrade(&rc));
-        service.insert(external, binding);
-        let bindings = service
-            .iter()
-            .flat_map(|(ext, int)| {
-                int.iter()
-                    .find(|(_, rc)| rc.strong_count() > 0)
-                    .map(|(addr, _)| (*ext, SocketAddr::from(*addr)))
-            })
-            .collect::<Vec<_>>();
-        self.services.insert(onion_base, service);
-        rm_res?;
-        self.connection
-            .add_onion_v3(key, false, false, false, None, &mut bindings.iter())
-            .await?;
-        Ok(rc)
-    }
+        reply: oneshot::Sender<Arc<()>>,
+    },
+    GC {
+        key: Option<TorSecretKeyV3>,
+        external: Option<u16>,
+    },
+    GetInfo {
+        query: String,
+        reply: oneshot::Sender<Result<String, Error>>,
+    },
+    Reset {
+        wipe_state: bool,
+        context: Error,
+    },
+}
 
-    #[instrument(skip_all)]
-    async fn gc(&mut self, key: &TorSecretKeyV3, external: u16) -> Result<(), Error> {
-        let onion_base = key
-            .public()
-            .get_onion_address()
-            .get_address_without_dot_onion();
-        if let Some(mut service) = self.services.remove(&onion_base) {
-            if let Some(mut binding) = service.remove(&external) {
-                binding = binding
-                    .into_iter()
-                    .filter(|(_, rc)| rc.strong_count() > 0)
-                    .collect();
-                if !binding.is_empty() {
-                    service.insert(external, binding);
+#[instrument(skip_all)]
+async fn torctl(
+    tor_control: SocketAddr,
+    tor_socks: SocketAddr,
+    recv: &mut mpsc::UnboundedReceiver<TorCommand>,
+    services: &mut BTreeMap<[u8; 64], BTreeMap<u16, BTreeMap<SocketAddr, Weak<()>>>>,
+    wipe_state: &AtomicBool,
+    health_timeout: &mut Duration,
+) -> Result<(), Error> {
+    let bootstrap = async {
+        if Command::new("systemctl")
+            .arg("is-active")
+            .arg("--quiet")
+            .arg("tor")
+            .invoke(ErrorKind::Tor)
+            .await
+            .is_ok()
+        {
+            Command::new("systemctl")
+                .arg("stop")
+                .arg("tor")
+                .invoke(ErrorKind::Tor)
+                .await?;
+            for _ in 0..30 {
+                if TcpStream::connect(tor_control).await.is_err() {
+                    break;
                 }
+                tokio::time::sleep(Duration::from_secs(1)).await;
             }
-            let rm_res = self.connection.del_onion(&onion_base).await;
-            if !service.is_empty() {
-                let bindings = service
-                    .iter()
-                    .flat_map(|(ext, int)| {
-                        int.iter()
-                            .find(|(_, rc)| rc.strong_count() > 0)
-                            .map(|(addr, _)| (*ext, SocketAddr::from(*addr)))
-                    })
-                    .collect::<Vec<_>>();
-                self.services.insert(onion_base, service);
-                rm_res?;
-                self.connection
-                    .add_onion_v3(&key, false, false, false, None, &mut bindings.iter())
-                    .await?;
-            } else {
-                rm_res?;
+            if TcpStream::connect(tor_control).await.is_ok() {
+                return Err(Error::new(
+                    eyre!("Tor is failing to shut down"),
+                    ErrorKind::Tor,
+                ));
             }
         }
+        if wipe_state.load(std::sync::atomic::Ordering::SeqCst) {
+            tokio::fs::remove_dir_all("/var/lib/tor").await?;
+            wipe_state.store(false, std::sync::atomic::Ordering::SeqCst);
+        }
+        tokio::fs::create_dir_all("/var/lib/tor").await?;
+        Command::new("chown")
+            .arg("-R")
+            .arg("debian-tor")
+            .arg("/var/lib/tor")
+            .invoke(ErrorKind::Filesystem)
+            .await?;
+        Command::new("systemctl")
+            .arg("start")
+            .arg("tor")
+            .invoke(ErrorKind::Tor)
+            .await?;
 
-        Ok(())
-    }
+        let logs = journalctl(LogSource::Service(SYSTEMD_UNIT), 0, None, false, true).await?;
 
-    #[instrument(skip_all)]
-    async fn init(tor_control: SocketAddr) -> Result<Self, Error> {
-        let mut conn = torut::control::UnauthenticatedConn::new(
-            TcpStream::connect(tor_control).await?, // TODO
-        );
+        let mut tcp_stream = None;
+        for _ in 0..60 {
+            if let Ok(conn) = TcpStream::connect(tor_control).await {
+                tcp_stream = Some(conn);
+                break;
+            }
+            tokio::time::sleep(Duration::from_secs(1)).await;
+        }
+        let tcp_stream = tcp_stream.ok_or_else(|| {
+            Error::new(eyre!("Timed out waiting for tor to start"), ErrorKind::Tor)
+        })?;
+        tracing::info!("Tor is started");
+
+        let mut conn = torut::control::UnauthenticatedConn::new(tcp_stream);
         let auth = conn
             .load_protocol_info()
             .await?
@@ -187,25 +329,356 @@ impl TorControllerInner {
             .with_kind(crate::ErrorKind::Tor)?;
         conn.authenticate(&auth).await?;
         let mut connection: AuthenticatedConnection = conn.into_authenticated().await;
-        connection.set_async_event_handler(Some(event_handler));
+        connection.set_async_event_handler(Some(Box::new(|event| event_handler(event))));
 
-        Ok(Self {
-            control_addr: tor_control,
-            connection,
-            services: BTreeMap::new(),
-        })
+        let mut bootstrapped = false;
+        let mut last_increment = (String::new(), Instant::now());
+        for _ in 0..300 {
+            match connection.get_info("status/bootstrap-phase").await {
+                Ok(a) => {
+                    if a.contains("TAG=done") {
+                        bootstrapped = true;
+                        break;
+                    }
+                    if let Some(p) = PROGRESS_REGEX.captures(&a) {
+                        if let Some(p) = p.get(1) {
+                            if p.as_str() != &*last_increment.0 {
+                                last_increment = (p.as_str().into(), Instant::now());
+                            }
+                        }
+                    }
+                }
+                Err(e) => {
+                    let e = Error::from(e);
+                    tracing::error!("{}", e);
+                    tracing::debug!("{:?}", e);
+                }
+            }
+            if last_increment.1.elapsed() > Duration::from_secs(30) {
+                return Err(Error::new(
+                    eyre!("Tor stuck bootstrapping at {}%", last_increment.0),
+                    ErrorKind::Tor,
+                ));
+            }
+            tokio::time::sleep(Duration::from_secs(1)).await;
+        }
+        if !bootstrapped {
+            return Err(Error::new(
+                eyre!("Timed out waiting for tor to bootstrap"),
+                ErrorKind::Tor,
+            ));
+        }
+        Ok((connection, logs))
+    };
+    let pre_handler = async {
+        while let Some(command) = recv.recv().await {
+            match command {
+                TorCommand::AddOnion {
+                    key,
+                    external,
+                    target,
+                    reply,
+                } => {
+                    let mut service = if let Some(service) = services.remove(&key.as_bytes()) {
+                        service
+                    } else {
+                        BTreeMap::new()
+                    };
+                    let mut binding = service.remove(&external).unwrap_or_default();
+                    let rc = if let Some(rc) =
+                        Weak::upgrade(&binding.remove(&target).unwrap_or_default())
+                    {
+                        rc
+                    } else {
+                        Arc::new(())
+                    };
+                    binding.insert(target, Arc::downgrade(&rc));
+                    service.insert(external, binding);
+                    services.insert(key.as_bytes(), service);
+                    reply.send(rc).unwrap_or_default();
+                }
+                TorCommand::GetInfo { reply, .. } => {
+                    reply
+                        .send(Err(Error::new(
+                            eyre!("Tor has not finished bootstrapping..."),
+                            ErrorKind::Tor,
+                        )))
+                        .unwrap_or_default();
+                }
+                TorCommand::GC { .. } => (),
+                TorCommand::Reset {
+                    wipe_state: new_wipe_state,
+                    context,
+                } => {
+                    wipe_state.fetch_or(new_wipe_state, std::sync::atomic::Ordering::SeqCst);
+                    return Err(context);
+                }
+            }
+        }
+        Ok(())
+    };
+
+    let (mut connection, mut logs) = tokio::select! {
+        res = bootstrap => res?,
+        res = pre_handler => return res,
+    };
+
+    let hck_key = TorSecretKeyV3::generate();
+    connection
+        .add_onion_v3(
+            &hck_key,
+            false,
+            false,
+            false,
+            None,
+            &mut [(80, SocketAddr::from(([127, 0, 0, 1], 80)))].iter(),
+        )
+        .await?;
+
+    for (key, service) in std::mem::take(services) {
+        let key = TorSecretKeyV3::from(key);
+        let bindings = service
+            .iter()
+            .flat_map(|(ext, int)| {
+                int.iter()
+                    .find(|(_, rc)| rc.strong_count() > 0)
+                    .map(|(addr, _)| (*ext, SocketAddr::from(*addr)))
+            })
+            .collect::<Vec<_>>();
+        if !bindings.is_empty() {
+            services.insert(key.as_bytes(), service);
+            connection
+                .add_onion_v3(&key, false, false, false, None, &mut bindings.iter())
+                .await?;
+        }
     }
 
-    #[instrument(skip_all)]
-    async fn list_services(&mut self) -> Result<Vec<OnionAddressV3>, Error> {
-        self.connection
-            .get_info("onions/current")
-            .await?
-            .lines()
-            .map(|l| l.trim())
-            .filter(|l| !l.is_empty())
-            .map(|l| l.parse().with_kind(ErrorKind::Tor))
-            .collect()
+    let handler = async {
+        while let Some(command) = recv.recv().await {
+            match command {
+                TorCommand::AddOnion {
+                    key,
+                    external,
+                    target,
+                    reply,
+                } => {
+                    let mut rm_res = Ok(());
+                    let onion_base = key
+                        .public()
+                        .get_onion_address()
+                        .get_address_without_dot_onion();
+                    let mut service = if let Some(service) = services.remove(&key.as_bytes()) {
+                        rm_res = connection.del_onion(&onion_base).await;
+                        service
+                    } else {
+                        BTreeMap::new()
+                    };
+                    let mut binding = service.remove(&external).unwrap_or_default();
+                    let rc = if let Some(rc) =
+                        Weak::upgrade(&binding.remove(&target).unwrap_or_default())
+                    {
+                        rc
+                    } else {
+                        Arc::new(())
+                    };
+                    binding.insert(target, Arc::downgrade(&rc));
+                    service.insert(external, binding);
+                    let bindings = service
+                        .iter()
+                        .flat_map(|(ext, int)| {
+                            int.iter()
+                                .find(|(_, rc)| rc.strong_count() > 0)
+                                .map(|(addr, _)| (*ext, SocketAddr::from(*addr)))
+                        })
+                        .collect::<Vec<_>>();
+                    services.insert(key.as_bytes(), service);
+                    reply.send(rc).unwrap_or_default();
+                    rm_res?;
+                    connection
+                        .add_onion_v3(&key, false, false, false, None, &mut bindings.iter())
+                        .await?;
+                }
+                TorCommand::GC { key, external } => {
+                    for key in if key.is_some() {
+                        itertools::Either::Left(key.into_iter().map(|k| k.as_bytes()))
+                    } else {
+                        itertools::Either::Right(services.keys().cloned().collect_vec().into_iter())
+                    } {
+                        let key = TorSecretKeyV3::from(key);
+                        let onion_base = key
+                            .public()
+                            .get_onion_address()
+                            .get_address_without_dot_onion();
+                        if let Some(mut service) = services.remove(&key.as_bytes()) {
+                            for external in if external.is_some() {
+                                itertools::Either::Left(external.into_iter())
+                            } else {
+                                itertools::Either::Right(
+                                    service.keys().copied().collect_vec().into_iter(),
+                                )
+                            } {
+                                if let Some(mut binding) = service.remove(&external) {
+                                    binding = binding
+                                        .into_iter()
+                                        .filter(|(_, rc)| rc.strong_count() > 0)
+                                        .collect();
+                                    if !binding.is_empty() {
+                                        service.insert(external, binding);
+                                    }
+                                }
+                            }
+                            let rm_res = connection.del_onion(&onion_base).await;
+                            if !service.is_empty() {
+                                let bindings = service
+                                    .iter()
+                                    .flat_map(|(ext, int)| {
+                                        int.iter()
+                                            .find(|(_, rc)| rc.strong_count() > 0)
+                                            .map(|(addr, _)| (*ext, SocketAddr::from(*addr)))
+                                    })
+                                    .collect::<Vec<_>>();
+                                if !bindings.is_empty() {
+                                    services.insert(key.as_bytes(), service);
+                                }
+                                rm_res?;
+                                if !bindings.is_empty() {
+                                    connection
+                                        .add_onion_v3(
+                                            &key,
+                                            false,
+                                            false,
+                                            false,
+                                            None,
+                                            &mut bindings.iter(),
+                                        )
+                                        .await?;
+                                }
+                            } else {
+                                rm_res?;
+                            }
+                        }
+                    }
+                }
+                TorCommand::GetInfo { query, reply } => {
+                    reply
+                        .send(connection.get_info(&query).await.with_kind(ErrorKind::Tor))
+                        .unwrap_or_default();
+                }
+                TorCommand::Reset {
+                    wipe_state: new_wipe_state,
+                    context,
+                } => {
+                    wipe_state.fetch_or(new_wipe_state, std::sync::atomic::Ordering::SeqCst);
+                    return Err(context);
+                }
+            }
+        }
+        Ok(())
+    };
+    let log_parser = async {
+        while let Some(log) = logs.try_next().await? {
+            for (regex, severity) in &*LOG_REGEXES {
+                if regex.is_match(&log.message) {
+                    let (check, wipe_state) = match severity {
+                        ErrorLogSeverity::Fatal { wipe_state } => (false, *wipe_state),
+                        ErrorLogSeverity::Unknown { wipe_state } => (true, *wipe_state),
+                    };
+                    if !check
+                        || tokio::time::timeout(
+                            Duration::from_secs(30),
+                            tokio_socks::tcp::Socks5Stream::connect(
+                                tor_socks,
+                                (hck_key.public().get_onion_address().to_string(), 80),
+                            ),
+                        )
+                        .await
+                        .map_err(|e| tracing::warn!("Tor is confirmed to be down: {e}"))
+                        .and_then(|a| {
+                            a.map_err(|e| tracing::warn!("Tor is confirmed to be down: {e}"))
+                        })
+                        .is_err()
+                    {
+                        if wipe_state {
+                            Command::new("systemctl")
+                                .arg("stop")
+                                .arg("tor")
+                                .invoke(ErrorKind::Tor)
+                                .await?;
+                            tokio::fs::remove_dir_all("/var/lib/tor").await?;
+                        }
+                        return Err(Error::new(eyre!("{}", log.message), ErrorKind::Tor));
+                    }
+                }
+            }
+        }
+        Err(Error::new(eyre!("Log stream terminated"), ErrorKind::Tor))
+    };
+    let health_checker = async {
+        let mut last_success = Instant::now();
+        loop {
+            tokio::time::sleep(Duration::from_secs(30)).await;
+            if let Err(e) = tokio::time::timeout(
+                Duration::from_secs(30),
+                tokio_socks::tcp::Socks5Stream::connect(
+                    tor_socks,
+                    (hck_key.public().get_onion_address().to_string(), 80),
+                ),
+            )
+            .await
+            .map_err(|e| e.to_string())
+            .and_then(|e| e.map_err(|e| e.to_string()))
+            {
+                if last_success.elapsed() > *health_timeout {
+                    let err = Error::new(eyre!("Tor health check failed for longer than current timeout ({health_timeout:?})"), crate::ErrorKind::Tor);
+                    *health_timeout *= 2;
+                    wipe_state.store(true, std::sync::atomic::Ordering::SeqCst);
+                    return Err(err);
+                }
+            } else {
+                last_success = Instant::now();
+            }
+        }
+    };
+
+    tokio::select! {
+        res = handler => res?,
+        res = log_parser => res?,
+        res = health_checker => res?,
+    }
+
+    Ok(())
+}
+
+struct TorControl {
+    _thread: NonDetachingJoinHandle<()>,
+    send: mpsc::UnboundedSender<TorCommand>,
+}
+impl TorControl {
+    pub fn new(tor_control: SocketAddr, tor_socks: SocketAddr) -> Self {
+        let (send, mut recv) = mpsc::unbounded_channel();
+        Self {
+            _thread: tokio::spawn(async move {
+                let mut services = BTreeMap::new();
+                let wipe_state = AtomicBool::new(false);
+                let mut health_timeout = Duration::from_secs(STARTING_HEALTH_TIMEOUT);
+                while let Err(e) = torctl(
+                    tor_control,
+                    tor_socks,
+                    &mut recv,
+                    &mut services,
+                    &wipe_state,
+                    &mut health_timeout,
+                )
+                .await
+                {
+                    tracing::error!("{e}: Restarting tor");
+                    tracing::debug!("{e:?}");
+                }
+                tracing::info!("TorControl is shut down.")
+            })
+            .into(),
+            send,
+        }
     }
 }
 

backend/src/net/utils.rs

@@ -1,5 +1,5 @@
 use std::convert::Infallible;
-use std::net::{Ipv4Addr, Ipv6Addr};
+use std::net::{Ipv4Addr, Ipv6Addr, SocketAddr};
 use std::path::Path;
 
 use async_stream::try_stream;
@@ -7,24 +7,29 @@ use color_eyre::eyre::eyre;
 use futures::stream::BoxStream;
 use futures::{StreamExt, TryStreamExt};
 use ipnet::{Ipv4Net, Ipv6Net};
+use tokio::net::{TcpListener, TcpStream};
 use tokio::process::Command;
 
 use crate::util::Invoke;
 use crate::Error;
 
-fn parse_iface_ip(output: &str) -> Result<Option<&str>, Error> {
+fn parse_iface_ip(output: &str) -> Result<Vec<&str>, Error> {
     let output = output.trim();
     if output.is_empty() {
-        return Ok(None);
+        return Ok(Vec::new());
     }
-    if let Some(ip) = output.split_ascii_whitespace().nth(3) {
-        Ok(Some(ip))
-    } else {
-        Err(Error::new(
-            eyre!("malformed output from `ip`"),
-            crate::ErrorKind::Network,
-        ))
+    let mut res = Vec::new();
+    for line in output.lines() {
+        if let Some(ip) = line.split_ascii_whitespace().nth(3) {
+            res.push(ip)
+        } else {
+            return Err(Error::new(
+                eyre!("malformed output from `ip`"),
+                crate::ErrorKind::Network,
+            ));
+        }
     }
+    Ok(res)
 }
 
 pub async fn get_iface_ipv4_addr(iface: &str) -> Result<Option<(Ipv4Addr, Ipv4Net)>, Error> {
@@ -38,7 +43,9 @@ pub async fn get_iface_ipv4_addr(iface: &str) -> Result<Option<(Ipv4Addr, Ipv4Ne
             .invoke(crate::ErrorKind::Network)
             .await?,
     )?)?
+    .into_iter()
     .map(|s| Ok::<_, Error>((s.split("/").next().unwrap().parse()?, s.parse()?)))
+    .next()
     .transpose()?)
 }
 
@@ -53,6 +60,8 @@ pub async fn get_iface_ipv6_addr(iface: &str) -> Result<Option<(Ipv6Addr, Ipv6Ne
             .invoke(crate::ErrorKind::Network)
             .await?,
     )?)?
+    .into_iter()
+    .find(|ip| !ip.starts_with("fe80::"))
     .map(|s| Ok::<_, Error>((s.split("/").next().unwrap().parse()?, s.parse()?)))
     .transpose()?)
 }
@@ -121,3 +130,37 @@ impl<T> hyper::server::accept::Accept for SingleAccept<T> {
         std::task::Poll::Ready(self.project().0.take().map(Ok))
     }
 }
+
+pub struct TcpListeners {
+    listeners: Vec<TcpListener>,
+}
+impl TcpListeners {
+    pub fn new(listeners: impl IntoIterator<Item = TcpListener>) -> Self {
+        Self {
+            listeners: listeners.into_iter().collect(),
+        }
+    }
+
+    pub async fn accept(&self) -> std::io::Result<(TcpStream, SocketAddr)> {
+        futures::future::select_all(self.listeners.iter().map(|l| Box::pin(l.accept())))
+            .await
+            .0
+    }
+}
+impl hyper::server::accept::Accept for TcpListeners {
+    type Conn = TcpStream;
+    type Error = std::io::Error;
+
+    fn poll_accept(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Option<Result<Self::Conn, Self::Error>>> {
+        for listener in self.listeners.iter() {
+            let poll = listener.poll_accept(cx);
+            if poll.is_ready() {
+                return poll.map(|a| a.map(|a| a.0)).map(Some);
+            }
+        }
+        std::task::Poll::Pending
+    }
+}

backend/src/net/vhost.rs

@@ -1,8 +1,9 @@
 use std::collections::BTreeMap;
 use std::convert::Infallible;
-use std::net::{IpAddr, SocketAddr};
+use std::net::{IpAddr, Ipv6Addr, SocketAddr};
 use std::str::FromStr;
 use std::sync::{Arc, Weak};
+use std::time::Duration;
 
 use color_eyre::eyre::eyre;
 use helpers::NonDetachingJoinHandle;
@@ -19,7 +20,7 @@ use tokio_rustls::{LazyConfigAcceptor, TlsConnector};
 use crate::net::keys::Key;
 use crate::net::ssl::SslManager;
 use crate::net::utils::SingleAccept;
-use crate::util::io::BackTrackingReader;
+use crate::util::io::{BackTrackingReader, TimeoutStream};
 use crate::Error;
 
 // not allowed: <=1024, >=32768, 5355, 5432, 9050, 6010, 9051, 5353
@@ -41,7 +42,7 @@ impl VHostController {
         hostname: Option<String>,
         external: u16,
         target: SocketAddr,
-        connect_ssl: bool,
+        connect_ssl: Result<(), AlpnInfo>,
     ) -> Result<Arc<()>, Error> {
         let mut writable = self.servers.lock().await;
         let server = if let Some(server) = writable.remove(&external) {
@@ -77,10 +78,16 @@ impl VHostController {
 #[derive(Clone, PartialEq, Eq, PartialOrd, Ord)]
 struct TargetInfo {
     addr: SocketAddr,
-    connect_ssl: bool,
+    connect_ssl: Result<(), AlpnInfo>,
     key: Key,
 }
 
+#[derive(Clone, PartialEq, Eq, PartialOrd, Ord)]
+pub enum AlpnInfo {
+    Reflect,
+    Specified(Vec<Vec<u8>>),
+}
+
 struct VHostServer {
     mapping: Weak<RwLock<BTreeMap<Option<String>, BTreeMap<TargetInfo, Weak<()>>>>>,
     _thread: NonDetachingJoinHandle<()>,
@@ -88,7 +95,7 @@ struct VHostServer {
 impl VHostServer {
     async fn new(port: u16, ssl: Arc<SslManager>) -> Result<Self, Error> {
         // check if port allowed
-        let listener = TcpListener::bind(SocketAddr::new([0, 0, 0, 0].into(), port))
+        let listener = TcpListener::bind(SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), port))
             .await
             .with_kind(crate::ErrorKind::Network)?;
         let mapping = Arc::new(RwLock::new(BTreeMap::new()));
@@ -98,6 +105,8 @@ impl VHostServer {
                 loop {
                     match listener.accept().await {
                         Ok((stream, _)) => {
+                            let stream =
+                                Box::pin(TimeoutStream::new(stream, Duration::from_secs(300)));
                             let mut stream = BackTrackingReader::new(stream);
                             stream.start_buffering();
                             let mapping = mapping.clone();
@@ -178,7 +187,7 @@ impl VHostServer {
                                         let cfg = ServerConfig::builder()
                                             .with_safe_defaults()
                                             .with_no_client_auth();
-                                        let cfg =
+                                        let mut cfg =
                                             if mid.client_hello().signature_schemes().contains(
                                                 &tokio_rustls::rustls::SignatureScheme::ED25519,
                                             ) {
@@ -213,49 +222,94 @@ impl VHostServer {
                                                             .private_key_to_der()?,
                                                     ),
                                                 )
-                                            };
-                                        let mut tls_stream = mid
-                                            .into_stream(Arc::new(
-                                                cfg.with_kind(crate::ErrorKind::OpenSsl)?,
-                                            ))
-                                            .await?;
-                                        tls_stream.get_mut().0.stop_buffering();
-                                        if target.connect_ssl {
-                                            tokio::io::copy_bidirectional(
-                                                &mut tls_stream,
-                                                &mut TlsConnector::from(Arc::new(
+                                            }
+                                            .with_kind(crate::ErrorKind::OpenSsl)?;
+                                        match target.connect_ssl {
+                                            Ok(()) => {
+                                                let mut client_cfg =
                                                     tokio_rustls::rustls::ClientConfig::builder()
                                                         .with_safe_defaults()
                                                         .with_root_certificates({
                                                             let mut store = RootCertStore::empty();
                                                             store.add(
-                                                                &tokio_rustls::rustls::Certificate(
-                                                                    key.root_ca().to_der()?,
-                                                                ),
-                                                            ).with_kind(crate::ErrorKind::OpenSsl)?;
+                                                        &tokio_rustls::rustls::Certificate(
+                                                            key.root_ca().to_der()?,
+                                                        ),
+                                                    ).with_kind(crate::ErrorKind::OpenSsl)?;
                                                             store
                                                         })
-                                                        .with_no_client_auth(),
-                                                ))
-                                                .connect(
-                                                    key.key()
-                                                        .internal_address()
-                                                        .as_str()
-                                                        .try_into()
-                                                        .with_kind(crate::ErrorKind::OpenSsl)?,
-                                                    tcp_stream,
+                                                        .with_no_client_auth();
+                                                client_cfg.alpn_protocols = mid
+                                                    .client_hello()
+                                                    .alpn()
+                                                    .into_iter()
+                                                    .flatten()
+                                                    .map(|x| x.to_vec())
+                                                    .collect();
+                                                let mut target_stream =
+                                                    TlsConnector::from(Arc::new(client_cfg))
+                                                        .connect_with(
+                                                            key.key()
+                                                                .internal_address()
+                                                                .as_str()
+                                                                .try_into()
+                                                                .with_kind(
+                                                                    crate::ErrorKind::OpenSsl,
+                                                                )?,
+                                                            tcp_stream,
+                                                            |conn| {
+                                                                cfg.alpn_protocols.extend(
+                                                                    conn.alpn_protocol()
+                                                                        .into_iter()
+                                                                        .map(|p| p.to_vec()),
+                                                                )
+                                                            },
+                                                        )
+                                                        .await
+                                                        .with_kind(crate::ErrorKind::OpenSsl)?;
+                                                let mut tls_stream =
+                                                    mid.into_stream(Arc::new(cfg)).await?;
+                                                tls_stream.get_mut().0.stop_buffering();
+                                                tokio::io::copy_bidirectional(
+                                                    &mut tls_stream,
+                                                    &mut target_stream,
                                                 )
                                                 .await
-                                                .with_kind(crate::ErrorKind::OpenSsl)?,
-                                            )
-                                            .await?;
-                                        } else {
-                                            tokio::io::copy_bidirectional(
-                                                &mut tls_stream,
-                                                &mut tcp_stream,
-                                            )
-                                            .await?;
+                                            }
+                                            Err(AlpnInfo::Reflect) => {
+                                                for proto in
+                                                    mid.client_hello().alpn().into_iter().flatten()
+                                                {
+                                                    cfg.alpn_protocols.push(proto.into());
+                                                }
+                                                let mut tls_stream =
+                                                    mid.into_stream(Arc::new(cfg)).await?;
+                                                tls_stream.get_mut().0.stop_buffering();
+                                                tokio::io::copy_bidirectional(
+                                                    &mut tls_stream,
+                                                    &mut tcp_stream,
+                                                )
+                                                .await
+                                            }
+                                            Err(AlpnInfo::Specified(alpn)) => {
+                                                cfg.alpn_protocols = alpn;
+                                                let mut tls_stream =
+                                                    mid.into_stream(Arc::new(cfg)).await?;
+                                                tls_stream.get_mut().0.stop_buffering();
+                                                tokio::io::copy_bidirectional(
+                                                    &mut tls_stream,
+                                                    &mut tcp_stream,
+                                                )
+                                                .await
+                                            }
                                         }
+                                        .map_or_else(
+                                            |e| match e.kind() {
+                                                std::io::ErrorKind::UnexpectedEof => Ok(()),
+                                                _ => Err(e),
+                                            },
+                                            |_| Ok(()),
+                                        )?;
                                     } else {
                                         // 503
                                     }

backend/src/net/wifi.rs

@@ -173,7 +173,7 @@ pub async fn delete(#[context] ctx: RpcContext, #[arg] ssid: String) -> Result<(
     let is_current_removed_and_no_hardwire =
         is_current_being_removed && !interface_connected(&ctx.ethernet_interface).await?;
     if is_current_removed_and_no_hardwire {
-        return Err(Error::new(color_eyre::eyre::eyre!("Forbidden: Deleting this Network would make your Embassy Unreachable. Either connect to ethernet or connect to a different WiFi network to remedy this."), ErrorKind::Wifi));
+        return Err(Error::new(color_eyre::eyre::eyre!("Forbidden: Deleting this network would make your server unreachable. Either connect to ethernet or connect to a different WiFi network to remedy this."), ErrorKind::Wifi));
     }
 
     wpa_supplicant

backend/src/os_install/gpt.rs

@@ -1,3 +1,5 @@
+use std::path::Path;
+
 use color_eyre::eyre::eyre;
 use gpt::disk::LogicalBlockSize;
 use gpt::GptConfig;
@@ -8,9 +10,10 @@ use crate::os_install::partition_for;
 use crate::Error;
 
 pub async fn partition(disk: &DiskInfo, overwrite: bool) -> Result<OsPartitionInfo, Error> {
-    {
+    let efi = {
         let disk = disk.clone();
         tokio::task::spawn_blocking(move || {
+            let use_efi = Path::new("/sys/firmware/efi").exists();
             let mut device = Box::new(
                 std::fs::File::options()
                     .read(true)
@@ -44,17 +47,15 @@ pub async fn partition(disk: &DiskInfo, overwrite: bool) -> Result<OsPartitionIn
                     .map(|(idx, x)| (idx + 1, x))
                 {
                     if let Some(entry) = gpt.partitions().get(&(idx as u32)) {
-                        if entry.first_lba >= 33556480 {
-                            if idx < 3 {
-                                guid_part = Some(entry.clone())
-                            }
-                            break;
-                        }
                         if part_info.guid.is_some() {
-                            return Err(Error::new(
-                                eyre!("Not enough space before embassy data"),
-                                crate::ErrorKind::InvalidRequest,
-                            ));
+                            if entry.first_lba < if use_efi { 33759266 } else { 33570850 } {
+                                return Err(Error::new(
+                                    eyre!("Not enough space before embassy data"),
+                                    crate::ErrorKind::InvalidRequest,
+                                ));
+                            }
+                            guid_part = Some(entry.clone());
+                            break;
                         }
                     }
                 }
@@ -63,7 +64,19 @@ pub async fn partition(disk: &DiskInfo, overwrite: bool) -> Result<OsPartitionIn
 
             gpt.update_partitions(Default::default())?;
 
-            gpt.add_partition("efi", 100 * 1024 * 1024, gpt::partition_types::EFI, 0, None)?;
+            let efi = if use_efi {
+                gpt.add_partition("efi", 100 * 1024 * 1024, gpt::partition_types::EFI, 0, None)?;
+                true
+            } else {
+                gpt.add_partition(
+                    "bios-grub",
+                    8 * 1024 * 1024,
+                    gpt::partition_types::BIOS,
+                    0,
+                    None,
+                )?;
+                false
+            };
             gpt.add_partition(
                 "boot",
                 1024 * 1024 * 1024,
@@ -108,14 +121,15 @@ pub async fn partition(disk: &DiskInfo, overwrite: bool) -> Result<OsPartitionIn
 
             gpt.write()?;
 
-            Ok(())
+            Ok(efi)
         })
         .await
-        .unwrap()?;
-    }
+        .unwrap()?
+    };
 
     Ok(OsPartitionInfo {
-        efi: Some(partition_for(&disk.logicalname, 1)),
+        efi: efi.then(|| partition_for(&disk.logicalname, 1)),
+        bios: (!efi).then(|| partition_for(&disk.logicalname, 1)),
         boot: partition_for(&disk.logicalname, 2),
         root: partition_for(&disk.logicalname, 3),
     })

backend/src/os_install/mbr.rs

@@ -27,18 +27,14 @@ pub async fn partition(disk: &DiskInfo, overwrite: bool) -> Result<OsPartitionIn
                     .map(|(idx, x)| (idx + 1, x))
                 {
                     if let Some(entry) = mbr.get_mut(idx) {
-                        if entry.starting_lba >= 33556480 {
-                            if idx < 3 {
-                                guid_part =
-                                    Some(std::mem::replace(entry, MBRPartitionEntry::empty()))
-                            }
-                            break;
-                        }
                         if part_info.guid.is_some() {
-                            return Err(Error::new(
-                                eyre!("Not enough space before embassy data"),
-                                crate::ErrorKind::InvalidRequest,
-                            ));
+                            if entry.starting_lba < 33556480 {
+                                return Err(Error::new(
+                                    eyre!("Not enough space before embassy data"),
+                                    crate::ErrorKind::InvalidRequest,
+                                ));
+                            }
+                            guid_part = Some(std::mem::replace(entry, MBRPartitionEntry::empty()));
                         }
                         *entry = MBRPartitionEntry::empty();
                     }
@@ -85,6 +81,7 @@ pub async fn partition(disk: &DiskInfo, overwrite: bool) -> Result<OsPartitionIn
 
     Ok(OsPartitionInfo {
         efi: None,
+        bios: None,
         boot: partition_for(&disk.logicalname, 1),
         root: partition_for(&disk.logicalname, 2),
     })

backend/src/os_install/mod.rs

@@ -10,7 +10,7 @@ use crate::context::InstallContext;
 use crate::disk::mount::filesystem::bind::Bind;
 use crate::disk::mount::filesystem::block_dev::BlockDev;
 use crate::disk::mount::filesystem::efivarfs::EfiVarFs;
-use crate::disk::mount::filesystem::ReadWrite;
+use crate::disk::mount::filesystem::{MountType, ReadWrite};
 use crate::disk::mount::guard::{MountGuard, TmpMountGuard};
 use crate::disk::util::{DiskInfo, PartitionTable};
 use crate::disk::OsPartitionInfo;
@@ -49,7 +49,7 @@ pub async fn list() -> Result<Vec<DiskInfo>, Error> {
                     Command::new("grub-probe-default")
                         .arg("-t")
                         .arg("disk")
-                        .arg("/cdrom")
+                        .arg("/run/live/medium")
                         .invoke(crate::ErrorKind::Grub)
                         .await?,
                 )?
@@ -93,13 +93,7 @@ pub fn partition_for(disk: impl AsRef<Path>, idx: usize) -> PathBuf {
 
 async fn partition(disk: &mut DiskInfo, overwrite: bool) -> Result<OsPartitionInfo, Error> {
     let partition_type = match (overwrite, disk.partition_table) {
-        (true, _) | (_, None) => {
-            if tokio::fs::metadata("/sys/firmware/efi").await.is_ok() {
-                PartitionTable::Gpt
-            } else {
-                PartitionTable::Mbr
-            }
-        }
+        (true, _) | (_, None) => PartitionTable::Gpt,
         (_, Some(t)) => t,
     };
     disk.partition_table = Some(partition_type);
@@ -153,18 +147,66 @@ pub async fn execute(
         .invoke(crate::ErrorKind::DiskManagement)
         .await?;
 
-    Command::new("mkfs.ext4")
+    if !overwrite {
+        if let Ok(guard) =
+            TmpMountGuard::mount(&BlockDev::new(part_info.root.clone()), MountType::ReadWrite).await
+        {
+            if let Err(e) = async {
+                // cp -r ${guard}/config /tmp/config
+                if tokio::fs::metadata(guard.as_ref().join("config/upgrade"))
+                    .await
+                    .is_ok()
+                {
+                    tokio::fs::remove_file(guard.as_ref().join("config/upgrade")).await?;
+                }
+                if tokio::fs::metadata(guard.as_ref().join("config/disk.guid"))
+                    .await
+                    .is_ok()
+                {
+                    tokio::fs::remove_file(guard.as_ref().join("config/disk.guid")).await?;
+                }
+                Command::new("cp")
+                    .arg("-r")
+                    .arg(guard.as_ref().join("config"))
+                    .arg("/tmp/config.bak")
+                    .invoke(crate::ErrorKind::Filesystem)
+                    .await?;
+                Ok::<_, Error>(())
+            }
+            .await
+            {
+                tracing::error!("Error recovering previous config: {e}");
+                tracing::debug!("{e:?}");
+            }
+            guard.unmount().await?;
+        }
+    }
+
+    Command::new("mkfs.btrfs")
+        .arg("-f")
         .arg(&part_info.root)
         .invoke(crate::ErrorKind::DiskManagement)
         .await?;
-    Command::new("e2label")
+    Command::new("btrfs")
+        .arg("property")
+        .arg("set")
         .arg(&part_info.root)
+        .arg("label")
         .arg("rootfs")
         .invoke(crate::ErrorKind::DiskManagement)
         .await?;
 
     let rootfs = TmpMountGuard::mount(&BlockDev::new(&part_info.root), ReadWrite).await?;
-    tokio::fs::create_dir(rootfs.as_ref().join("config")).await?;
+    if tokio::fs::metadata("/tmp/config.bak").await.is_ok() {
+        Command::new("cp")
+            .arg("-r")
+            .arg("/tmp/config.bak")
+            .arg(rootfs.as_ref().join("config"))
+            .invoke(crate::ErrorKind::Filesystem)
+            .await?;
+    } else {
+        tokio::fs::create_dir(rootfs.as_ref().join("config")).await?;
+    }
     tokio::fs::create_dir(rootfs.as_ref().join("next")).await?;
     let current = rootfs.as_ref().join("current");
     tokio::fs::create_dir(&current).await?;
@@ -188,7 +230,7 @@ pub async fn execute(
         .arg("-f")
         .arg("-d")
         .arg(&current)
-        .arg("/cdrom/casper/filesystem.squashfs")
+        .arg("/run/live/medium/live/filesystem.squashfs")
         .invoke(crate::ErrorKind::Filesystem)
         .await?;
 
@@ -230,10 +272,16 @@ pub async fn execute(
         .invoke(crate::ErrorKind::OpenSsh)
         .await?;
 
+    let dev = MountGuard::mount(
+        &Bind::new(rootfs.as_ref()),
+        current.join("media/embassy/embassyfs"),
+        MountType::ReadOnly,
+    )
+    .await?;
     let dev = MountGuard::mount(&Bind::new("/dev"), current.join("dev"), ReadWrite).await?;
     let proc = MountGuard::mount(&Bind::new("/proc"), current.join("proc"), ReadWrite).await?;
     let sys = MountGuard::mount(&Bind::new("/sys"), current.join("sys"), ReadWrite).await?;
-    let efivarfs = if let Some(efi) = &part_info.efi {
+    let efivarfs = if tokio::fs::metadata("/sys/firmware/efi").await.is_ok() {
         Some(
             MountGuard::mount(
                 &EfiVarFs,
@@ -246,14 +294,9 @@ pub async fn execute(
         None
     };
 
-    Command::new("chroot")
-        .arg(&current)
-        .arg("update-grub")
-        .invoke(crate::ErrorKind::Grub)
-        .await?;
     let mut install = Command::new("chroot");
     install.arg(&current).arg("grub-install");
-    if part_info.efi.is_none() {
+    if tokio::fs::metadata("/sys/firmware/efi").await.is_err() {
         install.arg("--target=i386-pc");
     } else {
         match *ARCH {
@@ -267,6 +310,12 @@ pub async fn execute(
         .invoke(crate::ErrorKind::Grub)
         .await?;
 
+    Command::new("chroot")
+        .arg(&current)
+        .arg("update-grub2")
+        .invoke(crate::ErrorKind::Grub)
+        .await?;
+
     dev.unmount(false).await?;
     if let Some(efivarfs) = efivarfs {
         efivarfs.unmount(false).await?;

backend/src/procedure/docker.rs

@@ -2,15 +2,17 @@ use std::borrow::Cow;
 use std::collections::{BTreeMap, BTreeSet, VecDeque};
 use std::ffi::{OsStr, OsString};
 use std::net::Ipv4Addr;
+use std::os::unix::prelude::FileTypeExt;
 use std::path::{Path, PathBuf};
 use std::time::Duration;
 
 use async_stream::stream;
 use bollard::container::RemoveContainerOptions;
+use chrono::format::Item;
 use color_eyre::eyre::eyre;
 use color_eyre::Report;
-use futures::future::Either as EitherFuture;
-use futures::TryStreamExt;
+use futures::future::{BoxFuture, Either as EitherFuture};
+use futures::{FutureExt, TryStreamExt};
 use helpers::{NonDetachingJoinHandle, UnixRpcClient};
 use models::{Id, ImageId};
 use nix::sys::signal;
@@ -18,10 +20,8 @@ use nix::unistd::Pid;
 use serde::de::DeserializeOwned;
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
-use tokio::{
-    io::{AsyncBufRead, AsyncBufReadExt, BufReader},
-    time::timeout,
-};
+use tokio::io::{AsyncBufRead, AsyncBufReadExt, BufReader};
+use tokio::time::timeout;
 use tracing::instrument;
 
 use super::ProcedureName;
@@ -68,6 +68,8 @@ pub struct DockerContainer {
     pub sigterm_timeout: Option<SerdeDuration>,
     #[serde(default)]
     pub system: bool,
+    #[serde(default)]
+    pub gpu_acceleration: bool,
 }
 
 impl DockerContainer {
@@ -154,6 +156,8 @@ pub struct DockerProcedure {
     pub sigterm_timeout: Option<SerdeDuration>,
     #[serde(default)]
     pub shm_size_mb: Option<usize>, // TODO: use postfix sizing? like 1k vs 1m vs 1g
+    #[serde(default)]
+    pub gpu_acceleration: bool,
 }
 
 #[derive(Clone, Debug, Deserialize, Serialize, Default)]
@@ -184,6 +188,7 @@ impl DockerProcedure {
             io_format: injectable.io_format,
             sigterm_timeout: injectable.sigterm_timeout,
             shm_size_mb: container.shm_size_mb,
+            gpu_acceleration: container.gpu_acceleration,
         }
     }
 
@@ -226,7 +231,6 @@ impl DockerProcedure {
         let name = name.docker_name();
         let name: Option<&str> = name.as_ref().map(|x| &**x);
         let mut cmd = tokio::process::Command::new("docker");
-        tracing::debug!("{:?} is run", name);
         let container_name = Self::container_name(pkg_id, name);
         cmd.arg("run")
             .arg("--rm")
@@ -408,7 +412,6 @@ impl DockerProcedure {
         let name: Option<&str> = name.as_deref();
         let mut cmd = tokio::process::Command::new("docker");
 
-        tracing::debug!("{:?} is exec", name);
         cmd.arg("exec");
 
         cmd.args(self.docker_args_inject(pkg_id).await?);
@@ -711,6 +714,32 @@ impl DockerProcedure {
             res.push(OsStr::new("--shm-size").into());
             res.push(OsString::from(format!("{}m", shm_size_mb)).into());
         }
+        if self.gpu_acceleration {
+            fn get_devices<'a>(
+                path: &'a Path,
+                res: &'a mut Vec<PathBuf>,
+            ) -> BoxFuture<'a, Result<(), Error>> {
+                async move {
+                    let mut read_dir = tokio::fs::read_dir(path).await?;
+                    while let Some(entry) = read_dir.next_entry().await? {
+                        let fty = entry.metadata().await?.file_type();
+                        if fty.is_block_device() || fty.is_char_device() {
+                            res.push(entry.path());
+                        } else if fty.is_dir() {
+                            get_devices(&*entry.path(), res).await?;
+                        }
+                    }
+                    Ok(())
+                }
+                .boxed()
+            }
+            let mut devices = Vec::new();
+            get_devices(Path::new("/dev/dri"), &mut devices).await?;
+            for device in devices {
+                res.push(OsStr::new("--device").into());
+                res.push(OsString::from(device).into());
+            }
+        }
         res.push(OsStr::new("--interactive").into());
         res.push(OsStr::new("--log-driver=journald").into());
         res.push(OsStr::new("--entrypoint").into());

backend/src/procedure/js_scripts.rs

@@ -690,3 +690,49 @@ async fn js_rsync() {
         .unwrap()
         .unwrap();
 }
+
+#[tokio::test]
+async fn js_disk_usage() {
+    let js_action = JsProcedure { args: vec![] };
+    let path: PathBuf = "test/js_action_execute/"
+        .parse::<PathBuf>()
+        .unwrap()
+        .canonicalize()
+        .unwrap();
+    let package_id = "test-package".parse().unwrap();
+    let package_version: Version = "0.3.0.3".parse().unwrap();
+    let name = ProcedureName::Action("test-disk-usage".parse().unwrap());
+    let volumes: Volumes = serde_json::from_value(serde_json::json!({
+        "main": {
+            "type": "data"
+        },
+        "compat": {
+            "type": "assets"
+        },
+        "filebrowser" :{
+            "package-id": "filebrowser",
+            "path": "data",
+            "readonly": true,
+            "type": "pointer",
+            "volume-id": "main",
+        }
+    }))
+    .unwrap();
+    let input: Option<serde_json::Value> = None;
+    let timeout = Some(Duration::from_secs(10));
+    dbg!(js_action
+        .execute::<serde_json::Value, serde_json::Value>(
+            &path,
+            &package_id,
+            &package_version,
+            name,
+            &volumes,
+            input,
+            timeout,
+            ProcessGroupId(0),
+            None,
+        )
+        .await
+        .unwrap()
+        .unwrap());
+}

backend/src/s9pk/manifest.rs

@@ -1,3 +1,4 @@
+use std::collections::BTreeMap;
 use std::path::{Path, PathBuf};
 
 use color_eyre::eyre::eyre;
@@ -16,6 +17,7 @@ use crate::net::interface::Interfaces;
 use crate::procedure::docker::DockerContainers;
 use crate::procedure::PackageProcedure;
 use crate::status::health_check::HealthChecks;
+use crate::util::serde::Regex;
 use crate::util::Version;
 use crate::version::{Current, VersionT};
 use crate::volume::Volumes;
@@ -79,6 +81,9 @@ pub struct Manifest {
 
     #[serde(default)]
     pub replaces: Vec<String>,
+
+    #[serde(default)]
+    pub hardware_requirements: HardwareRequirements,
 }
 
 impl Manifest {
@@ -109,6 +114,15 @@ impl Manifest {
     }
 }
 
+#[derive(Clone, Debug, Default, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct HardwareRequirements {
+    #[serde(default)]
+    device: BTreeMap<String, Regex>,
+    ram: Option<u64>,
+    arch: Option<Vec<String>>,
+}
+
 #[derive(Clone, Debug, Default, Deserialize, Serialize)]
 #[serde(rename_all = "kebab-case")]
 pub struct Assets {

backend/src/setup.rs

@@ -1,9 +1,9 @@
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 use std::sync::Arc;
+use std::time::Duration;
 
 use color_eyre::eyre::eyre;
 use futures::StreamExt;
-use helpers::{Rsync, RsyncOptions};
 use josekit::jwk::Jwk;
 use openssl::x509::X509;
 use patch_db::DbHandle;
@@ -13,6 +13,7 @@ use serde::{Deserialize, Serialize};
 use sqlx::Connection;
 use tokio::fs::File;
 use tokio::io::AsyncWriteExt;
+use tokio::try_join;
 use torut::onion::OnionAddressV3;
 use tracing::instrument;
 
@@ -32,6 +33,7 @@ use crate::disk::REPAIR_DISK_PATH;
 use crate::hostname::Hostname;
 use crate::init::{init, InitResult};
 use crate::middleware::encrypt::EncryptedWire;
+use crate::util::io::{dir_copy, dir_size, Counter};
 use crate::{Error, ErrorKind, ResultExt};
 
 #[command(subcommands(status, disk, attach, execute, cifs, complete, get_pubkey, exit))]
@@ -123,7 +125,7 @@ pub async fn attach(
                 } else {
                     RepairStrategy::Preen
                 },
-                DEFAULT_PASSWORD,
+                if guid.ends_with("_UNENC") { None } else { Some(DEFAULT_PASSWORD) },
             )
             .await?;
             if tokio::fs::metadata(REPAIR_DISK_PATH).await.is_ok() {
@@ -135,14 +137,14 @@ pub async fn attach(
                 crate::disk::main::export(&*guid, &ctx.datadir).await?;
                 return Err(Error::new(
                     eyre!(
-                        "Errors were corrected with your disk, but the Embassy must be restarted in order to proceed"
+                        "Errors were corrected with your disk, but the server must be restarted in order to proceed"
                     ),
                     ErrorKind::DiskManagement,
                 ));
             }
             let (hostname, tor_addr, root_ca) = setup_init(&ctx, password).await?;
             *ctx.setup_result.write().await = Some((guid, SetupResult {
-                tor_address: format!("http://{}", tor_addr),
+                tor_address: format!("https://{}", tor_addr),
                 lan_address: hostname.lan_address(),
                 root_ca: String::from_utf8(root_ca.to_pem()?)?,
             }));
@@ -279,7 +281,7 @@ pub async fn execute(
                 *ctx.setup_result.write().await = Some((
                     guid,
                     SetupResult {
-                        tor_address: format!("http://{}", tor_addr),
+                        tor_address: format!("https://{}", tor_addr),
                         lan_address: hostname.lan_address(),
                         root_ca: String::from_utf8(
                             root_ca.to_pem().expect("failed to serialize root ca"),
@@ -294,7 +296,7 @@ pub async fn execute(
                 }));
             }
             Err(e) => {
-                tracing::error!("Error Setting Up Embassy: {}", e);
+                tracing::error!("Error Setting Up Server: {}", e);
                 tracing::debug!("{:?}", e);
                 *ctx.setup_status.write().await = Some(Err(e.into()));
             }
@@ -335,12 +337,17 @@ pub async fn execute_inner(
     recovery_source: Option<RecoverySource>,
     recovery_password: Option<String>,
 ) -> Result<(Arc<String>, Hostname, OnionAddressV3, X509), Error> {
+    let encryption_password = if ctx.disable_encryption {
+        None
+    } else {
+        Some(DEFAULT_PASSWORD)
+    };
     let guid = Arc::new(
         crate::disk::main::create(
             &[embassy_logicalname],
             &pvscan().await?,
             &ctx.datadir,
-            DEFAULT_PASSWORD,
+            encryption_password,
         )
         .await?,
     );
@@ -348,7 +355,7 @@ pub async fn execute_inner(
         &*guid,
         &ctx.datadir,
         RepairStrategy::Preen,
-        DEFAULT_PASSWORD,
+        encryption_password,
     )
     .await?;
 
@@ -416,72 +423,78 @@ async fn migrate(
         &old_guid,
         "/media/embassy/migrate",
         RepairStrategy::Preen,
-        DEFAULT_PASSWORD,
-    )
-    .await?;
-
-    let mut main_transfer = Rsync::new(
-        "/media/embassy/migrate/main/",
-        "/embassy-data/main/",
-        RsyncOptions {
-            delete: true,
-            force: true,
-            ignore_existing: false,
-            exclude: Vec::new(),
-            no_permissions: false,
+        if guid.ends_with("_UNENC") {
+            None
+        } else {
+            Some(DEFAULT_PASSWORD)
         },
     )
     .await?;
-    let mut package_data_transfer = Rsync::new(
+
+    let main_transfer_args = ("/media/embassy/migrate/main/", "/embassy-data/main/");
+    let package_data_transfer_args = (
         "/media/embassy/migrate/package-data/",
         "/embassy-data/package-data/",
-        RsyncOptions {
-            delete: true,
-            force: true,
-            ignore_existing: false,
-            exclude: vec!["tmp".to_owned()],
-            no_permissions: false,
-        },
-    )
-    .await?;
+    );
 
-    let mut main_prog = 0.0;
-    let mut main_complete = false;
-    let mut pkg_prog = 0.0;
-    let mut pkg_complete = false;
-    loop {
-        tokio::select! {
-            p = main_transfer.progress.next() => {
-                if let Some(p) = p {
-                    main_prog = p;
-                } else {
-                    main_prog = 1.0;
-                    main_complete = true;
-                }
-            }
-            p = package_data_transfer.progress.next() => {
-                if let Some(p) = p {
-                    pkg_prog = p;
-                } else {
-                    pkg_prog = 1.0;
-                    pkg_complete = true;
-                }
-            }
-        }
-        if main_prog > 0.0 && pkg_prog > 0.0 {
-            *ctx.setup_status.write().await = Some(Ok(SetupStatus {
-                bytes_transferred: ((main_prog * 50.0) + (pkg_prog * 950.0)) as u64,
-                total_bytes: Some(1000),
-                complete: false,
-            }));
-        }
-        if main_complete && pkg_complete {
-            break;
-        }
+    let tmpdir = Path::new(package_data_transfer_args.0).join("tmp");
+    if tokio::fs::metadata(&tmpdir).await.is_ok() {
+        tokio::fs::remove_dir_all(&tmpdir).await?;
     }
 
-    main_transfer.wait().await?;
-    package_data_transfer.wait().await?;
+    let ordering = std::sync::atomic::Ordering::Relaxed;
+
+    let main_transfer_size = Counter::new(0, ordering);
+    let package_data_transfer_size = Counter::new(0, ordering);
+
+    let size = tokio::select! {
+        res = async {
+            let (main_size, package_data_size) = try_join!(
+                dir_size(main_transfer_args.0, Some(&main_transfer_size)),
+                dir_size(package_data_transfer_args.0, Some(&package_data_transfer_size))
+            )?;
+            Ok::<_, Error>(main_size + package_data_size)
+        } => { res? },
+        res = async {
+            loop {
+                tokio::time::sleep(Duration::from_secs(1)).await;
+                *ctx.setup_status.write().await = Some(Ok(SetupStatus {
+                    bytes_transferred: 0,
+                    total_bytes: Some(main_transfer_size.load() + package_data_transfer_size.load()),
+                    complete: false,
+                }));
+            }
+        } => res,
+    };
+
+    *ctx.setup_status.write().await = Some(Ok(SetupStatus {
+        bytes_transferred: 0,
+        total_bytes: Some(size),
+        complete: false,
+    }));
+
+    let main_transfer_progress = Counter::new(0, ordering);
+    let package_data_transfer_progress = Counter::new(0, ordering);
+
+    tokio::select! {
+        res = async {
+            try_join!(
+                dir_copy(main_transfer_args.0, main_transfer_args.1, Some(&main_transfer_progress)),
+                dir_copy(package_data_transfer_args.0, package_data_transfer_args.1, Some(&package_data_transfer_progress))
+            )?;
+            Ok::<_, Error>(())
+        } => { res? },
+        res = async {
+            loop {
+                tokio::time::sleep(Duration::from_secs(1)).await;
+                *ctx.setup_status.write().await = Some(Ok(SetupStatus {
+                    bytes_transferred: main_transfer_progress.load() + package_data_transfer_progress.load(),
+                    total_bytes: Some(size),
+                    complete: false,
+                }));
+            }
+        } => res,
+    }
 
     let (hostname, tor_addr, root_ca) = setup_init(&ctx, Some(embassy_password)).await?;
 

backend/src/shutdown.rs

@@ -8,7 +8,7 @@ use crate::disk::main::export;
 use crate::init::{STANDBY_MODE_PATH, SYSTEM_REBUILD_PATH};
 use crate::sound::SHUTDOWN;
 use crate::util::{display_none, Invoke};
-use crate::{Error, ErrorKind, IS_RASPBERRY_PI};
+use crate::{Error, ErrorKind, OS_ARCH};
 
 #[derive(Debug, Clone)]
 pub struct Shutdown {
@@ -58,7 +58,7 @@ impl Shutdown {
                     tracing::debug!("{:?}", e);
                 }
             }
-            if !*IS_RASPBERRY_PI || self.restart {
+            if OS_ARCH != "raspberrypi" || self.restart {
                 if let Err(e) = SHUTDOWN.play().await {
                     tracing::error!("Error Playing Shutdown Song: {}", e);
                     tracing::debug!("{:?}", e);
@@ -66,7 +66,7 @@ impl Shutdown {
             }
         });
         drop(rt);
-        if *IS_RASPBERRY_PI {
+        if OS_ARCH == "raspberrypi" {
             if !self.restart {
                 std::fs::write(STANDBY_MODE_PATH, "").unwrap();
                 Command::new("sync").spawn().unwrap().wait().unwrap();

backend/src/system.rs

@@ -6,6 +6,7 @@ use futures::FutureExt;
 use rpc_toolkit::command;
 use rpc_toolkit::yajrc::RpcError;
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
+use tokio::process::Command;
 use tokio::sync::broadcast::Receiver;
 use tokio::sync::RwLock;
 use tracing::instrument;
@@ -17,11 +18,71 @@ use crate::logs::{
     LogResponse, LogSource,
 };
 use crate::shutdown::Shutdown;
-use crate::util::display_none;
 use crate::util::serde::{display_serializable, IoFormat};
+use crate::util::{display_none, Invoke};
 use crate::{Error, ErrorKind, ResultExt};
 
-pub const SYSTEMD_UNIT: &'static str = "embassyd";
+pub const SYSTEMD_UNIT: &'static str = "startd";
+
+#[command(subcommands(zram))]
+pub async fn experimental() -> Result<(), Error> {
+    Ok(())
+}
+
+pub async fn enable_zram() -> Result<(), Error> {
+    let mem_info = get_mem_info().await?;
+    Command::new("modprobe")
+        .arg("zram")
+        .invoke(ErrorKind::Zram)
+        .await?;
+    tokio::fs::write("/sys/block/zram0/comp_algorithm", "lz4")
+        .await
+        .with_kind(ErrorKind::Zram)?;
+    tokio::fs::write(
+        "/sys/block/zram0/disksize",
+        format!("{}M", mem_info.total.0 as u64 / 4),
+    )
+    .await
+    .with_kind(ErrorKind::Zram)?;
+    Command::new("mkswap")
+        .arg("/dev/zram0")
+        .invoke(ErrorKind::Zram)
+        .await?;
+    Command::new("swapon")
+        .arg("-p")
+        .arg("5")
+        .arg("/dev/zram0")
+        .invoke(ErrorKind::Zram)
+        .await?;
+    Ok(())
+}
+
+#[command(display(display_none))]
+pub async fn zram(#[context] ctx: RpcContext, #[arg] enable: bool) -> Result<(), Error> {
+    let mut db = ctx.db.handle();
+    let mut zram = crate::db::DatabaseModel::new()
+        .server_info()
+        .zram()
+        .get_mut(&mut db)
+        .await?;
+    if enable == *zram {
+        return Ok(());
+    }
+    *zram = enable;
+    if enable {
+        enable_zram().await?;
+    } else {
+        Command::new("swapoff")
+            .arg("/dev/zram0")
+            .invoke(ErrorKind::Zram)
+            .await?;
+        tokio::fs::write("/sys/block/zram0/reset", "1")
+            .await
+            .with_kind(ErrorKind::Zram)?;
+    }
+    zram.save(&mut db).await?;
+    Ok(())
+}
 
 #[command]
 pub async fn time() -> Result<String, Error> {
@@ -190,7 +251,7 @@ impl<'de> Deserialize<'de> for Percentage {
 }
 
 #[derive(Clone, Debug)]
-pub struct MebiBytes(f64);
+pub struct MebiBytes(pub f64);
 impl Serialize for MebiBytes {
     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
     where
@@ -249,19 +310,19 @@ pub struct MetricsGeneral {
 #[derive(Deserialize, Serialize, Clone, Debug)]
 pub struct MetricsMemory {
     #[serde(rename = "Percentage Used")]
-    percentage_used: Percentage,
+    pub percentage_used: Percentage,
     #[serde(rename = "Total")]
-    total: MebiBytes,
+    pub total: MebiBytes,
     #[serde(rename = "Available")]
-    available: MebiBytes,
+    pub available: MebiBytes,
     #[serde(rename = "Used")]
-    used: MebiBytes,
+    pub used: MebiBytes,
     #[serde(rename = "Swap Total")]
-    swap_total: MebiBytes,
+    pub swap_total: MebiBytes,
     #[serde(rename = "Swap Free")]
-    swap_free: MebiBytes,
+    pub swap_free: MebiBytes,
     #[serde(rename = "Swap Used")]
-    swap_used: MebiBytes,
+    pub swap_used: MebiBytes,
 }
 #[derive(Deserialize, Serialize, Clone, Debug)]
 pub struct MetricsCpu {
@@ -512,13 +573,30 @@ async fn launch_disk_task(
 
 #[instrument(skip_all)]
 async fn get_temp() -> Result<Celsius, Error> {
-    let temp_file = "/sys/class/thermal/thermal_zone0/temp";
-    let milli = tokio::fs::read_to_string(temp_file)
-        .await
-        .with_ctx(|_| (crate::ErrorKind::Filesystem, temp_file))?
-        .trim()
-        .parse::<f64>()?;
-    Ok(Celsius(milli / 1000.0))
+    let temp = serde_json::from_slice::<serde_json::Value>(
+        &Command::new("sensors")
+            .arg("-j")
+            .invoke(ErrorKind::Filesystem)
+            .await?,
+    )
+    .with_kind(ErrorKind::Deserialization)?
+    .as_object()
+    .into_iter()
+    .flatten()
+    .flat_map(|(_, v)| v.as_object())
+    .flatten()
+    .flat_map(|(_, v)| v.as_object())
+    .flatten()
+    .filter_map(|(k, v)| {
+        if k.ends_with("_input") {
+            v.as_f64()
+        } else {
+            None
+        }
+    })
+    .reduce(f64::max)
+    .ok_or_else(|| Error::new(eyre!("No temperatures available"), ErrorKind::Filesystem))?;
+    Ok(Celsius(temp))
 }
 
 #[derive(Debug, Clone)]
@@ -620,7 +698,7 @@ pub struct MemInfo {
     swap_free: Option<u64>,
 }
 #[instrument(skip_all)]
-async fn get_mem_info() -> Result<MetricsMemory, Error> {
+pub async fn get_mem_info() -> Result<MetricsMemory, Error> {
     let contents = tokio::fs::read_to_string("/proc/meminfo").await?;
     let mut mem_info = MemInfo {
         mem_total: None,
@@ -681,7 +759,7 @@ async fn get_mem_info() -> Result<MetricsMemory, Error> {
     let swap_total = MebiBytes(swap_total_k as f64 / 1024.0);
     let swap_free = MebiBytes(swap_free_k as f64 / 1024.0);
     let swap_used = MebiBytes((swap_total_k - swap_free_k) as f64 / 1024.0);
-    let percentage_used = Percentage(used.0 / total.0 * 100.0);
+    let percentage_used = Percentage((total.0 - available.0) / total.0 * 100.0);
     Ok(MetricsMemory {
         percentage_used,
         total,

backend/src/update/mod.rs

@@ -19,6 +19,7 @@ use crate::db::model::UpdateProgress;
 use crate::disk::mount::filesystem::bind::Bind;
 use crate::disk::mount::filesystem::ReadWrite;
 use crate::disk::mount::guard::MountGuard;
+use crate::marketplace::with_query_params;
 use crate::notifications::NotificationLevel;
 use crate::sound::{
     CIRCLE_OF_5THS_SHORT, UPDATE_FAILED_1, UPDATE_FAILED_2, UPDATE_FAILED_3, UPDATE_FAILED_4,
@@ -26,7 +27,7 @@ use crate::sound::{
 use crate::update::latest_information::LatestInformation;
 use crate::util::Invoke;
 use crate::version::{Current, VersionT};
-use crate::{Error, ErrorKind, ResultExt, IS_RASPBERRY_PI};
+use crate::{Error, ErrorKind, ResultExt, OS_ARCH};
 
 mod latest_information;
 
@@ -81,23 +82,19 @@ async fn maybe_do_update(
     marketplace_url: Url,
 ) -> Result<Option<Arc<Revision>>, Error> {
     let mut db = ctx.db.handle();
-    let arch = if *IS_RASPBERRY_PI {
-        "raspberrypi"
-    } else {
-        *crate::ARCH
-    };
-    let latest_version: Version = reqwest::get(format!(
-        "{}/eos/v0/latest?eos-version={}&arch={}",
-        marketplace_url,
-        Current::new().semver(),
-        arch,
-    ))
-    .await
-    .with_kind(ErrorKind::Network)?
-    .json::<LatestInformation>()
-    .await
-    .with_kind(ErrorKind::Network)?
-    .version;
+    let latest_version: Version = ctx
+        .client
+        .get(with_query_params(
+            &ctx,
+            format!("{}/eos/v0/latest", marketplace_url,).parse()?,
+        ))
+        .send()
+        .await
+        .with_kind(ErrorKind::Network)?
+        .json::<LatestInformation>()
+        .await
+        .with_kind(ErrorKind::Network)?
+        .version;
     crate::db::DatabaseModel::new()
         .server_info()
         .lock(&mut db, LockType::Write)
@@ -241,12 +238,7 @@ impl EosUrl {
             .host_str()
             .ok_or_else(|| Error::new(eyre!("Could not get host of base"), ErrorKind::ParseUrl))?;
         let version: &Version = &self.version;
-        let arch = if *IS_RASPBERRY_PI {
-            "raspberrypi"
-        } else {
-            *crate::ARCH
-        };
-        Ok(format!("{host}::{version}/{arch}/")
+        Ok(format!("{host}::{version}/{OS_ARCH}/")
             .parse()
             .map_err(|_| Error::new(eyre!("Could not parse path"), ErrorKind::ParseUrl))?)
     }
@@ -306,12 +298,13 @@ async fn sync_boot() -> Result<(), Error> {
             ignore_existing: false,
             exclude: Vec::new(),
             no_permissions: false,
+            no_owner: false,
         },
     )
     .await?
     .wait()
     .await?;
-    if !*IS_RASPBERRY_PI {
+    if OS_ARCH != "raspberrypi" {
         let dev_mnt =
             MountGuard::mount(&Bind::new("/dev"), "/media/embassy/next/dev", ReadWrite).await?;
         let sys_mnt =
@@ -322,7 +315,7 @@ async fn sync_boot() -> Result<(), Error> {
             MountGuard::mount(&Bind::new("/boot"), "/media/embassy/next/boot", ReadWrite).await?;
         Command::new("chroot")
             .arg("/media/embassy/next")
-            .arg("update-grub")
+            .arg("update-grub2")
             .invoke(ErrorKind::MigrationFailed)
             .await?;
         boot_mnt.unmount(false).await?;

backend/src/util/io.rs

@@ -1,14 +1,20 @@
 use std::future::Future;
 use std::io::Cursor;
+use std::os::unix::prelude::MetadataExt;
 use std::path::Path;
+use std::sync::atomic::AtomicU64;
 use std::task::Poll;
+use std::time::Duration;
 
 use futures::future::{BoxFuture, Fuse};
 use futures::{AsyncSeek, FutureExt, TryStreamExt};
 use helpers::NonDetachingJoinHandle;
+use nix::unistd::{Gid, Uid};
 use tokio::io::{
     duplex, AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, DuplexStream, ReadBuf, WriteHalf,
 };
+use tokio::net::TcpStream;
+use tokio::time::{Instant, Sleep};
 
 use crate::ResultExt;
 
@@ -222,6 +228,7 @@ pub async fn copy_and_shutdown<R: AsyncRead + Unpin, W: AsyncWrite + Unpin>(
 
 pub fn dir_size<'a, P: AsRef<Path> + 'a + Send + Sync>(
     path: P,
+    ctr: Option<&'a Counter>,
 ) -> BoxFuture<'a, Result<u64, std::io::Error>> {
     async move {
         tokio_stream::wrappers::ReadDirStream::new(tokio::fs::read_dir(path.as_ref()).await?)
@@ -229,9 +236,12 @@ pub fn dir_size<'a, P: AsRef<Path> + 'a + Send + Sync>(
                 let m = e.metadata().await?;
                 Ok(acc
                     + if m.is_file() {
+                        if let Some(ctr) = ctr {
+                            ctr.add(m.len());
+                        }
                         m.len()
                     } else if m.is_dir() {
-                        dir_size(e.path()).await?
+                        dir_size(e.path(), ctr).await?
                     } else {
                         0
                     })
@@ -416,3 +426,245 @@ impl<T: AsyncWrite> AsyncWrite for BackTrackingReader<T> {
         self.project().reader.poll_write_vectored(cx, bufs)
     }
 }
+
+pub struct Counter {
+    atomic: AtomicU64,
+    ordering: std::sync::atomic::Ordering,
+}
+impl Counter {
+    pub fn new(init: u64, ordering: std::sync::atomic::Ordering) -> Self {
+        Self {
+            atomic: AtomicU64::new(init),
+            ordering,
+        }
+    }
+    pub fn load(&self) -> u64 {
+        self.atomic.load(self.ordering)
+    }
+    pub fn add(&self, value: u64) {
+        self.atomic.fetch_add(value, self.ordering);
+    }
+}
+
+#[pin_project::pin_project]
+pub struct CountingReader<'a, R> {
+    ctr: &'a Counter,
+    #[pin]
+    rdr: R,
+}
+impl<'a, R> CountingReader<'a, R> {
+    pub fn new(rdr: R, ctr: &'a Counter) -> Self {
+        Self { ctr, rdr }
+    }
+    pub fn into_inner(self) -> R {
+        self.rdr
+    }
+}
+impl<'a, R: AsyncRead> AsyncRead for CountingReader<'a, R> {
+    fn poll_read(
+        self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+        buf: &mut ReadBuf<'_>,
+    ) -> Poll<std::io::Result<()>> {
+        let this = self.project();
+        let start = buf.filled().len();
+        let res = this.rdr.poll_read(cx, buf);
+        let len = buf.filled().len() - start;
+        if len > 0 {
+            this.ctr.add(len as u64);
+        }
+        res
+    }
+}
+
+pub fn dir_copy<'a, P0: AsRef<Path> + 'a + Send + Sync, P1: AsRef<Path> + 'a + Send + Sync>(
+    src: P0,
+    dst: P1,
+    ctr: Option<&'a Counter>,
+) -> BoxFuture<'a, Result<(), crate::Error>> {
+    async move {
+        let m = tokio::fs::metadata(&src).await?;
+        let dst_path = dst.as_ref();
+        tokio::fs::create_dir_all(&dst_path).await.with_ctx(|_| {
+            (
+                crate::ErrorKind::Filesystem,
+                format!("mkdir {}", dst_path.display()),
+            )
+        })?;
+        tokio::fs::set_permissions(&dst_path, m.permissions())
+            .await
+            .with_ctx(|_| {
+                (
+                    crate::ErrorKind::Filesystem,
+                    format!("chmod {}", dst_path.display()),
+                )
+            })?;
+        let tmp_dst_path = dst_path.to_owned();
+        tokio::task::spawn_blocking(move || {
+            nix::unistd::chown(
+                &tmp_dst_path,
+                Some(Uid::from_raw(m.uid())),
+                Some(Gid::from_raw(m.gid())),
+            )
+        })
+        .await
+        .with_kind(crate::ErrorKind::Unknown)?
+        .with_ctx(|_| {
+            (
+                crate::ErrorKind::Filesystem,
+                format!("chown {}", dst_path.display()),
+            )
+        })?;
+        tokio_stream::wrappers::ReadDirStream::new(tokio::fs::read_dir(src.as_ref()).await?)
+            .map_err(|e| crate::Error::new(e, crate::ErrorKind::Filesystem))
+            .try_for_each(|e| async move {
+                let m = e.metadata().await?;
+                let src_path = e.path();
+                let dst_path = dst_path.join(e.file_name());
+                if m.is_file() {
+                    let len = m.len();
+                    let mut dst_file = tokio::fs::File::create(&dst_path).await.with_ctx(|_| {
+                        (
+                            crate::ErrorKind::Filesystem,
+                            format!("create {}", dst_path.display()),
+                        )
+                    })?;
+                    let mut rdr = tokio::fs::File::open(&src_path).await.with_ctx(|_| {
+                        (
+                            crate::ErrorKind::Filesystem,
+                            format!("open {}", src_path.display()),
+                        )
+                    })?;
+                    if let Some(ctr) = ctr {
+                        tokio::io::copy(&mut CountingReader::new(rdr, ctr), &mut dst_file).await
+                    } else {
+                        tokio::io::copy(&mut rdr, &mut dst_file).await
+                    }
+                    .with_ctx(|_| {
+                        (
+                            crate::ErrorKind::Filesystem,
+                            format!("cp {} -> {}", src_path.display(), dst_path.display()),
+                        )
+                    })?;
+                    dst_file.flush().await?;
+                    dst_file.shutdown().await?;
+                    dst_file.sync_all().await?;
+                    drop(dst_file);
+                    let tmp_dst_path = dst_path.clone();
+                    tokio::task::spawn_blocking(move || {
+                        nix::unistd::chown(
+                            &tmp_dst_path,
+                            Some(Uid::from_raw(m.uid())),
+                            Some(Gid::from_raw(m.gid())),
+                        )
+                    })
+                    .await
+                    .with_kind(crate::ErrorKind::Unknown)?
+                    .with_ctx(|_| {
+                        (
+                            crate::ErrorKind::Filesystem,
+                            format!("chown {}", dst_path.display()),
+                        )
+                    })?;
+                } else if m.is_dir() {
+                    dir_copy(src_path, dst_path, ctr).await?;
+                } else if m.file_type().is_symlink() {
+                    tokio::fs::symlink(
+                        tokio::fs::read_link(&src_path).await.with_ctx(|_| {
+                            (
+                                crate::ErrorKind::Filesystem,
+                                format!("readlink {}", src_path.display()),
+                            )
+                        })?,
+                        &dst_path,
+                    )
+                    .await
+                    .with_ctx(|_| {
+                        (
+                            crate::ErrorKind::Filesystem,
+                            format!("cp -P {} -> {}", src_path.display(), dst_path.display()),
+                        )
+                    })?;
+                    // Do not set permissions (see https://unix.stackexchange.com/questions/87200/change-permissions-for-a-symbolic-link)
+                }
+                Ok(())
+            })
+            .await?;
+        Ok(())
+    }
+    .boxed()
+}
+
+#[pin_project::pin_project]
+pub struct TimeoutStream<S: AsyncRead + AsyncWrite = TcpStream> {
+    timeout: Duration,
+    #[pin]
+    sleep: Sleep,
+    #[pin]
+    stream: S,
+}
+impl<S: AsyncRead + AsyncWrite> TimeoutStream<S> {
+    pub fn new(stream: S, timeout: Duration) -> Self {
+        Self {
+            timeout,
+            sleep: tokio::time::sleep(timeout),
+            stream,
+        }
+    }
+}
+impl<S: AsyncRead + AsyncWrite> AsyncRead for TimeoutStream<S> {
+    fn poll_read(
+        self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+        buf: &mut tokio::io::ReadBuf<'_>,
+    ) -> std::task::Poll<std::io::Result<()>> {
+        let mut this = self.project();
+        if let std::task::Poll::Ready(_) = this.sleep.as_mut().poll(cx) {
+            return std::task::Poll::Ready(Err(std::io::Error::new(
+                std::io::ErrorKind::TimedOut,
+                "timed out",
+            )));
+        }
+        let res = this.stream.poll_read(cx, buf);
+        if res.is_ready() {
+            this.sleep.reset(Instant::now() + *this.timeout);
+        }
+        res
+    }
+}
+impl<S: AsyncRead + AsyncWrite> AsyncWrite for TimeoutStream<S> {
+    fn poll_write(
+        self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+        buf: &[u8],
+    ) -> std::task::Poll<Result<usize, std::io::Error>> {
+        let mut this = self.project();
+        let res = this.stream.poll_write(cx, buf);
+        if res.is_ready() {
+            this.sleep.reset(Instant::now() + *this.timeout);
+        }
+        res
+    }
+    fn poll_flush(
+        self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Result<(), std::io::Error>> {
+        let mut this = self.project();
+        let res = this.stream.poll_flush(cx);
+        if res.is_ready() {
+            this.sleep.reset(Instant::now() + *this.timeout);
+        }
+        res
+    }
+    fn poll_shutdown(
+        self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Result<(), std::io::Error>> {
+        let mut this = self.project();
+        let res = this.stream.poll_shutdown(cx);
+        if res.is_ready() {
+            this.sleep.reset(Instant::now() + *this.timeout);
+        }
+        res
+    }
+}

backend/src/util/lshw.rs

@@ -0,0 +1,63 @@
+use models::{Error, ResultExt};
+use serde::{Deserialize, Serialize};
+use tokio::process::Command;
+
+use crate::util::Invoke;
+
+const KNOWN_CLASSES: &[&str] = &["processor", "display"];
+
+#[derive(Debug, Deserialize, Serialize)]
+#[serde(tag = "class")]
+#[serde(rename_all = "kebab-case")]
+pub enum LshwDevice {
+    Processor(LshwProcessor),
+    Display(LshwDisplay),
+}
+impl LshwDevice {
+    pub fn class(&self) -> &'static str {
+        match self {
+            Self::Processor(_) => "processor",
+            Self::Display(_) => "display",
+        }
+    }
+    pub fn product(&self) -> &str {
+        match self {
+            Self::Processor(hw) => hw.product.as_str(),
+            Self::Display(hw) => hw.product.as_str(),
+        }
+    }
+}
+
+#[derive(Debug, Deserialize, Serialize)]
+pub struct LshwProcessor {
+    pub product: String,
+}
+
+#[derive(Debug, Deserialize, Serialize)]
+pub struct LshwDisplay {
+    pub product: String,
+}
+
+pub async fn lshw() -> Result<Vec<LshwDevice>, Error> {
+    let mut cmd = Command::new("lshw");
+    cmd.arg("-json");
+    for class in KNOWN_CLASSES {
+        cmd.arg("-class").arg(*class);
+    }
+    Ok(
+        serde_json::from_slice::<Vec<serde_json::Value>>(
+            &cmd.invoke(crate::ErrorKind::Lshw).await?,
+        )
+        .with_kind(crate::ErrorKind::Deserialization)?
+        .into_iter()
+        .filter_map(|v| match serde_json::from_value(v) {
+            Ok(a) => Some(a),
+            Err(e) => {
+                tracing::error!("Failed to parse lshw output: {e}");
+                tracing::debug!("{e:?}");
+                None
+            }
+        })
+        .collect(),
+    )
+}

backend/src/util/mod.rs

@@ -27,6 +27,7 @@ pub mod config;
 pub mod http_reader;
 pub mod io;
 pub mod logger;
+pub mod lshw;
 pub mod serde;
 
 #[derive(Clone, Copy, Debug)]
@@ -58,7 +59,12 @@ impl Invoke for tokio::process::Command {
             res.status.success(),
             error_kind,
             "{}",
-            std::str::from_utf8(&res.stderr).unwrap_or("Unknown Error")
+            Some(&res.stderr)
+                .filter(|a| !a.is_empty())
+                .or(Some(&res.stdout))
+                .filter(|a| !a.is_empty())
+                .and_then(|a| std::str::from_utf8(a).ok())
+                .unwrap_or(&format!("Unknown Error ({})", res.status))
         );
         Ok(res.stdout)
     }

backend/src/util/serde.rs

@@ -793,3 +793,42 @@ impl<T: AsRef<[u8]>> Serialize for Base64<T> {
         serializer.serialize_str(&base64::encode(self.0.as_ref()))
     }
 }
+
+#[derive(Clone, Debug)]
+pub struct Regex(regex::Regex);
+impl From<Regex> for regex::Regex {
+    fn from(value: Regex) -> Self {
+        value.0
+    }
+}
+impl From<regex::Regex> for Regex {
+    fn from(value: regex::Regex) -> Self {
+        Regex(value)
+    }
+}
+impl AsRef<regex::Regex> for Regex {
+    fn as_ref(&self) -> &regex::Regex {
+        &self.0
+    }
+}
+impl AsMut<regex::Regex> for Regex {
+    fn as_mut(&mut self) -> &mut regex::Regex {
+        &mut self.0
+    }
+}
+impl<'de> Deserialize<'de> for Regex {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        deserialize_from_str(deserializer).map(Self)
+    }
+}
+impl Serialize for Regex {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        serialize_display(&self.0, serializer)
+    }
+}

backend/src/version/mod.rs

@@ -20,8 +20,12 @@ mod v0_3_2;
 mod v0_3_2_1;
 mod v0_3_3;
 mod v0_3_4;
+mod v0_3_4_1;
+mod v0_3_4_2;
+mod v0_3_4_3;
+mod v0_3_4_4;
 
-pub type Current = v0_3_4::Version;
+pub type Current = v0_3_4_4::Version;
 
 #[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]
 #[serde(untagged)]
@@ -37,6 +41,10 @@ enum Version {
     V0_3_2_1(Wrapper<v0_3_2_1::Version>),
     V0_3_3(Wrapper<v0_3_3::Version>),
     V0_3_4(Wrapper<v0_3_4::Version>),
+    V0_3_4_1(Wrapper<v0_3_4_1::Version>),
+    V0_3_4_2(Wrapper<v0_3_4_2::Version>),
+    V0_3_4_3(Wrapper<v0_3_4_3::Version>),
+    V0_3_4_4(Wrapper<v0_3_4_4::Version>),
     Other(emver::Version),
 }
 
@@ -63,6 +71,10 @@ impl Version {
             Version::V0_3_2_1(Wrapper(x)) => x.semver(),
             Version::V0_3_3(Wrapper(x)) => x.semver(),
             Version::V0_3_4(Wrapper(x)) => x.semver(),
+            Version::V0_3_4_1(Wrapper(x)) => x.semver(),
+            Version::V0_3_4_2(Wrapper(x)) => x.semver(),
+            Version::V0_3_4_3(Wrapper(x)) => x.semver(),
+            Version::V0_3_4_4(Wrapper(x)) => x.semver(),
             Version::Other(x) => x.clone(),
         }
     }
@@ -244,6 +256,22 @@ pub async fn init<Db: DbHandle>(
             v.0.migrate_to(&Current::new(), db, secrets, receipts)
                 .await?
         }
+        Version::V0_3_4_1(v) => {
+            v.0.migrate_to(&Current::new(), db, secrets, receipts)
+                .await?
+        }
+        Version::V0_3_4_2(v) => {
+            v.0.migrate_to(&Current::new(), db, secrets, receipts)
+                .await?
+        }
+        Version::V0_3_4_3(v) => {
+            v.0.migrate_to(&Current::new(), db, secrets, receipts)
+                .await?
+        }
+        Version::V0_3_4_4(v) => {
+            v.0.migrate_to(&Current::new(), db, secrets, receipts)
+                .await?
+        }
         Version::Other(_) => {
             return Err(Error::new(
                 eyre!("Cannot downgrade"),
@@ -287,6 +315,9 @@ mod tests {
             Just(Version::V0_3_2_1(Wrapper(v0_3_2_1::Version::new()))),
             Just(Version::V0_3_3(Wrapper(v0_3_3::Version::new()))),
             Just(Version::V0_3_4(Wrapper(v0_3_4::Version::new()))),
+            Just(Version::V0_3_4_1(Wrapper(v0_3_4_1::Version::new()))),
+            Just(Version::V0_3_4_2(Wrapper(v0_3_4_2::Version::new()))),
+            Just(Version::V0_3_4_3(Wrapper(v0_3_4_3::Version::new()))),
             em_version().prop_map(Version::Other),
         ]
     }

backend/src/version/v0_3_4.rs

@@ -5,11 +5,10 @@ use openssl::hash::MessageDigest;
 use serde_json::{json, Value};
 use ssh_key::public::Ed25519PublicKey;
 
-use crate::account::AccountInfo;
-use crate::hostname::{generate_hostname, sync_hostname, Hostname};
-
 use super::v0_3_0::V0_3_0_COMPAT;
 use super::*;
+use crate::account::AccountInfo;
+use crate::hostname::{generate_hostname, sync_hostname, Hostname};
 
 const V0_3_4: emver::Version = emver::Version::new(0, 3, 4, 0);
 
@@ -79,7 +78,7 @@ impl VersionT for Version {
             .unwrap_or_else(generate_hostname);
         account.server_id = server_info.id;
         account.save(secrets).await?;
-        sync_hostname(&account).await?;
+        sync_hostname(&account.hostname).await?;
 
         let parsed_url = Some(COMMUNITY_URL.parse().unwrap());
         let mut ui = crate::db::DatabaseModel::new().ui().get_mut(db).await?;