update readme layout and assets (#2382)

* update readme layout and assets

* Update README.md

---------

Co-authored-by: Matt Hill <MattDHill@users.noreply.github.com>

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -12,25 +12,23 @@ body:
       options:
         - label: I have searched for [existing issues](https://github.com/start9labs/start-os/issues) that already report this problem.
           required: true
+  - type: input
+    attributes:
+      label: Server Hardware
+      description: On what hardware are you running StartOS? Please be as detailed as possible!
+      placeholder: Pi (8GB) w/ 32GB microSD & Samsung T7 SSD
+    validations:
+      required: true
   - type: input
     attributes:
       label: StartOS Version
       description: What version of StartOS are you running?
-      placeholder: e.g. 0.3.4.2
+      placeholder: e.g. 0.3.4.3
     validations:
       required: true
   - type: dropdown
     attributes:
-      label: Device
-      description: What device are you using to connect to your server?
-      options:
-        - Phone/tablet
-        - Laptop/Desktop
-    validations:
-      required: true
-  - type: dropdown
-    attributes:
-      label: Device OS
+      label: Client OS
       description: What operating system is your device running?
       options:
         - MacOS
@@ -45,7 +43,7 @@ body:
       required: true
   - type: input
     attributes:
-      label: Device OS Version
+      label: Client OS Version
       description: What version is your device OS?
     validations:
       required: true

.github/workflows/README.md

@@ -1,29 +0,0 @@
-# This folder contains GitHub Actions workflows for building the project
-
-## backend
-Runs: manually (on: workflow_dispatch) or called by product-pipeline (on: workflow_call)
-
-This workflow uses the actions and docker/setup-buildx-action@v1 to prepare the environment for aarch64 cross complilation using docker buildx.
-When execution of aarch64 containers is required the action docker/setup-qemu-action@v1 is added.
-A matrix-strategy has been used to build for both x86_64 and aarch64 platforms in parallel.
-
-### Running unittests
-
-Unittests are run using [cargo-nextest]( https://nexte.st/). First the sources are (cross-)compiled and archived. The archive is then run on the correct platform.
-
-## frontend
-Runs: manually (on: workflow_dispatch) or called by product-pipeline (on: workflow_call)
-
-This workflow builds the frontends.
-
-## product
-Runs: when a pull request targets the master or next branch and when a change to the master or next branch is made
-
-This workflow builds everything, re-using the backend and frontend workflows.
-The download and extraction order of artifacts is relevant to `make`, as it checks the file timestamps to decide which targets need to be executed.
-
-Result: eos.img
-
-## a note on uploading artifacts
-
-Artifacts are used to share data between jobs. File permissions are not maintained during artifact upload. Where file permissions are relevant, the workaround using tar has been used. See (here)[https://github.com/actions/upload-artifact#maintaining-file-permissions-and-case-sensitive-files].

.github/workflows/backend.yaml

@@ -1,233 +0,0 @@
-name: Backend
-
-on:
-  workflow_call:
-  workflow_dispatch:
-
-env:
-  RUST_VERSION: "1.67.1"
-  ENVIRONMENT: "dev"
-
-jobs:
-  build_libs:
-    name: Build libs
-    strategy:
-      fail-fast: false
-      matrix:
-        target: [x86_64, aarch64]
-        include:
-          - target: x86_64
-            snapshot_command: ./build-v8-snapshot.sh
-            artifact_name: js_snapshot
-            artifact_path: libs/js_engine/src/artifacts/JS_SNAPSHOT.bin
-          - target: aarch64
-            snapshot_command: ./build-arm-v8-snapshot.sh
-            artifact_name: arm_js_snapshot
-            artifact_path: libs/js_engine/src/artifacts/ARM_JS_SNAPSHOT.bin
-    runs-on: ubuntu-latest
-    timeout-minutes: 60
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        submodules: recursive
-        
-    - name: Set up QEMU
-      uses: docker/setup-qemu-action@v2
-      if: ${{ matrix.target == 'aarch64' }}
-      
-    - name: Set up Docker Buildx        
-      uses: docker/setup-buildx-action@v2
-      if: ${{ matrix.target == 'aarch64' }}
-
-    - name: "Install Rust"
-      run: |
-        rustup toolchain install ${{ env.RUST_VERSION }} --profile minimal --no-self-update
-        rustup default ${{ inputs.rust }}
-      shell: bash
-      if: ${{ matrix.target == 'x86_64' }}
-    
-    - uses: actions/cache@v3
-      with:
-        path: |
-          ~/.cargo/bin/
-          ~/.cargo/registry/index/
-          ~/.cargo/registry/cache/
-          ~/.cargo/git/db/
-          libs/target/
-        key: ${{ runner.os }}-cargo-libs-${{ matrix.target }}-${{ hashFiles('libs/Cargo.lock') }}
-
-    - name: Build v8 snapshot
-      run: ${{ matrix.snapshot_command }}
-      working-directory: libs
-
-    - uses: actions/upload-artifact@v3
-      with:
-        name: ${{ matrix.artifact_name }}
-        path: ${{ matrix.artifact_path }}
-
-  build_backend:
-    name: Build backend
-    strategy:
-      fail-fast: false
-      matrix:
-        target: [x86_64, aarch64]
-        include:
-          - target: x86_64
-            snapshot_download: js_snapshot
-          - target: aarch64
-            snapshot_download: arm_js_snapshot
-    runs-on: ubuntu-latest
-    timeout-minutes: 120
-    needs: build_libs
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        submodules: recursive
-    
-    - name: Download ${{ matrix.snapshot_download }} artifact
-      uses: actions/download-artifact@v3
-      with:
-        name: ${{ matrix.snapshot_download }}
-        path: libs/js_engine/src/artifacts/
-        
-    - name: "Install Rust"
-      run: |
-        rustup toolchain install ${{ env.RUST_VERSION }} --profile minimal --no-self-update
-        rustup default ${{ inputs.rust }}
-      shell: bash
-      if: ${{ matrix.target == 'x86_64' }}
-    
-    - uses: actions/cache@v3
-      with:
-        path: |
-          ~/.cargo/bin/
-          ~/.cargo/registry/index/
-          ~/.cargo/registry/cache/
-          ~/.cargo/git/db/
-          backend/target/
-        key: ${{ runner.os }}-cargo-backend-${{ matrix.target }}-${{ hashFiles('backend/Cargo.lock') }}
-
-    - name: Install dependencies
-      run: |
-        sudo apt-get update
-        sudo apt-get install libavahi-client-dev
-      if: ${{ matrix.target == 'x86_64' }}
-
-    - name: Check Git Hash
-      run: ./check-git-hash.sh
-
-    - name: Check Environment
-      run: ./check-environment.sh
-
-    - name: Build backend
-      run: make ARCH=${{ matrix.target }} backend
-
-    - name: 'Tar files to preserve file permissions'
-      run: make ARCH=${{ matrix.target }} backend-${{ matrix.target }}.tar
-
-    - uses: actions/upload-artifact@v3
-      with:
-        name: backend-${{ matrix.target }}
-        path: backend-${{ matrix.target }}.tar
-    
-    - name: Install nextest
-      uses: taiki-e/install-action@nextest
-
-    - name: Build and archive tests
-      run: cargo nextest archive --archive-file nextest-archive-${{ matrix.target }}.tar.zst --target ${{ matrix.target }}-unknown-linux-gnu
-      working-directory: backend
-      if: ${{ matrix.target == 'x86_64' }}
-
-    - name: Build and archive tests
-      run: |
-        docker run --rm \
-        -v "$HOME/.cargo/registry":/root/.cargo/registry \
-        -v "$(pwd)":/home/rust/src \
-        -P start9/rust-arm-cross:aarch64 \
-        sh -c 'cd /home/rust/src/backend &&
-          rustup install ${{ env.RUST_VERSION }} &&
-          rustup override set ${{ env.RUST_VERSION }} &&
-          rustup target add aarch64-unknown-linux-gnu &&
-          curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin &&
-          cargo nextest archive --archive-file nextest-archive-${{ matrix.target }}.tar.zst --target ${{ matrix.target }}-unknown-linux-gnu'
-      if: ${{ matrix.target == 'aarch64' }}
-
-    - name: Reset permissions
-      run: sudo chown -R $USER target
-      working-directory: backend
-      if: ${{ matrix.target == 'aarch64' }}
-
-    - name: Upload archive to workflow
-      uses: actions/upload-artifact@v3
-      with:
-        name: nextest-archive-${{ matrix.target }}
-        path: backend/nextest-archive-${{ matrix.target }}.tar.zst
-
-  run_tests_backend:
-    name: Test backend
-    strategy:
-      fail-fast: false
-      matrix:
-        target: [x86_64, aarch64]
-        include:
-          - target: x86_64
-          - target: aarch64
-    runs-on: ubuntu-latest
-    timeout-minutes: 60
-    needs: build_backend
-    env:
-      CARGO_TERM_COLOR: always
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        submodules: recursive
-
-    - name: Set up QEMU
-      uses: docker/setup-qemu-action@v2
-      if: ${{ matrix.target == 'aarch64' }}
-
-    - name: Set up Docker Buildx        
-      uses: docker/setup-buildx-action@v2
-      if: ${{ matrix.target == 'aarch64' }}
-
-    - run: mkdir -p ~/.cargo/bin
-      if: ${{ matrix.target == 'x86_64' }}
-
-    - name: Install nextest
-      uses: taiki-e/install-action@v2
-      with:
-        tool: nextest@0.9.47
-      if: ${{ matrix.target == 'x86_64' }}
-
-    - name: Download archive
-      uses: actions/download-artifact@v3
-      with:
-        name: nextest-archive-${{ matrix.target }}
-
-    - name: Download nextest (aarch64)
-      run: wget -O nextest-aarch64.tar.gz https://get.nexte.st/0.9.47/linux-arm
-      if: ${{ matrix.target == 'aarch64' }}
-
-    - name: Run tests
-      run: |
-        ${CARGO_HOME:-~/.cargo}/bin/cargo-nextest nextest run --no-fail-fast --archive-file nextest-archive-${{ matrix.target }}.tar.zst \
-        --filter-expr 'not (test(system::test_get_temp) | test(net::tor::test) | test(system::test_get_disk_usage) | test(net::ssl::certificate_details_persist) | test(net::ssl::ca_details_persist))'
-      if: ${{ matrix.target == 'x86_64' }}
-
-    - name: Run tests
-      run: |
-        docker run --rm --platform linux/arm64/v8 \
-        -v "/home/runner/.cargo/registry":/usr/local/cargo/registry \
-        -v "$(pwd)":/home/rust/src \
-        -e CARGO_TERM_COLOR=${{ env.CARGO_TERM_COLOR }} \
-        -P ubuntu:20.04 \
-        sh -c '
-          apt-get update &&
-          apt-get install -y ca-certificates &&
-          apt-get install -y rsync &&
-          cd /home/rust/src &&
-          mkdir -p ~/.cargo/bin &&
-          tar -zxvf nextest-aarch64.tar.gz -C ${CARGO_HOME:-~/.cargo}/bin &&
-          ${CARGO_HOME:-~/.cargo}/bin/cargo-nextest nextest run --archive-file nextest-archive-${{ matrix.target }}.tar.zst \
-          --filter-expr "not (test(system::test_get_temp) | test(net::tor::test) | test(system::test_get_disk_usage) | test(net::ssl::certificate_details_persist) | test(net::ssl::ca_details_persist))"'
-      if: ${{ matrix.target == 'aarch64' }}

.github/workflows/frontend.yaml

@@ -1,46 +0,0 @@
-name: Frontend
-
-on:
-  workflow_call:
-  workflow_dispatch:
-
-env:
-  NODEJS_VERSION: '18.15.0'
-  ENVIRONMENT: "dev"
-
-jobs:
-  frontend:
-    name: Build frontend
-    runs-on: ubuntu-latest
-    timeout-minutes: 60
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        submodules: recursive
-
-    - uses: actions/setup-node@v3
-      with:
-        node-version: ${{ env.NODEJS_VERSION }}
-    
-    - name: Get npm cache directory
-      id: npm-cache-dir
-      run: |
-        echo "dir=$(npm config get cache)" >> $GITHUB_OUTPUT
-    - uses: actions/cache@v3
-      id: npm-cache
-      with:
-        path: ${{ steps.npm-cache-dir.outputs.dir }}
-        key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
-        restore-keys: |
-          ${{ runner.os }}-node-
-
-    - name: Build frontends
-      run: make frontends
-
-    - name: 'Tar files to preserve file permissions'
-      run: tar -cvf frontend.tar ENVIRONMENT.txt GIT_HASH.txt VERSION.txt frontend/dist frontend/config.json
-
-    - uses: actions/upload-artifact@v3
-      with:
-        name: frontend
-        path: frontend.tar

.github/workflows/reusable-workflow.yaml

@@ -1,37 +0,0 @@
-name: Reusable Workflow
-
-on:
-  workflow_call:
-    inputs:
-      build_command:
-        required: true
-        type: string
-      artifact_name:
-        required: true
-        type: string
-      artifact_path:
-        required: true
-        type: string
-
-env:
-  ENVIRONMENT: "dev"
-
-jobs:
-  generic_build_job:
-    runs-on: ubuntu-latest
-    timeout-minutes: 60
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        submodules: recursive
-
-    - name: Set up Docker Buildx        
-      uses: docker/setup-buildx-action@v2
-
-    - name: Build image
-      run: ${{ inputs.build_command }}
-
-    - uses: actions/upload-artifact@v3
-      with:
-        name: ${{ inputs.artifact_name }}
-        path: ${{ inputs.artifact_path }}

.github/workflows/startos-iso.yaml

@@ -8,10 +8,26 @@ on:
         type: choice
         description: Environment
         options:
-          - "<NONE>"
+          - NONE
           - dev
           - unstable
           - dev-unstable
+      runner:
+        type: choice
+        description: Runner
+        options:
+          - standard
+          - fast
+      platform:
+        type: choice
+        description: Platform
+        options:
+          - ALL
+          - x86_64
+          - x86_64-nonfree
+          - aarch64
+          - aarch64-nonfree
+          - raspberrypi
   push:
     branches:
       - master
@@ -23,31 +39,65 @@ on:
 
 env:
   NODEJS_VERSION: "18.15.0"
-  ENVIRONMENT: '${{ fromJson(format(''["{0}", ""]'', github.event.inputs.environment || ''dev''))[github.event.inputs.environment == ''<NONE>''] }}'
+  ENVIRONMENT: '${{ fromJson(format(''["{0}", ""]'', github.event.inputs.environment || ''dev''))[github.event.inputs.environment == ''NONE''] }}'
 
 jobs:
-  dpkg:
-    name: Build dpkg
+  all:
+    name: Build
     strategy:
       fail-fast: false
       matrix:
-        platform:
-          [x86_64, x86_64-nonfree, aarch64, aarch64-nonfree, raspberrypi]
-    runs-on: ubuntu-22.04
+        platform: >-
+          ${{
+            fromJson(
+              format(
+                '[
+                  ["{0}"],
+                  ["x86_64", "x86_64-nonfree", "aarch64", "aarch64-nonfree", "raspberrypi"]
+                ]',
+                github.event.inputs.platform || 'ALL'
+              )
+            )[(github.event.inputs.platform || 'ALL') == 'ALL']
+          }}
+    runs-on: >-
+      ${{
+        fromJson(
+          format(
+            '["ubuntu-22.04", "{0}"]',
+            fromJson('{
+              "x86_64": ["buildjet-32vcpu-ubuntu-2204", "buildjet-32vcpu-ubuntu-2204"],
+              "x86_64-nonfree": ["buildjet-32vcpu-ubuntu-2204", "buildjet-32vcpu-ubuntu-2204"],
+              "aarch64": ["buildjet-16vcpu-ubuntu-2204-arm", "buildjet-32vcpu-ubuntu-2204-arm"],
+              "aarch64-nonfree": ["buildjet-16vcpu-ubuntu-2204-arm", "buildjet-32vcpu-ubuntu-2204-arm"],
+              "raspberrypi": ["buildjet-16vcpu-ubuntu-2204-arm", "buildjet-32vcpu-ubuntu-2204-arm"],
+            }')[matrix.platform][github.event.inputs.platform == matrix.platform]
+          )
+        )[github.event.inputs.runner == 'fast']
+      }}
     steps:
+      - name: Free space
+        run: df -h && rm -rf /opt/hostedtoolcache* && df -h
+        if: ${{ github.event.inputs.runner != 'fast' }}
+
+      - run: |
+          sudo mount -t tmpfs tmpfs .
+        if: ${{ github.event.inputs.runner == 'fast' && (matrix.platform == 'x86_64' || matrix.platform == 'x86_64-nonfree' || github.event.inputs.platform == matrix.platform) }}
+
       - uses: actions/checkout@v3
         with:
           repository: Start9Labs/embassy-os-deb
+          path: embassy-os-deb
 
       - uses: actions/checkout@v3
         with:
           submodules: recursive
-          path: embassyos-0.3.x
+          path: embassy-os-deb/embassyos-0.3.x
+
       - run: |
           cp -r debian embassyos-0.3.x/
           VERSION=0.3.x ./control.sh
-          cp embassyos-0.3.x/backend/embassyd.service embassyos-0.3.x/debian/embassyos.embassyd.service
-          cp embassyos-0.3.x/backend/embassy-init.service embassyos-0.3.x/debian/embassyos.embassy-init.service
+          cp embassyos-0.3.x/backend/startd.service embassyos-0.3.x/debian/embassyos.startd.service
+        working-directory: embassy-os-deb
 
       - uses: actions/setup-node@v3
         with:
@@ -79,95 +129,80 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
 
-      - name: Run build
+      - name: Run dpkg build
+        working-directory: embassy-os-deb
         run: "make VERSION=0.3.x TAG=${{ github.ref_name }}"
         env:
           OS_ARCH: ${{ matrix.platform }}
 
-      - uses: actions/upload-artifact@v3
-        with:
-          name: ${{ matrix.platform }}.deb
-          path: embassyos_0.3.x-1_*.deb
-
-  iso:
-    name: Build iso
-    strategy:
-      fail-fast: false
-      matrix:
-        platform:
-          [x86_64, x86_64-nonfree, aarch64, aarch64-nonfree, raspberrypi]
-    runs-on: ubuntu-22.04
-    needs: [dpkg]
-    steps:
       - uses: actions/checkout@v3
         with:
           repository: Start9Labs/startos-image-recipes
+          path: startos-image-recipes
 
       - name: Install dependencies
         run: |
           sudo apt-get update
           sudo apt-get install -y qemu-user-static
-          wget http://ftp.us.debian.org/debian/pool/main/d/debspawn/debspawn_0.6.1-1_all.deb
-          sha256sum ./debspawn_0.6.1-1_all.deb | grep fb8a3f588438ff9ef51e713ec1d83306db893f0aa97447565e28bbba9c6e90c6
-          sudo apt-get install -y ./debspawn_0.6.1-1_all.deb
+          wget https://deb.debian.org/debian/pool/main/d/debspawn/debspawn_0.6.2-1_all.deb
+          sha256sum ./debspawn_0.6.2-1_all.deb | grep 37ef27458cb1e35e8bce4d4f639b06b4b3866fc0b9191ec6b9bd157afd06a817
+          sudo apt-get install -y ./debspawn_0.6.2-1_all.deb
 
       - name: Configure debspawn
         run: |
           sudo mkdir -p /etc/debspawn/
           echo "AllowUnsafePermissions=true" | sudo tee /etc/debspawn/global.toml
+          sudo mkdir -p /var/tmp/debspawn
+
+      - run: sudo mount -t tmpfs tmpfs /var/tmp/debspawn
+        if: ${{ github.event.inputs.runner == 'fast' && (matrix.platform == 'x86_64' || matrix.platform == 'x86_64-nonfree') }}
 
       - uses: actions/cache@v3
         with:
           path: /var/lib/debspawn
-          key: ${{ runner.os }}-debspawn-init-bullseye
+          key: ${{ runner.os }}-${{ matrix.platform }}-debspawn-init
 
-      - name: Make build container
-        run: "debspawn list | grep bullseye || debspawn create bullseye"
+      - run: "mkdir -p startos-image-recipes/overlays/deb"
 
-      - run: "mkdir -p overlays/deb"
+      - run: "mv embassy-os-deb/embassyos_0.3.x-1_*.deb startos-image-recipes/overlays/deb/"
 
-      - name: Download dpkg
-        uses: actions/download-artifact@v3
-        with:
-          name: ${{ matrix.platform }}.deb
-          path: overlays/deb
+      - run: "rm -rf embassy-os-deb ${{ steps.npm-cache-dir.outputs.dir }} $HOME/.cargo"
 
-      - name: Run build
+      - name: Run iso build
+        working-directory: startos-image-recipes
         run: |
           ./run-local-build.sh ${{ matrix.platform }}
 
       - uses: actions/upload-artifact@v3
         with:
           name: ${{ matrix.platform }}.squashfs
-          path: results/*.squashfs
+          path: startos-image-recipes/results/*.squashfs
 
       - uses: actions/upload-artifact@v3
         with:
           name: ${{ matrix.platform }}.iso
-          path: results/*.iso
+          path: startos-image-recipes/results/*.iso
         if: ${{ matrix.platform != 'raspberrypi' }}
 
-  image:
-    name: Build image
-    runs-on: ubuntu-22.04
-    timeout-minutes: 60
-    needs: [iso]
-    steps:
       - uses: actions/checkout@v3
         with:
           submodules: recursive
+          path: start-os
+        if: ${{ matrix.platform == 'raspberrypi' }}
 
-      - name: Download raspberrypi.squashfs artifact
-        uses: actions/download-artifact@v3
-        with:
-          name: raspberrypi.squashfs
+      - run: "mv startos-image-recipes/results/startos-*_raspberrypi.squashfs start-os/startos.raspberrypi.squashfs"
+        if: ${{ matrix.platform == 'raspberrypi' }}
 
-      - run: mv startos-*_raspberrypi.squashfs startos.raspberrypi.squashfs
+      - run: rm -rf startos-image-recipes
+        if: ${{ matrix.platform == 'raspberrypi' }}
 
       - name: Build image
+        working-directory: start-os
         run: make startos_raspberrypi.img
+        if: ${{ matrix.platform == 'raspberrypi' }}
 
       - uses: actions/upload-artifact@v3
         with:
           name: raspberrypi.img
-          path: startos-*_raspberrypi.img
+          path: start-os/startos-*_raspberrypi.img
+        if: ${{ matrix.platform == 'raspberrypi' }}

Makefile

@@ -3,20 +3,20 @@ ARCH := $(shell if [ "$(OS_ARCH)" = "raspberrypi" ]; then echo aarch64; else ech
 ENVIRONMENT_FILE = $(shell ./check-environment.sh)
 GIT_HASH_FILE = $(shell ./check-git-hash.sh)
 VERSION_FILE = $(shell ./check-version.sh)
-EMBASSY_BINS := backend/target/$(ARCH)-unknown-linux-gnu/release/embassyd backend/target/$(ARCH)-unknown-linux-gnu/release/embassy-init backend/target/$(ARCH)-unknown-linux-gnu/release/embassy-cli backend/target/$(ARCH)-unknown-linux-gnu/release/embassy-sdk backend/target/$(ARCH)-unknown-linux-gnu/release/avahi-alias libs/target/aarch64-unknown-linux-musl/release/embassy_container_init libs/target/x86_64-unknown-linux-musl/release/embassy_container_init
-EMBASSY_UIS := frontend/dist/ui frontend/dist/setup-wizard frontend/dist/diagnostic-ui frontend/dist/install-wizard
+EMBASSY_BINS := backend/target/$(ARCH)-unknown-linux-gnu/release/startbox libs/target/aarch64-unknown-linux-musl/release/embassy_container_init libs/target/x86_64-unknown-linux-musl/release/embassy_container_init
+EMBASSY_UIS := frontend/dist/raw/ui frontend/dist/raw/setup-wizard frontend/dist/raw/diagnostic-ui frontend/dist/raw/install-wizard
 BUILD_SRC := $(shell find build)
-EMBASSY_SRC := backend/embassyd.service backend/embassy-init.service $(EMBASSY_UIS) $(BUILD_SRC)
+EMBASSY_SRC := backend/startd.service $(BUILD_SRC)
 COMPAT_SRC := $(shell find system-images/compat/ -not -path 'system-images/compat/target/*' -and -not -name *.tar -and -not -name target)
 UTILS_SRC := $(shell find system-images/utils/ -not -name *.tar)
 BINFMT_SRC := $(shell find system-images/binfmt/ -not -name *.tar)
-BACKEND_SRC := $(shell find backend/src) $(shell find backend/migrations) $(shell find patch-db/*/src) $(shell find libs/*/src) libs/*/Cargo.toml backend/Cargo.toml backend/Cargo.lock
+BACKEND_SRC := $(shell find backend/src) $(shell find backend/migrations) $(shell find patch-db/*/src) $(shell find libs/*/src) libs/*/Cargo.toml backend/Cargo.toml backend/Cargo.lock frontend/dist/static
 FRONTEND_SHARED_SRC := $(shell find frontend/projects/shared) $(shell ls -p frontend/ | grep -v / | sed 's/^/frontend\//g') frontend/package.json frontend/node_modules frontend/config.json patch-db/client/dist frontend/patchdb-ui-seed.json
 FRONTEND_UI_SRC := $(shell find frontend/projects/ui)
 FRONTEND_SETUP_WIZARD_SRC := $(shell find frontend/projects/setup-wizard)
 FRONTEND_DIAGNOSTIC_UI_SRC := $(shell find frontend/projects/diagnostic-ui)
 FRONTEND_INSTALL_WIZARD_SRC := $(shell find frontend/projects/install-wizard)
-PATCH_DB_CLIENT_SRC := $(shell find patch-db/client -not -path patch-db/client/dist)
+PATCH_DB_CLIENT_SRC := $(shell find patch-db/client -not -path patch-db/client/dist -and -not -path patch-db/client/node_modules)
 GZIP_BIN := $(shell which pigz || which gzip)
 ALL_TARGETS := $(EMBASSY_BINS) system-images/compat/docker-images/$(ARCH).tar system-images/utils/docker-images/$(ARCH).tar system-images/binfmt/docker-images/$(ARCH).tar $(EMBASSY_SRC) $(shell if [ "$(OS_ARCH)" = "raspberrypi" ]; then echo cargo-deps/aarch64-unknown-linux-gnu/release/pi-beep; fi) $(ENVIRONMENT_FILE) $(GIT_HASH_FILE) $(VERSION_FILE)
 
@@ -24,9 +24,11 @@ ifeq ($(REMOTE),)
 	mkdir = mkdir -p $1
 	rm = rm -rf $1
 	cp = cp -r $1 $2
+	ln = ln -sf $1 $2
 else
 	mkdir = ssh $(REMOTE) 'mkdir -p $1'
 	rm  = ssh $(REMOTE) 'sudo rm -rf $1'
+	ln = ssh $(REMOTE) 'sudo ln -sf $1 $2'
 define cp
 	tar --transform "s|^$1|x|" -czv -f- $1 | ssh $(REMOTE) "sudo tar --transform 's|^x|$2|' -xzv -f- -C /"
 endef
@@ -71,10 +73,12 @@ startos_raspberrypi.img: $(BUILD_SRC) startos.raspberrypi.squashfs $(VERSION_FIL
 # For creating os images. DO NOT USE
 install: $(ALL_TARGETS)
 	$(call mkdir,$(DESTDIR)/usr/bin)
-	$(call cp,backend/target/$(ARCH)-unknown-linux-gnu/release/embassy-init,$(DESTDIR)/usr/bin/embassy-init)
-	$(call cp,backend/target/$(ARCH)-unknown-linux-gnu/release/embassyd,$(DESTDIR)/usr/bin/embassyd)
-	$(call cp,backend/target/$(ARCH)-unknown-linux-gnu/release/embassy-cli,$(DESTDIR)/usr/bin/embassy-cli)
-	$(call cp,backend/target/$(ARCH)-unknown-linux-gnu/release/avahi-alias,$(DESTDIR)/usr/bin/avahi-alias)
+	$(call cp,backend/target/$(ARCH)-unknown-linux-gnu/release/startbox,$(DESTDIR)/usr/bin/startbox)
+	$(call ln,/usr/bin/startbox,$(DESTDIR)/usr/bin/startd)
+	$(call ln,/usr/bin/startbox,$(DESTDIR)/usr/bin/start-cli)
+	$(call ln,/usr/bin/startbox,$(DESTDIR)/usr/bin/start-sdk)
+	$(call ln,/usr/bin/startbox,$(DESTDIR)/usr/bin/avahi-alias)
+	$(call ln,/usr/bin/startbox,$(DESTDIR)/usr/bin/embassy-cli)
 	if [ "$(OS_ARCH)" = "raspberrypi" ]; then $(call cp,cargo-deps/aarch64-unknown-linux-gnu/release/pi-beep,$(DESTDIR)/usr/bin/pi-beep); fi
 	
 	$(call mkdir,$(DESTDIR)/usr/lib)
@@ -94,22 +98,14 @@ install: $(ALL_TARGETS)
 	$(call cp,system-images/utils/docker-images/$(ARCH).tar,$(DESTDIR)/usr/lib/embassy/system-images/utils.tar)
 	$(call cp,system-images/binfmt/docker-images/$(ARCH).tar,$(DESTDIR)/usr/lib/embassy/system-images/binfmt.tar)
 
-	$(call mkdir,$(DESTDIR)/var/www/html)
-	$(call cp,frontend/dist/diagnostic-ui,$(DESTDIR)/var/www/html/diagnostic)
-	$(call cp,frontend/dist/setup-wizard,$(DESTDIR)/var/www/html/setup)
-	$(call cp,frontend/dist/install-wizard,$(DESTDIR)/var/www/html/install)
-	$(call cp,frontend/dist/ui,$(DESTDIR)/var/www/html/main)
-	$(call cp,index.html,$(DESTDIR)/var/www/html/index.html)
-
 update-overlay:
 	@echo "\033[33m!!! THIS WILL ONLY REFLASH YOUR DEVICE IN MEMORY !!!\033[0m"
 	@echo "\033[33mALL CHANGES WILL BE REVERTED IF YOU RESTART THE DEVICE\033[0m"
 	@if [ -z "$(REMOTE)" ]; then >&2 echo "Must specify REMOTE" && false; fi
-	@if [ "`ssh $(REMOTE) 'cat /usr/lib/embassy/VERSION.txt'`" != "`cat ./VERSION.txt`" ]; then >&2 echo "Embassy requires migrations: update-overlay is unavailable." && false; fi
-	@if ssh $(REMOTE) "pidof embassy-init"; then >&2 echo "Embassy in INIT: update-overlay is unavailable." && false; fi
-	ssh $(REMOTE) "sudo systemctl stop embassyd"
+	@if [ "`ssh $(REMOTE) 'cat /usr/lib/embassy/VERSION.txt'`" != "`cat ./VERSION.txt`" ]; then >&2 echo "StartOS requires migrations: update-overlay is unavailable." && false; fi
+	ssh $(REMOTE) "sudo systemctl stop startd"
 	$(MAKE) install REMOTE=$(REMOTE) OS_ARCH=$(OS_ARCH)
-	ssh $(REMOTE) "sudo systemctl start embassyd"
+	ssh $(REMOTE) "sudo systemctl start startd"
 
 update:
 	@if [ -z "$(REMOTE)" ]; then >&2 echo "Must specify REMOTE" && false; fi
@@ -132,11 +128,6 @@ system-images/utils/docker-images/aarch64.tar system-images/utils/docker-images/
 system-images/binfmt/docker-images/aarch64.tar system-images/binfmt/docker-images/x86_64.tar: $(BINFMT_SRC)
 	cd system-images/binfmt && make
 
-raspios.img:
-	wget --continue https://downloads.raspberrypi.org/raspios_lite_arm64/images/raspios_lite_arm64-2022-01-28/2022-01-28-raspios-bullseye-arm64-lite.zip
-	unzip 2022-01-28-raspios-bullseye-arm64-lite.zip
-	mv 2022-01-28-raspios-bullseye-arm64-lite.img raspios.img
-
 snapshots: libs/snapshot_creator/Cargo.toml
 	cd libs/  && ./build-v8-snapshot.sh
 	cd libs/  && ./build-arm-v8-snapshot.sh
@@ -148,18 +139,21 @@ $(EMBASSY_BINS): $(BACKEND_SRC) $(ENVIRONMENT_FILE) $(GIT_HASH_FILE) frontend/pa
 frontend/node_modules: frontend/package.json
 	npm --prefix frontend ci
 
-frontend/dist/ui: $(FRONTEND_UI_SRC) $(FRONTEND_SHARED_SRC) $(ENVIRONMENT_FILE)
+frontend/dist/raw/ui: $(FRONTEND_UI_SRC) $(FRONTEND_SHARED_SRC) $(ENVIRONMENT_FILE)
 	npm --prefix frontend run build:ui
 
-frontend/dist/setup-wizard: $(FRONTEND_SETUP_WIZARD_SRC) $(FRONTEND_SHARED_SRC) $(ENVIRONMENT_FILE)
+frontend/dist/raw/setup-wizard: $(FRONTEND_SETUP_WIZARD_SRC) $(FRONTEND_SHARED_SRC) $(ENVIRONMENT_FILE)
 	npm --prefix frontend run build:setup
 
-frontend/dist/diagnostic-ui: $(FRONTEND_DIAGNOSTIC_UI_SRC) $(FRONTEND_SHARED_SRC) $(ENVIRONMENT_FILE)
+frontend/dist/raw/diagnostic-ui: $(FRONTEND_DIAGNOSTIC_UI_SRC) $(FRONTEND_SHARED_SRC) $(ENVIRONMENT_FILE)
 	npm --prefix frontend run build:dui
 
-frontend/dist/install-wizard: $(FRONTEND_INSTALL_WIZARD_SRC) $(FRONTEND_SHARED_SRC) $(ENVIRONMENT_FILE)
+frontend/dist/raw/install-wizard: $(FRONTEND_INSTALL_WIZARD_SRC) $(FRONTEND_SHARED_SRC) $(ENVIRONMENT_FILE)
 	npm --prefix frontend run build:install-wiz
 
+frontend/dist/static: $(EMBASSY_UIS)
+	./compress-uis.sh
+
 frontend/config.json: $(GIT_HASH_FILE) frontend/config-sample.json
 	jq '.useMocks = false' frontend/config-sample.json > frontend/config.json
 	jq '.packageArch = "$(ARCH)"' frontend/config.json > frontend/config.json.tmp
@@ -186,13 +180,10 @@ backend-$(ARCH).tar: $(EMBASSY_BINS)
 frontends: $(EMBASSY_UIS) 
 
 # this is a convenience step to build the UI
-ui: frontend/dist/ui
+ui: frontend/dist/raw/ui
 
 # used by github actions
 backend: $(EMBASSY_BINS)
 
-cargo-deps/aarch64-unknown-linux-gnu/release/nc-broadcast:
-	./build-cargo-dep.sh nc-broadcast
-
 cargo-deps/aarch64-unknown-linux-gnu/release/pi-beep:
-	./build-cargo-dep.sh pi-beep
+	ARCH=aarch64 ./build-cargo-dep.sh pi-beep

README.md

@@ -1,51 +1,81 @@
-# StartOS
-[![version](https://img.shields.io/github/v/tag/Start9Labs/start-os?color=success)](https://github.com/Start9Labs/start-os/releases)
-[![build](https://github.com/Start9Labs/start-os/actions/workflows/startos-iso.yaml/badge.svg)](https://github.com/Start9Labs/start-os/actions/workflows/startos-iso.yaml)
-[![community](https://img.shields.io/badge/community-matrix-yellow)](https://matrix.to/#/#community:matrix.start9labs.com)
-[![community](https://img.shields.io/badge/community-telegram-informational)](https://t.me/start9_labs)
-[![support](https://img.shields.io/badge/support-docs-important)](https://docs.start9.com)
-[![developer](https://img.shields.io/badge/developer-matrix-blueviolet)](https://matrix.to/#/#community-dev:matrix.start9labs.com)
-[![website](https://img.shields.io/website?down_color=lightgrey&down_message=offline&up_color=green&up_message=online&url=https%3A%2F%2Fstart9.com)](https://start9.com)
-
-[![mastodon](https://img.shields.io/mastodon/follow/000000001?domain=https%3A%2F%2Fmastodon.start9labs.com&label=Follow&style=social)](http://mastodon.start9labs.com)
-[![twitter](https://img.shields.io/twitter/follow/start9labs?label=Follow)](https://twitter.com/start9labs)
-
-### _Welcome to the era of Sovereign Computing_ ###
-
-StartOS is a browser-based, graphical operating system for a personal server. StartOS facilitates the discovery, installation, network configuration, service configuration, data backup, dependency management, and health monitoring of self-hosted software services. It is the most advanced, secure, reliable, and user friendly personal server OS in the world.
-
-## Running StartOS
-There are multiple ways to get your hands on StartOS.
-
-### :moneybag: Buy a Start9 server
-This is the most convenient option. Simply [buy a server](https://start9.com) from Start9 and plug it in. Depending on where you live, shipping costs and import duties will vary.
-
-### :construction_worker: Build your own server
-This option is easier than you might imagine, and there are 4 reasons why you might prefer it:
-1. You already have your own hardware.
-1. You want to save on shipping costs.
-1. You prefer not to divulge your physical address.
-1. You just like building things.
-
-To pursue this option, follow one of our [DIY guides](https://start9.com/latest/diy).
-
-### :hammer_and_wrench: Build StartOS from Source
-
-StartOS can be built from source, for personal use, for free.
-A detailed guide for doing so can be found [here](https://github.com/Start9Labs/start-os/blob/master/build/README.md).
-
-## :heart: Contributing
-There are multiple ways to contribute: work directly on StartOS, package a service for the marketplace, or help with documentation and guides. To learn more about contributing, see [here](https://docs.start9.com/latest/contribute/) or [here](https://github.com/Start9Labs/start-os/blob/master/CONTRIBUTING.md).
-
-## UI Screenshots
+<div align="center">
+  <img src="frontend/projects/shared/assets/img/icon_pwa.png" alt="StartOS Logo" width="16%" />
+  <h1 style="margin-top: 0;">StartOS</h1>
+  <a href="https://github.com/Start9Labs/start-os/releases">
+    <img src="https://img.shields.io/github/v/tag/Start9Labs/start-os?color=success" />
+  </a>
+  <a href="https://github.com/Start9Labs/start-os/actions/workflows/startos-iso.yaml">
+    <img src="https://github.com/Start9Labs/start-os/actions/workflows/startos-iso.yaml/badge.svg">
+  </a>
+  <a href="https://twitter.com/start9labs">
+    <img src="https://img.shields.io/twitter/follow/start9labs?label=Follow">
+  </a>
+  <a href="http://mastodon.start9labs.com">
+    <img src="https://img.shields.io/mastodon/follow/000000001?domain=https%3A%2F%2Fmastodon.start9labs.com&label=Follow&style=social">
+  </a>
+  <a href="https://matrix.to/#/#community:matrix.start9labs.com">
+    <img src="https://img.shields.io/badge/community-matrix-yellow">
+  </a>
+  <a href="https://t.me/start9_labs">
+    <img src="https://img.shields.io/badge/community-telegram-informational">
+  </a>
+  <a href="https://docs.start9.com">
+    <img src="https://img.shields.io/badge/support-docs-important">
+  </a>
+  <a href="https://matrix.to/#/#community-dev:matrix.start9labs.com">
+    <img src="https://img.shields.io/badge/developer-matrix-blueviolet">
+  </a>
+  <a href="https://start9.com">
+    <img src="https://img.shields.io/website?down_color=lightgrey&down_message=offline&up_color=green&up_message=online&url=https%3A%2F%2Fstart9.com">
+  </a>
+</div>
+<br />
+<div align="center">
+  <h3>
+    Welcome to the era of Sovereign Computing
+  </h3>
+  <p>
+    StartOS is a Debian-based Linux distro optimized for running a personal server. It facilitates the discovery,       installation, network configuration, service configuration, data backup, dependency management, and health monitoring of self-hosted software services.
+  </p>
+</div>
+<br />
 <p align="center">
 <img src="assets/StartOS.png" alt="StartOS" width="85%">
 </p>
+<br />
+
+## Running StartOS
+There are multiple ways to get started with StartOS:
+
+### 💰 Buy a Start9 server
+This is the most convenient option. Simply [buy a server](https://store.start9.com) from Start9 and plug it in.
+
+### 👷 Build your own server
+This option is easier than you might imagine, and there are 4 reasons why you might prefer it:
+1. You already have hardware
+1. You want to save on shipping costs
+1. You prefer not to divulge your physical address
+1. You just like building things
+
+To pursue this option, follow one of our [DIY guides](https://start9.com/latest/diy).
+
+## ❤️ Contributing
+There are multiple ways to contribute: work directly on StartOS, package a service for the marketplace, or help with documentation and guides. To learn more about contributing, see [here](https://start9.com/contribute/).
+
+To report security issues, please email our security team - security@start9.com.
+
+## 🌎 Marketplace
+There are dozens of service available for StartOS, and new ones are being added all the time. Check out the full list of available services [here](https://marketplace.start9.com/marketplace). To read more about the Marketplace ecosystem, check out this [blog post](https://blog.start9.com/start9-marketplace-strategy/)
+
+## 🖥️ User Interface Screenshots
+
 <p align="center">
-<img src="assets/preferences.png" alt="StartOS Preferences" width="49%">
-<img src="assets/ghost.png" alt="StartOS Ghost Service" width="49%">
-</p>
-<p align="center">
-<img src="assets/synapse-health-check.png" alt="StartOS Synapse Health Checks" width="49%">
-<img src="assets/sideload.png" alt="StartOS Sideload Service" width="49%">
+<img src="assets/registry.png" alt="StartOS Marketplace" width="49%">
+<img src="assets/community.png" alt="StartOS Community Registry" width="49%">
+<img src="assets/c-lightning.png" alt="StartOS NextCloud Service" width="49%">
+<img src="assets/btcpay.png" alt="StartOS BTCPay Service" width="49%">
+<img src="assets/nextcloud.png" alt="StartOS System Settings" width="49%">
+<img src="assets/system.png" alt="StartOS System Settings" width="49%">
+<img src="assets/welcome.png" alt="StartOS System Settings" width="49%">
+<img src="assets/logs.png" alt="StartOS System Settings" width="49%">
 </p>

backend/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 authors = ["Aiden McClelland <me@drbonez.dev>"]
 description = "The core of StartOS"
-documentation = "https://docs.rs/embassy-os"
+documentation = "https://docs.rs/start-os"
 edition = "2021"
 keywords = [
   "self-hosted",
@@ -11,40 +11,28 @@ keywords = [
   "full-node",
   "lightning",
 ]
-name = "embassy-os"
+name = "start-os"
 readme = "README.md"
 repository = "https://github.com/Start9Labs/start-os"
-version = "0.3.4-rev.2"
+version = "0.3.4-rev.4"
 
 [lib]
-name = "embassy"
+name = "startos"
 path = "src/lib.rs"
 
 [[bin]]
-name = "embassyd"
-path = "src/bin/embassyd.rs"
-
-[[bin]]
-name = "embassy-init"
-path = "src/bin/embassy-init.rs"
-
-[[bin]]
-name = "embassy-sdk"
-path = "src/bin/embassy-sdk.rs"
-
-[[bin]]
-name = "embassy-cli"
-path = "src/bin/embassy-cli.rs"
-
-[[bin]]
-name = "avahi-alias"
-path = "src/bin/avahi-alias.rs"
+name = "startbox"
+path = "src/main.rs"
 
 [features]
 avahi = ["avahi-sys"]
-default = ["avahi", "js_engine"]
+default = ["avahi-alias", "cli", "sdk", "daemon", "js_engine"]
 dev = []
 unstable = ["patch-db/unstable"]
+avahi-alias = ["avahi"]
+cli = []
+sdk = []
+daemon = []
 
 [dependencies]
 aes = { version = "0.7.5", features = ["ctr"] }
@@ -90,11 +78,14 @@ http = "0.2.8"
 hyper = { version = "0.14.20", features = ["full"] }
 hyper-ws-listener = "0.2.0"
 imbl = "2.0.0"
+include_dir = "0.7.3"
 indexmap = { version = "1.9.1", features = ["serde"] }
 ipnet = { version = "2.7.1", features = ["serde"] }
 iprange = { version = "0.6.7", features = ["serde"] }
 isocountry = "0.3.2"
 itertools = "0.10.3"
+jaq-core = "0.10.0"
+jaq-std = "0.10.0"
 josekit = "0.8.1"
 js_engine = { path = '../libs/js_engine', optional = true }
 jsonpath_lib = "0.3.0"
@@ -103,6 +94,7 @@ libc = "0.2.126"
 log = "0.4.17"
 mbrman = "0.5.0"
 models = { version = "*", path = "../libs/models" }
+new_mime_guess = "4"
 nix = "0.25.0"
 nom = "7.1.1"
 num = "0.4.0"
@@ -152,6 +144,7 @@ tokio-stream = { version = "0.1.11", features = ["io-util", "sync", "net"] }
 tokio-tar = { git = "https://github.com/dr-bonez/tokio-tar.git" }
 tokio-tungstenite = { version = "0.17.1", features = ["native-tls"] }
 tokio-rustls = "0.23.4"
+tokio-socks = "0.5.1"
 tokio-util = { version = "0.7.3", features = ["io"] }
 torut = "0.2.1"
 tracing = "0.1.35"
@@ -161,6 +154,7 @@ tracing-subscriber = { version = "0.3.14", features = ["env-filter"] }
 trust-dns-server = "0.22.0"
 typed-builder = "0.10.0"
 url = { version = "2.2.2", features = ["serde"] }
+urlencoding = "2.1.2"
 uuid = { version = "1.1.2", features = ["v4"] }
 zeroize = "1.5.7"
 

backend/README.md

@@ -12,18 +12,17 @@
 
 ## Structure
 
-The StartOS backend is broken up into 4 different binaries:
+The StartOS backend is packed into a single binary `startbox` that is symlinked under 
+several different names for different behaviour:
 
-- embassyd: This is the main workhorse of StartOS - any new functionality you
+- startd: This is the main workhorse of StartOS - any new functionality you
   want will likely go here
-- embassy-init: This is the component responsible for allowing you to set up
-  your device, and handles system initialization on startup
-- embassy-cli: This is a CLI tool that will allow you to issue commands to
-  embassyd and control it similarly to the UI
-- embassy-sdk: This is a CLI tool that aids in building and packaging services
+- start-cli: This is a CLI tool that will allow you to issue commands to
+  startd and control it similarly to the UI
+- start-sdk: This is a CLI tool that aids in building and packaging services
   you wish to deploy to StartOS
 
-Finally there is a library `embassy` that supports all four of these tools.
+Finally there is a library `startos` that supports all of these tools.
 
 See [here](/backend/Cargo.toml) for details.
 

backend/build-dev.sh

@@ -1,24 +0,0 @@
-#!/bin/bash
-
-set -e
-shopt -s expand_aliases
-
-if [ "$0" != "./build-dev.sh" ]; then
-	>&2 echo "Must be run from backend directory"
-	exit 1
-fi
-
-USE_TTY=
-if tty -s; then
-	USE_TTY="-it"
-fi
-
-alias 'rust-arm64-builder'='docker run $USE_TTY --rm -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$(pwd)":/home/rust/src start9/rust-arm-cross:aarch64'
-
-cd ..
-rust-arm64-builder sh -c "(cd backend && cargo build --locked)"
-cd backend
-
-sudo chown -R $USER target
-sudo chown -R $USER ~/.cargo
-#rust-arm64-builder aarch64-linux-gnu-strip target/aarch64-unknown-linux-gnu/release/embassyd

backend/build-portable-dev.sh

@@ -1,23 +0,0 @@
-#!/bin/bash
-
-set -e
-shopt -s expand_aliases
-
-if [ "$0" != "./build-portable-dev.sh" ]; then
-	>&2 echo "Must be run from backend directory"
-	exit 1
-fi
-
-USE_TTY=
-if tty -s; then
-	USE_TTY="-it"
-fi
-
-alias 'rust-musl-builder'='docker run $USE_TTY --rm -v "$HOME"/.cargo/registry:/root/.cargo/registry -v "$(pwd)":/home/rust/src start9/rust-musl-cross:x86_64-musl'
-
-cd ..
-rust-musl-builder sh -c "(cd backend && cargo +beta build --target=x86_64-unknown-linux-musl --no-default-features --locked)"
-cd backend
-
-sudo chown -R $USER target
-sudo chown -R $USER ~/.cargo

backend/build-prod.sh

@@ -22,7 +22,7 @@ if tty -s; then
 	USE_TTY="-it"
 fi
 
-alias 'rust-gnu-builder'='docker run $USE_TTY --rm -e "OS_ARCH=$OS_ARCH" -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$(pwd)":/home/rust/src -P start9/rust-arm-cross:aarch64'
+alias 'rust-gnu-builder'='docker run $USE_TTY --rm -e "OS_ARCH=$OS_ARCH" -v "$HOME/.cargo/registry":/usr/local/cargo/registry -v "$(pwd)":/home/rust/src -w /home/rust/src -P start9/rust-arm-cross:aarch64'
 alias 'rust-musl-builder'='docker run $USE_TTY --rm -e "OS_ARCH=$OS_ARCH" -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$(pwd)":/home/rust/src -P messense/rust-musl-cross:$ARCH-musl'
 
 cd ..
@@ -37,26 +37,26 @@ fi
 set +e
 fail=
 if [[ "$FLAGS" = "" ]]; then
-	rust-gnu-builder sh -c "(git config --global --add safe.directory '*'; cd backend && cargo build --release --locked  --target=$ARCH-unknown-linux-gnu)"
+	rust-gnu-builder sh -c "(cd backend && cargo build --release --locked  --target=$ARCH-unknown-linux-gnu)"
 	if test $? -ne 0; then 
 		fail=true
 	fi
 	for ARCH in x86_64 aarch64
 	do
-		rust-musl-builder sh -c "(git config --global --add safe.directory '*'; cd libs && cargo build --release --locked --bin embassy_container_init )"
+		rust-musl-builder sh -c "(cd libs && cargo build --release --locked --bin embassy_container_init )"
 		if test $? -ne 0; then 
 			fail=true
 		fi
 	done
 else
 	echo "FLAGS=$FLAGS"
-	rust-gnu-builder sh -c "(git config --global --add safe.directory '*'; cd backend && cargo build --release --features $FLAGS --locked --target=$ARCH-unknown-linux-gnu)"
+	rust-gnu-builder sh -c "(cd backend && cargo build --release --features $FLAGS --locked --target=$ARCH-unknown-linux-gnu)"
 	if test $? -ne 0; then 
 		fail=true
 	fi
 	for ARCH in x86_64 aarch64
 	do
-		rust-musl-builder sh -c "(git config --global --add safe.directory '*'; cd libs && cargo build --release --features $FLAGS --locked --bin embassy_container_init)"
+		rust-musl-builder sh -c "(cd libs && cargo build --release --features $FLAGS --locked --bin embassy_container_init)"
 		if test $? -ne 0; then 
 			fail=true
 		fi
@@ -72,5 +72,3 @@ sudo chown -R $USER ../libs/target
 if [ -n "$fail" ]; then
 	exit 1
 fi
-
-#rust-arm64-builder aarch64-linux-gnu-strip target/aarch64-unknown-linux-gnu/release/embassyd

backend/embassy-init.service

@@ -1,15 +0,0 @@
-[Unit]
-Description=Embassy Init
-After=network-online.target
-Requires=network-online.target
-Wants=avahi-daemon.service
-
-[Service]
-Type=oneshot
-Environment=RUST_LOG=embassy_init=debug,embassy=debug,js_engine=debug,patch_db=warn
-ExecStart=/usr/bin/embassy-init
-RemainAfterExit=true
-StandardOutput=append:/var/log/embassy-init.log
-
-[Install]
-WantedBy=embassyd.service

backend/embassyd.service

@@ -1,17 +0,0 @@
-[Unit]
-Description=Embassy Daemon
-After=embassy-init.service
-Requires=embassy-init.service
-
-[Service]
-Type=simple
-Environment=RUST_LOG=embassyd=debug,embassy=debug,js_engine=debug,patch_db=warn
-ExecStart=/usr/bin/embassyd
-Restart=always
-RestartSec=3
-ManagedOOMPreference=avoid
-CPUAccounting=true
-CPUWeight=1000
-
-[Install]
-WantedBy=multi-user.target

backend/install-sdk.sh

@@ -8,4 +8,11 @@ if [ "$0" != "./install-sdk.sh" ]; then
 	exit 1
 fi
 
-cargo install --bin=embassy-sdk --bin=embassy-cli --path=. --no-default-features --features=js_engine --locked
+if [ -z "$OS_ARCH" ]; then
+  export OS_ARCH=$(uname -m)
+fi
+
+cargo install --path=. --no-default-features --features=js_engine,sdk,cli --locked
+startbox_loc=$(which startbox)
+ln -sf $startbox_loc $(dirname $startbox_loc)/start-cli
+ln -sf $startbox_loc $(dirname $startbox_loc)/start-sdk

backend/src/backup/backup_bulk.rs

@@ -1,5 +1,5 @@
 use std::collections::{BTreeMap, BTreeSet};
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 
 use chrono::Utc;
 use clap::ArgMatches;
@@ -8,6 +8,7 @@ use helpers::AtomicFile;
 use patch_db::{DbHandle, LockType, PatchDbHandle};
 use rpc_toolkit::command;
 use tokio::io::AsyncWriteExt;
+use tokio::process::Command;
 use tracing::instrument;
 
 use super::target::BackupTargetId;
@@ -23,8 +24,9 @@ use crate::disk::mount::guard::TmpMountGuard;
 use crate::notifications::NotificationLevel;
 use crate::s9pk::manifest::PackageId;
 use crate::status::MainStatus;
-use crate::util::display_none;
+use crate::util::io::dir_copy;
 use crate::util::serde::IoFormat;
+use crate::util::{display_none, Invoke};
 use crate::version::VersionT;
 use crate::{Error, ErrorKind, ResultExt};
 
@@ -358,6 +360,19 @@ async fn perform_backup<Db: DbHandle>(
         .await
         .with_kind(ErrorKind::Filesystem)?;
 
+    let luks_folder_old = backup_guard.as_ref().join("luks.old");
+    if tokio::fs::metadata(&luks_folder_old).await.is_ok() {
+        tokio::fs::remove_dir_all(&luks_folder_old).await?;
+    }
+    let luks_folder_bak = backup_guard.as_ref().join("luks");
+    if tokio::fs::metadata(&luks_folder_bak).await.is_ok() {
+        tokio::fs::rename(&luks_folder_bak, &luks_folder_old).await?;
+    }
+    let luks_folder = Path::new("/media/embassy/config/luks");
+    if tokio::fs::metadata(&luks_folder).await.is_ok() {
+        dir_copy(&luks_folder, &luks_folder_bak, None).await?;
+    }
+
     let timestamp = Some(Utc::now());
 
     backup_guard.unencrypted_metadata.version = crate::version::Current::new().semver().into();

backend/src/backup/os.rs

@@ -1,12 +1,13 @@
+use openssl::pkey::PKey;
+use openssl::x509::X509;
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+
 use crate::account::AccountInfo;
 use crate::hostname::{generate_hostname, generate_id, Hostname};
 use crate::net::keys::Key;
 use crate::util::serde::Base64;
 use crate::Error;
-use openssl::pkey::PKey;
-use openssl::x509::X509;
-use serde::{Deserialize, Serialize};
-use serde_json::Value;
 
 pub struct OsBackup {
     pub account: AccountInfo,

backend/src/backup/restore.rs

@@ -6,8 +6,8 @@ use std::time::Duration;
 
 use clap::ArgMatches;
 use color_eyre::eyre::eyre;
-use futures::{future::BoxFuture, stream};
-use futures::{FutureExt, StreamExt};
+use futures::future::BoxFuture;
+use futures::{stream, FutureExt, StreamExt};
 use openssl::x509::X509;
 use patch_db::{DbHandle, PatchDbHandle};
 use rpc_toolkit::command;
@@ -109,7 +109,7 @@ async fn approximate_progress(
         if tokio::fs::metadata(&dir).await.is_err() {
             *size = 0;
         } else {
-            *size = dir_size(&dir).await?;
+            *size = dir_size(&dir, None).await?;
         }
     }
     Ok(())
@@ -285,7 +285,7 @@ async fn restore_packages(
         progress_info.package_installs.insert(id.clone(), progress);
         progress_info
             .src_volume_size
-            .insert(id.clone(), dir_size(backup_dir(&id)).await?);
+            .insert(id.clone(), dir_size(backup_dir(&id), None).await?);
         progress_info.target_volume_size.insert(id.clone(), 0);
         let package_id = id.clone();
         tasks.push(
@@ -443,7 +443,7 @@ async fn restore_package<'a>(
     Ok((
         progress.clone(),
         async move {
-            download_install_s9pk(&ctx, &manifest, None, progress, file).await?;
+            download_install_s9pk(&ctx, &manifest, None, progress, file, None).await?;
 
             guard.unmount().await?;
 

backend/src/backup/target/mod.rs

@@ -7,10 +7,12 @@ use clap::ArgMatches;
 use color_eyre::eyre::eyre;
 use digest::generic_array::GenericArray;
 use digest::OutputSizeUser;
+use lazy_static::lazy_static;
 use rpc_toolkit::command;
 use serde::{Deserialize, Serialize};
 use sha2::Sha256;
 use sqlx::{Executor, Postgres};
+use tokio::sync::Mutex;
 use tracing::instrument;
 
 use self::cifs::CifsBackupTarget;
@@ -23,7 +25,7 @@ use crate::disk::mount::guard::TmpMountGuard;
 use crate::disk::util::PartitionInfo;
 use crate::s9pk::manifest::PackageId;
 use crate::util::serde::{deserialize_from_str, display_serializable, serialize_display};
-use crate::util::Version;
+use crate::util::{display_none, Version};
 use crate::Error;
 
 pub mod cifs;
@@ -42,7 +44,7 @@ pub enum BackupTarget {
     Cifs(CifsBackupTarget),
 }
 
-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)]
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
 pub enum BackupTargetId {
     Disk { logicalname: PathBuf },
     Cifs { id: i32 },
@@ -129,7 +131,7 @@ impl FileSystem for BackupTargetFS {
     }
 }
 
-#[command(subcommands(cifs::cifs, list, info))]
+#[command(subcommands(cifs::cifs, list, info, mount, umount))]
 pub fn target() -> Result<(), Error> {
     Ok(())
 }
@@ -247,3 +249,61 @@ pub async fn info(
 
     Ok(res)
 }
+
+lazy_static! {
+    static ref USER_MOUNTS: Mutex<BTreeMap<BackupTargetId, BackupMountGuard<TmpMountGuard>>> =
+        Mutex::new(BTreeMap::new());
+}
+
+#[command]
+#[instrument(skip_all)]
+pub async fn mount(
+    #[context] ctx: RpcContext,
+    #[arg(rename = "target-id")] target_id: BackupTargetId,
+    #[arg] password: String,
+) -> Result<String, Error> {
+    let mut mounts = USER_MOUNTS.lock().await;
+
+    if let Some(existing) = mounts.get(&target_id) {
+        return Ok(existing.as_ref().display().to_string());
+    }
+
+    let guard = BackupMountGuard::mount(
+        TmpMountGuard::mount(
+            &target_id
+                .clone()
+                .load(&mut ctx.secret_store.acquire().await?)
+                .await?,
+            ReadWrite,
+        )
+        .await?,
+        &password,
+    )
+    .await?;
+
+    let res = guard.as_ref().display().to_string();
+
+    mounts.insert(target_id, guard);
+
+    Ok(res)
+}
+
+#[command(display(display_none))]
+#[instrument(skip_all)]
+pub async fn umount(
+    #[context] ctx: RpcContext,
+    #[arg(rename = "target-id")] target_id: Option<BackupTargetId>,
+) -> Result<(), Error> {
+    let mut mounts = USER_MOUNTS.lock().await;
+    if let Some(target_id) = target_id {
+        if let Some(existing) = mounts.remove(&target_id) {
+            existing.unmount().await?;
+        }
+    } else {
+        for (_, existing) in std::mem::take(&mut *mounts) {
+            existing.unmount().await?;
+        }
+    }
+
+    Ok(())
+}

backend/src/bins/avahi_alias.rs

@@ -14,7 +14,7 @@ fn log_str_error(action: &str, e: i32) {
     }
 }
 
-fn main() {
+pub fn main() {
     let aliases: Vec<_> = std::env::args().skip(1).collect();
     unsafe {
         let simple_poll = avahi_sys::avahi_simple_poll_new();

backend/src/bins/deprecated.rs

@@ -0,0 +1,9 @@
+pub fn renamed(old: &str, new: &str) -> ! {
+    eprintln!("{old} has been renamed to {new}");
+    std::process::exit(1)
+}
+
+pub fn removed(name: &str) -> ! {
+    eprintln!("{name} has been removed");
+    std::process::exit(1)
+}

backend/src/bins/mod.rs

@@ -0,0 +1,55 @@
+use std::path::Path;
+
+#[cfg(feature = "avahi-alias")]
+pub mod avahi_alias;
+pub mod deprecated;
+#[cfg(feature = "cli")]
+pub mod start_cli;
+#[cfg(feature = "daemon")]
+pub mod start_init;
+#[cfg(feature = "sdk")]
+pub mod start_sdk;
+#[cfg(feature = "daemon")]
+pub mod startd;
+
+fn select_executable(name: &str) -> Option<fn()> {
+    match name {
+        #[cfg(feature = "avahi-alias")]
+        "avahi-alias" => Some(avahi_alias::main),
+        #[cfg(feature = "cli")]
+        "start-cli" => Some(start_cli::main),
+        #[cfg(feature = "sdk")]
+        "start-sdk" => Some(start_sdk::main),
+        #[cfg(feature = "daemon")]
+        "startd" => Some(startd::main),
+        "embassy-cli" => Some(|| deprecated::renamed("embassy-cli", "start-cli")),
+        "embassy-sdk" => Some(|| deprecated::renamed("embassy-sdk", "start-sdk")),
+        "embassyd" => Some(|| deprecated::renamed("embassyd", "startd")),
+        "embassy-init" => Some(|| deprecated::removed("embassy-init")),
+        _ => None,
+    }
+}
+
+pub fn startbox() {
+    let args = std::env::args().take(2).collect::<Vec<_>>();
+    if let Some(x) = args
+        .get(0)
+        .and_then(|s| Path::new(&*s).file_name())
+        .and_then(|s| s.to_str())
+        .and_then(|s| select_executable(&s))
+    {
+        x()
+    } else if let Some(x) = args.get(1).and_then(|s| select_executable(&s)) {
+        x()
+    } else {
+        eprintln!(
+            "unknown executable: {}",
+            args.get(0)
+                .filter(|x| &**x != "startbox")
+                .or_else(|| args.get(1))
+                .map(|s| s.as_str())
+                .unwrap_or("N/A")
+        );
+        std::process::exit(1);
+    }
+}

backend/src/bins/start_cli.rs

@@ -1,21 +1,22 @@
 use clap::Arg;
-use embassy::context::CliContext;
-use embassy::util::logger::EmbassyLogger;
-use embassy::version::{Current, VersionT};
-use embassy::Error;
 use rpc_toolkit::run_cli;
 use rpc_toolkit::yajrc::RpcError;
 use serde_json::Value;
 
+use crate::context::CliContext;
+use crate::util::logger::EmbassyLogger;
+use crate::version::{Current, VersionT};
+use crate::Error;
+
 lazy_static::lazy_static! {
     static ref VERSION_STRING: String = Current::new().semver().to_string();
 }
 
 fn inner_main() -> Result<(), Error> {
     run_cli!({
-        command: embassy::main_api,
+        command: crate::main_api,
         app: app => app
-            .name("Embassy CLI")
+            .name("StartOS CLI")
             .version(&**VERSION_STRING)
             .arg(
                 clap::Arg::with_name("config")
@@ -48,7 +49,7 @@ fn inner_main() -> Result<(), Error> {
     Ok(())
 }
 
-fn main() {
+pub fn main() {
     match inner_main() {
         Ok(_) => (),
         Err(e) => {

backend/src/bins/start_init.rs

@@ -1,22 +1,24 @@
+use std::net::{Ipv6Addr, SocketAddr};
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
 use std::time::Duration;
 
-use embassy::context::rpc::RpcContextConfig;
-use embassy::context::{DiagnosticContext, InstallContext, SetupContext};
-use embassy::disk::fsck::RepairStrategy;
-use embassy::disk::main::DEFAULT_PASSWORD;
-use embassy::disk::REPAIR_DISK_PATH;
-use embassy::init::STANDBY_MODE_PATH;
-use embassy::net::web_server::WebServer;
-use embassy::shutdown::Shutdown;
-use embassy::sound::CHIME;
-use embassy::util::logger::EmbassyLogger;
-use embassy::util::Invoke;
-use embassy::{Error, ErrorKind, ResultExt, OS_ARCH};
 use tokio::process::Command;
 use tracing::instrument;
 
+use crate::context::rpc::RpcContextConfig;
+use crate::context::{DiagnosticContext, InstallContext, SetupContext};
+use crate::disk::fsck::RepairStrategy;
+use crate::disk::main::DEFAULT_PASSWORD;
+use crate::disk::REPAIR_DISK_PATH;
+use crate::init::STANDBY_MODE_PATH;
+use crate::net::web_server::WebServer;
+use crate::shutdown::Shutdown;
+use crate::sound::CHIME;
+use crate::util::logger::EmbassyLogger;
+use crate::util::Invoke;
+use crate::{Error, ErrorKind, ResultExt, OS_ARCH};
+
 #[instrument(skip_all)]
 async fn setup_or_init(cfg_path: Option<PathBuf>) -> Result<(), Error> {
     Command::new("ln")
@@ -59,7 +61,11 @@ async fn setup_or_init(cfg_path: Option<PathBuf>) -> Result<(), Error> {
 
         let ctx = InstallContext::init(cfg_path).await?;
 
-        let server = WebServer::install(([0, 0, 0, 0], 80).into(), ctx.clone()).await?;
+        let server = WebServer::install(
+            SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 80),
+            ctx.clone(),
+        )
+        .await?;
 
         tokio::time::sleep(Duration::from_secs(1)).await; // let the record state that I hate this
         CHIME.play().await?;
@@ -73,7 +79,7 @@ async fn setup_or_init(cfg_path: Option<PathBuf>) -> Result<(), Error> {
         server.shutdown().await;
 
         Command::new("reboot")
-            .invoke(embassy::ErrorKind::Unknown)
+            .invoke(crate::ErrorKind::Unknown)
             .await?;
     } else if tokio::fs::metadata("/media/embassy/config/disk.guid")
         .await
@@ -81,7 +87,11 @@ async fn setup_or_init(cfg_path: Option<PathBuf>) -> Result<(), Error> {
     {
         let ctx = SetupContext::init(cfg_path).await?;
 
-        let server = WebServer::setup(([0, 0, 0, 0], 80).into(), ctx.clone()).await?;
+        let server = WebServer::setup(
+            SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 80),
+            ctx.clone(),
+        )
+        .await?;
 
         tokio::time::sleep(Duration::from_secs(1)).await; // let the record state that I hate this
         CHIME.play().await?;
@@ -107,7 +117,7 @@ async fn setup_or_init(cfg_path: Option<PathBuf>) -> Result<(), Error> {
         let guid_string = tokio::fs::read_to_string("/media/embassy/config/disk.guid") // unique identifier for volume group - keeps track of the disk that goes with your embassy
             .await?;
         let guid = guid_string.trim();
-        let requires_reboot = embassy::disk::main::import(
+        let requires_reboot = crate::disk::main::import(
             guid,
             cfg.datadir(),
             if tokio::fs::metadata(REPAIR_DISK_PATH).await.is_ok() {
@@ -115,22 +125,26 @@ async fn setup_or_init(cfg_path: Option<PathBuf>) -> Result<(), Error> {
             } else {
                 RepairStrategy::Preen
             },
-            DEFAULT_PASSWORD,
+            if guid.ends_with("_UNENC") {
+                None
+            } else {
+                Some(DEFAULT_PASSWORD)
+            },
         )
         .await?;
         if tokio::fs::metadata(REPAIR_DISK_PATH).await.is_ok() {
             tokio::fs::remove_file(REPAIR_DISK_PATH)
                 .await
-                .with_ctx(|_| (embassy::ErrorKind::Filesystem, REPAIR_DISK_PATH))?;
+                .with_ctx(|_| (crate::ErrorKind::Filesystem, REPAIR_DISK_PATH))?;
         }
         if requires_reboot.0 {
-            embassy::disk::main::export(guid, cfg.datadir()).await?;
+            crate::disk::main::export(guid, cfg.datadir()).await?;
             Command::new("reboot")
-                .invoke(embassy::ErrorKind::Unknown)
+                .invoke(crate::ErrorKind::Unknown)
                 .await?;
         }
         tracing::info!("Loaded Disk");
-        embassy::init::init(&cfg).await?;
+        crate::init::init(&cfg).await?;
     }
 
     Ok(())
@@ -159,11 +173,11 @@ async fn inner_main(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Error
     if OS_ARCH == "raspberrypi" && tokio::fs::metadata(STANDBY_MODE_PATH).await.is_ok() {
         tokio::fs::remove_file(STANDBY_MODE_PATH).await?;
         Command::new("sync").invoke(ErrorKind::Filesystem).await?;
-        embassy::sound::SHUTDOWN.play().await?;
+        crate::sound::SHUTDOWN.play().await?;
         futures::future::pending::<()>().await;
     }
 
-    embassy::sound::BEP.play().await?;
+    crate::sound::BEP.play().await?;
 
     run_script_if_exists("/media/embassy/config/preinit.sh").await;
 
@@ -171,7 +185,7 @@ async fn inner_main(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Error
         async move {
             tracing::error!("{}", e.source);
             tracing::debug!("{}", e.source);
-            embassy::sound::BEETHOVEN.play().await?;
+            crate::sound::BEETHOVEN.play().await?;
 
             let ctx = DiagnosticContext::init(
                 cfg_path,
@@ -192,7 +206,11 @@ async fn inner_main(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Error
             )
             .await?;
 
-            let server = WebServer::diagnostic(([0, 0, 0, 0], 80).into(), ctx.clone()).await?;
+            let server = WebServer::diagnostic(
+                SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 80),
+                ctx.clone(),
+            )
+            .await?;
 
             let shutdown = ctx.shutdown.subscribe().recv().await.unwrap();
 
@@ -210,8 +228,8 @@ async fn inner_main(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Error
     res
 }
 
-fn main() {
-    let matches = clap::App::new("embassy-init")
+pub fn main() {
+    let matches = clap::App::new("start-init")
         .arg(
             clap::Arg::with_name("config")
                 .short('c')
@@ -220,8 +238,6 @@ fn main() {
         )
         .get_matches();
 
-    EmbassyLogger::init();
-
     let cfg_path = matches.value_of("config").map(|p| Path::new(p).to_owned());
     let res = {
         let rt = tokio::runtime::Builder::new_multi_thread()

backend/src/bins/start_sdk.rs

@@ -1,20 +1,21 @@
-use embassy::context::SdkContext;
-use embassy::util::logger::EmbassyLogger;
-use embassy::version::{Current, VersionT};
-use embassy::Error;
 use rpc_toolkit::run_cli;
 use rpc_toolkit::yajrc::RpcError;
 use serde_json::Value;
 
+use crate::context::SdkContext;
+use crate::util::logger::EmbassyLogger;
+use crate::version::{Current, VersionT};
+use crate::Error;
+
 lazy_static::lazy_static! {
     static ref VERSION_STRING: String = Current::new().semver().to_string();
 }
 
 fn inner_main() -> Result<(), Error> {
     run_cli!({
-        command: embassy::portable_api,
+        command: crate::portable_api,
         app: app => app
-            .name("Embassy SDK")
+            .name("StartOS SDK")
             .version(&**VERSION_STRING)
             .arg(
                 clap::Arg::with_name("config")
@@ -47,7 +48,7 @@ fn inner_main() -> Result<(), Error> {
     Ok(())
 }
 
-fn main() {
+pub fn main() {
     match inner_main() {
         Ok(_) => (),
         Err(e) => {

backend/src/bins/startd.rs

@@ -1,17 +1,19 @@
+use std::net::{Ipv6Addr, SocketAddr};
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
 
 use color_eyre::eyre::eyre;
-use embassy::context::{DiagnosticContext, RpcContext};
-use embassy::net::web_server::WebServer;
-use embassy::shutdown::Shutdown;
-use embassy::system::launch_metrics_task;
-use embassy::util::logger::EmbassyLogger;
-use embassy::{Error, ErrorKind, ResultExt};
 use futures::{FutureExt, TryFutureExt};
 use tokio::signal::unix::signal;
 use tracing::instrument;
 
+use crate::context::{DiagnosticContext, RpcContext};
+use crate::net::web_server::WebServer;
+use crate::shutdown::Shutdown;
+use crate::system::launch_metrics_task;
+use crate::util::logger::EmbassyLogger;
+use crate::{Error, ErrorKind, ResultExt};
+
 #[instrument(skip_all)]
 async fn inner_main(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Error> {
     let (rpc_ctx, server, shutdown) = {
@@ -25,8 +27,12 @@ async fn inner_main(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Error
             ),
         )
         .await?;
-        embassy::hostname::sync_hostname(&rpc_ctx.account.read().await.hostname).await?;
-        let server = WebServer::main(([0, 0, 0, 0], 80).into(), rpc_ctx.clone()).await?;
+        crate::hostname::sync_hostname(&rpc_ctx.account.read().await.hostname).await?;
+        let server = WebServer::main(
+            SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 80),
+            rpc_ctx.clone(),
+        )
+        .await?;
 
         let mut shutdown_recv = rpc_ctx.shutdown.subscribe();
 
@@ -66,7 +72,7 @@ async fn inner_main(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Error
             .await
         });
 
-        embassy::sound::CHIME.play().await?;
+        crate::sound::CHIME.play().await?;
 
         metrics_task
             .map_err(|e| {
@@ -95,8 +101,15 @@ async fn inner_main(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Error
     Ok(shutdown)
 }
 
-fn main() {
-    let matches = clap::App::new("embassyd")
+pub fn main() {
+    EmbassyLogger::init();
+
+    if !Path::new("/run/embassy/initialized").exists() {
+        super::start_init::main();
+        std::fs::write("/run/embassy/initialized", "").unwrap();
+    }
+
+    let matches = clap::App::new("startd")
         .arg(
             clap::Arg::with_name("config")
                 .short('c')
@@ -105,8 +118,6 @@ fn main() {
         )
         .get_matches();
 
-    EmbassyLogger::init();
-
     let cfg_path = matches.value_of("config").map(|p| Path::new(p).to_owned());
 
     let res = {
@@ -121,7 +132,7 @@ fn main() {
                     async {
                         tracing::error!("{}", e.source);
                         tracing::debug!("{:?}", e.source);
-                        embassy::sound::BEETHOVEN.play().await?;
+                        crate::sound::BEETHOVEN.play().await?;
                         let ctx = DiagnosticContext::init(
                             cfg_path,
                             if tokio::fs::metadata("/media/embassy/config/disk.guid")
@@ -141,8 +152,11 @@ fn main() {
                         )
                         .await?;
 
-                        let server =
-                            WebServer::diagnostic(([0, 0, 0, 0], 80).into(), ctx.clone()).await?;
+                        let server = WebServer::diagnostic(
+                            SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 80),
+                            ctx.clone(),
+                        )
+                        .await?;
 
                         let mut shutdown = ctx.shutdown.subscribe();
 

backend/src/config/mod.rs

@@ -503,19 +503,27 @@ pub fn configure_rec<'a, Db: DbHandle>(
             .config_actions
             .get(db, id)
             .await?
-            .ok_or_else(not_found)?;
+            .ok_or_else(|| not_found!(id))?;
         let dependencies = receipts
             .dependencies
             .get(db, id)
             .await?
-            .ok_or_else(not_found)?;
-        let volumes = receipts.volumes.get(db, id).await?.ok_or_else(not_found)?;
+            .ok_or_else(|| not_found!(id))?;
+        let volumes = receipts
+            .volumes
+            .get(db, id)
+            .await?
+            .ok_or_else(|| not_found!(id))?;
         let is_needs_config = !receipts
             .configured
             .get(db, id)
             .await?
-            .ok_or_else(not_found)?;
-        let version = receipts.version.get(db, id).await?.ok_or_else(not_found)?;
+            .ok_or_else(|| not_found!(id))?;
+        let version = receipts
+            .version
+            .get(db, id)
+            .await?
+            .ok_or_else(|| not_found!(id))?;
 
         // get current config and current spec
         let ConfigRes {
@@ -530,7 +538,11 @@ pub fn configure_rec<'a, Db: DbHandle>(
             spec.gen(&mut rand::rngs::StdRng::from_entropy(), timeout)?
         };
 
-        let manifest = receipts.manifest.get(db, id).await?.ok_or_else(not_found)?;
+        let manifest = receipts
+            .manifest
+            .get(db, id)
+            .await?
+            .ok_or_else(|| not_found!(id))?;
 
         spec.validate(&manifest)?;
         spec.matches(&config)?; // check that new config matches spec
@@ -549,7 +561,7 @@ pub fn configure_rec<'a, Db: DbHandle>(
             .system_pointers
             .get(db, &id)
             .await?
-            .ok_or_else(not_found)?;
+            .ok_or_else(|| not_found!(id))?;
         sys.truncate(0);
         let mut current_dependencies: CurrentDependencies = CurrentDependencies(
             dependencies
@@ -655,7 +667,7 @@ pub fn configure_rec<'a, Db: DbHandle>(
             .dependency_errors
             .get(db, &id)
             .await?
-            .ok_or_else(not_found)?;
+            .ok_or_else(|| not_found!(id))?;
         tracing::warn!("Dependency Errors: {:?}", errs);
         let errs = DependencyErrors::init(
             ctx,
@@ -675,7 +687,7 @@ pub fn configure_rec<'a, Db: DbHandle>(
             .current_dependents
             .get(db, id)
             .await?
-            .ok_or_else(not_found)?;
+            .ok_or_else(|| not_found!(id))?;
         let prev = if is_needs_config { None } else { old_config }
             .map(Value::Object)
             .unwrap_or_default();
@@ -693,7 +705,7 @@ pub fn configure_rec<'a, Db: DbHandle>(
                     .manifest
                     .get(db, &dependent)
                     .await?
-                    .ok_or_else(not_found)?;
+                    .ok_or_else(|| not_found!(id))?;
                 if let Err(error) = cfg
                     .check(
                         ctx,
@@ -771,10 +783,16 @@ pub fn configure_rec<'a, Db: DbHandle>(
     }
     .boxed()
 }
-#[instrument(skip_all)]
-pub fn not_found() -> Error {
-    Error::new(eyre!("Could not find"), crate::ErrorKind::Incoherent)
+
+macro_rules! not_found {
+    ($x:expr) => {
+        crate::Error::new(
+            color_eyre::eyre::eyre!("Could not find {} at {}:{}", $x, module_path!(), line!()),
+            crate::ErrorKind::Incoherent,
+        )
+    };
 }
+pub(crate) use not_found;
 
 /// We want to have a double check that the paths are what we expect them to be.
 /// Found that earlier the paths where not what we expected them to be.

backend/src/context/cli.rs

@@ -17,12 +17,11 @@ use rpc_toolkit::Context;
 use serde::Deserialize;
 use tracing::instrument;
 
+use super::setup::CURRENT_SECRET;
 use crate::middleware::auth::LOCAL_AUTH_COOKIE_PATH;
 use crate::util::config::{load_config_from_paths, local_config_path};
 use crate::ResultExt;
 
-use super::setup::CURRENT_SECRET;
-
 #[derive(Debug, Default, Deserialize)]
 #[serde(rename_all = "kebab-case")]
 pub struct CliContextConfig {

backend/src/context/rpc.rs

@@ -11,7 +11,7 @@ use helpers::to_tmp_path;
 use josekit::jwk::Jwk;
 use patch_db::json_ptr::JsonPointer;
 use patch_db::{DbHandle, LockReceipt, LockType, PatchDb};
-use reqwest::Url;
+use reqwest::{Client, Proxy, Url};
 use rpc_toolkit::Context;
 use serde::Deserialize;
 use sqlx::postgres::PgConnectOptions;
@@ -19,11 +19,12 @@ use sqlx::PgPool;
 use tokio::sync::{broadcast, oneshot, Mutex, RwLock};
 use tracing::instrument;
 
+use super::setup::CURRENT_SECRET;
 use crate::account::AccountInfo;
 use crate::core::rpc_continuations::{RequestGuid, RestHandler, RpcContinuation};
-use crate::db::model::{Database, InstalledPackageDataEntry, PackageDataEntry};
+use crate::db::model::{CurrentDependents, Database, InstalledPackageDataEntry, PackageDataEntry};
 use crate::disk::OsPartitionInfo;
-use crate::init::{init_postgres, pgloader};
+use crate::init::init_postgres;
 use crate::install::cleanup::{cleanup_failed, uninstall, CleanupFailedReceipts};
 use crate::manager::ManagerMap;
 use crate::middleware::auth::HashSessionToken;
@@ -33,11 +34,11 @@ use crate::net::wifi::WpaCli;
 use crate::notifications::NotificationManager;
 use crate::shutdown::Shutdown;
 use crate::status::{MainStatus, Status};
+use crate::system::get_mem_info;
 use crate::util::config::load_config_from_paths;
+use crate::util::lshw::{lshw, LshwDevice};
 use crate::{Error, ErrorKind, ResultExt};
 
-use super::setup::CURRENT_SECRET;
-
 #[derive(Debug, Default, Deserialize)]
 #[serde(rename_all = "kebab-case")]
 pub struct RpcContextConfig {
@@ -96,15 +97,6 @@ impl RpcContextConfig {
             .run(&secret_store)
             .await
             .with_kind(crate::ErrorKind::Database)?;
-        let old_db_path = self.datadir().join("main/secrets.db");
-        if tokio::fs::metadata(&old_db_path).await.is_ok() {
-            pgloader(
-                &old_db_path,
-                self.migration_batch_rows.unwrap_or(25000),
-                self.migration_prefetch_rows.unwrap_or(100_000),
-            )
-            .await?;
-        }
         Ok(secret_store)
     }
 }
@@ -130,6 +122,13 @@ pub struct RpcContextSeed {
     pub rpc_stream_continuations: Mutex<BTreeMap<RequestGuid, RpcContinuation>>,
     pub wifi_manager: Option<Arc<RwLock<WpaCli>>>,
     pub current_secret: Arc<Jwk>,
+    pub client: Client,
+    pub hardware: Hardware,
+}
+
+pub struct Hardware {
+    pub devices: Vec<LshwDevice>,
+    pub ram: u64,
 }
 
 pub struct RpcCleanReceipts {
@@ -197,6 +196,7 @@ impl RpcContext {
             NetController::init(
                 base.tor_control
                     .unwrap_or(SocketAddr::from(([127, 0, 0, 1], 9051))),
+                tor_proxy,
                 base.dns_bind
                     .as_ref()
                     .map(|v| v.as_slice())
@@ -212,6 +212,9 @@ impl RpcContext {
         let metrics_cache = RwLock::new(None);
         let notification_manager = NotificationManager::new(secret_store.clone());
         tracing::info!("Initialized Notification Manager");
+        let tor_proxy_url = format!("socks5h://{tor_proxy}");
+        let devices = lshw().await?;
+        let ram = get_mem_info().await?.total.0 as u64 * 1024 * 1024;
         let seed = Arc::new(RpcContextSeed {
             is_closed: AtomicBool::new(false),
             datadir: base.datadir().to_path_buf(),
@@ -244,6 +247,17 @@ impl RpcContext {
                     )
                 })?,
             ),
+            client: Client::builder()
+                .proxy(Proxy::custom(move |url| {
+                    if url.host_str().map_or(false, |h| h.ends_with(".onion")) {
+                        Some(tor_proxy_url.clone())
+                    } else {
+                        None
+                    }
+                }))
+                .build()
+                .with_kind(crate::ErrorKind::ParseUrl)?,
+            hardware: Hardware { devices, ram },
         });
 
         let res = Self(seed);
@@ -274,6 +288,45 @@ impl RpcContext {
     pub async fn cleanup(&self) -> Result<(), Error> {
         let mut db = self.db.handle();
         let receipts = RpcCleanReceipts::new(&mut db).await?;
+        let packages = receipts.packages.get(&mut db).await?.0;
+        let mut current_dependents = packages
+            .keys()
+            .map(|k| (k.clone(), BTreeMap::new()))
+            .collect::<BTreeMap<_, _>>();
+        for (package_id, package) in packages {
+            for (k, v) in package
+                .into_installed()
+                .into_iter()
+                .flat_map(|i| i.current_dependencies.0)
+            {
+                let mut entry: BTreeMap<_, _> = current_dependents.remove(&k).unwrap_or_default();
+                entry.insert(package_id.clone(), v);
+                current_dependents.insert(k, entry);
+            }
+        }
+        for (package_id, current_dependents) in current_dependents {
+            if let Some(deps) = crate::db::DatabaseModel::new()
+                .package_data()
+                .idx_model(&package_id)
+                .and_then(|pde| pde.installed())
+                .map::<_, CurrentDependents>(|i| i.current_dependents())
+                .check(&mut db)
+                .await?
+            {
+                deps.put(&mut db, &CurrentDependents(current_dependents))
+                    .await?;
+            } else if let Some(deps) = crate::db::DatabaseModel::new()
+                .package_data()
+                .idx_model(&package_id)
+                .and_then(|pde| pde.removing())
+                .map::<_, CurrentDependents>(|i| i.current_dependents())
+                .check(&mut db)
+                .await?
+            {
+                deps.put(&mut db, &CurrentDependents(current_dependents))
+                    .await?;
+            }
+        }
         for (package_id, package) in receipts.packages.get(&mut db).await?.0 {
             if let Err(e) = async {
                 match package {

backend/src/context/setup.rs

@@ -17,7 +17,7 @@ use tracing::instrument;
 use crate::account::AccountInfo;
 use crate::db::model::Database;
 use crate::disk::OsPartitionInfo;
-use crate::init::{init_postgres, pgloader};
+use crate::init::init_postgres;
 use crate::setup::SetupStatus;
 use crate::util::config::load_config_from_paths;
 use crate::{Error, ResultExt};
@@ -45,6 +45,8 @@ pub struct SetupContextConfig {
     pub migration_batch_rows: Option<usize>,
     pub migration_prefetch_rows: Option<usize>,
     pub datadir: Option<PathBuf>,
+    #[serde(default)]
+    pub disable_encryption: bool,
 }
 impl SetupContextConfig {
     #[instrument(skip_all)]
@@ -75,6 +77,7 @@ pub struct SetupContextSeed {
     pub config_path: Option<PathBuf>,
     pub migration_batch_rows: usize,
     pub migration_prefetch_rows: usize,
+    pub disable_encryption: bool,
     pub shutdown: Sender<()>,
     pub datadir: PathBuf,
     pub selected_v2_drive: RwLock<Option<PathBuf>>,
@@ -102,6 +105,7 @@ impl SetupContext {
             config_path: path.as_ref().map(|p| p.as_ref().to_owned()),
             migration_batch_rows: cfg.migration_batch_rows.unwrap_or(25000),
             migration_prefetch_rows: cfg.migration_prefetch_rows.unwrap_or(100_000),
+            disable_encryption: cfg.disable_encryption,
             shutdown,
             datadir,
             selected_v2_drive: RwLock::new(None),
@@ -132,15 +136,6 @@ impl SetupContext {
             .run(&secret_store)
             .await
             .with_kind(crate::ErrorKind::Database)?;
-        let old_db_path = self.datadir.join("main/secrets.db");
-        if tokio::fs::metadata(&old_db_path).await.is_ok() {
-            pgloader(
-                &old_db_path,
-                self.migration_batch_rows,
-                self.migration_prefetch_rows,
-            )
-            .await?;
-        }
         Ok(secret_store)
     }
 }

backend/src/db/mod.rs

@@ -4,9 +4,10 @@ pub mod package;
 use std::future::Future;
 use std::sync::Arc;
 
+use color_eyre::eyre::eyre;
 use futures::{FutureExt, SinkExt, StreamExt};
 use patch_db::json_ptr::JsonPointer;
-use patch_db::{Dump, Revision};
+use patch_db::{DbHandle, Dump, LockType, Revision};
 use rpc_toolkit::command;
 use rpc_toolkit::hyper::upgrade::Upgraded;
 use rpc_toolkit::hyper::{Body, Error as HyperError, Request, Response};
@@ -24,6 +25,7 @@ use tracing::instrument;
 pub use self::model::DatabaseModel;
 use crate::context::RpcContext;
 use crate::middleware::auth::{HasValidSession, HashSessionToken};
+use crate::util::display_none;
 use crate::util::serde::{display_serializable, IoFormat};
 use crate::{Error, ResultExt};
 
@@ -163,7 +165,7 @@ pub async fn subscribe(ctx: RpcContext, req: Request<Body>) -> Result<Response<B
     Ok(res)
 }
 
-#[command(subcommands(revisions, dump, put))]
+#[command(subcommands(revisions, dump, put, apply))]
 pub fn db() -> Result<(), RpcError> {
     Ok(())
 }
@@ -199,6 +201,85 @@ pub async fn dump(
     Ok(ctx.db.dump().await?)
 }
 
+fn apply_expr(input: jaq_core::Val, expr: &str) -> Result<jaq_core::Val, Error> {
+    let (expr, errs) = jaq_core::parse::parse(expr, jaq_core::parse::main());
+
+    let Some(expr) = expr else {
+        return Err(Error::new(
+            eyre!("Failed to parse expression: {:?}", errs),
+            crate::ErrorKind::InvalidRequest,
+        ));
+    };
+
+    let mut errs = Vec::new();
+
+    let mut defs = jaq_core::Definitions::core();
+    for def in jaq_std::std() {
+        defs.insert(def, &mut errs);
+    }
+
+    let filter = defs.finish(expr, Vec::new(), &mut errs);
+
+    if !errs.is_empty() {
+        return Err(Error::new(
+            eyre!("Failed to compile expression: {:?}", errs),
+            crate::ErrorKind::InvalidRequest,
+        ));
+    };
+
+    let inputs = jaq_core::RcIter::new(std::iter::empty());
+    let mut res_iter = filter.run(jaq_core::Ctx::new([], &inputs), input);
+
+    let Some(res) = res_iter
+        .next()
+        .transpose()
+        .map_err(|e| eyre!("{e}"))
+        .with_kind(crate::ErrorKind::Deserialization)?
+    else {
+        return Err(Error::new(
+            eyre!("expr returned no results"),
+            crate::ErrorKind::InvalidRequest,
+        ));
+    };
+
+    if res_iter.next().is_some() {
+        return Err(Error::new(
+            eyre!("expr returned too many results"),
+            crate::ErrorKind::InvalidRequest,
+        ));
+    }
+
+    Ok(res)
+}
+
+#[command(display(display_none))]
+pub async fn apply(#[context] ctx: RpcContext, #[arg] expr: String) -> Result<(), Error> {
+    let mut db = ctx.db.handle();
+
+    DatabaseModel::new().lock(&mut db, LockType::Write).await?;
+
+    let root_ptr = JsonPointer::<String>::default();
+
+    let input = db.get_value(&root_ptr, None).await?;
+
+    let res = (|| {
+        let res = apply_expr(input.into(), &expr)?;
+
+        serde_json::from_value::<model::Database>(res.clone().into()).with_ctx(|_| {
+            (
+                crate::ErrorKind::Deserialization,
+                "result does not match database model",
+            )
+        })?;
+
+        Ok::<serde_json::Value, Error>(res.into())
+    })()?;
+
+    db.put_value(&root_ptr, &res).await?;
+
+    Ok(())
+}
+
 #[command(subcommands(ui))]
 pub fn put() -> Result<(), RpcError> {
     Ok(())

backend/src/db/model.rs

@@ -49,7 +49,7 @@ impl Database {
                 last_wifi_region: None,
                 eos_version_compat: Current::new().compat().clone(),
                 lan_address,
-                tor_address: format!("http://{}", account.key.tor_address())
+                tor_address: format!("https://{}", account.key.tor_address())
                     .parse()
                     .unwrap(),
                 ip_info: BTreeMap::new(),
@@ -80,6 +80,7 @@ impl Database {
                     .map(|x| format!("{x:X}"))
                     .join(":"),
                 system_start_time: Utc::now().to_rfc3339(),
+                zram: false,
             },
             package_data: AllPackageData::default(),
             ui: serde_json::from_str(include_str!("../../../frontend/patchdb-ui-seed.json"))
@@ -106,6 +107,7 @@ pub struct ServerInfo {
     pub lan_address: Url,
     pub tor_address: Url,
     #[model]
+    #[serde(default)]
     pub ip_info: BTreeMap<String, IpInfo>,
     #[model]
     #[serde(default)]
@@ -117,6 +119,8 @@ pub struct ServerInfo {
     pub pubkey: String,
     pub ca_fingerprint: String,
     pub system_start_time: String,
+    #[serde(default)]
+    pub zram: bool,
 }
 
 #[derive(Debug, Deserialize, Serialize, HasModel)]

backend/src/dependencies.rs

@@ -237,13 +237,16 @@ impl DependencyError {
                     }
                 }
                 DependencyError::ConfigUnsatisfied { .. } => {
-                    let dependent_manifest =
-                        receipts.manifest.get(db, id).await?.ok_or_else(not_found)?;
+                    let dependent_manifest = receipts
+                        .manifest
+                        .get(db, id)
+                        .await?
+                        .ok_or_else(|| not_found!(id))?;
                     let dependency_manifest = receipts
                         .manifest
                         .get(db, dependency)
                         .await?
-                        .ok_or_else(not_found)?;
+                        .ok_or_else(|| not_found!(dependency))?;
 
                     let dependency_config = if let Some(cfg) = dependency_config.take() {
                         cfg
@@ -294,7 +297,7 @@ impl DependencyError {
                         .status
                         .get(db, dependency)
                         .await?
-                        .ok_or_else(not_found)?;
+                        .ok_or_else(|| not_found!(dependency))?;
                     if status.main.running() {
                         DependencyError::HealthChecksFailed {
                             failures: BTreeMap::new(),
@@ -310,7 +313,7 @@ impl DependencyError {
                         .status
                         .get(db, dependency)
                         .await?
-                        .ok_or_else(not_found)?;
+                        .ok_or_else(|| not_found!(dependency))?;
                     match status.main {
                         MainStatus::BackingUp {
                             started: Some(_),
@@ -324,7 +327,7 @@ impl DependencyError {
                                         .current_dependencies
                                         .get(db, id)
                                         .await?
-                                        .ok_or_else(not_found)?
+                                        .ok_or_else(|| not_found!(id))?
                                         .get(dependency)
                                         .map(|x| x.health_checks.contains(&check))
                                         .unwrap_or(false)
@@ -934,7 +937,7 @@ pub fn break_transitive<'a, Db: DbHandle>(
             .dependency_errors
             .get(&mut tx, id)
             .await?
-            .ok_or_else(not_found)?;
+            .ok_or_else(|| not_found!(id))?;
 
         let old = dependency_errors.0.remove(dependency);
         let newly_broken = if let Some(e) = &old {
@@ -997,7 +1000,7 @@ pub async fn heal_all_dependents_transitive<'a, Db: DbHandle>(
         .current_dependents
         .get(db, id)
         .await?
-        .ok_or_else(not_found)?;
+        .ok_or_else(|| not_found!(id))?;
     for dependent in dependents.0.keys().filter(|dependent| id != *dependent) {
         heal_transitive(ctx, db, dependent, id, locks).await?;
     }
@@ -1013,7 +1016,11 @@ pub fn heal_transitive<'a, Db: DbHandle>(
     receipts: &'a DependencyReceipt,
 ) -> BoxFuture<'a, Result<(), Error>> {
     async move {
-        let mut status = receipts.status.get(db, id).await?.ok_or_else(not_found)?;
+        let mut status = receipts
+            .status
+            .get(db, id)
+            .await?
+            .ok_or_else(|| not_found!(id))?;
 
         let old = status.dependency_errors.0.remove(dependency);
 
@@ -1022,7 +1029,7 @@ pub fn heal_transitive<'a, Db: DbHandle>(
                 .dependency
                 .get(db, (id, dependency))
                 .await?
-                .ok_or_else(not_found)?;
+                .ok_or_else(|| not_found!(format!("{id}'s dependency: {dependency}")))?;
             if let Some(new) = old
                 .try_heal(ctx, db, id, dependency, None, &info, &receipts.try_heal)
                 .await?

backend/src/diagnostic.rs

@@ -9,11 +9,10 @@ use crate::disk::repair;
 use crate::init::SYSTEM_REBUILD_PATH;
 use crate::logs::{fetch_logs, LogResponse, LogSource};
 use crate::shutdown::Shutdown;
+use crate::system::SYSTEMD_UNIT;
 use crate::util::display_none;
 use crate::Error;
 
-pub const SYSTEMD_UNIT: &'static str = "embassy-init";
-
 #[command(subcommands(error, logs, exit, restart, forget_disk, disk, rebuild))]
 pub fn diagnostic() -> Result<(), Error> {
     Ok(())

backend/src/disk/fsck/btrfs.rs

@@ -0,0 +1,31 @@
+use std::path::Path;
+
+use tokio::process::Command;
+use tracing::instrument;
+
+use crate::disk::fsck::RequiresReboot;
+use crate::util::Invoke;
+use crate::Error;
+
+#[instrument(skip_all)]
+pub async fn btrfs_check_readonly(logicalname: impl AsRef<Path>) -> Result<RequiresReboot, Error> {
+    Command::new("btrfs")
+        .arg("check")
+        .arg("--readonly")
+        .arg(logicalname.as_ref())
+        .invoke(crate::ErrorKind::DiskManagement)
+        .await?;
+
+    Ok(RequiresReboot(false))
+}
+
+pub async fn btrfs_check_repair(logicalname: impl AsRef<Path>) -> Result<RequiresReboot, Error> {
+    Command::new("btrfs")
+        .arg("check")
+        .arg("--repair")
+        .arg(logicalname.as_ref())
+        .invoke(crate::ErrorKind::DiskManagement)
+        .await?;
+
+    Ok(RequiresReboot(false))
+}

backend/src/disk/fsck/ext4.rs

@@ -7,34 +7,9 @@ use futures::FutureExt;
 use tokio::process::Command;
 use tracing::instrument;
 
+use crate::disk::fsck::RequiresReboot;
 use crate::Error;
 
-#[derive(Debug, Clone, Copy)]
-#[must_use]
-pub struct RequiresReboot(pub bool);
-impl std::ops::BitOrAssign for RequiresReboot {
-    fn bitor_assign(&mut self, rhs: Self) {
-        self.0 |= rhs.0
-    }
-}
-
-#[derive(Debug, Clone, Copy)]
-pub enum RepairStrategy {
-    Preen,
-    Aggressive,
-}
-impl RepairStrategy {
-    pub async fn e2fsck(
-        &self,
-        logicalname: impl AsRef<Path> + std::fmt::Debug,
-    ) -> Result<RequiresReboot, Error> {
-        match self {
-            RepairStrategy::Preen => e2fsck_preen(logicalname).await,
-            RepairStrategy::Aggressive => e2fsck_aggressive(logicalname).await,
-        }
-    }
-}
-
 #[instrument(skip_all)]
 pub async fn e2fsck_preen(
     logicalname: impl AsRef<Path> + std::fmt::Debug,

backend/src/disk/fsck/mod.rs

@@ -0,0 +1,70 @@
+use std::path::Path;
+
+use color_eyre::eyre::eyre;
+use tokio::process::Command;
+
+use crate::disk::fsck::btrfs::{btrfs_check_readonly, btrfs_check_repair};
+use crate::disk::fsck::ext4::{e2fsck_aggressive, e2fsck_preen};
+use crate::util::Invoke;
+use crate::Error;
+
+pub mod btrfs;
+pub mod ext4;
+
+#[derive(Debug, Clone, Copy)]
+#[must_use]
+pub struct RequiresReboot(pub bool);
+impl std::ops::BitOrAssign for RequiresReboot {
+    fn bitor_assign(&mut self, rhs: Self) {
+        self.0 |= rhs.0
+    }
+}
+
+#[derive(Debug, Clone, Copy)]
+pub enum RepairStrategy {
+    Preen,
+    Aggressive,
+}
+impl RepairStrategy {
+    pub async fn fsck(
+        &self,
+        logicalname: impl AsRef<Path> + std::fmt::Debug,
+    ) -> Result<RequiresReboot, Error> {
+        match &*String::from_utf8(
+            Command::new("grub-probe")
+                .arg("-d")
+                .arg(logicalname.as_ref())
+                .invoke(crate::ErrorKind::DiskManagement)
+                .await?,
+        )?
+        .trim()
+        {
+            "ext2" => self.e2fsck(logicalname).await,
+            "btrfs" => self.btrfs_check(logicalname).await,
+            fs => {
+                return Err(Error::new(
+                    eyre!("Unknown filesystem {fs}"),
+                    crate::ErrorKind::DiskManagement,
+                ))
+            }
+        }
+    }
+    pub async fn e2fsck(
+        &self,
+        logicalname: impl AsRef<Path> + std::fmt::Debug,
+    ) -> Result<RequiresReboot, Error> {
+        match self {
+            RepairStrategy::Preen => e2fsck_preen(logicalname).await,
+            RepairStrategy::Aggressive => e2fsck_aggressive(logicalname).await,
+        }
+    }
+    pub async fn btrfs_check(
+        &self,
+        logicalname: impl AsRef<Path> + std::fmt::Debug,
+    ) -> Result<RequiresReboot, Error> {
+        match self {
+            RepairStrategy::Preen => btrfs_check_readonly(logicalname).await,
+            RepairStrategy::Aggressive => btrfs_check_repair(logicalname).await,
+        }
+    }
+}

backend/src/disk/main.rs

@@ -13,7 +13,7 @@ use crate::disk::mount::util::unmount;
 use crate::util::Invoke;
 use crate::{Error, ErrorKind, ResultExt};
 
-pub const PASSWORD_PATH: &'static str = "/etc/embassy/password";
+pub const PASSWORD_PATH: &'static str = "/run/embassy/password";
 pub const DEFAULT_PASSWORD: &'static str = "password";
 pub const MAIN_FS_SIZE: FsSize = FsSize::Gigabytes(8);
 
@@ -22,13 +22,13 @@ pub async fn create<I, P>(
     disks: &I,
     pvscan: &BTreeMap<PathBuf, Option<String>>,
     datadir: impl AsRef<Path>,
-    password: &str,
+    password: Option<&str>,
 ) -> Result<String, Error>
 where
     for<'a> &'a I: IntoIterator<Item = &'a P>,
     P: AsRef<Path>,
 {
-    let guid = create_pool(disks, pvscan).await?;
+    let guid = create_pool(disks, pvscan, password.is_some()).await?;
     create_all_fs(&guid, &datadir, password).await?;
     export(&guid, datadir).await?;
     Ok(guid)
@@ -38,6 +38,7 @@ where
 pub async fn create_pool<I, P>(
     disks: &I,
     pvscan: &BTreeMap<PathBuf, Option<String>>,
+    encrypted: bool,
 ) -> Result<String, Error>
 where
     for<'a> &'a I: IntoIterator<Item = &'a P>,
@@ -62,13 +63,16 @@ where
             .invoke(crate::ErrorKind::DiskManagement)
             .await?;
     }
-    let guid = format!(
+    let mut guid = format!(
         "EMBASSY_{}",
         base32::encode(
             base32::Alphabet::RFC4648 { padding: false },
             &rand::random::<[u8; 32]>(),
         )
     );
+    if !encrypted {
+        guid += "_UNENC";
+    }
     let mut cmd = Command::new("vgcreate");
     cmd.arg("-y").arg(&guid);
     for disk in disks {
@@ -90,11 +94,8 @@ pub async fn create_fs<P: AsRef<Path>>(
     datadir: P,
     name: &str,
     size: FsSize,
-    password: &str,
+    password: Option<&str>,
 ) -> Result<(), Error> {
-    tokio::fs::write(PASSWORD_PATH, password)
-        .await
-        .with_ctx(|_| (crate::ErrorKind::Filesystem, PASSWORD_PATH))?;
     let mut cmd = Command::new("lvcreate");
     match size {
         FsSize::Gigabytes(a) => cmd.arg("-L").arg(format!("{}G", a)),
@@ -106,36 +107,41 @@ pub async fn create_fs<P: AsRef<Path>>(
         .arg(guid)
         .invoke(crate::ErrorKind::DiskManagement)
         .await?;
-    Command::new("cryptsetup")
-        .arg("-q")
-        .arg("luksFormat")
-        .arg(format!("--key-file={}", PASSWORD_PATH))
-        .arg(format!("--keyfile-size={}", password.len()))
-        .arg(Path::new("/dev").join(guid).join(name))
+    let mut blockdev_path = Path::new("/dev").join(guid).join(name);
+    if let Some(password) = password {
+        if let Some(parent) = Path::new(PASSWORD_PATH).parent() {
+            tokio::fs::create_dir_all(parent).await?;
+        }
+        tokio::fs::write(PASSWORD_PATH, password)
+            .await
+            .with_ctx(|_| (crate::ErrorKind::Filesystem, PASSWORD_PATH))?;
+        Command::new("cryptsetup")
+            .arg("-q")
+            .arg("luksFormat")
+            .arg(format!("--key-file={}", PASSWORD_PATH))
+            .arg(format!("--keyfile-size={}", password.len()))
+            .arg(&blockdev_path)
+            .invoke(crate::ErrorKind::DiskManagement)
+            .await?;
+        Command::new("cryptsetup")
+            .arg("-q")
+            .arg("luksOpen")
+            .arg(format!("--key-file={}", PASSWORD_PATH))
+            .arg(format!("--keyfile-size={}", password.len()))
+            .arg(&blockdev_path)
+            .arg(format!("{}_{}", guid, name))
+            .invoke(crate::ErrorKind::DiskManagement)
+            .await?;
+        tokio::fs::remove_file(PASSWORD_PATH)
+            .await
+            .with_ctx(|_| (crate::ErrorKind::Filesystem, PASSWORD_PATH))?;
+        blockdev_path = Path::new("/dev/mapper").join(format!("{}_{}", guid, name));
+    }
+    Command::new("mkfs.btrfs")
+        .arg(&blockdev_path)
         .invoke(crate::ErrorKind::DiskManagement)
         .await?;
-    Command::new("cryptsetup")
-        .arg("-q")
-        .arg("luksOpen")
-        .arg(format!("--key-file={}", PASSWORD_PATH))
-        .arg(format!("--keyfile-size={}", password.len()))
-        .arg(Path::new("/dev").join(guid).join(name))
-        .arg(format!("{}_{}", guid, name))
-        .invoke(crate::ErrorKind::DiskManagement)
-        .await?;
-    Command::new("mkfs.ext4")
-        .arg(Path::new("/dev/mapper").join(format!("{}_{}", guid, name)))
-        .invoke(crate::ErrorKind::DiskManagement)
-        .await?;
-    mount(
-        Path::new("/dev/mapper").join(format!("{}_{}", guid, name)),
-        datadir.as_ref().join(name),
-        ReadWrite,
-    )
-    .await?;
-    tokio::fs::remove_file(PASSWORD_PATH)
-        .await
-        .with_ctx(|_| (crate::ErrorKind::Filesystem, PASSWORD_PATH))?;
+    mount(&blockdev_path, datadir.as_ref().join(name), ReadWrite).await?;
     Ok(())
 }
 
@@ -143,7 +149,7 @@ pub async fn create_fs<P: AsRef<Path>>(
 pub async fn create_all_fs<P: AsRef<Path>>(
     guid: &str,
     datadir: P,
-    password: &str,
+    password: Option<&str>,
 ) -> Result<(), Error> {
     create_fs(guid, &datadir, "main", MAIN_FS_SIZE, password).await?;
     create_fs(
@@ -160,12 +166,14 @@ pub async fn create_all_fs<P: AsRef<Path>>(
 #[instrument(skip_all)]
 pub async fn unmount_fs<P: AsRef<Path>>(guid: &str, datadir: P, name: &str) -> Result<(), Error> {
     unmount(datadir.as_ref().join(name)).await?;
-    Command::new("cryptsetup")
-        .arg("-q")
-        .arg("luksClose")
-        .arg(format!("{}_{}", guid, name))
-        .invoke(crate::ErrorKind::DiskManagement)
-        .await?;
+    if !guid.ends_with("_UNENC") {
+        Command::new("cryptsetup")
+            .arg("-q")
+            .arg("luksClose")
+            .arg(format!("{}_{}", guid, name))
+            .invoke(crate::ErrorKind::DiskManagement)
+            .await?;
+    }
 
     Ok(())
 }
@@ -202,7 +210,7 @@ pub async fn import<P: AsRef<Path>>(
     guid: &str,
     datadir: P,
     repair: RepairStrategy,
-    password: &str,
+    password: Option<&str>,
 ) -> Result<RequiresReboot, Error> {
     let scan = pvscan().await?;
     if scan
@@ -260,27 +268,56 @@ pub async fn mount_fs<P: AsRef<Path>>(
     datadir: P,
     name: &str,
     repair: RepairStrategy,
-    password: &str,
+    password: Option<&str>,
 ) -> Result<RequiresReboot, Error> {
-    tokio::fs::write(PASSWORD_PATH, password)
-        .await
-        .with_ctx(|_| (crate::ErrorKind::Filesystem, PASSWORD_PATH))?;
-    Command::new("cryptsetup")
-        .arg("-q")
-        .arg("luksOpen")
-        .arg(format!("--key-file={}", PASSWORD_PATH))
-        .arg(format!("--keyfile-size={}", password.len()))
-        .arg(Path::new("/dev").join(guid).join(name))
-        .arg(format!("{}_{}", guid, name))
-        .invoke(crate::ErrorKind::DiskManagement)
-        .await?;
-    let mapper_path = Path::new("/dev/mapper").join(format!("{}_{}", guid, name));
-    let reboot = repair.e2fsck(&mapper_path).await?;
-    mount(&mapper_path, datadir.as_ref().join(name), ReadWrite).await?;
+    let orig_path = Path::new("/dev").join(guid).join(name);
+    let mut blockdev_path = orig_path.clone();
+    let full_name = format!("{}_{}", guid, name);
+    if !guid.ends_with("_UNENC") {
+        let password = password.unwrap_or(DEFAULT_PASSWORD);
+        if let Some(parent) = Path::new(PASSWORD_PATH).parent() {
+            tokio::fs::create_dir_all(parent).await?;
+        }
+        tokio::fs::write(PASSWORD_PATH, password)
+            .await
+            .with_ctx(|_| (crate::ErrorKind::Filesystem, PASSWORD_PATH))?;
+        Command::new("cryptsetup")
+            .arg("-q")
+            .arg("luksOpen")
+            .arg(format!("--key-file={}", PASSWORD_PATH))
+            .arg(format!("--keyfile-size={}", password.len()))
+            .arg(&blockdev_path)
+            .arg(&full_name)
+            .invoke(crate::ErrorKind::DiskManagement)
+            .await?;
+        tokio::fs::remove_file(PASSWORD_PATH)
+            .await
+            .with_ctx(|_| (crate::ErrorKind::Filesystem, PASSWORD_PATH))?;
+        blockdev_path = Path::new("/dev/mapper").join(&full_name);
+    }
+    let reboot = repair.fsck(&blockdev_path).await?;
 
-    tokio::fs::remove_file(PASSWORD_PATH)
-        .await
-        .with_ctx(|_| (crate::ErrorKind::Filesystem, PASSWORD_PATH))?;
+    if !guid.ends_with("_UNENC") {
+        // Backup LUKS header if e2fsck succeeded
+        let luks_folder = Path::new("/media/embassy/config/luks");
+        tokio::fs::create_dir_all(luks_folder).await?;
+        let tmp_luks_bak = luks_folder.join(format!(".{full_name}.luks.bak.tmp"));
+        if tokio::fs::metadata(&tmp_luks_bak).await.is_ok() {
+            tokio::fs::remove_file(&tmp_luks_bak).await?;
+        }
+        let luks_bak = luks_folder.join(format!("{full_name}.luks.bak"));
+        Command::new("cryptsetup")
+            .arg("-q")
+            .arg("luksHeaderBackup")
+            .arg("--header-backup-file")
+            .arg(&tmp_luks_bak)
+            .arg(&orig_path)
+            .invoke(crate::ErrorKind::DiskManagement)
+            .await?;
+        tokio::fs::rename(&tmp_luks_bak, &luks_bak).await?;
+    }
+
+    mount(&blockdev_path, datadir.as_ref().join(name), ReadWrite).await?;
 
     Ok(reboot)
 }
@@ -290,7 +327,7 @@ pub async fn mount_all_fs<P: AsRef<Path>>(
     guid: &str,
     datadir: P,
     repair: RepairStrategy,
-    password: &str,
+    password: Option<&str>,
 ) -> Result<RequiresReboot, Error> {
     let mut reboot = RequiresReboot(false);
     reboot |= mount_fs(guid, &datadir, "main", repair, password).await?;

backend/src/disk/mount/filesystem/cifs.rs

@@ -16,6 +16,9 @@ use crate::util::Invoke;
 use crate::Error;
 
 async fn resolve_hostname(hostname: &str) -> Result<IpAddr, Error> {
+    if let Ok(addr) = hostname.parse() {
+        return Ok(addr);
+    }
     #[cfg(feature = "avahi")]
     if hostname.ends_with(".local") {
         return Ok(IpAddr::V4(crate::net::mdns::resolve_mdns(hostname).await?));

backend/src/disk/mount/filesystem/efivarfs.rs

@@ -1,4 +1,3 @@
-use std::os::unix::ffi::OsStrExt;
 use std::path::Path;
 
 use async_trait::async_trait;

backend/src/disk/util.rs

@@ -1,4 +1,4 @@
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, BTreeSet};
 use std::path::{Path, PathBuf};
 
 use color_eyre::eyre::{self, eyre};
@@ -251,7 +251,7 @@ pub async fn recovery_info(
 #[instrument(skip_all)]
 pub async fn list(os: &OsPartitionInfo) -> Result<Vec<DiskInfo>, Error> {
     struct DiskIndex {
-        parts: IndexSet<PathBuf>,
+        parts: BTreeSet<PathBuf>,
         internal: bool,
     }
     let disk_guids = pvscan().await?;
@@ -301,7 +301,7 @@ pub async fn list(os: &OsPartitionInfo) -> Result<Vec<DiskInfo>, Error> {
                     disks.insert(
                         disk.clone(),
                         DiskIndex {
-                            parts: IndexSet::new(),
+                            parts: BTreeSet::new(),
                             internal: false,
                         },
                     );
@@ -324,11 +324,13 @@ pub async fn list(os: &OsPartitionInfo) -> Result<Vec<DiskInfo>, Error> {
         if index.internal {
             for part in index.parts {
                 let mut disk_info = disk_info(disk.clone()).await;
-                disk_info.logicalname = part;
+                let part_info = part_info(part).await;
+                disk_info.logicalname = part_info.logicalname.clone();
+                disk_info.capacity = part_info.capacity;
                 if let Some(g) = disk_guids.get(&disk_info.logicalname) {
                     disk_info.guid = g.clone();
                 } else {
-                    disk_info.partitions = vec![part_info(disk_info.logicalname.clone()).await];
+                    disk_info.partitions = vec![part_info];
                 }
                 res.push(disk_info);
             }

backend/src/hostname.rs

@@ -56,7 +56,8 @@ pub async fn get_current_hostname() -> Result<Hostname, Error> {
 #[instrument(skip_all)]
 pub async fn set_hostname(hostname: &Hostname) -> Result<(), Error> {
     let hostname: &String = &hostname.0;
-    let _out = Command::new("hostnamectl")
+    Command::new("hostnamectl")
+        .arg("--static")
         .arg("set-hostname")
         .arg(hostname)
         .invoke(ErrorKind::ParseSysInfo)

backend/src/init.rs

@@ -1,9 +1,7 @@
-use std::collections::{BTreeMap, HashMap};
+use std::collections::HashMap;
 use std::fs::Permissions;
-use std::net::SocketAddr;
 use std::os::unix::fs::PermissionsExt;
 use std::path::Path;
-use std::process::Stdio;
 use std::time::Duration;
 
 use color_eyre::eyre::eyre;
@@ -16,7 +14,8 @@ use tokio::process::Command;
 
 use crate::account::AccountInfo;
 use crate::context::rpc::RpcContextConfig;
-use crate::db::model::{IpInfo, ServerStatus};
+use crate::db::model::{ServerInfo, ServerStatus};
+use crate::disk::mount::util::unmount;
 use crate::install::PKG_ARCHIVE_DIR;
 use crate::middleware::auth::LOCAL_AUTH_COOKIE_PATH;
 use crate::sound::BEP;
@@ -40,22 +39,19 @@ pub async fn check_time_is_synchronized() -> Result<bool, Error> {
         == "NTPSynchronized=yes")
 }
 
-pub async fn check_tor_is_ready(tor_control: SocketAddr) -> bool {
-    tokio::net::TcpStream::connect(tor_control).await.is_ok()
-}
-
 pub struct InitReceipts {
+    pub server_info: LockReceipt<ServerInfo, ()>,
     pub server_version: LockReceipt<crate::util::Version, ()>,
     pub version_range: LockReceipt<emver::VersionRange, ()>,
-    pub last_wifi_region: LockReceipt<Option<isocountry::CountryCode>, ()>,
-    pub status_info: LockReceipt<ServerStatus, ()>,
-    pub ip_info: LockReceipt<BTreeMap<String, IpInfo>, ()>,
-    pub system_start_time: LockReceipt<String, ()>,
 }
 impl InitReceipts {
     pub async fn new(db: &mut impl DbHandle) -> Result<Self, Error> {
         let mut locks = Vec::new();
 
+        let server_info = crate::db::DatabaseModel::new()
+            .server_info()
+            .make_locker(LockType::Write)
+            .add_to_keys(&mut locks);
         let server_version = crate::db::DatabaseModel::new()
             .server_info()
             .version()
@@ -66,112 +62,29 @@ impl InitReceipts {
             .eos_version_compat()
             .make_locker(LockType::Write)
             .add_to_keys(&mut locks);
-        let last_wifi_region = crate::db::DatabaseModel::new()
-            .server_info()
-            .last_wifi_region()
-            .make_locker(LockType::Write)
-            .add_to_keys(&mut locks);
-        let ip_info = crate::db::DatabaseModel::new()
-            .server_info()
-            .ip_info()
-            .make_locker(LockType::Write)
-            .add_to_keys(&mut locks);
-        let status_info = crate::db::DatabaseModel::new()
-            .server_info()
-            .status_info()
-            .into_model()
-            .make_locker(LockType::Write)
-            .add_to_keys(&mut locks);
-        let system_start_time = crate::db::DatabaseModel::new()
-            .server_info()
-            .system_start_time()
-            .make_locker(LockType::Write)
-            .add_to_keys(&mut locks);
 
         let skeleton_key = db.lock_all(locks).await?;
         Ok(Self {
+            server_info: server_info.verify(&skeleton_key)?,
             server_version: server_version.verify(&skeleton_key)?,
             version_range: version_range.verify(&skeleton_key)?,
-            ip_info: ip_info.verify(&skeleton_key)?,
-            status_info: status_info.verify(&skeleton_key)?,
-            last_wifi_region: last_wifi_region.verify(&skeleton_key)?,
-            system_start_time: system_start_time.verify(&skeleton_key)?,
         })
     }
 }
 
-pub async fn pgloader(
-    old_db_path: impl AsRef<Path>,
-    batch_rows: usize,
-    prefetch_rows: usize,
-) -> Result<(), Error> {
-    tokio::fs::write(
-        "/etc/embassy/migrate.load",
-        format!(
-            include_str!("migrate.load"),
-            sqlite_path = old_db_path.as_ref().display(),
-            batch_rows = batch_rows,
-            prefetch_rows = prefetch_rows
-        ),
-    )
-    .await?;
-    match tokio::fs::remove_dir_all("/tmp/pgloader").await {
-        Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
-        a => a,
-    }?;
-    tracing::info!("Running pgloader");
-    let out = Command::new("pgloader")
-        .arg("-v")
-        .arg("/etc/embassy/migrate.load")
-        .stdout(Stdio::piped())
-        .stderr(Stdio::piped())
-        .output()
-        .await?;
-    let stdout = String::from_utf8(out.stdout)?;
-    for line in stdout.lines() {
-        tracing::debug!("pgloader: {}", line);
-    }
-    let stderr = String::from_utf8(out.stderr)?;
-    for line in stderr.lines() {
-        tracing::debug!("pgloader err: {}", line);
-    }
-    tracing::debug!("pgloader exited with code {:?}", out.status);
-    if let Some(err) = stdout.lines().chain(stderr.lines()).find_map(|l| {
-        if l.split_ascii_whitespace()
-            .any(|word| word == "ERROR" || word == "FATAL")
-        {
-            Some(l)
-        } else {
-            None
-        }
-    }) {
-        return Err(Error::new(
-            eyre!("pgloader error: {}", err),
-            crate::ErrorKind::Database,
-        ));
-    }
-    tokio::fs::rename(
-        old_db_path.as_ref(),
-        old_db_path.as_ref().with_extension("bak"),
-    )
-    .await?;
-    Ok(())
-}
-
 // must be idempotent
 pub async fn init_postgres(datadir: impl AsRef<Path>) -> Result<(), Error> {
     let db_dir = datadir.as_ref().join("main/postgresql");
-    let is_mountpoint = || async {
-        Ok::<_, Error>(
-            tokio::process::Command::new("mountpoint")
-                .arg("/var/lib/postgresql")
-                .stdout(std::process::Stdio::null())
-                .stderr(std::process::Stdio::null())
-                .status()
-                .await?
-                .success(),
-        )
-    };
+    if tokio::process::Command::new("mountpoint")
+        .arg("/var/lib/postgresql")
+        .stdout(std::process::Stdio::null())
+        .stderr(std::process::Stdio::null())
+        .status()
+        .await?
+        .success()
+    {
+        unmount("/var/lib/postgresql").await?;
+    }
     let exists = tokio::fs::metadata(&db_dir).await.is_ok();
     if !exists {
         Command::new("cp")
@@ -181,18 +94,73 @@ pub async fn init_postgres(datadir: impl AsRef<Path>) -> Result<(), Error> {
             .invoke(crate::ErrorKind::Filesystem)
             .await?;
     }
-    if !is_mountpoint().await? {
-        crate::disk::mount::util::bind(&db_dir, "/var/lib/postgresql", false).await?;
-    }
     Command::new("chown")
         .arg("-R")
-        .arg("postgres")
-        .arg("/var/lib/postgresql")
+        .arg("postgres:postgres")
+        .arg(&db_dir)
         .invoke(crate::ErrorKind::Database)
         .await?;
+
+    let mut pg_paths = tokio::fs::read_dir("/usr/lib/postgresql").await?;
+    let mut pg_version = None;
+    while let Some(pg_path) = pg_paths.next_entry().await? {
+        let pg_path_version = pg_path
+            .file_name()
+            .to_str()
+            .map(|v| v.parse())
+            .transpose()?
+            .unwrap_or(0);
+        if pg_path_version > pg_version.unwrap_or(0) {
+            pg_version = Some(pg_path_version)
+        }
+    }
+    let pg_version = pg_version.ok_or_else(|| {
+        Error::new(
+            eyre!("could not determine postgresql version"),
+            crate::ErrorKind::Database,
+        )
+    })?;
+
+    crate::disk::mount::util::bind(&db_dir, "/var/lib/postgresql", false).await?;
+
+    let pg_version_string = pg_version.to_string();
+    let pg_version_path = db_dir.join(&pg_version_string);
+    if tokio::fs::metadata(&pg_version_path).await.is_err() {
+        let conf_dir = Path::new("/etc/postgresql").join(pg_version.to_string());
+        let conf_dir_tmp = {
+            let mut tmp = conf_dir.clone();
+            tmp.set_extension("tmp");
+            tmp
+        };
+        if tokio::fs::metadata(&conf_dir).await.is_ok() {
+            tokio::fs::rename(&conf_dir, &conf_dir_tmp).await?;
+        }
+        let mut old_version = pg_version;
+        while old_version > 13
+        /* oldest pg version included in startos */
+        {
+            old_version -= 1;
+            let old_datadir = db_dir.join(old_version.to_string());
+            if tokio::fs::metadata(&old_datadir).await.is_ok() {
+                Command::new("pg_upgradecluster")
+                    .arg(old_version.to_string())
+                    .arg("main")
+                    .invoke(crate::ErrorKind::Database)
+                    .await?;
+                break;
+            }
+        }
+        if tokio::fs::metadata(&conf_dir).await.is_ok() {
+            if tokio::fs::metadata(&conf_dir).await.is_ok() {
+                tokio::fs::remove_dir_all(&conf_dir).await?;
+            }
+            tokio::fs::rename(&conf_dir_tmp, &conf_dir).await?;
+        }
+    }
+
     Command::new("systemctl")
         .arg("start")
-        .arg("postgresql")
+        .arg(format!("postgresql@{pg_version}-main.service"))
         .invoke(crate::ErrorKind::Database)
         .await?;
     if !exists {
@@ -213,6 +181,7 @@ pub async fn init_postgres(datadir: impl AsRef<Path>) -> Result<(), Error> {
             .invoke(crate::ErrorKind::Database)
             .await?;
     }
+
     Ok(())
 }
 
@@ -255,9 +224,9 @@ pub async fn init(cfg: &RpcContextConfig) -> Result<InitResult, Error> {
     let db = cfg.db(&account).await?;
     tracing::info!("Opened PatchDB");
     let mut handle = db.handle();
-    crate::db::DatabaseModel::new()
+    let mut server_info = crate::db::DatabaseModel::new()
         .server_info()
-        .lock(&mut handle, LockType::Write)
+        .get_mut(&mut handle)
         .await?;
     let receipts = InitReceipts::new(&mut handle).await?;
 
@@ -275,17 +244,15 @@ pub async fn init(cfg: &RpcContextConfig) -> Result<InitResult, Error> {
         crate::net::wifi::synchronize_wpa_supplicant_conf(
             &cfg.datadir().join("main"),
             wifi_interface,
-            &receipts.last_wifi_region.get(&mut handle).await?,
+            &server_info.last_wifi_region,
         )
         .await?;
         tracing::info!("Synchronized WiFi");
     }
 
     let should_rebuild = tokio::fs::metadata(SYSTEM_REBUILD_PATH).await.is_ok()
-        || &*receipts.server_version.get(&mut handle).await? < &emver::Version::new(0, 3, 2, 0)
-        || (*ARCH == "x86_64"
-            && &*receipts.server_version.get(&mut handle).await?
-                < &emver::Version::new(0, 3, 4, 0));
+        || &*server_info.version < &emver::Version::new(0, 3, 2, 0)
+        || (*ARCH == "x86_64" && &*server_info.version < &emver::Version::new(0, 3, 4, 0));
 
     let song = if should_rebuild {
         Some(NonDetachingJoinHandle::from(tokio::spawn(async {
@@ -403,46 +370,19 @@ pub async fn init(cfg: &RpcContextConfig) -> Result<InitResult, Error> {
         tracing::info!("Syncronized system clock");
     }
 
-    Command::new("systemctl")
-        .arg("start")
-        .arg("tor")
-        .invoke(crate::ErrorKind::Tor)
-        .await?;
-
-    let mut warn_tor_not_ready = true;
-    for _ in 0..60 {
-        if check_tor_is_ready(cfg.tor_control.unwrap_or(([127, 0, 0, 1], 9051).into())).await {
-            warn_tor_not_ready = false;
-            break;
-        }
-        tokio::time::sleep(Duration::from_secs(1)).await;
-    }
-    if warn_tor_not_ready {
-        tracing::warn!("Timed out waiting for tor to start");
-    } else {
-        tracing::info!("Tor is started");
+    if server_info.zram {
+        crate::system::enable_zram().await?
     }
+    server_info.ip_info = crate::net::dhcp::init_ips().await?;
+    server_info.status_info = ServerStatus {
+        updated: false,
+        update_progress: None,
+        backup_progress: None,
+    };
 
-    receipts
-        .ip_info
-        .set(&mut handle, crate::net::dhcp::init_ips().await?)
-        .await?;
-    receipts
-        .status_info
-        .set(
-            &mut handle,
-            ServerStatus {
-                updated: false,
-                update_progress: None,
-                backup_progress: None,
-            },
-        )
-        .await?;
+    server_info.system_start_time = time().await?;
 
-    receipts
-        .system_start_time
-        .set(&mut handle, time().await?)
-        .await?;
+    server_info.save(&mut handle).await?;
 
     crate::version::init(&mut handle, &secret_store, &receipts).await?;
 

backend/src/install/cleanup.rs

@@ -82,7 +82,7 @@ pub async fn update_dependency_errors_of_dependents<'a, Db: DbHandle>(
                     .dependency_errors
                     .get(db, dep)
                     .await?
-                    .ok_or_else(not_found)?;
+                    .ok_or_else(|| not_found!(dep))?;
                 errs.0.insert(id.clone(), e);
                 receipts.dependency_errors.set(db, errs, dep).await?
             } else {
@@ -90,7 +90,7 @@ pub async fn update_dependency_errors_of_dependents<'a, Db: DbHandle>(
                     .dependency_errors
                     .get(db, dep)
                     .await?
-                    .ok_or_else(not_found)?;
+                    .ok_or_else(|| not_found!(dep))?;
                 errs.0.remove(id);
                 receipts.dependency_errors.set(db, errs, dep).await?
             }
@@ -215,7 +215,7 @@ pub async fn cleanup_failed<Db: DbHandle>(
         .package_data_entry
         .get(db, id)
         .await?
-        .ok_or_else(not_found)?;
+        .ok_or_else(|| not_found!(id))?;
     if let Some(manifest) = match &pde {
         PackageDataEntry::Installing { manifest, .. }
         | PackageDataEntry::Restoring { manifest, .. } => Some(manifest),

backend/src/install/mod.rs

@@ -21,6 +21,7 @@ use rpc_toolkit::yajrc::RpcError;
 use tokio::fs::{File, OpenOptions};
 use tokio::io::{AsyncRead, AsyncSeek, AsyncSeekExt};
 use tokio::process::Command;
+use tokio::sync::oneshot;
 use tokio_stream::wrappers::ReadDirStream;
 use tracing::instrument;
 
@@ -39,13 +40,14 @@ use crate::dependencies::{
 };
 use crate::install::cleanup::{cleanup, update_dependency_errors_of_dependents};
 use crate::install::progress::{InstallProgress, InstallProgressTracker};
+use crate::marketplace::with_query_params;
 use crate::notifications::NotificationLevel;
 use crate::s9pk::manifest::{Manifest, PackageId};
 use crate::s9pk::reader::S9pkReader;
 use crate::status::{MainStatus, Status};
 use crate::util::io::{copy_and_shutdown, response_to_reader};
 use crate::util::serde::{display_serializable, Port};
-use crate::util::{assure_send, display_none, AsyncFileExt, Version};
+use crate::util::{display_none, AsyncFileExt, Version};
 use crate::version::{Current, VersionT};
 use crate::volume::{asset_dir, script_dir};
 use crate::{Error, ErrorKind, ResultExt};
@@ -135,35 +137,39 @@ pub async fn install(
     let marketplace_url =
         marketplace_url.unwrap_or_else(|| crate::DEFAULT_MARKETPLACE.parse().unwrap());
     let version_priority = version_priority.unwrap_or_default();
-    let man: Manifest = reqwest::get(format!(
-        "{}/package/v0/manifest/{}?spec={}&version-priority={}&eos-version-compat={}&arch={}",
-        marketplace_url,
-        id,
-        version,
-        version_priority,
-        Current::new().compat(),
-        &*crate::ARCH,
-    ))
-    .await
-    .with_kind(crate::ErrorKind::Registry)?
-    .error_for_status()
-    .with_kind(crate::ErrorKind::Registry)?
-    .json()
-    .await
-    .with_kind(crate::ErrorKind::Registry)?;
-    let s9pk = reqwest::get(format!(
-        "{}/package/v0/{}.s9pk?spec=={}&version-priority={}&eos-version-compat={}&arch={}",
-        marketplace_url,
-        id,
-        man.version,
-        version_priority,
-        Current::new().compat(),
-        &*crate::ARCH,
-    ))
-    .await
-    .with_kind(crate::ErrorKind::Registry)?
-    .error_for_status()
-    .with_kind(crate::ErrorKind::Registry)?;
+    let man: Manifest = ctx
+        .client
+        .get(with_query_params(
+            &ctx,
+            format!(
+                "{}/package/v0/manifest/{}?spec={}&version-priority={}",
+                marketplace_url, id, version, version_priority,
+            )
+            .parse()?,
+        ))
+        .send()
+        .await
+        .with_kind(crate::ErrorKind::Registry)?
+        .error_for_status()
+        .with_kind(crate::ErrorKind::Registry)?
+        .json()
+        .await
+        .with_kind(crate::ErrorKind::Registry)?;
+    let s9pk = ctx
+        .client
+        .get(with_query_params(
+            &ctx,
+            format!(
+                "{}/package/v0/{}.s9pk?spec=={}&version-priority={}",
+                marketplace_url, id, man.version, version_priority,
+            )
+            .parse()?,
+        ))
+        .send()
+        .await
+        .with_kind(crate::ErrorKind::Registry)?
+        .error_for_status()
+        .with_kind(crate::ErrorKind::Registry)?;
 
     if man.id.as_str() != id || !man.version.satisfies(&version) {
         return Err(Error::new(
@@ -184,16 +190,18 @@ pub async fn install(
         async {
             tokio::io::copy(
                 &mut response_to_reader(
-                    reqwest::get(format!(
-                        "{}/package/v0/license/{}?spec=={}&eos-version-compat={}&arch={}",
-                        marketplace_url,
-                        id,
-                        man.version,
-                        Current::new().compat(),
-                        &*crate::ARCH,
-                    ))
-                    .await?
-                    .error_for_status()?,
+                    ctx.client
+                        .get(with_query_params(
+                            &ctx,
+                            format!(
+                                "{}/package/v0/license/{}?spec=={}",
+                                marketplace_url, id, man.version,
+                            )
+                            .parse()?,
+                        ))
+                        .send()
+                        .await?
+                        .error_for_status()?,
                 ),
                 &mut File::create(public_dir_path.join("LICENSE.md")).await?,
             )
@@ -203,16 +211,18 @@ pub async fn install(
         async {
             tokio::io::copy(
                 &mut response_to_reader(
-                    reqwest::get(format!(
-                        "{}/package/v0/instructions/{}?spec=={}&eos-version-compat={}&arch={}",
-                        marketplace_url,
-                        id,
-                        man.version,
-                        Current::new().compat(),
-                        &*crate::ARCH,
-                    ))
-                    .await?
-                    .error_for_status()?,
+                    ctx.client
+                        .get(with_query_params(
+                            &ctx,
+                            format!(
+                                "{}/package/v0/instructions/{}?spec=={}",
+                                marketplace_url, id, man.version,
+                            )
+                            .parse()?,
+                        ))
+                        .send()
+                        .await?
+                        .error_for_status()?,
                 ),
                 &mut File::create(public_dir_path.join("INSTRUCTIONS.md")).await?,
             )
@@ -222,16 +232,18 @@ pub async fn install(
         async {
             tokio::io::copy(
                 &mut response_to_reader(
-                    reqwest::get(format!(
-                        "{}/package/v0/icon/{}?spec=={}&eos-version-compat={}&arch={}",
-                        marketplace_url,
-                        id,
-                        man.version,
-                        Current::new().compat(),
-                        &*crate::ARCH,
-                    ))
-                    .await?
-                    .error_for_status()?,
+                    ctx.client
+                        .get(with_query_params(
+                            &ctx,
+                            format!(
+                                "{}/package/v0/icon/{}?spec=={}",
+                                marketplace_url, id, man.version,
+                            )
+                            .parse()?,
+                        ))
+                        .send()
+                        .await?
+                        .error_for_status()?,
                 ),
                 &mut File::create(public_dir_path.join(format!("icon.{}", icon_type))).await?,
             )
@@ -297,6 +309,7 @@ pub async fn install(
             Some(marketplace_url),
             InstallProgress::new(s9pk.content_length()),
             response_to_reader(s9pk),
+            None,
         )
         .await
         {
@@ -425,52 +438,64 @@ pub async fn sideload(
             pde.save(&mut tx).await?;
             tx.commit().await?;
 
-            if let Err(e) = download_install_s9pk(
-                &new_ctx,
-                &manifest,
-                None,
-                progress,
-                tokio_util::io::StreamReader::new(req.into_body().map_err(|e| {
-                    std::io::Error::new(
-                        match &e {
-                            e if e.is_connect() => std::io::ErrorKind::ConnectionRefused,
-                            e if e.is_timeout() => std::io::ErrorKind::TimedOut,
-                            _ => std::io::ErrorKind::Other,
-                        },
-                        e,
-                    )
-                })),
-            )
-            .await
-            {
-                let err_str = format!(
-                    "Install of {}@{} Failed: {}",
-                    manifest.id, manifest.version, e
-                );
-                tracing::error!("{}", err_str);
-                tracing::debug!("{:?}", e);
-                if let Err(e) = new_ctx
-                    .notification_manager
-                    .notify(
-                        &mut hdl,
-                        Some(manifest.id),
-                        NotificationLevel::Error,
-                        String::from("Install Failed"),
-                        err_str,
-                        (),
-                        None,
-                    )
-                    .await
-                {
-                    tracing::error!("Failed to issue Notification: {}", e);
-                    tracing::debug!("{:?}", e);
-                }
-            }
+            let (send, recv) = oneshot::channel();
 
-            Response::builder()
-                .status(StatusCode::OK)
-                .body(Body::empty())
-                .with_kind(ErrorKind::Network)
+            tokio::spawn(async move {
+                if let Err(e) = download_install_s9pk(
+                    &new_ctx,
+                    &manifest,
+                    None,
+                    progress,
+                    tokio_util::io::StreamReader::new(req.into_body().map_err(|e| {
+                        std::io::Error::new(
+                            match &e {
+                                e if e.is_connect() => std::io::ErrorKind::ConnectionRefused,
+                                e if e.is_timeout() => std::io::ErrorKind::TimedOut,
+                                _ => std::io::ErrorKind::Other,
+                            },
+                            e,
+                        )
+                    })),
+                    Some(send),
+                )
+                .await
+                {
+                    let err_str = format!(
+                        "Install of {}@{} Failed: {}",
+                        manifest.id, manifest.version, e
+                    );
+                    tracing::error!("{}", err_str);
+                    tracing::debug!("{:?}", e);
+                    if let Err(e) = new_ctx
+                        .notification_manager
+                        .notify(
+                            &mut hdl,
+                            Some(manifest.id),
+                            NotificationLevel::Error,
+                            String::from("Install Failed"),
+                            err_str,
+                            (),
+                            None,
+                        )
+                        .await
+                    {
+                        tracing::error!("Failed to issue Notification: {}", e);
+                        tracing::debug!("{:?}", e);
+                    }
+                }
+            });
+
+            if let Ok(_) = recv.await {
+                Response::builder()
+                    .status(StatusCode::OK)
+                    .body(Body::empty())
+                    .with_kind(ErrorKind::Network)
+            } else {
+                Response::builder()
+                    .status(StatusCode::INTERNAL_SERVER_ERROR)
+                    .body(Body::from("installation aborted before upload completed"))
+                    .with_kind(ErrorKind::Network)
+            }
         }
         .boxed()
     });
@@ -707,6 +732,7 @@ pub async fn download_install_s9pk(
     marketplace_url: Option<Url>,
     progress: Arc<InstallProgress>,
     mut s9pk: impl AsyncRead + Unpin,
+    download_complete: Option<oneshot::Sender<()>>,
 ) -> Result<(), Error> {
     let pkg_id = &temp_manifest.id;
     let version = &temp_manifest.version;
@@ -799,6 +825,9 @@ pub async fn download_install_s9pk(
                 let mut progress_writer = InstallProgressTracker::new(&mut dst, progress.clone());
                 tokio::io::copy(&mut s9pk, &mut progress_writer).await?;
                 progress.download_complete();
+                if let Some(complete) = download_complete {
+                    complete.send(()).unwrap_or_default();
+                }
                 Ok(())
             })
             .await?;
@@ -910,17 +939,20 @@ pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin + Send + Sync>(
         {
             Some(local_man)
         } else if let Some(marketplace_url) = &marketplace_url {
-            match reqwest::get(format!(
-                "{}/package/v0/manifest/{}?spec={}&eos-version-compat={}&arch={}",
-                marketplace_url,
-                dep,
-                info.version,
-                Current::new().compat(),
-                &*crate::ARCH,
-            ))
-            .await
-            .with_kind(crate::ErrorKind::Registry)?
-            .error_for_status()
+            match ctx
+                .client
+                .get(with_query_params(
+                    ctx,
+                    format!(
+                        "{}/package/v0/manifest/{}?spec={}",
+                        marketplace_url, dep, info.version,
+                    )
+                    .parse()?,
+                ))
+                .send()
+                .await
+                .with_kind(crate::ErrorKind::Registry)?
+                .error_for_status()
             {
                 Ok(a) => Ok(Some(
                     a.json()
@@ -945,16 +977,19 @@ pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin + Send + Sync>(
                 let icon_path = dir.join(format!("icon.{}", manifest.assets.icon_type()));
                 if tokio::fs::metadata(&icon_path).await.is_err() {
                     tokio::fs::create_dir_all(&dir).await?;
-                    let icon = reqwest::get(format!(
-                        "{}/package/v0/icon/{}?spec={}&eos-version-compat={}&arch={}",
-                        marketplace_url,
-                        dep,
-                        info.version,
-                        Current::new().compat(),
-                        &*crate::ARCH,
-                    ))
-                    .await
-                    .with_kind(crate::ErrorKind::Registry)?;
+                    let icon = ctx
+                        .client
+                        .get(with_query_params(
+                            ctx,
+                            format!(
+                                "{}/package/v0/icon/{}?spec={}",
+                                marketplace_url, dep, info.version,
+                            )
+                            .parse()?,
+                        ))
+                        .send()
+                        .await
+                        .with_kind(crate::ErrorKind::Registry)?;
                     let mut dst = File::create(&icon_path).await?;
                     tokio::io::copy(&mut response_to_reader(icon), &mut dst).await?;
                     dst.sync_all().await?;
@@ -1279,6 +1314,14 @@ pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin + Send + Sync>(
             migration.or(prev_migration)
         };
 
+        remove_from_current_dependents_lists(
+            &mut tx,
+            pkg_id,
+            &prev.current_dependencies,
+            &receipts.config.current_dependents,
+        )
+        .await?; // remove previous
+
         let configured = if let Some(f) = viable_migration {
             f.await?.configured && prev_is_configured
         } else {
@@ -1298,13 +1341,6 @@ pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin + Send + Sync>(
             )
             .await?;
         } else {
-            remove_from_current_dependents_lists(
-                &mut tx,
-                pkg_id,
-                &prev.current_dependencies,
-                &receipts.config.current_dependents,
-            )
-            .await?; // remove previous
             add_dependent_to_current_dependents_lists(
                 &mut tx,
                 pkg_id,

backend/src/install/update.rs

@@ -76,7 +76,7 @@ pub async fn dry(
         .current_dependents
         .get(&mut tx, &id)
         .await?
-        .ok_or_else(not_found)?
+        .ok_or_else(|| not_found!(id))?
         .0
         .keys()
         .into_iter()

backend/src/lib.rs

@@ -17,6 +17,7 @@ pub mod account;
 pub mod action;
 pub mod auth;
 pub mod backup;
+pub mod bins;
 pub mod config;
 pub mod context;
 pub mod control;
@@ -85,6 +86,7 @@ pub fn main_api() -> Result<(), RpcError> {
 
 #[command(subcommands(
     system::time,
+    system::experimental,
     system::logs,
     system::kernel_logs,
     system::metrics,

backend/src/logs.rs

@@ -33,7 +33,7 @@ use crate::util::serde::Reversible;
 use crate::{Error, ErrorKind};
 
 #[pin_project::pin_project]
-struct LogStream {
+pub struct LogStream {
     _child: Child,
     #[pin]
     entries: BoxStream<'static, Result<JournalctlEntry, Error>>,
@@ -141,14 +141,14 @@ impl std::fmt::Display for LogEntry {
 }
 
 #[derive(Serialize, Deserialize, Debug)]
-struct JournalctlEntry {
+pub struct JournalctlEntry {
     #[serde(rename = "__REALTIME_TIMESTAMP")]
-    timestamp: String,
+    pub timestamp: String,
     #[serde(rename = "MESSAGE")]
     #[serde(deserialize_with = "deserialize_string_or_utf8_array")]
-    message: String,
+    pub message: String,
     #[serde(rename = "__CURSOR")]
-    cursor: String,
+    pub cursor: String,
 }
 impl JournalctlEntry {
     fn log_entry(self) -> Result<(String, LogEntry), Error> {
@@ -344,7 +344,7 @@ pub async fn cli_logs_generic_follow(
     Ok(())
 }
 
-async fn journalctl(
+pub async fn journalctl(
     id: LogSource,
     limit: usize,
     cursor: Option<&str>,

backend/src/main.rs

@@ -0,0 +1,3 @@
+fn main() {
+    startos::bins::startbox()
+}

backend/src/manager/mod.rs

@@ -21,6 +21,7 @@ use tracing::instrument;
 use crate::context::RpcContext;
 use crate::manager::sync::synchronizer;
 use crate::net::net_controller::NetService;
+use crate::net::vhost::AlpnInfo;
 use crate::procedure::docker::{DockerContainer, DockerProcedure, LongRunning};
 #[cfg(feature = "js_engine")]
 use crate::procedure::js_scripts::JsProcedure;
@@ -573,8 +574,14 @@ async fn add_network_for_main(
     let mut tx = secrets.begin().await?;
     for (id, interface) in &seed.manifest.interfaces.0 {
         for (external, internal) in interface.lan_config.iter().flatten() {
-            svc.add_lan(&mut tx, id.clone(), external.0, internal.internal, false)
-                .await?;
+            svc.add_lan(
+                &mut tx,
+                id.clone(),
+                external.0,
+                internal.internal,
+                Err(AlpnInfo::Specified(vec![])),
+            )
+            .await?;
         }
         for (external, internal) in interface.tor_config.iter().flat_map(|t| &t.port_mapping) {
             svc.add_tor(&mut tx, id.clone(), external.0, internal.0)

backend/src/marketplace.rs

@@ -3,6 +3,8 @@ use reqwest::{StatusCode, Url};
 use rpc_toolkit::command;
 use serde_json::Value;
 
+use crate::context::RpcContext;
+use crate::version::VersionT;
 use crate::{Error, ResultExt};
 
 #[command(subcommands(get))]
@@ -10,9 +12,34 @@ pub fn marketplace() -> Result<(), Error> {
     Ok(())
 }
 
+pub fn with_query_params(ctx: &RpcContext, mut url: Url) -> Url {
+    url.query_pairs_mut()
+        .append_pair(
+            "os.version",
+            &crate::version::Current::new().semver().to_string(),
+        )
+        .append_pair(
+            "os.compat",
+            &crate::version::Current::new().compat().to_string(),
+        )
+        .append_pair("os.arch", crate::OS_ARCH)
+        .append_pair("hardware.arch", &*crate::ARCH)
+        .append_pair("hardware.ram", &ctx.hardware.ram.to_string());
+
+    for hw in &ctx.hardware.devices {
+        url.query_pairs_mut()
+            .append_pair(&format!("hardware.device.{}", hw.class()), hw.product());
+    }
+
+    url
+}
+
 #[command]
-pub async fn get(#[arg] url: Url) -> Result<Value, Error> {
-    let mut response = reqwest::get(url)
+pub async fn get(#[context] ctx: RpcContext, #[arg] url: Url) -> Result<Value, Error> {
+    let mut response = ctx
+        .client
+        .get(with_query_params(&ctx, url))
+        .send()
         .await
         .with_kind(crate::ErrorKind::Network)?;
     let status = response.status();

backend/src/net/net_controller.rs

@@ -15,7 +15,7 @@ use crate::net::keys::Key;
 use crate::net::mdns::MdnsController;
 use crate::net::ssl::{export_cert, export_key, SslManager};
 use crate::net::tor::TorController;
-use crate::net::vhost::VHostController;
+use crate::net::vhost::{AlpnInfo, VHostController};
 use crate::s9pk::manifest::PackageId;
 use crate::volume::cert_dir;
 use crate::{Error, HOST_IP};
@@ -34,6 +34,7 @@ impl NetController {
     #[instrument(skip_all)]
     pub async fn init(
         tor_control: SocketAddr,
+        tor_socks: SocketAddr,
         dns_bind: &[SocketAddr],
         ssl: SslManager,
         hostname: &Hostname,
@@ -41,7 +42,7 @@ impl NetController {
     ) -> Result<Self, Error> {
         let ssl = Arc::new(ssl);
         let mut res = Self {
-            tor: TorController::init(tor_control).await?,
+            tor: TorController::new(tor_control, tor_socks),
             #[cfg(feature = "avahi")]
             mdns: MdnsController::init().await?,
             vhost: VHostController::new(ssl.clone()),
@@ -54,6 +55,8 @@ impl NetController {
     }
 
     async fn add_os_bindings(&mut self, hostname: &Hostname, key: &Key) -> Result<(), Error> {
+        let alpn = Err(AlpnInfo::Specified(vec!["http/1.1".into(), "h2".into()]));
+
         // Internal DNS
         self.vhost
             .add(
@@ -61,7 +64,7 @@ impl NetController {
                 Some("embassy".into()),
                 443,
                 ([127, 0, 0, 1], 80).into(),
-                false,
+                alpn.clone(),
             )
             .await?;
         self.os_bindings
@@ -70,7 +73,13 @@ impl NetController {
         // LAN IP
         self.os_bindings.push(
             self.vhost
-                .add(key.clone(), None, 443, ([127, 0, 0, 1], 80).into(), false)
+                .add(
+                    key.clone(),
+                    None,
+                    443,
+                    ([127, 0, 0, 1], 80).into(),
+                    alpn.clone(),
+                )
                 .await?,
         );
 
@@ -82,7 +91,7 @@ impl NetController {
                     Some("localhost".into()),
                     443,
                     ([127, 0, 0, 1], 80).into(),
-                    false,
+                    alpn.clone(),
                 )
                 .await?,
         );
@@ -93,7 +102,7 @@ impl NetController {
                     Some(hostname.no_dot_host_name()),
                     443,
                     ([127, 0, 0, 1], 80).into(),
-                    false,
+                    alpn.clone(),
                 )
                 .await?,
         );
@@ -106,7 +115,7 @@ impl NetController {
                     Some(hostname.local_domain_name()),
                     443,
                     ([127, 0, 0, 1], 80).into(),
-                    false,
+                    alpn.clone(),
                 )
                 .await?,
         );
@@ -114,7 +123,7 @@ impl NetController {
         // Tor (http)
         self.os_bindings.push(
             self.tor
-                .add(&key.tor_key(), 80, ([127, 0, 0, 1], 80).into())
+                .add(key.tor_key(), 80, ([127, 0, 0, 1], 80).into())
                 .await?,
         );
 
@@ -126,13 +135,13 @@ impl NetController {
                     Some(key.tor_address().to_string()),
                     443,
                     ([127, 0, 0, 1], 80).into(),
-                    false,
+                    alpn.clone(),
                 )
                 .await?,
         );
         self.os_bindings.push(
             self.tor
-                .add(&key.tor_key(), 443, ([127, 0, 0, 1], 443).into())
+                .add(key.tor_key(), 443, ([127, 0, 0, 1], 443).into())
                 .await?,
         );
 
@@ -164,13 +173,13 @@ impl NetController {
         target: SocketAddr,
     ) -> Result<Vec<Arc<()>>, Error> {
         let mut rcs = Vec::with_capacity(1);
-        rcs.push(self.tor.add(&key.tor_key(), external, target).await?);
+        rcs.push(self.tor.add(key.tor_key(), external, target).await?);
         Ok(rcs)
     }
 
     async fn remove_tor(&self, key: &Key, external: u16, rcs: Vec<Arc<()>>) -> Result<(), Error> {
         drop(rcs);
-        self.tor.gc(&key.tor_key(), external).await
+        self.tor.gc(Some(key.tor_key()), Some(external)).await
     }
 
     async fn add_lan(
@@ -178,7 +187,7 @@ impl NetController {
         key: Key,
         external: u16,
         target: SocketAddr,
-        connect_ssl: bool,
+        connect_ssl: Result<(), AlpnInfo>,
     ) -> Result<Vec<Arc<()>>, Error> {
         let mut rcs = Vec::with_capacity(2);
         rcs.push(
@@ -260,7 +269,7 @@ impl NetService {
         id: InterfaceId,
         external: u16,
         internal: u16,
-        connect_ssl: bool,
+        connect_ssl: Result<(), AlpnInfo>,
     ) -> Result<(), Error>
     where
         for<'a> &'a mut Ex: PgExecutor<'a>,

backend/src/net/static_server.rs

@@ -1,18 +1,19 @@
+use std::borrow::Cow;
 use std::fs::Metadata;
-use std::path::Path;
+use std::path::{Path, PathBuf};
 use std::sync::Arc;
 use std::time::UNIX_EPOCH;
 
-use async_compression::tokio::bufread::BrotliEncoder;
 use async_compression::tokio::bufread::GzipEncoder;
 use color_eyre::eyre::eyre;
 use digest::Digest;
 use futures::FutureExt;
 use http::header::ACCEPT_ENCODING;
-use http::header::CONTENT_ENCODING;
 use http::request::Parts as RequestParts;
 use http::response::Builder;
 use hyper::{Body, Method, Request, Response, StatusCode};
+use include_dir::{include_dir, Dir};
+use new_mime_guess::MimeGuess;
 use openssl::hash::MessageDigest;
 use openssl::x509::X509;
 use rpc_toolkit::rpc_handler;
@@ -35,10 +36,9 @@ static NOT_FOUND: &[u8] = b"Not Found";
 static METHOD_NOT_ALLOWED: &[u8] = b"Method Not Allowed";
 static NOT_AUTHORIZED: &[u8] = b"Not Authorized";
 
-pub const MAIN_UI_WWW_DIR: &str = "/var/www/html/main";
-pub const SETUP_UI_WWW_DIR: &str = "/var/www/html/setup";
-pub const DIAG_UI_WWW_DIR: &str = "/var/www/html/diagnostic";
-pub const INSTALL_UI_WWW_DIR: &str = "/var/www/html/install";
+static EMBEDDED_UIS: Dir<'_> = include_dir!("$CARGO_MANIFEST_DIR/../frontend/dist/static");
+
+const PROXY_STRIP_HEADERS: &[&str] = &["cookie", "host", "origin", "referer", "user-agent"];
 
 fn status_fn(_: i32) -> StatusCode {
     StatusCode::OK
@@ -52,6 +52,17 @@ pub enum UiMode {
     Main,
 }
 
+impl UiMode {
+    fn path(&self, path: &str) -> PathBuf {
+        match self {
+            Self::Setup => Path::new("setup-wizard").join(path),
+            Self::Diag => Path::new("diagnostic-ui").join(path),
+            Self::Install => Path::new("install-wizard").join(path),
+            Self::Main => Path::new("ui").join(path),
+        }
+    }
+}
+
 pub async fn setup_ui_file_router(ctx: SetupContext) -> Result<HttpHandler, Error> {
     let handler: HttpHandler = Arc::new(move |req| {
         let ctx = ctx.clone();
@@ -226,65 +237,35 @@ pub async fn main_ui_server_router(ctx: RpcContext) -> Result<HttpHandler, Error
 }
 
 async fn alt_ui(req: Request<Body>, ui_mode: UiMode) -> Result<Response<Body>, Error> {
-    let selected_root_dir = match ui_mode {
-        UiMode::Setup => SETUP_UI_WWW_DIR,
-        UiMode::Diag => DIAG_UI_WWW_DIR,
-        UiMode::Install => INSTALL_UI_WWW_DIR,
-        UiMode::Main => MAIN_UI_WWW_DIR,
-    };
-
     let (request_parts, _body) = req.into_parts();
-    let accept_encoding = request_parts
-        .headers
-        .get_all(ACCEPT_ENCODING)
-        .into_iter()
-        .filter_map(|h| h.to_str().ok())
-        .flat_map(|s| s.split(","))
-        .filter_map(|s| s.split(";").next())
-        .map(|s| s.trim())
-        .collect::<Vec<_>>();
     match &request_parts.method {
         &Method::GET => {
-            let uri_path = request_parts
-                .uri
-                .path()
-                .strip_prefix('/')
-                .unwrap_or(request_parts.uri.path());
+            let uri_path = ui_mode.path(
+                request_parts
+                    .uri
+                    .path()
+                    .strip_prefix('/')
+                    .unwrap_or(request_parts.uri.path()),
+            );
 
-            let full_path = Path::new(selected_root_dir).join(uri_path);
-            file_send(
-                &request_parts,
-                if tokio::fs::metadata(&full_path)
+            let file = EMBEDDED_UIS
+                .get_file(&*uri_path)
+                .or_else(|| EMBEDDED_UIS.get_file(&*ui_mode.path("index.html")));
+
+            if let Some(file) = file {
+                FileData::from_embedded(&request_parts, file)
+                    .into_response(&request_parts)
                     .await
-                    .ok()
-                    .map(|f| f.is_file())
-                    .unwrap_or(false)
-                {
-                    full_path
-                } else {
-                    Path::new(selected_root_dir).join("index.html")
-                },
-                &accept_encoding,
-            )
-            .await
+            } else {
+                Ok(not_found())
+            }
         }
         _ => Ok(method_not_allowed()),
     }
 }
 
 async fn main_embassy_ui(req: Request<Body>, ctx: RpcContext) -> Result<Response<Body>, Error> {
-    let selected_root_dir = MAIN_UI_WWW_DIR;
-
     let (request_parts, _body) = req.into_parts();
-    let accept_encoding = request_parts
-        .headers
-        .get_all(ACCEPT_ENCODING)
-        .into_iter()
-        .filter_map(|h| h.to_str().ok())
-        .flat_map(|s| s.split(","))
-        .filter_map(|s| s.split(";").next())
-        .map(|s| s.trim())
-        .collect::<Vec<_>>();
     match (
         &request_parts.method,
         request_parts
@@ -299,11 +280,12 @@ async fn main_embassy_ui(req: Request<Body>, ctx: RpcContext) -> Result<Response
                 Ok(_) => {
                     let sub_path = Path::new(path);
                     if let Ok(rest) = sub_path.strip_prefix("package-data") {
-                        file_send(
+                        FileData::from_path(
                             &request_parts,
-                            ctx.datadir.join(PKG_PUBLIC_DIR).join(rest),
-                            &accept_encoding,
+                            &ctx.datadir.join(PKG_PUBLIC_DIR).join(rest),
                         )
+                        .await?
+                        .into_response(&request_parts)
                         .await
                     } else if let Ok(rest) = sub_path.strip_prefix("eos") {
                         match rest.to_str() {
@@ -318,6 +300,40 @@ async fn main_embassy_ui(req: Request<Body>, ctx: RpcContext) -> Result<Response
                 Err(e) => un_authorized(e, &format!("public/{path}")),
             }
         }
+        (&Method::GET, Some(("proxy", target))) => {
+            match HasValidSession::from_request_parts(&request_parts, &ctx).await {
+                Ok(_) => {
+                    let target = urlencoding::decode(target)?;
+                    let res = ctx
+                        .client
+                        .get(target.as_ref())
+                        .headers(
+                            request_parts
+                                .headers
+                                .iter()
+                                .filter(|(h, _)| {
+                                    !PROXY_STRIP_HEADERS
+                                        .iter()
+                                        .any(|bad| h.as_str().eq_ignore_ascii_case(bad))
+                                })
+                                .map(|(h, v)| (h.clone(), v.clone()))
+                                .collect(),
+                        )
+                        .send()
+                        .await
+                        .with_kind(crate::ErrorKind::Network)?;
+                    let mut hres = Response::builder().status(res.status());
+                    for (h, v) in res.headers().clone() {
+                        if let Some(h) = h {
+                            hres = hres.header(h, v);
+                        }
+                    }
+                    hres.body(Body::wrap_stream(res.bytes_stream()))
+                        .with_kind(crate::ErrorKind::Network)
+                }
+                Err(e) => un_authorized(e, &format!("proxy/{target}")),
+            }
+        }
         (&Method::GET, Some(("eos", "local.crt"))) => {
             match HasValidSession::from_request_parts(&request_parts, &ctx).await {
                 Ok(_) => cert_send(&ctx.account.read().await.root_ca_cert),
@@ -325,28 +341,25 @@ async fn main_embassy_ui(req: Request<Body>, ctx: RpcContext) -> Result<Response
             }
         }
         (&Method::GET, _) => {
-            let uri_path = request_parts
-                .uri
-                .path()
-                .strip_prefix('/')
-                .unwrap_or(request_parts.uri.path());
+            let uri_path = UiMode::Main.path(
+                request_parts
+                    .uri
+                    .path()
+                    .strip_prefix('/')
+                    .unwrap_or(request_parts.uri.path()),
+            );
 
-            let full_path = Path::new(selected_root_dir).join(uri_path);
-            file_send(
-                &request_parts,
-                if tokio::fs::metadata(&full_path)
+            let file = EMBEDDED_UIS
+                .get_file(&*uri_path)
+                .or_else(|| EMBEDDED_UIS.get_file(&*UiMode::Main.path("index.html")));
+
+            if let Some(file) = file {
+                FileData::from_embedded(&request_parts, file)
+                    .into_response(&request_parts)
                     .await
-                    .ok()
-                    .map(|f| f.is_file())
-                    .unwrap_or(false)
-                {
-                    full_path
-                } else {
-                    Path::new(selected_root_dir).join("index.html")
-                },
-                &accept_encoding,
-            )
-            .await
+            } else {
+                Ok(not_found())
+            }
         }
         _ => Ok(method_not_allowed()),
     }
@@ -409,118 +422,158 @@ fn cert_send(cert: &X509) -> Result<Response<Body>, Error> {
         .with_kind(ErrorKind::Network)
 }
 
-async fn file_send(
-    req: &RequestParts,
-    path: impl AsRef<Path>,
-    accept_encoding: &[&str],
-) -> Result<Response<Body>, Error> {
-    // Serve a file by asynchronously reading it by chunks using tokio-util crate.
+struct FileData {
+    data: Body,
+    len: Option<u64>,
+    encoding: Option<&'static str>,
+    e_tag: String,
+    mime: Option<String>,
+}
+impl FileData {
+    fn from_embedded(req: &RequestParts, file: &'static include_dir::File<'static>) -> Self {
+        let path = file.path();
+        let (encoding, data) = req
+            .headers
+            .get_all(ACCEPT_ENCODING)
+            .into_iter()
+            .filter_map(|h| h.to_str().ok())
+            .flat_map(|s| s.split(","))
+            .filter_map(|s| s.split(";").next())
+            .map(|s| s.trim())
+            .fold((None, file.contents()), |acc, e| {
+                if let Some(file) = (e == "br")
+                    .then_some(())
+                    .and_then(|_| EMBEDDED_UIS.get_file(format!("{}.br", path.display())))
+                {
+                    (Some("br"), file.contents())
+                } else if let Some(file) = (e == "gzip" && acc.0 != Some("br"))
+                    .then_some(())
+                    .and_then(|_| EMBEDDED_UIS.get_file(format!("{}.gz", path.display())))
+                {
+                    (Some("gzip"), file.contents())
+                } else {
+                    acc
+                }
+            });
 
-    let path = path.as_ref();
-
-    let file = File::open(path)
-        .await
-        .with_ctx(|_| (ErrorKind::Filesystem, path.display().to_string()))?;
-    let metadata = file
-        .metadata()
-        .await
-        .with_ctx(|_| (ErrorKind::Filesystem, path.display().to_string()))?;
-
-    let e_tag = e_tag(path, &metadata)?;
-
-    let mut builder = Response::builder();
-    builder = with_content_type(path, builder);
-    builder = builder.header(http::header::ETAG, &e_tag);
-    builder = builder.header(
-        http::header::CACHE_CONTROL,
-        "public, max-age=21000000, immutable",
-    );
-
-    if req
-        .headers
-        .get_all(http::header::CONNECTION)
-        .iter()
-        .flat_map(|s| s.to_str().ok())
-        .flat_map(|s| s.split(","))
-        .any(|s| s.trim() == "keep-alive")
-    {
-        builder = builder.header(http::header::CONNECTION, "keep-alive");
+        Self {
+            len: Some(data.len() as u64),
+            encoding,
+            data: data.into(),
+            e_tag: e_tag(path, None),
+            mime: MimeGuess::from_path(path)
+                .first()
+                .map(|m| m.essence_str().to_owned()),
+        }
     }
 
-    if req
-        .headers
-        .get("if-none-match")
-        .and_then(|h| h.to_str().ok())
-        == Some(e_tag.as_str())
-    {
-        builder = builder.status(StatusCode::NOT_MODIFIED);
-        builder.body(Body::empty())
-    } else {
-        let body = if false && accept_encoding.contains(&"br") && metadata.len() > u16::MAX as u64 {
-            builder = builder.header(CONTENT_ENCODING, "br");
-            Body::wrap_stream(ReaderStream::new(BrotliEncoder::new(BufReader::new(file))))
-        } else if accept_encoding.contains(&"gzip") && metadata.len() > u16::MAX as u64 {
-            builder = builder.header(CONTENT_ENCODING, "gzip");
-            Body::wrap_stream(ReaderStream::new(GzipEncoder::new(BufReader::new(file))))
+    async fn from_path(req: &RequestParts, path: &Path) -> Result<Self, Error> {
+        let encoding = req
+            .headers
+            .get_all(ACCEPT_ENCODING)
+            .into_iter()
+            .filter_map(|h| h.to_str().ok())
+            .flat_map(|s| s.split(","))
+            .filter_map(|s| s.split(";").next())
+            .map(|s| s.trim())
+            .any(|e| e == "gzip")
+            .then_some("gzip");
+
+        let file = File::open(path)
+            .await
+            .with_ctx(|_| (ErrorKind::Filesystem, path.display().to_string()))?;
+        let metadata = file
+            .metadata()
+            .await
+            .with_ctx(|_| (ErrorKind::Filesystem, path.display().to_string()))?;
+
+        let e_tag = e_tag(path, Some(&metadata));
+
+        let (len, data) = if encoding == Some("gzip") {
+            (
+                None,
+                Body::wrap_stream(ReaderStream::new(GzipEncoder::new(BufReader::new(file)))),
+            )
         } else {
-            builder = with_content_length(&metadata, builder);
-            Body::wrap_stream(ReaderStream::new(file))
+            (
+                Some(metadata.len()),
+                Body::wrap_stream(ReaderStream::new(file)),
+            )
         };
-        builder.body(body)
+
+        Ok(Self {
+            data,
+            len,
+            encoding,
+            e_tag,
+            mime: MimeGuess::from_path(path)
+                .first()
+                .map(|m| m.essence_str().to_owned()),
+        })
+    }
+
+    async fn into_response(self, req: &RequestParts) -> Result<Response<Body>, Error> {
+        let mut builder = Response::builder();
+        if let Some(mime) = self.mime {
+            builder = builder.header(http::header::CONTENT_TYPE, &*mime);
+        }
+        builder = builder.header(http::header::ETAG, &*self.e_tag);
+        builder = builder.header(
+            http::header::CACHE_CONTROL,
+            "public, max-age=21000000, immutable",
+        );
+
+        if req
+            .headers
+            .get_all(http::header::CONNECTION)
+            .iter()
+            .flat_map(|s| s.to_str().ok())
+            .flat_map(|s| s.split(","))
+            .any(|s| s.trim() == "keep-alive")
+        {
+            builder = builder.header(http::header::CONNECTION, "keep-alive");
+        }
+
+        if req
+            .headers
+            .get("if-none-match")
+            .and_then(|h| h.to_str().ok())
+            == Some(self.e_tag.as_ref())
+        {
+            builder = builder.status(StatusCode::NOT_MODIFIED);
+            builder.body(Body::empty())
+        } else {
+            if let Some(len) = self.len {
+                builder = builder.header(http::header::CONTENT_LENGTH, len);
+            }
+            if let Some(encoding) = self.encoding {
+                builder = builder.header(http::header::CONTENT_ENCODING, encoding);
+            }
+
+            builder.body(self.data)
+        }
+        .with_kind(ErrorKind::Network)
     }
-    .with_kind(ErrorKind::Network)
 }
 
-fn e_tag(path: &Path, metadata: &Metadata) -> Result<String, Error> {
-    let modified = metadata.modified().with_kind(ErrorKind::Filesystem)?;
+fn e_tag(path: &Path, metadata: Option<&Metadata>) -> String {
     let mut hasher = sha2::Sha256::new();
     hasher.update(format!("{:?}", path).as_bytes());
-    hasher.update(
-        format!(
-            "{}",
-            modified
-                .duration_since(UNIX_EPOCH)
-                .unwrap_or_default()
-                .as_secs()
-        )
-        .as_bytes(),
-    );
+    if let Some(modified) = metadata.and_then(|m| m.modified().ok()) {
+        hasher.update(
+            format!(
+                "{}",
+                modified
+                    .duration_since(UNIX_EPOCH)
+                    .unwrap_or_default()
+                    .as_secs()
+            )
+            .as_bytes(),
+        );
+    }
     let res = hasher.finalize();
-    Ok(format!(
+    format!(
         "\"{}\"",
         base32::encode(base32::Alphabet::RFC4648 { padding: false }, res.as_slice()).to_lowercase()
-    ))
-}
-
-///https://en.wikipedia.org/wiki/Media_type
-fn with_content_type(path: &Path, builder: Builder) -> Builder {
-    let content_type = match path.extension() {
-        Some(os_str) => match os_str.to_str() {
-            Some("apng") => "image/apng",
-            Some("avif") => "image/avif",
-            Some("flif") => "image/flif",
-            Some("gif") => "image/gif",
-            Some("jpg") | Some("jpeg") | Some("jfif") | Some("pjpeg") | Some("pjp") => "image/jpeg",
-            Some("jxl") => "image/jxl",
-            Some("png") => "image/png",
-            Some("svg") => "image/svg+xml",
-            Some("webp") => "image/webp",
-            Some("mng") | Some("x-mng") => "image/x-mng",
-            Some("css") => "text/css",
-            Some("csv") => "text/csv",
-            Some("html") => "text/html",
-            Some("php") => "text/php",
-            Some("plain") | Some("md") | Some("txt") => "text/plain",
-            Some("xml") => "text/xml",
-            Some("js") => "text/javascript",
-            Some("wasm") => "application/wasm",
-            None | Some(_) => "text/plain",
-        },
-        None => "text/plain",
-    };
-    builder.header(http::header::CONTENT_TYPE, content_type)
-}
-
-fn with_content_length(metadata: &Metadata, builder: Builder) -> Builder {
-    builder.header(http::header::CONTENT_LENGTH, metadata.len())
+    )
 }

backend/src/net/tor.rs

@@ -1,32 +1,80 @@
 use std::collections::BTreeMap;
 use std::net::SocketAddr;
+use std::sync::atomic::AtomicBool;
 use std::sync::{Arc, Weak};
+use std::time::Duration;
 
 use clap::ArgMatches;
 use color_eyre::eyre::eyre;
 use futures::future::BoxFuture;
-use futures::FutureExt;
+use futures::{FutureExt, TryStreamExt};
+use helpers::NonDetachingJoinHandle;
+use itertools::Itertools;
+use lazy_static::lazy_static;
+use regex::Regex;
 use rpc_toolkit::command;
+use rpc_toolkit::yajrc::RpcError;
 use tokio::net::TcpStream;
-use tokio::sync::Mutex;
+use tokio::process::Command;
+use tokio::sync::{mpsc, oneshot};
+use tokio::time::Instant;
 use torut::control::{AsyncEvent, AuthenticatedConn, ConnError};
 use torut::onion::{OnionAddressV3, TorSecretKeyV3};
 use tracing::instrument;
 
-use crate::context::RpcContext;
+use crate::context::{CliContext, RpcContext};
+use crate::logs::{
+    cli_logs_generic_follow, cli_logs_generic_nofollow, fetch_logs, follow_logs, journalctl,
+    LogFollowResponse, LogResponse, LogSource,
+};
 use crate::util::serde::{display_serializable, IoFormat};
+use crate::util::{display_none, Invoke};
 use crate::{Error, ErrorKind, ResultExt as _};
 
+pub const SYSTEMD_UNIT: &str = "tor@default";
+const STARTING_HEALTH_TIMEOUT: u64 = 120; // 2min
+
+enum ErrorLogSeverity {
+    Fatal { wipe_state: bool },
+    Unknown { wipe_state: bool },
+}
+
+lazy_static! {
+    static ref LOG_REGEXES: Vec<(Regex, ErrorLogSeverity)> = vec![(
+        Regex::new("This could indicate a route manipulation attack, network overload, bad local network connectivity, or a bug\\.").unwrap(),
+        ErrorLogSeverity::Unknown { wipe_state: true }
+    ),(
+        Regex::new("died due to an invalid selected path").unwrap(),
+        ErrorLogSeverity::Fatal { wipe_state: false }
+    ),(
+        Regex::new("Tor has not observed any network activity for the past").unwrap(),
+        ErrorLogSeverity::Unknown { wipe_state: false }
+    )];
+    static ref PROGRESS_REGEX: Regex = Regex::new("PROGRESS=([0-9]+)").unwrap();
+}
+
 #[test]
 fn random_key() {
     println!("x'{}'", hex::encode(rand::random::<[u8; 32]>()));
 }
 
-#[command(subcommands(list_services))]
+#[command(subcommands(list_services, logs, reset))]
 pub fn tor() -> Result<(), Error> {
     Ok(())
 }
 
+#[command(display(display_none))]
+pub async fn reset(
+    #[context] ctx: RpcContext,
+    #[arg(rename = "wipe-state", short = 'w', long = "wipe-state")] wipe_state: bool,
+    #[arg] reason: String,
+) -> Result<(), Error> {
+    ctx.net_controller
+        .tor
+        .reset(wipe_state, Error::new(eyre!("{reason}"), ErrorKind::Tor))
+        .await
+}
+
 fn display_services(services: Vec<OnionAddressV3>, matches: &ArgMatches) {
     use prettytable::*;
 
@@ -52,133 +100,227 @@ pub async fn list_services(
     ctx.net_controller.tor.list_services().await
 }
 
+#[command(
+    custom_cli(cli_logs(async, context(CliContext))),
+    subcommands(self(logs_nofollow(async)), logs_follow),
+    display(display_none)
+)]
+pub async fn logs(
+    #[arg(short = 'l', long = "limit")] limit: Option<usize>,
+    #[arg(short = 'c', long = "cursor")] cursor: Option<String>,
+    #[arg(short = 'B', long = "before", default)] before: bool,
+    #[arg(short = 'f', long = "follow", default)] follow: bool,
+) -> Result<(Option<usize>, Option<String>, bool, bool), Error> {
+    Ok((limit, cursor, before, follow))
+}
+pub async fn cli_logs(
+    ctx: CliContext,
+    (limit, cursor, before, follow): (Option<usize>, Option<String>, bool, bool),
+) -> Result<(), RpcError> {
+    if follow {
+        if cursor.is_some() {
+            return Err(RpcError::from(Error::new(
+                eyre!("The argument '--cursor <cursor>' cannot be used with '--follow'"),
+                crate::ErrorKind::InvalidRequest,
+            )));
+        }
+        if before {
+            return Err(RpcError::from(Error::new(
+                eyre!("The argument '--before' cannot be used with '--follow'"),
+                crate::ErrorKind::InvalidRequest,
+            )));
+        }
+        cli_logs_generic_follow(ctx, "net.tor.logs.follow", None, limit).await
+    } else {
+        cli_logs_generic_nofollow(ctx, "net.tor.logs", None, limit, cursor, before).await
+    }
+}
+pub async fn logs_nofollow(
+    _ctx: (),
+    (limit, cursor, before, _): (Option<usize>, Option<String>, bool, bool),
+) -> Result<LogResponse, Error> {
+    fetch_logs(LogSource::Service(SYSTEMD_UNIT), limit, cursor, before).await
+}
+
+#[command(rpc_only, rename = "follow", display(display_none))]
+pub async fn logs_follow(
+    #[context] ctx: RpcContext,
+    #[parent_data] (limit, _, _, _): (Option<usize>, Option<String>, bool, bool),
+) -> Result<LogFollowResponse, Error> {
+    follow_logs(ctx, LogSource::Service(SYSTEMD_UNIT), limit).await
+}
+
 fn event_handler(_event: AsyncEvent<'static>) -> BoxFuture<'static, Result<(), ConnError>> {
     async move { Ok(()) }.boxed()
 }
 
-pub struct TorController(Mutex<TorControllerInner>);
+pub struct TorController(TorControl);
 impl TorController {
-    pub async fn init(tor_control: SocketAddr) -> Result<Self, Error> {
-        Ok(TorController(Mutex::new(
-            TorControllerInner::init(tor_control).await?,
-        )))
+    pub fn new(tor_control: SocketAddr, tor_socks: SocketAddr) -> Self {
+        TorController(TorControl::new(tor_control, tor_socks))
     }
 
     pub async fn add(
         &self,
-        key: &TorSecretKeyV3,
+        key: TorSecretKeyV3,
         external: u16,
         target: SocketAddr,
     ) -> Result<Arc<()>, Error> {
-        self.0.lock().await.add(key, external, target).await
+        let (reply, res) = oneshot::channel();
+        self.0
+            .send
+            .send(TorCommand::AddOnion {
+                key,
+                external,
+                target,
+                reply,
+            })
+            .ok()
+            .ok_or_else(|| Error::new(eyre!("TorControl died"), ErrorKind::Tor))?;
+        res.await
+            .ok()
+            .ok_or_else(|| Error::new(eyre!("TorControl died"), ErrorKind::Tor))
     }
 
-    pub async fn gc(&self, key: &TorSecretKeyV3, external: u16) -> Result<(), Error> {
-        self.0.lock().await.gc(key, external).await
+    pub async fn gc(
+        &self,
+        key: Option<TorSecretKeyV3>,
+        external: Option<u16>,
+    ) -> Result<(), Error> {
+        self.0
+            .send
+            .send(TorCommand::GC { key, external })
+            .ok()
+            .ok_or_else(|| Error::new(eyre!("TorControl died"), ErrorKind::Tor))
+    }
+
+    pub async fn reset(&self, wipe_state: bool, context: Error) -> Result<(), Error> {
+        self.0
+            .send
+            .send(TorCommand::Reset {
+                wipe_state,
+                context,
+            })
+            .ok()
+            .ok_or_else(|| Error::new(eyre!("TorControl died"), ErrorKind::Tor))
     }
 
     pub async fn list_services(&self) -> Result<Vec<OnionAddressV3>, Error> {
-        self.0.lock().await.list_services().await
+        let (reply, res) = oneshot::channel();
+        self.0
+            .send
+            .send(TorCommand::GetInfo {
+                query: "onions/current".into(),
+                reply,
+            })
+            .ok()
+            .ok_or_else(|| Error::new(eyre!("TorControl died"), ErrorKind::Tor))?;
+        res.await
+            .ok()
+            .ok_or_else(|| Error::new(eyre!("TorControl died"), ErrorKind::Tor))??
+            .lines()
+            .map(|l| l.trim())
+            .filter(|l| !l.is_empty())
+            .map(|l| l.parse().with_kind(ErrorKind::Tor))
+            .collect()
     }
 }
 
 type AuthenticatedConnection = AuthenticatedConn<
     TcpStream,
-    fn(AsyncEvent<'static>) -> BoxFuture<'static, Result<(), ConnError>>,
+    Box<dyn Fn(AsyncEvent<'static>) -> BoxFuture<'static, Result<(), ConnError>> + Send + Sync>,
 >;
 
-pub struct TorControllerInner {
-    control_addr: SocketAddr,
-    connection: AuthenticatedConnection,
-    services: BTreeMap<String, BTreeMap<u16, BTreeMap<SocketAddr, Weak<()>>>>,
-}
-impl TorControllerInner {
-    #[instrument(skip_all)]
-    async fn add(
-        &mut self,
-        key: &TorSecretKeyV3,
+enum TorCommand {
+    AddOnion {
+        key: TorSecretKeyV3,
         external: u16,
         target: SocketAddr,
-    ) -> Result<Arc<()>, Error> {
-        let mut rm_res = Ok(());
-        let onion_base = key
-            .public()
-            .get_onion_address()
-            .get_address_without_dot_onion();
-        let mut service = if let Some(service) = self.services.remove(&onion_base) {
-            rm_res = self.connection.del_onion(&onion_base).await;
-            service
-        } else {
-            BTreeMap::new()
-        };
-        let mut binding = service.remove(&external).unwrap_or_default();
-        let rc = if let Some(rc) = Weak::upgrade(&binding.remove(&target).unwrap_or_default()) {
-            rc
-        } else {
-            Arc::new(())
-        };
-        binding.insert(target, Arc::downgrade(&rc));
-        service.insert(external, binding);
-        let bindings = service
-            .iter()
-            .flat_map(|(ext, int)| {
-                int.iter()
-                    .find(|(_, rc)| rc.strong_count() > 0)
-                    .map(|(addr, _)| (*ext, SocketAddr::from(*addr)))
-            })
-            .collect::<Vec<_>>();
-        self.services.insert(onion_base, service);
-        rm_res?;
-        self.connection
-            .add_onion_v3(key, false, false, false, None, &mut bindings.iter())
-            .await?;
-        Ok(rc)
-    }
+        reply: oneshot::Sender<Arc<()>>,
+    },
+    GC {
+        key: Option<TorSecretKeyV3>,
+        external: Option<u16>,
+    },
+    GetInfo {
+        query: String,
+        reply: oneshot::Sender<Result<String, Error>>,
+    },
+    Reset {
+        wipe_state: bool,
+        context: Error,
+    },
+}
 
-    #[instrument(skip_all)]
-    async fn gc(&mut self, key: &TorSecretKeyV3, external: u16) -> Result<(), Error> {
-        let onion_base = key
-            .public()
-            .get_onion_address()
-            .get_address_without_dot_onion();
-        if let Some(mut service) = self.services.remove(&onion_base) {
-            if let Some(mut binding) = service.remove(&external) {
-                binding = binding
-                    .into_iter()
-                    .filter(|(_, rc)| rc.strong_count() > 0)
-                    .collect();
-                if !binding.is_empty() {
-                    service.insert(external, binding);
+#[instrument(skip_all)]
+async fn torctl(
+    tor_control: SocketAddr,
+    tor_socks: SocketAddr,
+    recv: &mut mpsc::UnboundedReceiver<TorCommand>,
+    services: &mut BTreeMap<[u8; 64], BTreeMap<u16, BTreeMap<SocketAddr, Weak<()>>>>,
+    wipe_state: &AtomicBool,
+    health_timeout: &mut Duration,
+) -> Result<(), Error> {
+    let bootstrap = async {
+        if Command::new("systemctl")
+            .arg("is-active")
+            .arg("--quiet")
+            .arg("tor")
+            .invoke(ErrorKind::Tor)
+            .await
+            .is_ok()
+        {
+            Command::new("systemctl")
+                .arg("stop")
+                .arg("tor")
+                .invoke(ErrorKind::Tor)
+                .await?;
+            for _ in 0..30 {
+                if TcpStream::connect(tor_control).await.is_err() {
+                    break;
                 }
+                tokio::time::sleep(Duration::from_secs(1)).await;
             }
-            let rm_res = self.connection.del_onion(&onion_base).await;
-            if !service.is_empty() {
-                let bindings = service
-                    .iter()
-                    .flat_map(|(ext, int)| {
-                        int.iter()
-                            .find(|(_, rc)| rc.strong_count() > 0)
-                            .map(|(addr, _)| (*ext, SocketAddr::from(*addr)))
-                    })
-                    .collect::<Vec<_>>();
-                self.services.insert(onion_base, service);
-                rm_res?;
-                self.connection
-                    .add_onion_v3(&key, false, false, false, None, &mut bindings.iter())
-                    .await?;
-            } else {
-                rm_res?;
+            if TcpStream::connect(tor_control).await.is_ok() {
+                return Err(Error::new(
+                    eyre!("Tor is failing to shut down"),
+                    ErrorKind::Tor,
+                ));
             }
         }
+        if wipe_state.load(std::sync::atomic::Ordering::SeqCst) {
+            tokio::fs::remove_dir_all("/var/lib/tor").await?;
+            wipe_state.store(false, std::sync::atomic::Ordering::SeqCst);
+        }
+        tokio::fs::create_dir_all("/var/lib/tor").await?;
+        Command::new("chown")
+            .arg("-R")
+            .arg("debian-tor")
+            .arg("/var/lib/tor")
+            .invoke(ErrorKind::Filesystem)
+            .await?;
+        Command::new("systemctl")
+            .arg("start")
+            .arg("tor")
+            .invoke(ErrorKind::Tor)
+            .await?;
 
-        Ok(())
-    }
+        let logs = journalctl(LogSource::Service(SYSTEMD_UNIT), 0, None, false, true).await?;
 
-    #[instrument(skip_all)]
-    async fn init(tor_control: SocketAddr) -> Result<Self, Error> {
-        let mut conn = torut::control::UnauthenticatedConn::new(
-            TcpStream::connect(tor_control).await?, // TODO
-        );
+        let mut tcp_stream = None;
+        for _ in 0..60 {
+            if let Ok(conn) = TcpStream::connect(tor_control).await {
+                tcp_stream = Some(conn);
+                break;
+            }
+            tokio::time::sleep(Duration::from_secs(1)).await;
+        }
+        let tcp_stream = tcp_stream.ok_or_else(|| {
+            Error::new(eyre!("Timed out waiting for tor to start"), ErrorKind::Tor)
+        })?;
+        tracing::info!("Tor is started");
+
+        let mut conn = torut::control::UnauthenticatedConn::new(tcp_stream);
         let auth = conn
             .load_protocol_info()
             .await?
@@ -187,25 +329,356 @@ impl TorControllerInner {
             .with_kind(crate::ErrorKind::Tor)?;
         conn.authenticate(&auth).await?;
         let mut connection: AuthenticatedConnection = conn.into_authenticated().await;
-        connection.set_async_event_handler(Some(event_handler));
+        connection.set_async_event_handler(Some(Box::new(|event| event_handler(event))));
 
-        Ok(Self {
-            control_addr: tor_control,
-            connection,
-            services: BTreeMap::new(),
-        })
+        let mut bootstrapped = false;
+        let mut last_increment = (String::new(), Instant::now());
+        for _ in 0..300 {
+            match connection.get_info("status/bootstrap-phase").await {
+                Ok(a) => {
+                    if a.contains("TAG=done") {
+                        bootstrapped = true;
+                        break;
+                    }
+                    if let Some(p) = PROGRESS_REGEX.captures(&a) {
+                        if let Some(p) = p.get(1) {
+                            if p.as_str() != &*last_increment.0 {
+                                last_increment = (p.as_str().into(), Instant::now());
+                            }
+                        }
+                    }
+                }
+                Err(e) => {
+                    let e = Error::from(e);
+                    tracing::error!("{}", e);
+                    tracing::debug!("{:?}", e);
+                }
+            }
+            if last_increment.1.elapsed() > Duration::from_secs(30) {
+                return Err(Error::new(
+                    eyre!("Tor stuck bootstrapping at {}%", last_increment.0),
+                    ErrorKind::Tor,
+                ));
+            }
+            tokio::time::sleep(Duration::from_secs(1)).await;
+        }
+        if !bootstrapped {
+            return Err(Error::new(
+                eyre!("Timed out waiting for tor to bootstrap"),
+                ErrorKind::Tor,
+            ));
+        }
+        Ok((connection, logs))
+    };
+    let pre_handler = async {
+        while let Some(command) = recv.recv().await {
+            match command {
+                TorCommand::AddOnion {
+                    key,
+                    external,
+                    target,
+                    reply,
+                } => {
+                    let mut service = if let Some(service) = services.remove(&key.as_bytes()) {
+                        service
+                    } else {
+                        BTreeMap::new()
+                    };
+                    let mut binding = service.remove(&external).unwrap_or_default();
+                    let rc = if let Some(rc) =
+                        Weak::upgrade(&binding.remove(&target).unwrap_or_default())
+                    {
+                        rc
+                    } else {
+                        Arc::new(())
+                    };
+                    binding.insert(target, Arc::downgrade(&rc));
+                    service.insert(external, binding);
+                    services.insert(key.as_bytes(), service);
+                    reply.send(rc).unwrap_or_default();
+                }
+                TorCommand::GetInfo { reply, .. } => {
+                    reply
+                        .send(Err(Error::new(
+                            eyre!("Tor has not finished bootstrapping..."),
+                            ErrorKind::Tor,
+                        )))
+                        .unwrap_or_default();
+                }
+                TorCommand::GC { .. } => (),
+                TorCommand::Reset {
+                    wipe_state: new_wipe_state,
+                    context,
+                } => {
+                    wipe_state.fetch_or(new_wipe_state, std::sync::atomic::Ordering::SeqCst);
+                    return Err(context);
+                }
+            }
+        }
+        Ok(())
+    };
+
+    let (mut connection, mut logs) = tokio::select! {
+        res = bootstrap => res?,
+        res = pre_handler => return res,
+    };
+
+    let hck_key = TorSecretKeyV3::generate();
+    connection
+        .add_onion_v3(
+            &hck_key,
+            false,
+            false,
+            false,
+            None,
+            &mut [(80, SocketAddr::from(([127, 0, 0, 1], 80)))].iter(),
+        )
+        .await?;
+
+    for (key, service) in std::mem::take(services) {
+        let key = TorSecretKeyV3::from(key);
+        let bindings = service
+            .iter()
+            .flat_map(|(ext, int)| {
+                int.iter()
+                    .find(|(_, rc)| rc.strong_count() > 0)
+                    .map(|(addr, _)| (*ext, SocketAddr::from(*addr)))
+            })
+            .collect::<Vec<_>>();
+        if !bindings.is_empty() {
+            services.insert(key.as_bytes(), service);
+            connection
+                .add_onion_v3(&key, false, false, false, None, &mut bindings.iter())
+                .await?;
+        }
     }
 
-    #[instrument(skip_all)]
-    async fn list_services(&mut self) -> Result<Vec<OnionAddressV3>, Error> {
-        self.connection
-            .get_info("onions/current")
-            .await?
-            .lines()
-            .map(|l| l.trim())
-            .filter(|l| !l.is_empty())
-            .map(|l| l.parse().with_kind(ErrorKind::Tor))
-            .collect()
+    let handler = async {
+        while let Some(command) = recv.recv().await {
+            match command {
+                TorCommand::AddOnion {
+                    key,
+                    external,
+                    target,
+                    reply,
+                } => {
+                    let mut rm_res = Ok(());
+                    let onion_base = key
+                        .public()
+                        .get_onion_address()
+                        .get_address_without_dot_onion();
+                    let mut service = if let Some(service) = services.remove(&key.as_bytes()) {
+                        rm_res = connection.del_onion(&onion_base).await;
+                        service
+                    } else {
+                        BTreeMap::new()
+                    };
+                    let mut binding = service.remove(&external).unwrap_or_default();
+                    let rc = if let Some(rc) =
+                        Weak::upgrade(&binding.remove(&target).unwrap_or_default())
+                    {
+                        rc
+                    } else {
+                        Arc::new(())
+                    };
+                    binding.insert(target, Arc::downgrade(&rc));
+                    service.insert(external, binding);
+                    let bindings = service
+                        .iter()
+                        .flat_map(|(ext, int)| {
+                            int.iter()
+                                .find(|(_, rc)| rc.strong_count() > 0)
+                                .map(|(addr, _)| (*ext, SocketAddr::from(*addr)))
+                        })
+                        .collect::<Vec<_>>();
+                    services.insert(key.as_bytes(), service);
+                    reply.send(rc).unwrap_or_default();
+                    rm_res?;
+                    connection
+                        .add_onion_v3(&key, false, false, false, None, &mut bindings.iter())
+                        .await?;
+                }
+                TorCommand::GC { key, external } => {
+                    for key in if key.is_some() {
+                        itertools::Either::Left(key.into_iter().map(|k| k.as_bytes()))
+                    } else {
+                        itertools::Either::Right(services.keys().cloned().collect_vec().into_iter())
+                    } {
+                        let key = TorSecretKeyV3::from(key);
+                        let onion_base = key
+                            .public()
+                            .get_onion_address()
+                            .get_address_without_dot_onion();
+                        if let Some(mut service) = services.remove(&key.as_bytes()) {
+                            for external in if external.is_some() {
+                                itertools::Either::Left(external.into_iter())
+                            } else {
+                                itertools::Either::Right(
+                                    service.keys().copied().collect_vec().into_iter(),
+                                )
+                            } {
+                                if let Some(mut binding) = service.remove(&external) {
+                                    binding = binding
+                                        .into_iter()
+                                        .filter(|(_, rc)| rc.strong_count() > 0)
+                                        .collect();
+                                    if !binding.is_empty() {
+                                        service.insert(external, binding);
+                                    }
+                                }
+                            }
+                            let rm_res = connection.del_onion(&onion_base).await;
+                            if !service.is_empty() {
+                                let bindings = service
+                                    .iter()
+                                    .flat_map(|(ext, int)| {
+                                        int.iter()
+                                            .find(|(_, rc)| rc.strong_count() > 0)
+                                            .map(|(addr, _)| (*ext, SocketAddr::from(*addr)))
+                                    })
+                                    .collect::<Vec<_>>();
+                                if !bindings.is_empty() {
+                                    services.insert(key.as_bytes(), service);
+                                }
+                                rm_res?;
+                                if !bindings.is_empty() {
+                                    connection
+                                        .add_onion_v3(
+                                            &key,
+                                            false,
+                                            false,
+                                            false,
+                                            None,
+                                            &mut bindings.iter(),
+                                        )
+                                        .await?;
+                                }
+                            } else {
+                                rm_res?;
+                            }
+                        }
+                    }
+                }
+                TorCommand::GetInfo { query, reply } => {
+                    reply
+                        .send(connection.get_info(&query).await.with_kind(ErrorKind::Tor))
+                        .unwrap_or_default();
+                }
+                TorCommand::Reset {
+                    wipe_state: new_wipe_state,
+                    context,
+                } => {
+                    wipe_state.fetch_or(new_wipe_state, std::sync::atomic::Ordering::SeqCst);
+                    return Err(context);
+                }
+            }
+        }
+        Ok(())
+    };
+    let log_parser = async {
+        while let Some(log) = logs.try_next().await? {
+            for (regex, severity) in &*LOG_REGEXES {
+                if regex.is_match(&log.message) {
+                    let (check, wipe_state) = match severity {
+                        ErrorLogSeverity::Fatal { wipe_state } => (false, *wipe_state),
+                        ErrorLogSeverity::Unknown { wipe_state } => (true, *wipe_state),
+                    };
+                    if !check
+                        || tokio::time::timeout(
+                            Duration::from_secs(30),
+                            tokio_socks::tcp::Socks5Stream::connect(
+                                tor_socks,
+                                (hck_key.public().get_onion_address().to_string(), 80),
+                            ),
+                        )
+                        .await
+                        .map_err(|e| tracing::warn!("Tor is confirmed to be down: {e}"))
+                        .and_then(|a| {
+                            a.map_err(|e| tracing::warn!("Tor is confirmed to be down: {e}"))
+                        })
+                        .is_err()
+                    {
+                        if wipe_state {
+                            Command::new("systemctl")
+                                .arg("stop")
+                                .arg("tor")
+                                .invoke(ErrorKind::Tor)
+                                .await?;
+                            tokio::fs::remove_dir_all("/var/lib/tor").await?;
+                        }
+                        return Err(Error::new(eyre!("{}", log.message), ErrorKind::Tor));
+                    }
+                }
+            }
+        }
+        Err(Error::new(eyre!("Log stream terminated"), ErrorKind::Tor))
+    };
+    let health_checker = async {
+        let mut last_success = Instant::now();
+        loop {
+            tokio::time::sleep(Duration::from_secs(30)).await;
+            if let Err(e) = tokio::time::timeout(
+                Duration::from_secs(30),
+                tokio_socks::tcp::Socks5Stream::connect(
+                    tor_socks,
+                    (hck_key.public().get_onion_address().to_string(), 80),
+                ),
+            )
+            .await
+            .map_err(|e| e.to_string())
+            .and_then(|e| e.map_err(|e| e.to_string()))
+            {
+                if last_success.elapsed() > *health_timeout {
+                    let err = Error::new(eyre!("Tor health check failed for longer than current timeout ({health_timeout:?})"), crate::ErrorKind::Tor);
+                    *health_timeout *= 2;
+                    wipe_state.store(true, std::sync::atomic::Ordering::SeqCst);
+                    return Err(err);
+                }
+            } else {
+                last_success = Instant::now();
+            }
+        }
+    };
+
+    tokio::select! {
+        res = handler => res?,
+        res = log_parser => res?,
+        res = health_checker => res?,
+    }
+
+    Ok(())
+}
+
+struct TorControl {
+    _thread: NonDetachingJoinHandle<()>,
+    send: mpsc::UnboundedSender<TorCommand>,
+}
+impl TorControl {
+    pub fn new(tor_control: SocketAddr, tor_socks: SocketAddr) -> Self {
+        let (send, mut recv) = mpsc::unbounded_channel();
+        Self {
+            _thread: tokio::spawn(async move {
+                let mut services = BTreeMap::new();
+                let wipe_state = AtomicBool::new(false);
+                let mut health_timeout = Duration::from_secs(STARTING_HEALTH_TIMEOUT);
+                while let Err(e) = torctl(
+                    tor_control,
+                    tor_socks,
+                    &mut recv,
+                    &mut services,
+                    &wipe_state,
+                    &mut health_timeout,
+                )
+                .await
+                {
+                    tracing::error!("{e}: Restarting tor");
+                    tracing::debug!("{e:?}");
+                }
+                tracing::info!("TorControl is shut down.")
+            })
+            .into(),
+            send,
+        }
     }
 }
 

backend/src/net/utils.rs

@@ -1,5 +1,5 @@
 use std::convert::Infallible;
-use std::net::{Ipv4Addr, Ipv6Addr};
+use std::net::{Ipv4Addr, Ipv6Addr, SocketAddr};
 use std::path::Path;
 
 use async_stream::try_stream;
@@ -7,24 +7,29 @@ use color_eyre::eyre::eyre;
 use futures::stream::BoxStream;
 use futures::{StreamExt, TryStreamExt};
 use ipnet::{Ipv4Net, Ipv6Net};
+use tokio::net::{TcpListener, TcpStream};
 use tokio::process::Command;
 
 use crate::util::Invoke;
 use crate::Error;
 
-fn parse_iface_ip(output: &str) -> Result<Option<&str>, Error> {
+fn parse_iface_ip(output: &str) -> Result<Vec<&str>, Error> {
     let output = output.trim();
     if output.is_empty() {
-        return Ok(None);
+        return Ok(Vec::new());
     }
-    if let Some(ip) = output.split_ascii_whitespace().nth(3) {
-        Ok(Some(ip))
-    } else {
-        Err(Error::new(
-            eyre!("malformed output from `ip`"),
-            crate::ErrorKind::Network,
-        ))
+    let mut res = Vec::new();
+    for line in output.lines() {
+        if let Some(ip) = line.split_ascii_whitespace().nth(3) {
+            res.push(ip)
+        } else {
+            return Err(Error::new(
+                eyre!("malformed output from `ip`"),
+                crate::ErrorKind::Network,
+            ));
+        }
     }
+    Ok(res)
 }
 
 pub async fn get_iface_ipv4_addr(iface: &str) -> Result<Option<(Ipv4Addr, Ipv4Net)>, Error> {
@@ -38,7 +43,9 @@ pub async fn get_iface_ipv4_addr(iface: &str) -> Result<Option<(Ipv4Addr, Ipv4Ne
             .invoke(crate::ErrorKind::Network)
             .await?,
     )?)?
+    .into_iter()
     .map(|s| Ok::<_, Error>((s.split("/").next().unwrap().parse()?, s.parse()?)))
+    .next()
     .transpose()?)
 }
 
@@ -53,6 +60,8 @@ pub async fn get_iface_ipv6_addr(iface: &str) -> Result<Option<(Ipv6Addr, Ipv6Ne
             .invoke(crate::ErrorKind::Network)
             .await?,
     )?)?
+    .into_iter()
+    .find(|ip| !ip.starts_with("fe80::"))
     .map(|s| Ok::<_, Error>((s.split("/").next().unwrap().parse()?, s.parse()?)))
     .transpose()?)
 }
@@ -121,3 +130,37 @@ impl<T> hyper::server::accept::Accept for SingleAccept<T> {
         std::task::Poll::Ready(self.project().0.take().map(Ok))
     }
 }
+
+pub struct TcpListeners {
+    listeners: Vec<TcpListener>,
+}
+impl TcpListeners {
+    pub fn new(listeners: impl IntoIterator<Item = TcpListener>) -> Self {
+        Self {
+            listeners: listeners.into_iter().collect(),
+        }
+    }
+
+    pub async fn accept(&self) -> std::io::Result<(TcpStream, SocketAddr)> {
+        futures::future::select_all(self.listeners.iter().map(|l| Box::pin(l.accept())))
+            .await
+            .0
+    }
+}
+impl hyper::server::accept::Accept for TcpListeners {
+    type Conn = TcpStream;
+    type Error = std::io::Error;
+
+    fn poll_accept(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Option<Result<Self::Conn, Self::Error>>> {
+        for listener in self.listeners.iter() {
+            let poll = listener.poll_accept(cx);
+            if poll.is_ready() {
+                return poll.map(|a| a.map(|a| a.0)).map(Some);
+            }
+        }
+        std::task::Poll::Pending
+    }
+}

backend/src/net/vhost.rs

@@ -1,8 +1,9 @@
 use std::collections::BTreeMap;
 use std::convert::Infallible;
-use std::net::{IpAddr, SocketAddr};
+use std::net::{IpAddr, Ipv6Addr, SocketAddr};
 use std::str::FromStr;
 use std::sync::{Arc, Weak};
+use std::time::Duration;
 
 use color_eyre::eyre::eyre;
 use helpers::NonDetachingJoinHandle;
@@ -19,7 +20,7 @@ use tokio_rustls::{LazyConfigAcceptor, TlsConnector};
 use crate::net::keys::Key;
 use crate::net::ssl::SslManager;
 use crate::net::utils::SingleAccept;
-use crate::util::io::BackTrackingReader;
+use crate::util::io::{BackTrackingReader, TimeoutStream};
 use crate::Error;
 
 // not allowed: <=1024, >=32768, 5355, 5432, 9050, 6010, 9051, 5353
@@ -41,7 +42,7 @@ impl VHostController {
         hostname: Option<String>,
         external: u16,
         target: SocketAddr,
-        connect_ssl: bool,
+        connect_ssl: Result<(), AlpnInfo>,
     ) -> Result<Arc<()>, Error> {
         let mut writable = self.servers.lock().await;
         let server = if let Some(server) = writable.remove(&external) {
@@ -77,10 +78,16 @@ impl VHostController {
 #[derive(Clone, PartialEq, Eq, PartialOrd, Ord)]
 struct TargetInfo {
     addr: SocketAddr,
-    connect_ssl: bool,
+    connect_ssl: Result<(), AlpnInfo>,
     key: Key,
 }
 
+#[derive(Clone, PartialEq, Eq, PartialOrd, Ord)]
+pub enum AlpnInfo {
+    Reflect,
+    Specified(Vec<Vec<u8>>),
+}
+
 struct VHostServer {
     mapping: Weak<RwLock<BTreeMap<Option<String>, BTreeMap<TargetInfo, Weak<()>>>>>,
     _thread: NonDetachingJoinHandle<()>,
@@ -88,7 +95,7 @@ struct VHostServer {
 impl VHostServer {
     async fn new(port: u16, ssl: Arc<SslManager>) -> Result<Self, Error> {
         // check if port allowed
-        let listener = TcpListener::bind(SocketAddr::new([0, 0, 0, 0].into(), port))
+        let listener = TcpListener::bind(SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), port))
             .await
             .with_kind(crate::ErrorKind::Network)?;
         let mapping = Arc::new(RwLock::new(BTreeMap::new()));
@@ -98,6 +105,8 @@ impl VHostServer {
                 loop {
                     match listener.accept().await {
                         Ok((stream, _)) => {
+                            let stream =
+                                Box::pin(TimeoutStream::new(stream, Duration::from_secs(300)));
                             let mut stream = BackTrackingReader::new(stream);
                             stream.start_buffering();
                             let mapping = mapping.clone();
@@ -178,7 +187,7 @@ impl VHostServer {
                                         let cfg = ServerConfig::builder()
                                             .with_safe_defaults()
                                             .with_no_client_auth();
-                                        let cfg =
+                                        let mut cfg =
                                             if mid.client_hello().signature_schemes().contains(
                                                 &tokio_rustls::rustls::SignatureScheme::ED25519,
                                             ) {
@@ -213,49 +222,94 @@ impl VHostServer {
                                                             .private_key_to_der()?,
                                                     ),
                                                 )
-                                            };
-                                        let mut tls_stream = mid
-                                            .into_stream(Arc::new(
-                                                cfg.with_kind(crate::ErrorKind::OpenSsl)?,
-                                            ))
-                                            .await?;
-                                        tls_stream.get_mut().0.stop_buffering();
-                                        if target.connect_ssl {
-                                            tokio::io::copy_bidirectional(
-                                                &mut tls_stream,
-                                                &mut TlsConnector::from(Arc::new(
+                                            }
+                                            .with_kind(crate::ErrorKind::OpenSsl)?;
+                                        match target.connect_ssl {
+                                            Ok(()) => {
+                                                let mut client_cfg =
                                                     tokio_rustls::rustls::ClientConfig::builder()
                                                         .with_safe_defaults()
                                                         .with_root_certificates({
                                                             let mut store = RootCertStore::empty();
                                                             store.add(
-                                                                &tokio_rustls::rustls::Certificate(
-                                                                    key.root_ca().to_der()?,
-                                                                ),
-                                                            ).with_kind(crate::ErrorKind::OpenSsl)?;
+                                                        &tokio_rustls::rustls::Certificate(
+                                                            key.root_ca().to_der()?,
+                                                        ),
+                                                    ).with_kind(crate::ErrorKind::OpenSsl)?;
                                                             store
                                                         })
-                                                        .with_no_client_auth(),
-                                                ))
-                                                .connect(
-                                                    key.key()
-                                                        .internal_address()
-                                                        .as_str()
-                                                        .try_into()
-                                                        .with_kind(crate::ErrorKind::OpenSsl)?,
-                                                    tcp_stream,
+                                                        .with_no_client_auth();
+                                                client_cfg.alpn_protocols = mid
+                                                    .client_hello()
+                                                    .alpn()
+                                                    .into_iter()
+                                                    .flatten()
+                                                    .map(|x| x.to_vec())
+                                                    .collect();
+                                                let mut target_stream =
+                                                    TlsConnector::from(Arc::new(client_cfg))
+                                                        .connect_with(
+                                                            key.key()
+                                                                .internal_address()
+                                                                .as_str()
+                                                                .try_into()
+                                                                .with_kind(
+                                                                    crate::ErrorKind::OpenSsl,
+                                                                )?,
+                                                            tcp_stream,
+                                                            |conn| {
+                                                                cfg.alpn_protocols.extend(
+                                                                    conn.alpn_protocol()
+                                                                        .into_iter()
+                                                                        .map(|p| p.to_vec()),
+                                                                )
+                                                            },
+                                                        )
+                                                        .await
+                                                        .with_kind(crate::ErrorKind::OpenSsl)?;
+                                                let mut tls_stream =
+                                                    mid.into_stream(Arc::new(cfg)).await?;
+                                                tls_stream.get_mut().0.stop_buffering();
+                                                tokio::io::copy_bidirectional(
+                                                    &mut tls_stream,
+                                                    &mut target_stream,
                                                 )
                                                 .await
-                                                .with_kind(crate::ErrorKind::OpenSsl)?,
-                                            )
-                                            .await?;
-                                        } else {
-                                            tokio::io::copy_bidirectional(
-                                                &mut tls_stream,
-                                                &mut tcp_stream,
-                                            )
-                                            .await?;
+                                            }
+                                            Err(AlpnInfo::Reflect) => {
+                                                for proto in
+                                                    mid.client_hello().alpn().into_iter().flatten()
+                                                {
+                                                    cfg.alpn_protocols.push(proto.into());
+                                                }
+                                                let mut tls_stream =
+                                                    mid.into_stream(Arc::new(cfg)).await?;
+                                                tls_stream.get_mut().0.stop_buffering();
+                                                tokio::io::copy_bidirectional(
+                                                    &mut tls_stream,
+                                                    &mut tcp_stream,
+                                                )
+                                                .await
+                                            }
+                                            Err(AlpnInfo::Specified(alpn)) => {
+                                                cfg.alpn_protocols = alpn;
+                                                let mut tls_stream =
+                                                    mid.into_stream(Arc::new(cfg)).await?;
+                                                tls_stream.get_mut().0.stop_buffering();
+                                                tokio::io::copy_bidirectional(
+                                                    &mut tls_stream,
+                                                    &mut tcp_stream,
+                                                )
+                                                .await
+                                            }
                                         }
+                                        .map_or_else(
+                                            |e| match e.kind() {
+                                                std::io::ErrorKind::UnexpectedEof => Ok(()),
+                                                _ => Err(e),
+                                            },
+                                            |_| Ok(()),
+                                        )?;
                                     } else {
                                         // 503
                                     }

backend/src/os_install/mod.rs

@@ -10,7 +10,7 @@ use crate::context::InstallContext;
 use crate::disk::mount::filesystem::bind::Bind;
 use crate::disk::mount::filesystem::block_dev::BlockDev;
 use crate::disk::mount::filesystem::efivarfs::EfiVarFs;
-use crate::disk::mount::filesystem::ReadWrite;
+use crate::disk::mount::filesystem::{MountType, ReadWrite};
 use crate::disk::mount::guard::{MountGuard, TmpMountGuard};
 use crate::disk::util::{DiskInfo, PartitionTable};
 use crate::disk::OsPartitionInfo;
@@ -147,18 +147,66 @@ pub async fn execute(
         .invoke(crate::ErrorKind::DiskManagement)
         .await?;
 
-    Command::new("mkfs.ext4")
+    if !overwrite {
+        if let Ok(guard) =
+            TmpMountGuard::mount(&BlockDev::new(part_info.root.clone()), MountType::ReadWrite).await
+        {
+            if let Err(e) = async {
+                // cp -r ${guard}/config /tmp/config
+                if tokio::fs::metadata(guard.as_ref().join("config/upgrade"))
+                    .await
+                    .is_ok()
+                {
+                    tokio::fs::remove_file(guard.as_ref().join("config/upgrade")).await?;
+                }
+                if tokio::fs::metadata(guard.as_ref().join("config/disk.guid"))
+                    .await
+                    .is_ok()
+                {
+                    tokio::fs::remove_file(guard.as_ref().join("config/disk.guid")).await?;
+                }
+                Command::new("cp")
+                    .arg("-r")
+                    .arg(guard.as_ref().join("config"))
+                    .arg("/tmp/config.bak")
+                    .invoke(crate::ErrorKind::Filesystem)
+                    .await?;
+                Ok::<_, Error>(())
+            }
+            .await
+            {
+                tracing::error!("Error recovering previous config: {e}");
+                tracing::debug!("{e:?}");
+            }
+            guard.unmount().await?;
+        }
+    }
+
+    Command::new("mkfs.btrfs")
+        .arg("-f")
         .arg(&part_info.root)
         .invoke(crate::ErrorKind::DiskManagement)
         .await?;
-    Command::new("e2label")
+    Command::new("btrfs")
+        .arg("property")
+        .arg("set")
         .arg(&part_info.root)
+        .arg("label")
         .arg("rootfs")
         .invoke(crate::ErrorKind::DiskManagement)
         .await?;
 
     let rootfs = TmpMountGuard::mount(&BlockDev::new(&part_info.root), ReadWrite).await?;
-    tokio::fs::create_dir(rootfs.as_ref().join("config")).await?;
+    if tokio::fs::metadata("/tmp/config.bak").await.is_ok() {
+        Command::new("cp")
+            .arg("-r")
+            .arg("/tmp/config.bak")
+            .arg(rootfs.as_ref().join("config"))
+            .invoke(crate::ErrorKind::Filesystem)
+            .await?;
+    } else {
+        tokio::fs::create_dir(rootfs.as_ref().join("config")).await?;
+    }
     tokio::fs::create_dir(rootfs.as_ref().join("next")).await?;
     let current = rootfs.as_ref().join("current");
     tokio::fs::create_dir(&current).await?;
@@ -224,6 +272,12 @@ pub async fn execute(
         .invoke(crate::ErrorKind::OpenSsh)
         .await?;
 
+    let dev = MountGuard::mount(
+        &Bind::new(rootfs.as_ref()),
+        current.join("media/embassy/embassyfs"),
+        MountType::ReadOnly,
+    )
+    .await?;
     let dev = MountGuard::mount(&Bind::new("/dev"), current.join("dev"), ReadWrite).await?;
     let proc = MountGuard::mount(&Bind::new("/proc"), current.join("proc"), ReadWrite).await?;
     let sys = MountGuard::mount(&Bind::new("/sys"), current.join("sys"), ReadWrite).await?;

backend/src/procedure/docker.rs

@@ -2,15 +2,17 @@ use std::borrow::Cow;
 use std::collections::{BTreeMap, BTreeSet, VecDeque};
 use std::ffi::{OsStr, OsString};
 use std::net::Ipv4Addr;
+use std::os::unix::prelude::FileTypeExt;
 use std::path::{Path, PathBuf};
 use std::time::Duration;
 
 use async_stream::stream;
 use bollard::container::RemoveContainerOptions;
+use chrono::format::Item;
 use color_eyre::eyre::eyre;
 use color_eyre::Report;
-use futures::future::Either as EitherFuture;
-use futures::TryStreamExt;
+use futures::future::{BoxFuture, Either as EitherFuture};
+use futures::{FutureExt, TryStreamExt};
 use helpers::{NonDetachingJoinHandle, UnixRpcClient};
 use models::{Id, ImageId};
 use nix::sys::signal;
@@ -18,10 +20,8 @@ use nix::unistd::Pid;
 use serde::de::DeserializeOwned;
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
-use tokio::{
-    io::{AsyncBufRead, AsyncBufReadExt, BufReader},
-    time::timeout,
-};
+use tokio::io::{AsyncBufRead, AsyncBufReadExt, BufReader};
+use tokio::time::timeout;
 use tracing::instrument;
 
 use super::ProcedureName;
@@ -68,6 +68,8 @@ pub struct DockerContainer {
     pub sigterm_timeout: Option<SerdeDuration>,
     #[serde(default)]
     pub system: bool,
+    #[serde(default)]
+    pub gpu_acceleration: bool,
 }
 
 impl DockerContainer {
@@ -154,6 +156,8 @@ pub struct DockerProcedure {
     pub sigterm_timeout: Option<SerdeDuration>,
     #[serde(default)]
     pub shm_size_mb: Option<usize>, // TODO: use postfix sizing? like 1k vs 1m vs 1g
+    #[serde(default)]
+    pub gpu_acceleration: bool,
 }
 
 #[derive(Clone, Debug, Deserialize, Serialize, Default)]
@@ -184,6 +188,7 @@ impl DockerProcedure {
             io_format: injectable.io_format,
             sigterm_timeout: injectable.sigterm_timeout,
             shm_size_mb: container.shm_size_mb,
+            gpu_acceleration: container.gpu_acceleration,
         }
     }
 
@@ -709,6 +714,32 @@ impl DockerProcedure {
             res.push(OsStr::new("--shm-size").into());
             res.push(OsString::from(format!("{}m", shm_size_mb)).into());
         }
+        if self.gpu_acceleration {
+            fn get_devices<'a>(
+                path: &'a Path,
+                res: &'a mut Vec<PathBuf>,
+            ) -> BoxFuture<'a, Result<(), Error>> {
+                async move {
+                    let mut read_dir = tokio::fs::read_dir(path).await?;
+                    while let Some(entry) = read_dir.next_entry().await? {
+                        let fty = entry.metadata().await?.file_type();
+                        if fty.is_block_device() || fty.is_char_device() {
+                            res.push(entry.path());
+                        } else if fty.is_dir() {
+                            get_devices(&*entry.path(), res).await?;
+                        }
+                    }
+                    Ok(())
+                }
+                .boxed()
+            }
+            let mut devices = Vec::new();
+            get_devices(Path::new("/dev/dri"), &mut devices).await?;
+            for device in devices {
+                res.push(OsStr::new("--device").into());
+                res.push(OsString::from(device).into());
+            }
+        }
         res.push(OsStr::new("--interactive").into());
         res.push(OsStr::new("--log-driver=journald").into());
         res.push(OsStr::new("--entrypoint").into());

backend/src/procedure/js_scripts.rs

@@ -690,3 +690,49 @@ async fn js_rsync() {
         .unwrap()
         .unwrap();
 }
+
+#[tokio::test]
+async fn js_disk_usage() {
+    let js_action = JsProcedure { args: vec![] };
+    let path: PathBuf = "test/js_action_execute/"
+        .parse::<PathBuf>()
+        .unwrap()
+        .canonicalize()
+        .unwrap();
+    let package_id = "test-package".parse().unwrap();
+    let package_version: Version = "0.3.0.3".parse().unwrap();
+    let name = ProcedureName::Action("test-disk-usage".parse().unwrap());
+    let volumes: Volumes = serde_json::from_value(serde_json::json!({
+        "main": {
+            "type": "data"
+        },
+        "compat": {
+            "type": "assets"
+        },
+        "filebrowser" :{
+            "package-id": "filebrowser",
+            "path": "data",
+            "readonly": true,
+            "type": "pointer",
+            "volume-id": "main",
+        }
+    }))
+    .unwrap();
+    let input: Option<serde_json::Value> = None;
+    let timeout = Some(Duration::from_secs(10));
+    dbg!(js_action
+        .execute::<serde_json::Value, serde_json::Value>(
+            &path,
+            &package_id,
+            &package_version,
+            name,
+            &volumes,
+            input,
+            timeout,
+            ProcessGroupId(0),
+            None,
+        )
+        .await
+        .unwrap()
+        .unwrap());
+}

backend/src/s9pk/manifest.rs

@@ -1,3 +1,4 @@
+use std::collections::BTreeMap;
 use std::path::{Path, PathBuf};
 
 use color_eyre::eyre::eyre;
@@ -16,6 +17,7 @@ use crate::net::interface::Interfaces;
 use crate::procedure::docker::DockerContainers;
 use crate::procedure::PackageProcedure;
 use crate::status::health_check::HealthChecks;
+use crate::util::serde::Regex;
 use crate::util::Version;
 use crate::version::{Current, VersionT};
 use crate::volume::Volumes;
@@ -79,6 +81,9 @@ pub struct Manifest {
 
     #[serde(default)]
     pub replaces: Vec<String>,
+
+    #[serde(default)]
+    pub hardware_requirements: HardwareRequirements,
 }
 
 impl Manifest {
@@ -109,6 +114,15 @@ impl Manifest {
     }
 }
 
+#[derive(Clone, Debug, Default, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct HardwareRequirements {
+    #[serde(default)]
+    device: BTreeMap<String, Regex>,
+    ram: Option<u64>,
+    arch: Option<Vec<String>>,
+}
+
 #[derive(Clone, Debug, Default, Deserialize, Serialize)]
 #[serde(rename_all = "kebab-case")]
 pub struct Assets {

backend/src/setup.rs

@@ -1,9 +1,9 @@
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 use std::sync::Arc;
+use std::time::Duration;
 
 use color_eyre::eyre::eyre;
 use futures::StreamExt;
-use helpers::{Rsync, RsyncOptions};
 use josekit::jwk::Jwk;
 use openssl::x509::X509;
 use patch_db::DbHandle;
@@ -13,6 +13,7 @@ use serde::{Deserialize, Serialize};
 use sqlx::Connection;
 use tokio::fs::File;
 use tokio::io::AsyncWriteExt;
+use tokio::try_join;
 use torut::onion::OnionAddressV3;
 use tracing::instrument;
 
@@ -32,6 +33,7 @@ use crate::disk::REPAIR_DISK_PATH;
 use crate::hostname::Hostname;
 use crate::init::{init, InitResult};
 use crate::middleware::encrypt::EncryptedWire;
+use crate::util::io::{dir_copy, dir_size, Counter};
 use crate::{Error, ErrorKind, ResultExt};
 
 #[command(subcommands(status, disk, attach, execute, cifs, complete, get_pubkey, exit))]
@@ -123,7 +125,7 @@ pub async fn attach(
                 } else {
                     RepairStrategy::Preen
                 },
-                DEFAULT_PASSWORD,
+                if guid.ends_with("_UNENC") { None } else { Some(DEFAULT_PASSWORD) },
             )
             .await?;
             if tokio::fs::metadata(REPAIR_DISK_PATH).await.is_ok() {
@@ -142,7 +144,7 @@ pub async fn attach(
             }
             let (hostname, tor_addr, root_ca) = setup_init(&ctx, password).await?;
             *ctx.setup_result.write().await = Some((guid, SetupResult {
-                tor_address: format!("http://{}", tor_addr),
+                tor_address: format!("https://{}", tor_addr),
                 lan_address: hostname.lan_address(),
                 root_ca: String::from_utf8(root_ca.to_pem()?)?,
             }));
@@ -279,7 +281,7 @@ pub async fn execute(
                 *ctx.setup_result.write().await = Some((
                     guid,
                     SetupResult {
-                        tor_address: format!("http://{}", tor_addr),
+                        tor_address: format!("https://{}", tor_addr),
                         lan_address: hostname.lan_address(),
                         root_ca: String::from_utf8(
                             root_ca.to_pem().expect("failed to serialize root ca"),
@@ -335,12 +337,17 @@ pub async fn execute_inner(
     recovery_source: Option<RecoverySource>,
     recovery_password: Option<String>,
 ) -> Result<(Arc<String>, Hostname, OnionAddressV3, X509), Error> {
+    let encryption_password = if ctx.disable_encryption {
+        None
+    } else {
+        Some(DEFAULT_PASSWORD)
+    };
     let guid = Arc::new(
         crate::disk::main::create(
             &[embassy_logicalname],
             &pvscan().await?,
             &ctx.datadir,
-            DEFAULT_PASSWORD,
+            encryption_password,
         )
         .await?,
     );
@@ -348,7 +355,7 @@ pub async fn execute_inner(
         &*guid,
         &ctx.datadir,
         RepairStrategy::Preen,
-        DEFAULT_PASSWORD,
+        encryption_password,
     )
     .await?;
 
@@ -416,74 +423,78 @@ async fn migrate(
         &old_guid,
         "/media/embassy/migrate",
         RepairStrategy::Preen,
-        DEFAULT_PASSWORD,
-    )
-    .await?;
-
-    let mut main_transfer = Rsync::new(
-        "/media/embassy/migrate/main/",
-        "/embassy-data/main/",
-        RsyncOptions {
-            delete: true,
-            force: true,
-            ignore_existing: false,
-            exclude: Vec::new(),
-            no_permissions: false,
-            no_owner: false,
+        if guid.ends_with("_UNENC") {
+            None
+        } else {
+            Some(DEFAULT_PASSWORD)
         },
     )
     .await?;
-    let mut package_data_transfer = Rsync::new(
+
+    let main_transfer_args = ("/media/embassy/migrate/main/", "/embassy-data/main/");
+    let package_data_transfer_args = (
         "/media/embassy/migrate/package-data/",
         "/embassy-data/package-data/",
-        RsyncOptions {
-            delete: true,
-            force: true,
-            ignore_existing: false,
-            exclude: vec!["tmp".to_owned()],
-            no_permissions: false,
-            no_owner: false,
-        },
-    )
-    .await?;
+    );
 
-    let mut main_prog = 0.0;
-    let mut main_complete = false;
-    let mut pkg_prog = 0.0;
-    let mut pkg_complete = false;
-    loop {
-        tokio::select! {
-            p = main_transfer.progress.next() => {
-                if let Some(p) = p {
-                    main_prog = p;
-                } else {
-                    main_prog = 1.0;
-                    main_complete = true;
-                }
-            }
-            p = package_data_transfer.progress.next() => {
-                if let Some(p) = p {
-                    pkg_prog = p;
-                } else {
-                    pkg_prog = 1.0;
-                    pkg_complete = true;
-                }
-            }
-        }
-        if main_prog > 0.0 && pkg_prog > 0.0 {
-            *ctx.setup_status.write().await = Some(Ok(SetupStatus {
-                bytes_transferred: ((main_prog * 50.0) + (pkg_prog * 950.0)) as u64,
-                total_bytes: Some(1000),
-                complete: false,
-            }));
-        }
-        if main_complete && pkg_complete {
-            break;
-        }
+    let tmpdir = Path::new(package_data_transfer_args.0).join("tmp");
+    if tokio::fs::metadata(&tmpdir).await.is_ok() {
+        tokio::fs::remove_dir_all(&tmpdir).await?;
     }
 
-    main_transfer.wait().await?;
-    package_data_transfer.wait().await?;
+    let ordering = std::sync::atomic::Ordering::Relaxed;
+
+    let main_transfer_size = Counter::new(0, ordering);
+    let package_data_transfer_size = Counter::new(0, ordering);
+
+    let size = tokio::select! {
+        res = async {
+            let (main_size, package_data_size) = try_join!(
+                dir_size(main_transfer_args.0, Some(&main_transfer_size)),
+                dir_size(package_data_transfer_args.0, Some(&package_data_transfer_size))
+            )?;
+            Ok::<_, Error>(main_size + package_data_size)
+        } => { res? },
+        res = async {
+            loop {
+                tokio::time::sleep(Duration::from_secs(1)).await;
+                *ctx.setup_status.write().await = Some(Ok(SetupStatus {
+                    bytes_transferred: 0,
+                    total_bytes: Some(main_transfer_size.load() + package_data_transfer_size.load()),
+                    complete: false,
+                }));
+            }
+        } => res,
+    };
+
+    *ctx.setup_status.write().await = Some(Ok(SetupStatus {
+        bytes_transferred: 0,
+        total_bytes: Some(size),
+        complete: false,
+    }));
+
+    let main_transfer_progress = Counter::new(0, ordering);
+    let package_data_transfer_progress = Counter::new(0, ordering);
+
+    tokio::select! {
+        res = async {
+            try_join!(
+                dir_copy(main_transfer_args.0, main_transfer_args.1, Some(&main_transfer_progress)),
+                dir_copy(package_data_transfer_args.0, package_data_transfer_args.1, Some(&package_data_transfer_progress))
+            )?;
+            Ok::<_, Error>(())
+        } => { res? },
+        res = async {
+            loop {
+                tokio::time::sleep(Duration::from_secs(1)).await;
+                *ctx.setup_status.write().await = Some(Ok(SetupStatus {
+                    bytes_transferred: main_transfer_progress.load() + package_data_transfer_progress.load(),
+                    total_bytes: Some(size),
+                    complete: false,
+                }));
+            }
+        } => res,
+    }
 
     let (hostname, tor_addr, root_ca) = setup_init(&ctx, Some(embassy_password)).await?;
 

backend/src/system.rs

@@ -22,7 +22,67 @@ use crate::util::serde::{display_serializable, IoFormat};
 use crate::util::{display_none, Invoke};
 use crate::{Error, ErrorKind, ResultExt};
 
-pub const SYSTEMD_UNIT: &'static str = "embassyd";
+pub const SYSTEMD_UNIT: &'static str = "startd";
+
+#[command(subcommands(zram))]
+pub async fn experimental() -> Result<(), Error> {
+    Ok(())
+}
+
+pub async fn enable_zram() -> Result<(), Error> {
+    let mem_info = get_mem_info().await?;
+    Command::new("modprobe")
+        .arg("zram")
+        .invoke(ErrorKind::Zram)
+        .await?;
+    tokio::fs::write("/sys/block/zram0/comp_algorithm", "lz4")
+        .await
+        .with_kind(ErrorKind::Zram)?;
+    tokio::fs::write(
+        "/sys/block/zram0/disksize",
+        format!("{}M", mem_info.total.0 as u64 / 4),
+    )
+    .await
+    .with_kind(ErrorKind::Zram)?;
+    Command::new("mkswap")
+        .arg("/dev/zram0")
+        .invoke(ErrorKind::Zram)
+        .await?;
+    Command::new("swapon")
+        .arg("-p")
+        .arg("5")
+        .arg("/dev/zram0")
+        .invoke(ErrorKind::Zram)
+        .await?;
+    Ok(())
+}
+
+#[command(display(display_none))]
+pub async fn zram(#[context] ctx: RpcContext, #[arg] enable: bool) -> Result<(), Error> {
+    let mut db = ctx.db.handle();
+    let mut zram = crate::db::DatabaseModel::new()
+        .server_info()
+        .zram()
+        .get_mut(&mut db)
+        .await?;
+    if enable == *zram {
+        return Ok(());
+    }
+    *zram = enable;
+    if enable {
+        enable_zram().await?;
+    } else {
+        Command::new("swapoff")
+            .arg("/dev/zram0")
+            .invoke(ErrorKind::Zram)
+            .await?;
+        tokio::fs::write("/sys/block/zram0/reset", "1")
+            .await
+            .with_kind(ErrorKind::Zram)?;
+    }
+    zram.save(&mut db).await?;
+    Ok(())
+}
 
 #[command]
 pub async fn time() -> Result<String, Error> {
@@ -191,7 +251,7 @@ impl<'de> Deserialize<'de> for Percentage {
 }
 
 #[derive(Clone, Debug)]
-pub struct MebiBytes(f64);
+pub struct MebiBytes(pub f64);
 impl Serialize for MebiBytes {
     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
     where
@@ -250,19 +310,19 @@ pub struct MetricsGeneral {
 #[derive(Deserialize, Serialize, Clone, Debug)]
 pub struct MetricsMemory {
     #[serde(rename = "Percentage Used")]
-    percentage_used: Percentage,
+    pub percentage_used: Percentage,
     #[serde(rename = "Total")]
-    total: MebiBytes,
+    pub total: MebiBytes,
     #[serde(rename = "Available")]
-    available: MebiBytes,
+    pub available: MebiBytes,
     #[serde(rename = "Used")]
-    used: MebiBytes,
+    pub used: MebiBytes,
     #[serde(rename = "Swap Total")]
-    swap_total: MebiBytes,
+    pub swap_total: MebiBytes,
     #[serde(rename = "Swap Free")]
-    swap_free: MebiBytes,
+    pub swap_free: MebiBytes,
     #[serde(rename = "Swap Used")]
-    swap_used: MebiBytes,
+    pub swap_used: MebiBytes,
 }
 #[derive(Deserialize, Serialize, Clone, Debug)]
 pub struct MetricsCpu {
@@ -638,7 +698,7 @@ pub struct MemInfo {
     swap_free: Option<u64>,
 }
 #[instrument(skip_all)]
-async fn get_mem_info() -> Result<MetricsMemory, Error> {
+pub async fn get_mem_info() -> Result<MetricsMemory, Error> {
     let contents = tokio::fs::read_to_string("/proc/meminfo").await?;
     let mut mem_info = MemInfo {
         mem_total: None,
@@ -699,7 +759,7 @@ async fn get_mem_info() -> Result<MetricsMemory, Error> {
     let swap_total = MebiBytes(swap_total_k as f64 / 1024.0);
     let swap_free = MebiBytes(swap_free_k as f64 / 1024.0);
     let swap_used = MebiBytes((swap_total_k - swap_free_k) as f64 / 1024.0);
-    let percentage_used = Percentage(used.0 / total.0 * 100.0);
+    let percentage_used = Percentage((total.0 - available.0) / total.0 * 100.0);
     Ok(MetricsMemory {
         percentage_used,
         total,

backend/src/update/mod.rs

@@ -19,6 +19,7 @@ use crate::db::model::UpdateProgress;
 use crate::disk::mount::filesystem::bind::Bind;
 use crate::disk::mount::filesystem::ReadWrite;
 use crate::disk::mount::guard::MountGuard;
+use crate::marketplace::with_query_params;
 use crate::notifications::NotificationLevel;
 use crate::sound::{
     CIRCLE_OF_5THS_SHORT, UPDATE_FAILED_1, UPDATE_FAILED_2, UPDATE_FAILED_3, UPDATE_FAILED_4,
@@ -81,18 +82,19 @@ async fn maybe_do_update(
     marketplace_url: Url,
 ) -> Result<Option<Arc<Revision>>, Error> {
     let mut db = ctx.db.handle();
-    let latest_version: Version = reqwest::get(format!(
-        "{}/eos/v0/latest?eos-version={}&arch={}",
-        marketplace_url,
-        Current::new().semver(),
-        OS_ARCH,
-    ))
-    .await
-    .with_kind(ErrorKind::Network)?
-    .json::<LatestInformation>()
-    .await
-    .with_kind(ErrorKind::Network)?
-    .version;
+    let latest_version: Version = ctx
+        .client
+        .get(with_query_params(
+            &ctx,
+            format!("{}/eos/v0/latest", marketplace_url,).parse()?,
+        ))
+        .send()
+        .await
+        .with_kind(ErrorKind::Network)?
+        .json::<LatestInformation>()
+        .await
+        .with_kind(ErrorKind::Network)?
+        .version;
     crate::db::DatabaseModel::new()
         .server_info()
         .lock(&mut db, LockType::Write)

backend/src/util/io.rs

@@ -1,14 +1,20 @@
 use std::future::Future;
 use std::io::Cursor;
+use std::os::unix::prelude::MetadataExt;
 use std::path::Path;
+use std::sync::atomic::AtomicU64;
 use std::task::Poll;
+use std::time::Duration;
 
 use futures::future::{BoxFuture, Fuse};
 use futures::{AsyncSeek, FutureExt, TryStreamExt};
 use helpers::NonDetachingJoinHandle;
+use nix::unistd::{Gid, Uid};
 use tokio::io::{
     duplex, AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, DuplexStream, ReadBuf, WriteHalf,
 };
+use tokio::net::TcpStream;
+use tokio::time::{Instant, Sleep};
 
 use crate::ResultExt;
 
@@ -222,6 +228,7 @@ pub async fn copy_and_shutdown<R: AsyncRead + Unpin, W: AsyncWrite + Unpin>(
 
 pub fn dir_size<'a, P: AsRef<Path> + 'a + Send + Sync>(
     path: P,
+    ctr: Option<&'a Counter>,
 ) -> BoxFuture<'a, Result<u64, std::io::Error>> {
     async move {
         tokio_stream::wrappers::ReadDirStream::new(tokio::fs::read_dir(path.as_ref()).await?)
@@ -229,9 +236,12 @@ pub fn dir_size<'a, P: AsRef<Path> + 'a + Send + Sync>(
                 let m = e.metadata().await?;
                 Ok(acc
                     + if m.is_file() {
+                        if let Some(ctr) = ctr {
+                            ctr.add(m.len());
+                        }
                         m.len()
                     } else if m.is_dir() {
-                        dir_size(e.path()).await?
+                        dir_size(e.path(), ctr).await?
                     } else {
                         0
                     })
@@ -416,3 +426,245 @@ impl<T: AsyncWrite> AsyncWrite for BackTrackingReader<T> {
         self.project().reader.poll_write_vectored(cx, bufs)
     }
 }
+
+pub struct Counter {
+    atomic: AtomicU64,
+    ordering: std::sync::atomic::Ordering,
+}
+impl Counter {
+    pub fn new(init: u64, ordering: std::sync::atomic::Ordering) -> Self {
+        Self {
+            atomic: AtomicU64::new(init),
+            ordering,
+        }
+    }
+    pub fn load(&self) -> u64 {
+        self.atomic.load(self.ordering)
+    }
+    pub fn add(&self, value: u64) {
+        self.atomic.fetch_add(value, self.ordering);
+    }
+}
+
+#[pin_project::pin_project]
+pub struct CountingReader<'a, R> {
+    ctr: &'a Counter,
+    #[pin]
+    rdr: R,
+}
+impl<'a, R> CountingReader<'a, R> {
+    pub fn new(rdr: R, ctr: &'a Counter) -> Self {
+        Self { ctr, rdr }
+    }
+    pub fn into_inner(self) -> R {
+        self.rdr
+    }
+}
+impl<'a, R: AsyncRead> AsyncRead for CountingReader<'a, R> {
+    fn poll_read(
+        self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+        buf: &mut ReadBuf<'_>,
+    ) -> Poll<std::io::Result<()>> {
+        let this = self.project();
+        let start = buf.filled().len();
+        let res = this.rdr.poll_read(cx, buf);
+        let len = buf.filled().len() - start;
+        if len > 0 {
+            this.ctr.add(len as u64);
+        }
+        res
+    }
+}
+
+pub fn dir_copy<'a, P0: AsRef<Path> + 'a + Send + Sync, P1: AsRef<Path> + 'a + Send + Sync>(
+    src: P0,
+    dst: P1,
+    ctr: Option<&'a Counter>,
+) -> BoxFuture<'a, Result<(), crate::Error>> {
+    async move {
+        let m = tokio::fs::metadata(&src).await?;
+        let dst_path = dst.as_ref();
+        tokio::fs::create_dir_all(&dst_path).await.with_ctx(|_| {
+            (
+                crate::ErrorKind::Filesystem,
+                format!("mkdir {}", dst_path.display()),
+            )
+        })?;
+        tokio::fs::set_permissions(&dst_path, m.permissions())
+            .await
+            .with_ctx(|_| {
+                (
+                    crate::ErrorKind::Filesystem,
+                    format!("chmod {}", dst_path.display()),
+                )
+            })?;
+        let tmp_dst_path = dst_path.to_owned();
+        tokio::task::spawn_blocking(move || {
+            nix::unistd::chown(
+                &tmp_dst_path,
+                Some(Uid::from_raw(m.uid())),
+                Some(Gid::from_raw(m.gid())),
+            )
+        })
+        .await
+        .with_kind(crate::ErrorKind::Unknown)?
+        .with_ctx(|_| {
+            (
+                crate::ErrorKind::Filesystem,
+                format!("chown {}", dst_path.display()),
+            )
+        })?;
+        tokio_stream::wrappers::ReadDirStream::new(tokio::fs::read_dir(src.as_ref()).await?)
+            .map_err(|e| crate::Error::new(e, crate::ErrorKind::Filesystem))
+            .try_for_each(|e| async move {
+                let m = e.metadata().await?;
+                let src_path = e.path();
+                let dst_path = dst_path.join(e.file_name());
+                if m.is_file() {
+                    let len = m.len();
+                    let mut dst_file = tokio::fs::File::create(&dst_path).await.with_ctx(|_| {
+                        (
+                            crate::ErrorKind::Filesystem,
+                            format!("create {}", dst_path.display()),
+                        )
+                    })?;
+                    let mut rdr = tokio::fs::File::open(&src_path).await.with_ctx(|_| {
+                        (
+                            crate::ErrorKind::Filesystem,
+                            format!("open {}", src_path.display()),
+                        )
+                    })?;
+                    if let Some(ctr) = ctr {
+                        tokio::io::copy(&mut CountingReader::new(rdr, ctr), &mut dst_file).await
+                    } else {
+                        tokio::io::copy(&mut rdr, &mut dst_file).await
+                    }
+                    .with_ctx(|_| {
+                        (
+                            crate::ErrorKind::Filesystem,
+                            format!("cp {} -> {}", src_path.display(), dst_path.display()),
+                        )
+                    })?;
+                    dst_file.flush().await?;
+                    dst_file.shutdown().await?;
+                    dst_file.sync_all().await?;
+                    drop(dst_file);
+                    let tmp_dst_path = dst_path.clone();
+                    tokio::task::spawn_blocking(move || {
+                        nix::unistd::chown(
+                            &tmp_dst_path,
+                            Some(Uid::from_raw(m.uid())),
+                            Some(Gid::from_raw(m.gid())),
+                        )
+                    })
+                    .await
+                    .with_kind(crate::ErrorKind::Unknown)?
+                    .with_ctx(|_| {
+                        (
+                            crate::ErrorKind::Filesystem,
+                            format!("chown {}", dst_path.display()),
+                        )
+                    })?;
+                } else if m.is_dir() {
+                    dir_copy(src_path, dst_path, ctr).await?;
+                } else if m.file_type().is_symlink() {
+                    tokio::fs::symlink(
+                        tokio::fs::read_link(&src_path).await.with_ctx(|_| {
+                            (
+                                crate::ErrorKind::Filesystem,
+                                format!("readlink {}", src_path.display()),
+                            )
+                        })?,
+                        &dst_path,
+                    )
+                    .await
+                    .with_ctx(|_| {
+                        (
+                            crate::ErrorKind::Filesystem,
+                            format!("cp -P {} -> {}", src_path.display(), dst_path.display()),
+                        )
+                    })?;
+                    // Do not set permissions (see https://unix.stackexchange.com/questions/87200/change-permissions-for-a-symbolic-link)
+                }
+                Ok(())
+            })
+            .await?;
+        Ok(())
+    }
+    .boxed()
+}
+
+#[pin_project::pin_project]
+pub struct TimeoutStream<S: AsyncRead + AsyncWrite = TcpStream> {
+    timeout: Duration,
+    #[pin]
+    sleep: Sleep,
+    #[pin]
+    stream: S,
+}
+impl<S: AsyncRead + AsyncWrite> TimeoutStream<S> {
+    pub fn new(stream: S, timeout: Duration) -> Self {
+        Self {
+            timeout,
+            sleep: tokio::time::sleep(timeout),
+            stream,
+        }
+    }
+}
+impl<S: AsyncRead + AsyncWrite> AsyncRead for TimeoutStream<S> {
+    fn poll_read(
+        self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+        buf: &mut tokio::io::ReadBuf<'_>,
+    ) -> std::task::Poll<std::io::Result<()>> {
+        let mut this = self.project();
+        if let std::task::Poll::Ready(_) = this.sleep.as_mut().poll(cx) {
+            return std::task::Poll::Ready(Err(std::io::Error::new(
+                std::io::ErrorKind::TimedOut,
+                "timed out",
+            )));
+        }
+        let res = this.stream.poll_read(cx, buf);
+        if res.is_ready() {
+            this.sleep.reset(Instant::now() + *this.timeout);
+        }
+        res
+    }
+}
+impl<S: AsyncRead + AsyncWrite> AsyncWrite for TimeoutStream<S> {
+    fn poll_write(
+        self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+        buf: &[u8],
+    ) -> std::task::Poll<Result<usize, std::io::Error>> {
+        let mut this = self.project();
+        let res = this.stream.poll_write(cx, buf);
+        if res.is_ready() {
+            this.sleep.reset(Instant::now() + *this.timeout);
+        }
+        res
+    }
+    fn poll_flush(
+        self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Result<(), std::io::Error>> {
+        let mut this = self.project();
+        let res = this.stream.poll_flush(cx);
+        if res.is_ready() {
+            this.sleep.reset(Instant::now() + *this.timeout);
+        }
+        res
+    }
+    fn poll_shutdown(
+        self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Result<(), std::io::Error>> {
+        let mut this = self.project();
+        let res = this.stream.poll_shutdown(cx);
+        if res.is_ready() {
+            this.sleep.reset(Instant::now() + *this.timeout);
+        }
+        res
+    }
+}

backend/src/util/lshw.rs

@@ -0,0 +1,49 @@
+use models::{Error, ResultExt};
+use serde::{Deserialize, Serialize};
+use tokio::process::Command;
+
+use crate::util::Invoke;
+
+const KNOWN_CLASSES: &[&str] = &["processor", "display"];
+
+#[derive(Debug, Deserialize, Serialize)]
+#[serde(tag = "class")]
+#[serde(rename_all = "kebab-case")]
+pub enum LshwDevice {
+    Processor(LshwProcessor),
+    Display(LshwDisplay),
+}
+impl LshwDevice {
+    pub fn class(&self) -> &'static str {
+        match self {
+            Self::Processor(_) => "processor",
+            Self::Display(_) => "display",
+        }
+    }
+    pub fn product(&self) -> &str {
+        match self {
+            Self::Processor(hw) => hw.product.as_str(),
+            Self::Display(hw) => hw.product.as_str(),
+        }
+    }
+}
+
+#[derive(Debug, Deserialize, Serialize)]
+pub struct LshwProcessor {
+    pub product: String,
+}
+
+#[derive(Debug, Deserialize, Serialize)]
+pub struct LshwDisplay {
+    pub product: String,
+}
+
+pub async fn lshw() -> Result<Vec<LshwDevice>, Error> {
+    let mut cmd = Command::new("lshw");
+    cmd.arg("-json");
+    for class in KNOWN_CLASSES {
+        cmd.arg("-class").arg(*class);
+    }
+    serde_json::from_slice(&cmd.invoke(crate::ErrorKind::Lshw).await?)
+        .with_kind(crate::ErrorKind::Deserialization)
+}

backend/src/util/mod.rs

@@ -27,6 +27,7 @@ pub mod config;
 pub mod http_reader;
 pub mod io;
 pub mod logger;
+pub mod lshw;
 pub mod serde;
 
 #[derive(Clone, Copy, Debug)]
@@ -58,7 +59,12 @@ impl Invoke for tokio::process::Command {
             res.status.success(),
             error_kind,
             "{}",
-            std::str::from_utf8(&res.stderr).unwrap_or("Unknown Error")
+            Some(&res.stderr)
+                .filter(|a| !a.is_empty())
+                .or(Some(&res.stdout))
+                .filter(|a| !a.is_empty())
+                .and_then(|a| std::str::from_utf8(a).ok())
+                .unwrap_or(&format!("Unknown Error ({})", res.status))
         );
         Ok(res.stdout)
     }

backend/src/util/serde.rs

@@ -793,3 +793,42 @@ impl<T: AsRef<[u8]>> Serialize for Base64<T> {
         serializer.serialize_str(&base64::encode(self.0.as_ref()))
     }
 }
+
+#[derive(Clone, Debug)]
+pub struct Regex(regex::Regex);
+impl From<Regex> for regex::Regex {
+    fn from(value: Regex) -> Self {
+        value.0
+    }
+}
+impl From<regex::Regex> for Regex {
+    fn from(value: regex::Regex) -> Self {
+        Regex(value)
+    }
+}
+impl AsRef<regex::Regex> for Regex {
+    fn as_ref(&self) -> &regex::Regex {
+        &self.0
+    }
+}
+impl AsMut<regex::Regex> for Regex {
+    fn as_mut(&mut self) -> &mut regex::Regex {
+        &mut self.0
+    }
+}
+impl<'de> Deserialize<'de> for Regex {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        deserialize_from_str(deserializer).map(Self)
+    }
+}
+impl Serialize for Regex {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        serialize_display(&self.0, serializer)
+    }
+}

backend/src/version/mod.rs

@@ -22,8 +22,10 @@ mod v0_3_3;
 mod v0_3_4;
 mod v0_3_4_1;
 mod v0_3_4_2;
+mod v0_3_4_3;
+mod v0_3_4_4;
 
-pub type Current = v0_3_4_2::Version;
+pub type Current = v0_3_4_4::Version;
 
 #[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]
 #[serde(untagged)]
@@ -41,6 +43,8 @@ enum Version {
     V0_3_4(Wrapper<v0_3_4::Version>),
     V0_3_4_1(Wrapper<v0_3_4_1::Version>),
     V0_3_4_2(Wrapper<v0_3_4_2::Version>),
+    V0_3_4_3(Wrapper<v0_3_4_3::Version>),
+    V0_3_4_4(Wrapper<v0_3_4_4::Version>),
     Other(emver::Version),
 }
 
@@ -69,6 +73,8 @@ impl Version {
             Version::V0_3_4(Wrapper(x)) => x.semver(),
             Version::V0_3_4_1(Wrapper(x)) => x.semver(),
             Version::V0_3_4_2(Wrapper(x)) => x.semver(),
+            Version::V0_3_4_3(Wrapper(x)) => x.semver(),
+            Version::V0_3_4_4(Wrapper(x)) => x.semver(),
             Version::Other(x) => x.clone(),
         }
     }
@@ -258,6 +264,14 @@ pub async fn init<Db: DbHandle>(
             v.0.migrate_to(&Current::new(), db, secrets, receipts)
                 .await?
         }
+        Version::V0_3_4_3(v) => {
+            v.0.migrate_to(&Current::new(), db, secrets, receipts)
+                .await?
+        }
+        Version::V0_3_4_4(v) => {
+            v.0.migrate_to(&Current::new(), db, secrets, receipts)
+                .await?
+        }
         Version::Other(_) => {
             return Err(Error::new(
                 eyre!("Cannot downgrade"),
@@ -303,6 +317,7 @@ mod tests {
             Just(Version::V0_3_4(Wrapper(v0_3_4::Version::new()))),
             Just(Version::V0_3_4_1(Wrapper(v0_3_4_1::Version::new()))),
             Just(Version::V0_3_4_2(Wrapper(v0_3_4_2::Version::new()))),
+            Just(Version::V0_3_4_3(Wrapper(v0_3_4_3::Version::new()))),
             em_version().prop_map(Version::Other),
         ]
     }

backend/src/version/v0_3_4.rs

@@ -5,11 +5,10 @@ use openssl::hash::MessageDigest;
 use serde_json::{json, Value};
 use ssh_key::public::Ed25519PublicKey;
 
-use crate::account::AccountInfo;
-use crate::hostname::{generate_hostname, sync_hostname, Hostname};
-
 use super::v0_3_0::V0_3_0_COMPAT;
 use super::*;
+use crate::account::AccountInfo;
+use crate::hostname::{generate_hostname, sync_hostname, Hostname};
 
 const V0_3_4: emver::Version = emver::Version::new(0, 3, 4, 0);
 

backend/src/version/v0_3_4_3.rs

@@ -0,0 +1,36 @@
+use async_trait::async_trait;
+use emver::VersionRange;
+
+use super::v0_3_0::V0_3_0_COMPAT;
+use super::*;
+
+const V0_3_4_3: emver::Version = emver::Version::new(0, 3, 4, 3);
+
+#[derive(Clone, Debug)]
+pub struct Version;
+
+#[async_trait]
+impl VersionT for Version {
+    type Previous = v0_3_4_2::Version;
+    fn new() -> Self {
+        Version
+    }
+    fn semver(&self) -> emver::Version {
+        V0_3_4_3
+    }
+    fn compat(&self) -> &'static VersionRange {
+        &*V0_3_0_COMPAT
+    }
+    async fn up<Db: DbHandle>(&self, db: &mut Db, _secrets: &PgPool) -> Result<(), Error> {
+        crate::db::DatabaseModel::new()
+            .server_info()
+            .get_mut(db)
+            .await?
+            .save(db)
+            .await?;
+        Ok(())
+    }
+    async fn down<Db: DbHandle>(&self, _db: &mut Db, _secrets: &PgPool) -> Result<(), Error> {
+        Ok(())
+    }
+}

backend/src/version/v0_3_4_4.rs

@@ -0,0 +1,41 @@
+use async_trait::async_trait;
+use emver::VersionRange;
+use models::ResultExt;
+
+use super::v0_3_0::V0_3_0_COMPAT;
+use super::*;
+
+const V0_3_4_4: emver::Version = emver::Version::new(0, 3, 4, 4);
+
+#[derive(Clone, Debug)]
+pub struct Version;
+
+#[async_trait]
+impl VersionT for Version {
+    type Previous = v0_3_4_3::Version;
+    fn new() -> Self {
+        Version
+    }
+    fn semver(&self) -> emver::Version {
+        V0_3_4_4
+    }
+    fn compat(&self) -> &'static VersionRange {
+        &*V0_3_0_COMPAT
+    }
+    async fn up<Db: DbHandle>(&self, db: &mut Db, _secrets: &PgPool) -> Result<(), Error> {
+        let mut tor_addr = crate::db::DatabaseModel::new()
+            .server_info()
+            .tor_address()
+            .get_mut(db)
+            .await?;
+        tor_addr
+            .set_scheme("https")
+            .map_err(|_| eyre!("unable to update url scheme to https"))
+            .with_kind(crate::ErrorKind::ParseUrl)?;
+        tor_addr.save(db).await?;
+        Ok(())
+    }
+    async fn down<Db: DbHandle>(&self, _db: &mut Db, _secrets: &PgPool) -> Result<(), Error> {
+        Ok(())
+    }
+}

backend/startd.service

@@ -0,0 +1,19 @@
+[Unit]
+Description=StartOS Daemon
+After=network-online.target
+Requires=network-online.target
+Wants=avahi-daemon.service
+
+[Service]
+Type=simple
+Environment=RUST_LOG=startos=debug,js_engine=debug,patch_db=warn
+ExecStart=/usr/bin/startd
+Restart=always
+RestartSec=3
+ManagedOOMPreference=avoid
+CPUAccounting=true
+CPUWeight=1000
+LimitNOFILE=65536
+
+[Install]
+WantedBy=multi-user.target

backend/test/js_action_execute/package-data/scripts/test-package/0.3.0.3/embassy.js

@@ -1039,4 +1039,16 @@ export const action = {
     }
   },
 
+  async "test-disk-usage"(effects, _input) {
+    const usage = await effects.diskUsage()
+    return {
+      result: {
+        copyable: false,
+        message: `${usage.used} / ${usage.total}`,
+        version: "0",
+        qr: false,
+      },
+    };
+  }
+
 };

build-cargo-dep.sh

@@ -13,9 +13,13 @@ if tty -s; then
 	USE_TTY="-it"
 fi
 
-mkdir -p cargo-deps
-alias 'rust-arm64-builder'='docker run $USE_TTY --rm -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$(pwd)"/cargo-deps:/home/rust/src -P start9/rust-arm-cross:aarch64'
+if [ -z "$ARCH" ]; then
+	ARCH=$(uname -m)
+fi
 
-rust-arm64-builder cargo install "$1" --target-dir /home/rust/src
+mkdir -p cargo-deps
+alias 'rust-arm64-builder'='docker run $USE_TTY --rm -v "$HOME/.cargo/registry":/usr/local/cargo/registry -v "$(pwd)"/cargo-deps:/home/rust/src -P start9/rust-arm-cross:aarch64'
+
+rust-arm64-builder cargo install "$1" --target-dir /home/rust/src --target=$ARCH-unknown-linux-gnu
 sudo chown -R $USER cargo-deps
 sudo chown -R $USER ~/.cargo

build/lib/depends

@@ -3,11 +3,11 @@ avahi-utils
 bash-completion
 beep
 bmon
+btrfs-progs
 ca-certificates
 cifs-utils
 containerd.io
 curl
-crda
 cryptsetup
 docker-ce
 docker-ce-cli
@@ -15,7 +15,8 @@ docker-compose-plugin
 dosfstools
 e2fsprogs
 ecryptfs-utils
-exfat-utils
+exfatprogs
+grub-common
 htop
 httpdirfs
 iotop
@@ -23,24 +24,30 @@ iw
 jq
 libavahi-client3
 lm-sensors
+lshw
 lvm2
 magic-wormhole
+man-db
 ncdu
 net-tools
 network-manager
+nvme-cli
 nyx
 openssh-server
-pgloader
 postgresql
 psmisc
+qemu-guest-agent
 rsync
 samba-common-bin
+smartmontools
 sqlite3
 squashfs-tools
+sudo
 systemd
 systemd-resolved
 systemd-sysv
 systemd-timesyncd
 tor
+util-linux
 vim
 wireless-tools

build/lib/motd

@@ -10,7 +10,7 @@ cat << "ASCII"
 ╰ ━ ━ ━ ╯ ╰ ━ ┻ ╯ ╰ ┻ ╯ ╰ ━ ┻ ━ ━ ━ ┻ ━ ━ ━ ╯
 ASCII
 printf "   %s (%s %s)\n" "$(uname -o)" "$(uname -r)" "$(uname -m)"
-printf "   $(embassy-cli --version | sed 's/Embassy CLI /StartOS v/g') - $(embassy-cli git-info)"
+printf "   $(start-cli --version | sed 's/StartOS CLI /StartOS v/g') - $(start-cli git-info)"
 if [ -n "$(cat /usr/lib/embassy/ENVIRONMENT.txt)" ]; then
     printf " ~ $(cat /usr/lib/embassy/ENVIRONMENT.txt)\n"
 else

build/lib/scripts/embassy-initramfs-module

@@ -31,6 +31,8 @@ local_mount_root()
 	modprobe ${FSTYPE}
 	checkfs ${ROOT} root "${FSTYPE}"
 
+	ROOTFLAGS="$(echo "${ROOTFLAGS}" | sed 's/subvol=\(next\|current\)//' | sed 's/^-o *$//')"
+
 	if [ "${FSTYPE}" != "unknown" ]; then
 		mount -t ${FSTYPE} ${ROOTFLAGS} ${ROOT} ${rootmnt}
 	else

build/lib/scripts/grub-probe-eos

@@ -3,30 +3,10 @@
 ARGS=
 
 for ARG in $@; do
- if [ "${ARG%%[!/]*}" = "/" ]; then
-
-  OPTIONS=
-
-  path="$ARG"
-  while true; do
-   if FSTYPE=$( findmnt -n -o FSTYPE "$path" ); then
-    if [ "$FSTYPE" = "overlay" ]; then
-  OPTIONS=$(findmnt -n -o OPTIONS "$path")
-  break
- else
-  break
-    fi
-   fi
-   if [ "$path" = "/" ]; then break; fi
-   path=$(dirname "$path")
-  done
-
-  if LOWERDIR=$(echo "$OPTIONS" | grep -m 1 -oP 'lowerdir=\K[^,]+'); then
-   #echo "[DEBUG] Overlay filesystem detected ${ARG} --> ${LOWERDIR}${ARG%*/}" 1>&2
-   ARG=/media/embassy/embassyfs"${ARG%*/}"
+  if [ -d "/media/embassy/embassyfs" ] && [ "$ARG" = "/" ]; then
+   ARG=/media/embassy/embassyfs
   fi
- fi
- ARGS="$ARGS $ARG"
+  ARGS="$ARGS $ARG"
 done
 
 grub-probe-default $ARGS

build/raspberrypi/config.yaml

@@ -1,5 +0,0 @@
-os-partitions:
-  boot: /dev/mmcblk0p1
-  root: /dev/mmcblk0p2
-ethernet-interface: end0
-wifi-interface: wlan0

build/raspberrypi/make-image.sh

@@ -71,6 +71,7 @@ sudo losetup -d $OUTPUT_DEVICE
 if [ "$ALLOW_VERSION_MISMATCH" != 1 ]; then
     if [ "$(cat GIT_HASH.txt)" != "$REAL_GIT_HASH" ]; then
         >&2 echo "startos.raspberrypi.squashfs GIT_HASH.txt mismatch"
+        >&2 echo "expected $REAL_GIT_HASH (dpkg) found $(cat GIT_HASH.txt) (repo)"
         exit 1
     fi
     if [ "$(cat VERSION.txt)" != "$REAL_VERSION" ]; then

check-version.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-FE_VERSION="$(cat frontend/package.json | grep -Po '"version":[ \t\n]*"\K[^"]*')"
+FE_VERSION="$(cat frontend/package.json | grep '"version"' | sed 's/[ \t]*"version":[ \t]*"\([^"]*\)",/\1/')"
 
 # TODO: Validate other version sources - backend/Cargo.toml, backend/src/version/mod.rs
 

compress-uis.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -e
+
+rm -rf frontend/dist/static
+
+find frontend/dist/raw -type f -not -name '*.gz' -and -not -name '*.br' | xargs -n 1 -P 0 gzip -kf
+find frontend/dist/raw -type f -not -name '*.gz' -and -not -name '*.br' | xargs -n 1 -P 0 brotli -kf
+
+for file in $(find frontend/dist/raw -type f -not -name '*.gz' -and -not -name '*.br'); do
+    raw_size=$(du --bytes $file | awk '{print $1}')
+    gz_size=$(du --bytes $file.gz | awk '{print $1}')
+    br_size=$(du --bytes $file.br | awk '{print $1}')
+    if [ $((gz_size * 100 / raw_size)) -gt 70 ]; then
+        rm $file.gz
+    fi
+    if [ $((br_size * 100 / raw_size)) -gt 70 ]; then
+        rm $file.br
+    fi
+done
+
+cp -r frontend/dist/raw frontend/dist/static

frontend/README.md

@@ -19,7 +19,7 @@ Check your versions
 
 ```sh
 node --version
-v16.10.0
+v18.15.0
 
 npm --version
 v8.0.0

frontend/angular.json

@@ -14,7 +14,7 @@
           "builder": "@angular-devkit/build-angular:browser",
           "options": {
             "preserveSymlinks": true,
-            "outputPath": "dist/ui",
+            "outputPath": "dist/raw/ui",
             "index": "projects/ui/src/index.html",
             "main": "projects/ui/src/main.ts",
             "polyfills": "projects/ui/src/polyfills.ts",
@@ -39,7 +39,7 @@
               "projects/ui/src/manifest.webmanifest",
               {
                 "glob": "ngsw.json",
-                "input": "dist/ui",
+                "input": "dist/raw/ui",
                 "output": "projects/ui/src"
               }
             ],
@@ -147,7 +147,7 @@
         "build": {
           "builder": "@angular-devkit/build-angular:browser",
           "options": {
-            "outputPath": "dist/install-wizard",
+            "outputPath": "dist/raw/install-wizard",
             "index": "projects/install-wizard/src/index.html",
             "main": "projects/install-wizard/src/main.ts",
             "polyfills": "projects/install-wizard/src/polyfills.ts",
@@ -277,7 +277,7 @@
         "build": {
           "builder": "@angular-devkit/build-angular:browser",
           "options": {
-            "outputPath": "dist/setup-wizard",
+            "outputPath": "dist/raw/setup-wizard",
             "index": "projects/setup-wizard/src/index.html",
             "main": "projects/setup-wizard/src/main.ts",
             "polyfills": "projects/setup-wizard/src/polyfills.ts",
@@ -397,7 +397,7 @@
         "build": {
           "builder": "@angular-devkit/build-angular:browser",
           "options": {
-            "outputPath": "dist/diagnostic-ui",
+            "outputPath": "dist/raw/diagnostic-ui",
             "index": "projects/diagnostic-ui/src/index.html",
             "main": "projects/diagnostic-ui/src/main.ts",
             "polyfills": "projects/diagnostic-ui/src/polyfills.ts",

frontend/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "startos-ui",
-  "version": "0.3.4.2",
+  "version": "0.3.4.4",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "startos-ui",
-      "version": "0.3.4.2",
+      "version": "0.3.4.4",
       "dependencies": {
         "@angular/animations": "^14.1.0",
         "@angular/common": "^14.1.0",