fix error and display of unknown font weight on success page (#2038)

* hack: Make the restores single restore at a time

* chore: Fix the build

* chore: Convert to using concurrent

* fix: Don't spawn for the tasks

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -1,21 +1,21 @@
 name: 🐛 Bug Report
-description: Create a report to help us improve EmbassyOS
+description: Create a report to help us improve embassyOS
 title: '[bug]: '
 labels: [Bug, Needs Triage]
 assignees:
-  - dr-bonez
+  - MattDHill
 body:
   - type: checkboxes
     attributes:
       label: Prerequisites
       description: Please confirm you have completed the following.
       options:
-        - label: I have searched for [existing issues](https://github.com/start9labs/embassy-os/issues) that already report this problem, without success.
+        - label: I have searched for [existing issues](https://github.com/start9labs/embassy-os/issues) that already report this problem.
           required: true
   - type: input
     attributes:
-      label: EmbassyOS Version
-      description: What version of EmbassyOS are you running?
+      label: embassyOS Version
+      description: What version of embassyOS are you running?
       placeholder: e.g. 0.3.0
     validations:
       required: true

.github/ISSUE_TEMPLATE/feature-request.yml

@@ -1,16 +1,16 @@
 name: 💡 Feature Request
-description: Suggest an idea for EmbassyOS
+description: Suggest an idea for embassyOS
 title: '[feat]: '
 labels: [Enhancement]
 assignees:
-  - dr-bonez
+  - MattDHill
 body:
   - type: checkboxes
     attributes:
       label: Prerequisites
       description: Please confirm you have completed the following.
       options:
-        - label: I have searched for [existing issues](https://github.com/start9labs/embassy-os/issues) that already suggest this feature, without success.
+        - label: I have searched for [existing issues](https://github.com/start9labs/embassy-os/issues) that already suggest this feature.
           required: true
   - type: textarea
     attributes:
@@ -27,7 +27,7 @@ body:
   - type: textarea
     attributes:
       label: Describe Preferred Solution
-      description: How you want this feature added to EmbassyOS?
+      description: How you want this feature added to embassyOS?
   - type: textarea
     attributes:
       label: Describe Alternatives

.github/workflows/README.md

@@ -0,0 +1,29 @@
+# This folder contains GitHub Actions workflows for building the project
+
+## backend
+Runs: manually (on: workflow_dispatch) or called by product-pipeline (on: workflow_call)
+
+This workflow uses the actions and docker/setup-buildx-action@v1 to prepare the environment for aarch64 cross complilation using docker buildx.
+When execution of aarch64 containers is required the action docker/setup-qemu-action@v1 is added.
+A matrix-strategy has been used to build for both x86_64 and aarch64 platforms in parallel.
+
+### Running unittests
+
+Unittests are run using [cargo-nextest]( https://nexte.st/). First the sources are (cross-)compiled and archived. The archive is then run on the correct platform.
+
+## frontend
+Runs: manually (on: workflow_dispatch) or called by product-pipeline (on: workflow_call)
+
+This workflow builds the frontends.
+
+## product
+Runs: when a pull request targets the master or next branch and when a change to the master or next branch is made
+
+This workflow builds everything, re-using the backend and frontend workflows.
+The download and extraction order of artifacts is relevant to `make`, as it checks the file timestamps to decide which targets need to be executed.
+
+Result: eos.img
+
+## a note on uploading artifacts
+
+Artifacts are used to share data between jobs. File permissions are not maintained during artifact upload. Where file permissions are relevant, the workaround using tar has been used. See (here)[https://github.com/actions/upload-artifact#maintaining-file-permissions-and-case-sensitive-files].

.github/workflows/backend.yaml

@@ -0,0 +1,231 @@
+name: Backend
+
+on:
+  workflow_call:
+  workflow_dispatch:
+
+env:
+  RUST_VERSION: "1.62.1"
+  ENVIRONMENT: "dev"
+
+jobs:
+  build_libs:
+    name: Build libs
+    strategy:
+      fail-fast: false
+      matrix:
+        target: [x86_64, aarch64]
+        include:
+          - target: x86_64
+            snapshot_command: ./build-v8-snapshot.sh
+            artifact_name: js_snapshot
+            artifact_path: libs/js_engine/src/artifacts/JS_SNAPSHOT.bin
+          - target: aarch64
+            snapshot_command: ./build-arm-v8-snapshot.sh
+            artifact_name: arm_js_snapshot
+            artifact_path: libs/js_engine/src/artifacts/ARM_JS_SNAPSHOT.bin
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: recursive
+        
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v2
+      if: ${{ matrix.target == 'aarch64' }}
+      
+    - name: Set up Docker Buildx        
+      uses: docker/setup-buildx-action@v2
+      if: ${{ matrix.target == 'aarch64' }}
+
+    - name: "Install Rust"
+      run: |
+        rustup toolchain install ${{ env.RUST_VERSION }} --profile minimal --no-self-update
+        rustup default ${{ inputs.rust }}
+      shell: bash
+      if: ${{ matrix.target == 'x86_64' }}
+    
+    - uses: actions/cache@v3
+      with:
+        path: |
+          ~/.cargo/bin/
+          ~/.cargo/registry/index/
+          ~/.cargo/registry/cache/
+          ~/.cargo/git/db/
+          libs/target/
+        key: ${{ runner.os }}-cargo-libs-${{ matrix.target }}-${{ hashFiles('libs/Cargo.lock') }}
+
+    - name: Build v8 snapshot
+      run: ${{ matrix.snapshot_command }}
+      working-directory: libs
+
+    - uses: actions/upload-artifact@v3
+      with:
+        name: ${{ matrix.artifact_name }}
+        path: ${{ matrix.artifact_path }}
+
+  build_backend:
+    name: Build backend
+    strategy:
+      fail-fast: false
+      matrix:
+        target: [x86_64, aarch64]
+        include:
+          - target: x86_64
+            snapshot_download: js_snapshot
+          - target: aarch64
+            snapshot_download: arm_js_snapshot
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    needs: build_libs
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: recursive
+    
+    - name: Download ${{ matrix.snapshot_download }} artifact
+      uses: actions/download-artifact@v3
+      with:
+        name: ${{ matrix.snapshot_download }}
+        path: libs/js_engine/src/artifacts/
+        
+    - name: "Install Rust"
+      run: |
+        rustup toolchain install ${{ env.RUST_VERSION }} --profile minimal --no-self-update
+        rustup default ${{ inputs.rust }}
+      shell: bash
+      if: ${{ matrix.target == 'x86_64' }}
+    
+    - uses: actions/cache@v3
+      with:
+        path: |
+          ~/.cargo/bin/
+          ~/.cargo/registry/index/
+          ~/.cargo/registry/cache/
+          ~/.cargo/git/db/
+          backend/target/
+        key: ${{ runner.os }}-cargo-backend-${{ matrix.target }}-${{ hashFiles('backend/Cargo.lock') }}
+
+    - name: Install dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install libavahi-client-dev
+      if: ${{ matrix.target == 'x86_64' }}
+
+    - name: Check Git Hash
+      run: ./check-git-hash.sh
+
+    - name: Check Environment
+      run: ./check-environment.sh
+
+    - name: Build backend
+      run: make ARCH=${{ matrix.target }} backend
+
+    - name: 'Tar files to preserve file permissions'
+      run: make ARCH=${{ matrix.target }} backend-${{ matrix.target }}.tar
+
+    - uses: actions/upload-artifact@v3
+      with:
+        name: backend-${{ matrix.target }}
+        path: backend-${{ matrix.target }}.tar
+    
+    - name: Install nextest
+      uses: taiki-e/install-action@nextest
+
+    - name: Build and archive tests
+      run: cargo nextest archive --archive-file nextest-archive-${{ matrix.target }}.tar.zst --target ${{ matrix.target }}-unknown-linux-gnu
+      working-directory: backend
+      if: ${{ matrix.target == 'x86_64' }}
+
+    - name: Build and archive tests
+      run: |
+        docker run --rm \
+        -v "$HOME/.cargo/registry":/root/.cargo/registry \
+        -v "$(pwd)":/home/rust/src \
+        -P start9/rust-arm-cross:aarch64 \
+        sh -c 'cd /home/rust/src/backend &&
+          rustup install ${{ env.RUST_VERSION }} &&
+          rustup override set ${{ env.RUST_VERSION }} &&
+          rustup target add aarch64-unknown-linux-gnu &&
+          curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin &&
+          cargo nextest archive --archive-file nextest-archive-${{ matrix.target }}.tar.zst --target ${{ matrix.target }}-unknown-linux-gnu'
+      if: ${{ matrix.target == 'aarch64' }}
+
+    - name: Reset permissions
+      run: sudo chown -R $USER target
+      working-directory: backend
+      if: ${{ matrix.target == 'aarch64' }}
+
+    - name: Upload archive to workflow
+      uses: actions/upload-artifact@v3
+      with:
+        name: nextest-archive-${{ matrix.target }}
+        path: backend/nextest-archive-${{ matrix.target }}.tar.zst
+
+  run_tests_backend:
+    name: Test backend
+    strategy:
+      fail-fast: false
+      matrix:
+        target: [x86_64, aarch64]
+        include:
+          - target: x86_64
+          - target: aarch64
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    needs: build_backend
+    env:
+      CARGO_TERM_COLOR: always
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: recursive
+
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v2
+      if: ${{ matrix.target == 'aarch64' }}
+
+    - name: Set up Docker Buildx        
+      uses: docker/setup-buildx-action@v2
+      if: ${{ matrix.target == 'aarch64' }}
+
+    - run: mkdir -p ~/.cargo/bin
+      if: ${{ matrix.target == 'x86_64' }}
+
+    - name: Install nextest
+      uses: taiki-e/install-action@nextest
+      if: ${{ matrix.target == 'x86_64' }}
+
+    - name: Download archive
+      uses: actions/download-artifact@v3
+      with:
+        name: nextest-archive-${{ matrix.target }}
+
+    - name: Download nextest (aarch64)
+      run: wget -O nextest-aarch64.tar.gz https://get.nexte.st/latest/linux-arm
+      if: ${{ matrix.target == 'aarch64' }}
+
+    - name: Run tests
+      run: |
+        ${CARGO_HOME:-~/.cargo}/bin/cargo-nextest nextest run --no-fail-fast --archive-file nextest-archive-${{ matrix.target }}.tar.zst \
+        --filter-expr 'not (test(system::test_get_temp) | test(net::tor::test) | test(system::test_get_disk_usage) | test(net::ssl::certificate_details_persist) | test(net::ssl::ca_details_persist))'
+      if: ${{ matrix.target == 'x86_64' }}
+
+    - name: Run tests
+      run: |
+        docker run --rm --platform linux/arm64/v8 \
+        -v "/home/runner/.cargo/registry":/usr/local/cargo/registry \
+        -v "$(pwd)":/home/rust/src \
+        -e CARGO_TERM_COLOR=${{ env.CARGO_TERM_COLOR }} \
+        -P ubuntu:20.04 \
+        sh -c '
+          apt-get update &&
+          apt-get install -y ca-certificates &&
+          apt-get install -y rsync &&
+          cd /home/rust/src &&
+          mkdir -p ~/.cargo/bin &&
+          tar -zxvf nextest-aarch64.tar.gz -C ${CARGO_HOME:-~/.cargo}/bin &&
+          ${CARGO_HOME:-~/.cargo}/bin/cargo-nextest nextest run --archive-file nextest-archive-${{ matrix.target }}.tar.zst \
+          --filter-expr "not (test(system::test_get_temp) | test(net::tor::test) | test(system::test_get_disk_usage) | test(net::ssl::certificate_details_persist) | test(net::ssl::ca_details_persist))"'
+      if: ${{ matrix.target == 'aarch64' }}

.github/workflows/debian.yaml

@@ -0,0 +1,63 @@
+name: Debian Package
+
+on:
+  workflow_call:
+  workflow_dispatch:
+
+env:
+  NODEJS_VERSION: '16'
+  ENVIRONMENT: "dev"
+
+jobs:
+  dpkg:
+    name: Build dpkg
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        repository: Start9Labs/embassy-os-deb
+
+    - uses: actions/checkout@v3
+      with:
+        submodules: recursive
+        path: embassyos-0.3.x
+    - run: |
+        cp -r debian embassyos-0.3.x/
+        VERSION=0.3.x ./control.sh
+        cp embassyos-0.3.x/backend/embassyd.service embassyos-0.3.x/debian/embassyos.embassyd.service
+        cp embassyos-0.3.x/backend/embassy-init.service embassyos-0.3.x/debian/embassyos.embassy-init.service
+    
+    - uses: actions/setup-node@v3
+      with:
+        node-version: ${{ env.NODEJS_VERSION }}
+    
+    - name: Get npm cache directory
+      id: npm-cache-dir
+      run: |
+        echo "dir=$(npm config get cache)" >> $GITHUB_OUTPUT
+    - uses: actions/cache@v3
+      id: npm-cache
+      with:
+        path: ${{ steps.npm-cache-dir.outputs.dir }}
+        key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+        restore-keys: |
+          ${{ runner.os }}-node-
+
+    - name: Install dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install debmake debhelper-compat
+
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v2
+      
+    - name: Set up Docker Buildx        
+      uses: docker/setup-buildx-action@v2
+
+    - name: Run build
+      run: "make VERSION=0.3.x TAG=${{ github.ref_name }}"
+
+    - uses: actions/upload-artifact@v3
+      with:
+        name: deb
+        path: embassyos_0.3.x-1_amd64.deb

.github/workflows/frontend.yaml

@@ -0,0 +1,46 @@
+name: Frontend
+
+on:
+  workflow_call:
+  workflow_dispatch:
+
+env:
+  NODEJS_VERSION: '16'
+  ENVIRONMENT: "dev"
+
+jobs:
+  frontend:
+    name: Build frontend
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: recursive
+
+    - uses: actions/setup-node@v3
+      with:
+        node-version: ${{ env.NODEJS_VERSION }}
+    
+    - name: Get npm cache directory
+      id: npm-cache-dir
+      run: |
+        echo "dir=$(npm config get cache)" >> $GITHUB_OUTPUT
+    - uses: actions/cache@v3
+      id: npm-cache
+      with:
+        path: ${{ steps.npm-cache-dir.outputs.dir }}
+        key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+        restore-keys: |
+          ${{ runner.os }}-node-
+
+    - name: Build frontends
+      run: make frontends
+
+    - name: 'Tar files to preserve file permissions'
+      run: tar -cvf frontend.tar ENVIRONMENT.txt GIT_HASH.txt VERSION.txt frontend/dist frontend/config.json
+
+    - uses: actions/upload-artifact@v3
+      with:
+        name: frontend
+        path: frontend.tar

.github/workflows/product.yaml

@@ -0,0 +1,142 @@
+name: Build Pipeline
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+    - master
+    - next
+  pull_request:
+    branches:
+    - master
+    - next
+
+env:
+  ENVIRONMENT: "dev"
+
+jobs:
+  compat:
+    uses: ./.github/workflows/reusable-workflow.yaml
+    with:
+      build_command: make system-images/compat/docker-images/aarch64.tar
+      artifact_name: compat.tar
+      artifact_path: system-images/compat/docker-images/aarch64.tar
+
+  utils:
+    uses: ./.github/workflows/reusable-workflow.yaml
+    with:
+      build_command: make system-images/utils/docker-images/aarch64.tar
+      artifact_name: utils.tar
+      artifact_path: system-images/utils/docker-images/aarch64.tar
+
+  binfmt:
+    uses: ./.github/workflows/reusable-workflow.yaml
+    with:
+      build_command: make system-images/binfmt/docker-images/aarch64.tar
+      artifact_name: binfmt.tar
+      artifact_path: system-images/binfmt/docker-images/aarch64.tar
+
+  nc-broadcast:
+    uses: ./.github/workflows/reusable-workflow.yaml
+    with:
+      build_command: make cargo-deps/aarch64-unknown-linux-gnu/release/nc-broadcast
+      artifact_name: nc-broadcast.tar
+      artifact_path: cargo-deps/aarch64-unknown-linux-gnu/release/nc-broadcast
+
+  backend:
+    uses: ./.github/workflows/backend.yaml
+
+  frontend:
+    uses: ./.github/workflows/frontend.yaml
+    
+  image:
+    name: Build image
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    needs: [compat,utils,binfmt,nc-broadcast,backend,frontend]
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: recursive
+
+    - name: Download compat.tar artifact
+      uses: actions/download-artifact@v3
+      with:
+        name: compat.tar
+        path: system-images/compat/docker-images/
+
+    - name: Download utils.tar artifact
+      uses: actions/download-artifact@v3
+      with:
+        name: utils.tar
+        path: system-images/utils/docker-images/
+    
+    - name: Download binfmt.tar artifact
+      uses: actions/download-artifact@v3
+      with:
+        name: binfmt.tar
+        path: system-images/binfmt/docker-images/
+
+    - name: Download nc-broadcast.tar artifact
+      uses: actions/download-artifact@v3
+      with:
+        name: nc-broadcast.tar
+        path: cargo-deps/aarch64-unknown-linux-gnu/release
+
+    - name: Download js_snapshot artifact
+      uses: actions/download-artifact@v3
+      with:
+        name: js_snapshot
+        path: libs/js_engine/src/artifacts/
+  
+    - name: Download arm_js_snapshot artifact
+      uses: actions/download-artifact@v3
+      with:
+        name: arm_js_snapshot
+        path: libs/js_engine/src/artifacts/
+
+    - name: Download backend artifact
+      uses: actions/download-artifact@v3
+      with:
+        name: backend-aarch64
+
+    - name: 'Extract backend'
+      run: 
+        tar -mxvf backend-aarch64.tar
+
+    - name: Download frontend artifact
+      uses: actions/download-artifact@v3
+      with:
+        name: frontend
+
+    - name: Skip frontend build
+      run: |
+        mkdir frontend/node_modules
+        mkdir frontend/dist
+        mkdir patch-db/client/node_modules
+        mkdir patch-db/client/dist
+
+    - name: 'Extract frontend'
+      run: |
+        tar -mxvf frontend.tar frontend/config.json
+        tar -mxvf frontend.tar frontend/dist
+        tar -xvf frontend.tar GIT_HASH.txt
+        tar -xvf frontend.tar ENVIRONMENT.txt
+        tar -xvf frontend.tar VERSION.txt
+        rm frontend.tar
+
+    - name: Cache raspiOS
+      id: cache-raspios
+      uses: actions/cache@v3
+      with:
+        path: raspios.img
+        key: cache-raspios
+    
+    - name: Build image
+      run: |
+        make V=1 embassyos-raspi.img --debug
+      
+    - uses: actions/upload-artifact@v3
+      with:
+        name: image
+        path: embassyos-raspi.img

.github/workflows/pureos-iso.yaml

@@ -0,0 +1,70 @@
+name: PureOS Based ISO
+
+on:
+  workflow_call:
+  workflow_dispatch:
+  push:
+    branches:
+    - master
+    - next
+  pull_request:
+    branches:
+    - master
+    - next
+
+env:
+  ENVIRONMENT: "dev"
+
+jobs:
+  dpkg:
+    uses: ./.github/workflows/debian.yaml
+
+  iso:
+    name: Build iso
+    runs-on: ubuntu-22.04
+    needs: [dpkg]
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        repository: Start9Labs/eos-image-recipes
+
+    - name: Install dependencies
+      run: |
+        sudo apt update
+        wget http://ftp.us.debian.org/debian/pool/main/d/debspawn/debspawn_0.6.0-1.1_all.deb
+        sha256sum ./debspawn_0.6.0-1.1_all.deb | grep 6cdb444844825b82ef388378f2c508fd8886f79a49800d4547a353ea6772d493
+        sudo apt-get install -y ./debspawn_0.6.0-1.1_all.deb
+        wget https://repo.pureos.net/pureos/pool/main/d/debootstrap/debootstrap_1.0.125pureos1_all.deb
+        sudo apt-get install -y --allow-downgrades ./debootstrap_1.0.125pureos1_all.deb
+        wget https://repo.pureos.net/pureos/pool/main/p/pureos-archive-keyring/pureos-archive-keyring_2021.11.0_all.deb
+        sudo apt-get install -y ./pureos-archive-keyring_2021.11.0_all.deb
+
+    - name: Configure debspawn
+      run: |
+        sudo mkdir -p /etc/debspawn/
+        echo "AllowUnsafePermissions=true" | sudo tee /etc/debspawn/global.toml
+
+    - uses: actions/cache@v3
+      with:
+        path: /var/lib/debspawn
+        key: ${{ runner.os }}-debspawn-init-byzantium
+
+    - name: Make build container
+      run: "debspawn list | grep byzantium || debspawn create --with-init byzantium"
+
+    - run: "mkdir -p overlays/vendor/root"
+
+    - name: Download dpkg
+      uses: actions/download-artifact@v3
+      with:
+        name: deb
+        path: overlays/vendor/root
+
+    - name: Run build
+      run: |
+        ./run-local-build.sh --no-fakemachine byzantium none custom "" true
+
+    - uses: actions/upload-artifact@v3
+      with:
+        name: iso
+        path: results/*.iso

.github/workflows/reusable-workflow.yaml

@@ -0,0 +1,37 @@
+name: Reusable Workflow
+
+on:
+  workflow_call:
+    inputs:
+      build_command:
+        required: true
+        type: string
+      artifact_name:
+        required: true
+        type: string
+      artifact_path:
+        required: true
+        type: string
+
+env:
+  ENVIRONMENT: "dev"
+
+jobs:
+  generic_build_job:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: recursive
+
+    - name: Set up Docker Buildx        
+      uses: docker/setup-buildx-action@v2
+
+    - name: Build image
+      run: ${{ inputs.build_command }}
+
+    - uses: actions/upload-artifact@v3
+      with:
+        name: ${{ inputs.artifact_name }}
+        path: ${{ inputs.artifact_path }}

.gitignore

@@ -1,5 +1,8 @@
 .DS_Store
 .idea
+system-images/binfmt/binfmt.tar
+system-images/compat/compat.tar
+system-images/util/util.tar
 /*.img
 /*.img.gz
 /*.img.xz
@@ -8,3 +11,18 @@
 /product_key.txt
 /*_product_key.txt
 .vscode/settings.json
+deploy_web.sh
+deploy_web.sh
+secrets.db
+.vscode/
+/cargo-deps/**/*
+/ENVIRONMENT.txt
+/GIT_HASH.txt
+/VERSION.txt
+/embassyos-*.tar.gz
+/eos-*.tar.gz
+/*.deb
+/target
+/*.squashfs
+/debian
+/DEBIAN

.gitmodules

@@ -1,6 +1,3 @@
-[submodule "rpc-toolkit"]
-	path = rpc-toolkit
-	url = https://github.com/Start9Labs/rpc-toolkit.git
 [submodule "patch-db"]
 	path = patch-db
 	url = https://github.com/Start9Labs/patch-db.git

CONTRIBUTING.md

@@ -1,11 +1,20 @@
 <!-- omit in toc -->
+
 # Contributing to Embassy OS
 
 First off, thanks for taking the time to contribute! ❤️
 
-All types of contributions are encouraged and valued. See the [Table of Contents](#table-of-contents) for different ways to help and details about how this project handles them. Please make sure to read the relevant section before making your contribution. It will make it a lot easier for us maintainers and smooth out the experience for all involved. The community looks forward to your contributions. 🎉
+All types of contributions are encouraged and valued. See the
+[Table of Contents](#table-of-contents) for different ways to help and details
+about how this project handles them. Please make sure to read the relevant
+section before making your contribution. It will make it a lot easier for us
+maintainers and smooth out the experience for all involved. The community looks
+forward to your contributions. 🎉
 
-> And if you like the project, but just don't have time to contribute, that's fine. There are other easy ways to support the project and show your appreciation, which we would also be very happy about:
+> And if you like the project, but just don't have time to contribute, that's
+> fine. There are other easy ways to support the project and show your
+> appreciation, which we would also be very happy about:
+>
 > - Star the project
 > - Tweet about it
 > - Refer this project in your project's readme
@@ -13,12 +22,14 @@ All types of contributions are encouraged and valued. See the [Table of Contents
 > - Buy an [Embassy](https://start9labs.com)
 
 <!-- omit in toc -->
+
 ## Table of Contents
 
 - [I Have a Question](#i-have-a-question)
 - [I Want To Contribute](#i-want-to-contribute)
   - [Reporting Bugs](#reporting-bugs)
   - [Suggesting Enhancements](#suggesting-enhancements)
+  - [Project Structure](#project-structure)
   - [Your First Code Contribution](#your-first-code-contribution)
     - [Setting Up Your Development Environment](#setting-up-your-development-environment)
     - [Building The Image](#building-the-image)
@@ -32,15 +43,19 @@ All types of contributions are encouraged and valued. See the [Table of Contents
 - [Join The Discussion](#join-the-discussion)
 - [Join The Project Team](#join-the-project-team)
 
-
-
 ## I Have a Question
 
-> If you want to ask a question, we assume that you have read the available [Documentation](https://docs.start9labs.com).
+> If you want to ask a question, we assume that you have read the available
+> [Documentation](https://docs.start9labs.com).
 
-Before you ask a question, it is best to search for existing [Issues](https://github.com/Start9Labs/embassy-os/issues) that might help you. In case you have found a suitable issue and still need clarification, you can write your question in this issue. It is also advisable to search the internet for answers first.
+Before you ask a question, it is best to search for existing
+[Issues](https://github.com/Start9Labs/embassy-os/issues) that might help you.
+In case you have found a suitable issue and still need clarification, you can
+write your question in this issue. It is also advisable to search the internet
+for answers first.
 
-If you then still feel the need to ask a question and need clarification, we recommend the following:
+If you then still feel the need to ask a question and need clarification, we
+recommend the following:
 
 - Open an [Issue](https://github.com/Start9Labs/embassy-os/issues/new).
 - Provide as much context as you can about what you're running into.
@@ -66,98 +81,180 @@ Depending on how large the project is, you may want to outsource the questioning
 ## I Want To Contribute
 
 > ### Legal Notice <!-- omit in toc -->
-> When contributing to this project, you must agree that you have authored 100% of the content, that you have the necessary rights to the content and that the content you contribute may be provided under the project license.
+>
+> When contributing to this project, you must agree that you have authored 100%
+> of the content, that you have the necessary rights to the content and that the
+> content you contribute may be provided under the project license.
 
 ### Reporting Bugs
 
 <!-- omit in toc -->
+
 #### Before Submitting a Bug Report
 
-A good bug report shouldn't leave others needing to chase you up for more information. Therefore, we ask you to investigate carefully, collect information and describe the issue in detail in your report. Please complete the following steps in advance to help us fix any potential bug as fast as possible.
+A good bug report shouldn't leave others needing to chase you up for more
+information. Therefore, we ask you to investigate carefully, collect information
+and describe the issue in detail in your report. Please complete the following
+steps in advance to help us fix any potential bug as fast as possible.
 
 - Make sure that you are using the latest version.
-- Determine if your bug is really a bug and not an error on your side e.g. using incompatible environment components/versions (Make sure that you have read the [documentation](https://start9.com/latest/user-manual). If you are looking for support, you might want to check [this section](#i-have-a-question)).
-- To see if other users have experienced (and potentially already solved) the same issue you are having, check if there is not already a bug report existing for your bug or error in the [bug tracker](https://github.com/Start9Labs/embassy-os/issues?q=label%3Abug).
-- Also make sure to search the internet (including Stack Overflow) to see if users outside of the GitHub community have discussed the issue.
+- Determine if your bug is really a bug and not an error on your side e.g. using
+  incompatible environment components/versions (Make sure that you have read the
+  [documentation](https://start9.com/latest/user-manual). If you are looking for
+  support, you might want to check [this section](#i-have-a-question)).
+- To see if other users have experienced (and potentially already solved) the
+  same issue you are having, check if there is not already a bug report existing
+  for your bug or error in the
+  [bug tracker](https://github.com/Start9Labs/embassy-os/issues?q=label%3Abug).
+- Also make sure to search the internet (including Stack Overflow) to see if
+  users outside of the GitHub community have discussed the issue.
 - Collect information about the bug:
   - Stack trace (Traceback)
-  - Client OS, Platform and Version (Windows/Linux/macOS/iOS/Android, Firefox/Tor Browser/Consulate)
-  - Version of the interpreter, compiler, SDK, runtime environment, package manager, depending on what seems relevant.
+  - Client OS, Platform and Version (Windows/Linux/macOS/iOS/Android,
+    Firefox/Tor Browser/Consulate)
+  - Version of the interpreter, compiler, SDK, runtime environment, package
+    manager, depending on what seems relevant.
   - Possibly your input and the output
-  - Can you reliably reproduce the issue? And can you also reproduce it with older versions?
+  - Can you reliably reproduce the issue? And can you also reproduce it with
+    older versions?
 
 <!-- omit in toc -->
+
 #### How Do I Submit a Good Bug Report?
 
-> You must never report security related issues, vulnerabilities or bugs to the issue tracker, or elsewhere in public. Instead sensitive bugs must be sent by email to <security@start9labs.com>.
+> You must never report security related issues, vulnerabilities or bugs to the
+> issue tracker, or elsewhere in public. Instead sensitive bugs must be sent by
+> email to <security@start9labs.com>.
+
 <!-- You may add a PGP key to allow the messages to be sent encrypted as well. -->
 
-We use GitHub issues to track bugs and errors. If you run into an issue with the project:
+We use GitHub issues to track bugs and errors. If you run into an issue with the
+project:
 
-- Open an [Issue](https://github.com/Start9Labs/embassy-os/issues/new/choose) selecting the appropriate type.
+- Open an [Issue](https://github.com/Start9Labs/embassy-os/issues/new/choose)
+  selecting the appropriate type.
 - Explain the behavior you would expect and the actual behavior.
-- Please provide as much context as possible and describe the *reproduction steps* that someone else can follow to recreate the issue on their own. This usually includes your code. For good bug reports you should isolate the problem and create a reduced test case.
+- Please provide as much context as possible and describe the _reproduction
+  steps_ that someone else can follow to recreate the issue on their own. This
+  usually includes your code. For good bug reports you should isolate the
+  problem and create a reduced test case.
 - Provide the information you collected in the previous section.
 
 Once it's filed:
 
 - The project team will label the issue accordingly.
-- A team member will try to reproduce the issue with your provided steps. If there are no reproduction steps or no obvious way to reproduce the issue, the team will ask you for those steps and mark the issue as `Question`. Bugs with the `Question` tag will not be addressed until they are answered.
-- If the team is able to reproduce the issue, it will be marked a scoping level tag, as well as possibly other tags (such as `Security`), and the issue will be left to be [implemented by someone](#your-first-code-contribution).
+- A team member will try to reproduce the issue with your provided steps. If
+  there are no reproduction steps or no obvious way to reproduce the issue, the
+  team will ask you for those steps and mark the issue as `Question`. Bugs with
+  the `Question` tag will not be addressed until they are answered.
+- If the team is able to reproduce the issue, it will be marked a scoping level
+  tag, as well as possibly other tags (such as `Security`), and the issue will
+  be left to be [implemented by someone](#your-first-code-contribution).
 
 <!-- You might want to create an issue template for bugs and errors that can be used as a guide and that defines the structure of the information to be included. If you do so, reference it here in the description. -->
 
-
 ### Suggesting Enhancements
 
-This section guides you through submitting an enhancement suggestion for Embassy OS, **including completely new features and minor improvements to existing functionality**. Following these guidelines will help maintainers and the community to understand your suggestion and find related suggestions.
+This section guides you through submitting an enhancement suggestion for Embassy
+OS, **including completely new features and minor improvements to existing
+functionality**. Following these guidelines will help maintainers and the
+community to understand your suggestion and find related suggestions.
 
 <!-- omit in toc -->
+
 #### Before Submitting an Enhancement
 
 - Make sure that you are using the latest version.
-- Read the [documentation](https://start9.com/latest/user-manual) carefully and find out if the functionality is already covered, maybe by an individual configuration.
-- Perform a [search](https://github.com/Start9Labs/embassy-os/issues) to see if the enhancement has already been suggested. If it has, add a comment to the existing issue instead of opening a new one.
-- Find out whether your idea fits with the scope and aims of the project. It's up to you to make a strong case to convince the project's developers of the merits of this feature. Keep in mind that we want features that will be useful to the majority of our users and not just a small subset. If you're just targeting a minority of users, consider writing an add-on/plugin library.
+- Read the [documentation](https://start9.com/latest/user-manual) carefully and
+  find out if the functionality is already covered, maybe by an individual
+  configuration.
+- Perform a [search](https://github.com/Start9Labs/embassy-os/issues) to see if
+  the enhancement has already been suggested. If it has, add a comment to the
+  existing issue instead of opening a new one.
+- Find out whether your idea fits with the scope and aims of the project. It's
+  up to you to make a strong case to convince the project's developers of the
+  merits of this feature. Keep in mind that we want features that will be useful
+  to the majority of our users and not just a small subset. If you're just
+  targeting a minority of users, consider writing an add-on/plugin library.
 
 <!-- omit in toc -->
+
 #### How Do I Submit a Good Enhancement Suggestion?
 
-Enhancement suggestions are tracked as [GitHub issues](https://github.com/Start9Labs/embassy-os/issues).
+Enhancement suggestions are tracked as
+[GitHub issues](https://github.com/Start9Labs/embassy-os/issues).
 
-- Use a **clear and descriptive title** for the issue to identify the suggestion.
-- Provide a **step-by-step description of the suggested enhancement** in as many details as possible.
-- **Describe the current behavior** and **explain which behavior you expected to see instead** and why. At this point you can also tell which alternatives do not work for you.
-- You may want to **include screenshots and animated GIFs** which help you demonstrate the steps or point out the part which the suggestion is related to. You can use [this tool](https://www.cockos.com/licecap/) to record GIFs on macOS and Windows, and [this tool](https://github.com/colinkeenan/silentcast) or [this tool](https://github.com/GNOME/byzanz) on Linux. <!-- this should only be included if the project has a GUI -->
-- **Explain why this enhancement would be useful** to most Embassy OS users. You may also want to point out the other projects that solved it better and which could serve as inspiration.
+- Use a **clear and descriptive title** for the issue to identify the
+  suggestion.
+- Provide a **step-by-step description of the suggested enhancement** in as many
+  details as possible.
+- **Describe the current behavior** and **explain which behavior you expected to
+  see instead** and why. At this point you can also tell which alternatives do
+  not work for you.
+- You may want to **include screenshots and animated GIFs** which help you
+  demonstrate the steps or point out the part which the suggestion is related
+  to. You can use [this tool](https://www.cockos.com/licecap/) to record GIFs on
+  macOS and Windows, and [this tool](https://github.com/colinkeenan/silentcast)
+  or [this tool](https://github.com/GNOME/byzanz) on Linux.
+  <!-- this should only be included if the project has a GUI -->
+- **Explain why this enhancement would be useful** to most Embassy OS users. You
+  may also want to point out the other projects that solved it better and which
+  could serve as inspiration.
 
 <!-- You might want to create an issue template for enhancement suggestions that can be used as a guide and that defines the structure of the information to be included. If you do so, reference it here in the description. -->
 
 ### Project Structure
-EmbassyOS is composed of the following components. Please visit the README for each component to understand the dependency requirements and installation instructions.
-- [`ui`](ui/README.md) (Typescript Ionic Angular) is the code that is deployed to the browser to provide the user interface for EmbassyOS.
-- [`backend`] (backend/README.md) (Rust) is a command line utility, daemon, and software development kit that sets up and manages services and their environments, provides the interface for the ui, manages system state, and provides utilities for packaging services for EmbassyOS.
-- `patch-db` - A diff based data store that is used to synchronize data between the front and backend.
-  - Notably, `patch-db` has a [client](patch-db/client/README.md) with its own dependency and installation requirements.
-- `rpc-toolkit` - A library for generating an rpc server with cli bindings from Rust functions.
-- `system-images` - (Docker, Rust) A suite of utility Docker images that are preloaded with EmbassyOS to assist with functions relating to services (eg. configuration, backups, health checks).
-- [`setup-wizard`] (ui/README.md)- Code for the user interface that is displayed during the setup and recovery process for EmbassyOS.
-- [`diagnostic-ui`] (diagnostic-ui/README.md) - Code for the user interface that is displayed when something has gone wrong with starting up EmbassyOS, which provides helpful debugging tools.
+
+embassyOS is composed of the following components. Please visit the README for
+each component to understand the dependency requirements and installation
+instructions.
+
+- [`backend`](backend/README.md) (Rust) is a command line utility, daemon, and
+  software development kit that sets up and manages services and their
+  environments, provides the interface for the ui, manages system state, and
+  provides utilities for packaging services for embassyOS.
+- [`build`](build/README.md) contains scripts and necessary for deploying
+  embassyOS to a debian/raspbian system.
+- [`frontend`](frontend/README.md) (Typescript Ionic Angular) is the code that
+  is deployed to the browser to provide the user interface for embassyOS.
+  - `projects/ui` - Code for the user interface that is displayed when embassyOS
+    is running normally.
+  - `projects/setup-wizard`(frontend/README.md) - Code for the user interface
+    that is displayed during the setup and recovery process for embassyOS.
+  - `projects/diagnostic-ui` - Code for the user interface that is displayed
+    when something has gone wrong with starting up embassyOS, which provides
+    helpful debugging tools.
+- `libs` (Rust) is a set of standalone crates that were separated out of
+  `backend` for the purpose of portability
+- `patch-db` - A diff based data store that is used to synchronize data between
+  the front and backend.
+  - Notably, `patch-db` has a
+    [client](https://github.com/Start9Labs/patch-db/tree/master/client) with its
+    own dependency and installation requirements.
+- `system-images` - (Docker, Rust) A suite of utility Docker images that are
+  preloaded with embassyOS to assist with functions relating to services (eg.
+  configuration, backups, health checks).
+
 ### Your First Code Contribution
 
-#### Setting up your development environment
+#### Setting Up Your Development Environment
 
-First, clone the EmbassyOS repository and from the project root, pull in the submodules for dependent libraries.
+First, clone the embassyOS repository and from the project root, pull in the
+submodules for dependent libraries.
 
-```
+```sh
 git clone https://github.com/Start9Labs/embassy-os.git
 git submodule update --init --recursive
 ```
 
-Depending on which component of the ecosystem you are interested in contributing to, follow the installation requirements listed in that component's README (linked [above](#project-structure))
+Depending on which component of the ecosystem you are interested in contributing
+to, follow the installation requirements listed in that component's README
+(linked [above](#project-structure))
 
-#### Building The Image
-This step is for setting up an environment in which to test your code changes if you do not yet have a EmbassyOS.
+#### Building The Raspberry Pi Image
+
+This step is for setting up an environment in which to test your code changes if
+you do not yet have a embassyOS.
 
 - Requirements
   - `ext4fs` (available if running on the Linux kernel)
@@ -165,41 +262,79 @@ This step is for setting up an environment in which to test your code changes if
   - GNU Make
 - Building
   - see setup instructions [here](build/README.md)
-  - run `make` from the project root
+  - run `make embassyos-raspi.img ARCH=aarch64` from the project root
 
 ### Improving The Documentation
-You can find the repository for Start9's documentation [here](https://github.com/Start9Labs/documentation). If there is something you would like to see added, let us know, or create an issue yourself. Welcome are contributions for lacking or incorrect information, broken links, requested additions, or general style improvements.
 
-Contributions in the form of setup guides for integrations with external applications are highly encouraged. If you struggled through a process and would like to share your steps with others, check out the docs for each [service](https://github.com/Start9Labs/documentation/blob/master/source/user-manuals/available-services/index.rst) we support. The wrapper repos contain sections for adding integration guides, such as this [one](https://github.com/Start9Labs/bitcoind-wrapper/tree/master/docs). These not only help out others in the community, but inform how we can create a more seamless and intuitive experience.
+You can find the repository for Start9's documentation
+[here](https://github.com/Start9Labs/documentation). If there is something you
+would like to see added, let us know, or create an issue yourself. Welcome are
+contributions for lacking or incorrect information, broken links, requested
+additions, or general style improvements.
+
+Contributions in the form of setup guides for integrations with external
+applications are highly encouraged. If you struggled through a process and would
+like to share your steps with others, check out the docs for each
+[service](https://github.com/Start9Labs/documentation/blob/master/source/user-manuals/available-services/index.rst)
+we support. The wrapper repos contain sections for adding integration guides,
+such as this
+[one](https://github.com/Start9Labs/bitcoind-wrapper/tree/master/docs). These
+not only help out others in the community, but inform how we can create a more
+seamless and intuitive experience.
 
 ## Styleguides
+
 ### Formatting
-Each component of EmbassyOS contains its own style guide. Code must be formatted with the formatter designated for each component. These are outlined within each component folder's README.
+
+Each component of embassyOS contains its own style guide. Code must be formatted
+with the formatter designated for each component. These are outlined within each
+component folder's README.
 
 ### Atomic Commits
-Commits [should be atomic](https://en.wikipedia.org/wiki/Atomic_commit#Atomic_commit_convention) and diffs should be easy to read.
-Do not mix any formatting fixes or code moves with actual code changes.
+
+Commits
+[should be atomic](https://en.wikipedia.org/wiki/Atomic_commit#Atomic_commit_convention)
+and diffs should be easy to read. Do not mix any formatting fixes or code moves
+with actual code changes.
 
 ### Commit Messages
-If a commit touches only 1 component, prefix the message with the affected component. i.e. `backend: update to tokio v0.3`.
+
+If a commit touches only 1 component, prefix the message with the affected
+component. i.e. `backend: update to tokio v0.3`.
 
 ### Pull Requests
-The body of a pull request should contain sufficient description of what the changes do, as well as a justification.
-You should include references to any relevant [issues](https://github.com/Start9Labs/embassy-os/issues).
+
+The body of a pull request should contain sufficient description of what the
+changes do, as well as a justification. You should include references to any
+relevant [issues](https://github.com/Start9Labs/embassy-os/issues).
 
 ### Rebasing Changes
-When a pull request conflicts with the target branch, you may be asked to rebase it on top of the current target branch. The git rebase command will take care of rebuilding your commits on top of the new base.
 
-This project aims to have a clean git history, where code changes are only made in non-merge commits. This simplifies auditability because merge commits can be assumed to not contain arbitrary code changes.
+When a pull request conflicts with the target branch, you may be asked to rebase
+it on top of the current target branch. The `git rebase` command will take care
+of rebuilding your commits on top of the new base.
+
+This project aims to have a clean git history, where code changes are only made
+in non-merge commits. This simplifies auditability because merge commits can be
+assumed to not contain arbitrary code changes.
 
 ## Join The Discussion
-Current or aspiring contributors? Join our community developer [Matrix channel](https://matrix.to/#/#community-dev:matrix.start9labs.com).
 
-Just interested in or using the project? Join our community [Telegram](https://t.me/start9_labs) or [Matrix](https://matrix.to/#/#community:matrix.start9labs.com).
+Current or aspiring contributors? Join our community developer
+[Matrix channel](https://matrix.to/#/#community-dev:matrix.start9labs.com).
+
+Just interested in or using the project? Join our community
+[Telegram](https://t.me/start9_labs) or
+[Matrix](https://matrix.to/#/#community:matrix.start9labs.com).
 
 ## Join The Project Team
-Interested in becoming a part of the Start9 Labs team? Send an email to <jobs@start9labs.com>
+
+Interested in becoming a part of the Start9 Labs team? Send an email to
+<jobs@start9labs.com>
 
 <!-- omit in toc -->
+
 ## Attribution
-This guide is based on the **contributing-gen**. [Make your own](https://github.com/bttger/contributing-gen)!
+
+This guide is based on the **contributing-gen**.
+[Make your own](https://github.com/bttger/contributing-gen)!

LICENSE.md

@@ -1,25 +1,42 @@
-# START9 PERSONAL USE LICENSE v1.0
+# START9 NON-COMMERCIAL LICENSE v1
+Version 1, 22 September 2022
 
-This license governs the use of the accompanying Software. If you use the Software, you accept this license. If you do not accept the license, do not use the Software.
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
 
-1. **Definitions.**
-    1. “Licensor” means the copyright owner, Start9 Labs, Inc, or its successor(s) in interest, or a future assignee of the copyright.
-    2. “Source Code” means the preferred form of the Software for making modifications to it.
-    3. “Object Code” means any non-source form of the Software, including the machine-language output by a compiler or assembler.
-    4. “Distribute” means to convey or to publish and generally has the same meaning here as under U.S. Copyright law.
-    5. “Sell” means practicing any or all of the rights granted to you under the License to provide to third parties, for a fee or other consideration (including without limitation fees for hosting or consulting/support services related to the Software), a product or service whose value derives, entirely or substantially, from the functionality of the Software.
+### 1.Definitions
 
-2. **Grant of Rights.** Subject to the terms of this license, the Licensor grants you, the licensee, a non-exclusive, worldwide, royalty-free copyright license to:
-    1. Access, audit, copy, modify, compile, or distribute the Source Code or modifications to the Source Code.
-    2. Run, test, or otherwise use the Object Code.
+"License" means version 1 of the Start9 Non-Commercial License.
 
-3. **Limitations.**
-    1. The grant of rights under the License will NOT include, and the License does NOT grant you the right to:
-        1. Sell the Software or any derivative works based thereon.
-        2. Distribute the Object Code.
-    2. If you Distribute the Source Code, or if permission is separately granted to Distribute the Object Code, you expressly undertake not to remove, or modify, in any manner, the copyright notices attached to the Source Code, and displayed in any output of the Object Code when run, and to reproduce these notices, in an identical manner, in any distributed copies of the Software together with a copy of this license. If you Distribute a modified copy of the Software, or a derivative work based thereon, the work must carry prominent notices stating that you modified it, and giving a relevant date.
-    3. The terms of this license will apply to anyone who comes into possession of a copy of the Software, and any modifications or derivative works based thereon, made by anyone.
+"Licensor" means the Start9 Labs, Inc, or its successor(s) in interest, or a future assignee of the copyright.
 
-4. **Contributions.** You hereby grant to Licensor a perpetual, irrevocable, worldwide, non-exclusive, royalty-free license to use and exploit any modifications or derivative works based on the Source Code of which you are the author.
+"You" (or "Your") means an individual or organization exercising permissions granted by this License.
 
-5. **Disclaimer.** THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. LICENSOR HAS NO OBLIGATION TO SUPPORT RECIPIENTS OF THE SOFTWARE.
+"Source Code" for a work means the preferred form of the work for making modifications to it.
+
+"Object Code" means any non-source form of a work, including the machine-language output by a compiler or assembler.
+
+"Work" means any work of authorship, whether in Source or Object form, made available under this License.
+
+"Derivative Work" means any work, whether in Source or Object form, that is based on (or derived from) the Work.
+
+"Distribute" means to convey or to publish and generally has the same meaning here as under U.S. Copyright law.
+
+"Sell" means practicing any or all of the rights granted to you under the License to provide to third parties, for a fee or other consideration (including, without limitation, fees for hosting, consulting, or support services), a product or service whose value derives, entirely or substantially, from the functionality of the Work or Derivative Work.
+
+### 2. Grant of Rights
+
+Subject to the terms of this license, the Licensor grants you, the licensee, a non-exclusive, worldwide, royalty-free copyright license to access, audit, copy, modify, compile, run, test, distribute, or otherwise use the Software.
+
+### 3. Limitations
+
+1. The grant of rights under the License does NOT include, and the License does NOT grant You the right to Sell the Work or Derivative Work.
+2. If you Distribute the Work or Derivative Work, you expressly undertake not to remove or modify, in any manner, the copyright notices attached to the Work or displayed in any output of the Work when run, and to reproduce these notices, in an identical manner, in any distributed copies of the Work or Derivative Work together with a copy of this License.
+3. If you Distribute a Derivative Work, it must carry prominent notices stating that it has been modified from the Work, providing a relevant date.
+
+### 4. Contributions
+
+You hereby grant to Licensor a perpetual, irrevocable, worldwide, non-exclusive, royalty-free license to use and exploit any Derivative Work of which you are the author.
+
+### 5. Disclaimer
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. LICENSOR HAS NO OBLIGATION TO SUPPORT RECIPIENTS OF THE SOFTWARE.

Makefile

@@ -1,73 +1,178 @@
-EMBASSY_BINS := backend/target/aarch64-unknown-linux-gnu/release/embassyd backend/target/aarch64-unknown-linux-gnu/release/embassy-init backend/target/aarch64-unknown-linux-gnu/release/embassy-cli backend/target/aarch64-unknown-linux-gnu/release/embassy-sdk
-EMBASSY_UIS := frontend/dist/ui frontend/dist/setup-wizard frontend/dist/diagnostic-ui
-EMBASSY_SRC := raspios.img product_key.txt $(EMBASSY_BINS) backend/embassyd.service backend/embassy-init.service $(EMBASSY_UIS) $(shell find build)
-COMPAT_SRC := $(shell find system-images/compat/src)
-UTILS_SRC := $(shell find system-images/utils/Dockerfile)
-BACKEND_SRC := $(shell find backend/src) $(shell find patch-db/*/src) $(shell find rpc-toolkit/*/src) backend/Cargo.toml backend/Cargo.lock
-FRONTEND_SRC := $(shell find frontend/projects) $(shell find frontend/styles) $(shell find frontend/assets)
-PATCH_DB_CLIENT_SRC = $(shell find patch-db/client -not -path patch-db/client/dist)
-GIT_REFS := $(shell find .git/refs/heads)
-TMP_FILE := $(shell mktemp)
+RASPI_TARGETS := embassyos-raspi.img embassyos-raspi.tar.gz gzip lite-upgrade.img
+ARCH := $(shell if echo $(RASPI_TARGETS) | grep -qw "$(MAKECMDGOALS)"; then echo aarch64; else uname -m; fi)
+OS_ARCH := $(shell if echo $(RASPI_TARGETS) | grep -qw "$(MAKECMDGOALS)"; then echo raspberrypi; else uname -m; fi)
+ENVIRONMENT_FILE = $(shell ./check-environment.sh)
+GIT_HASH_FILE = $(shell ./check-git-hash.sh)
+VERSION_FILE = $(shell ./check-version.sh)
+EMBASSY_BINS := backend/target/$(ARCH)-unknown-linux-gnu/release/embassyd backend/target/$(ARCH)-unknown-linux-gnu/release/embassy-init backend/target/$(ARCH)-unknown-linux-gnu/release/embassy-cli backend/target/$(ARCH)-unknown-linux-gnu/release/embassy-sdk backend/target/$(ARCH)-unknown-linux-gnu/release/avahi-alias libs/target/aarch64-unknown-linux-musl/release/embassy_container_init libs/target/x86_64-unknown-linux-musl/release/embassy_container_init
+EMBASSY_UIS := frontend/dist/ui frontend/dist/setup-wizard frontend/dist/diagnostic-ui frontend/dist/install-wizard
+BUILD_SRC := $(shell find build)
+EMBASSY_SRC := backend/embassyd.service backend/embassy-init.service $(EMBASSY_UIS) $(BUILD_SRC)
+COMPAT_SRC := $(shell find system-images/compat/ -not -path 'system-images/compat/target/*' -and -not -name *.tar -and -not -name target)
+UTILS_SRC := $(shell find system-images/utils/ -not -name *.tar)
+BINFMT_SRC := $(shell find system-images/binfmt/ -not -name *.tar)
+BACKEND_SRC := $(shell find backend/src) $(shell find backend/migrations) $(shell find patch-db/*/src) $(shell find libs/*/src) libs/*/Cargo.toml backend/Cargo.toml backend/Cargo.lock
+FRONTEND_SHARED_SRC := $(shell find frontend/projects/shared) $(shell ls -p frontend/ | grep -v / | sed 's/^/frontend\//g') frontend/package.json frontend/node_modules frontend/config.json patch-db/client/dist frontend/patchdb-ui-seed.json
+FRONTEND_UI_SRC := $(shell find frontend/projects/ui)
+FRONTEND_SETUP_WIZARD_SRC := $(shell find frontend/projects/setup-wizard)
+FRONTEND_DIAGNOSTIC_UI_SRC := $(shell find frontend/projects/diagnostic-ui)
+FRONTEND_INSTALL_WIZARD_SRC := $(shell find frontend/projects/install-wizard)
+PATCH_DB_CLIENT_SRC := $(shell find patch-db/client -not -path patch-db/client/dist)
+GZIP_BIN := $(shell which pigz || which gzip)
+$(shell sudo true)
 
 .DELETE_ON_ERROR:
 
-all: eos.img
+.PHONY: all gzip install clean format sdk snapshots frontends ui backend
 
-gzip: eos.img
-	gzip -k eos.img
+all: $(EMBASSY_SRC) $(EMBASSY_BINS) system-images/compat/docker-images/aarch64.tar system-images/utils/docker-images/$(ARCH).tar system-images/binfmt/docker-images/$(ARCH).tar $(ENVIRONMENT_FILE) $(GIT_HASH_FILE) $(VERSION_FILE)
+
+gzip: embassyos-raspi.tar.gz
+
+embassyos-raspi.tar.gz: embassyos-raspi.img
+	tar --format=posix -cS -f- embassyos-raspi.img | $(GZIP_BIN) > embassyos-raspi.tar.gz
 
 clean:
-	rm -f eos.img
+	rm -f 2022-01-28-raspios-bullseye-arm64-lite.zip
+	rm -f raspios.img
+	rm -f embassyos-raspi.img
+	rm -f embassyos-raspi.tar.gz
 	rm -f ubuntu.img
 	rm -f product_key.txt
 	rm -f system-images/**/*.tar
-	sudo rm -f $(EMBASSY_BINS)
+	rm -rf system-images/compat/target
+	rm -rf backend/target
+	rm -rf frontend/.angular
+	rm -f frontend/config.json
 	rm -rf frontend/node_modules
 	rm -rf frontend/dist
+	rm -rf libs/target
 	rm -rf patch-db/client/node_modules
 	rm -rf patch-db/client/dist
+	rm -rf patch-db/target
+	rm -rf cargo-deps
+	rm ENVIRONMENT.txt
+	rm GIT_HASH.txt
+	rm VERSION.txt
 
-eos.img: $(EMBASSY_SRC) system-images/compat/compat.tar system-images/utils/utils.tar
-	! test -f eos.img || rm eos.img
-	if [ "$(NO_KEY)" = "1" ]; then NO_KEY=1 ./build/make-image.sh; else ./build/make-image.sh; fi
+format:
+	cd backend && cargo +nightly fmt
+	cd libs && cargo +nightly fmt
 
-system-images/compat/compat.tar: $(COMPAT_SRC)
-	cd system-images/compat && ./build.sh
-	cd system-images/compat && DOCKER_CLI_EXPERIMENTAL=enabled docker buildx build --tag start9/x_system/compat --platform=linux/arm64 -o type=docker,dest=compat.tar .
+sdk:
+	cd backend/ && ./install-sdk.sh
 
-system-images/utils/utils.tar: $(UTILS_SRC)
-	cd system-images/utils && DOCKER_CLI_EXPERIMENTAL=enabled docker buildx build --tag start9/x_system/utils --platform=linux/arm64 -o type=docker,dest=utils.tar .
+embassyos-raspi.img: all raspios.img cargo-deps/aarch64-unknown-linux-gnu/release/nc-broadcast
+	! test -f embassyos-raspi.img || rm embassyos-raspi.img
+	./build/raspberry-pi/make-image.sh
+
+lite-upgrade.img: raspios.img cargo-deps/aarch64-unknown-linux-gnu/release/nc-broadcast $(BUILD_SRC) eos.raspberrypi.squashfs
+	! test -f lite-upgrade.img || rm lite-upgrade.img
+	./build/raspberry-pi/make-upgrade-image.sh
+
+eos_raspberrypi.img: raspios.img $(BUILD_SRC) eos.raspberrypi.squashfs $(VERSION_FILE) $(ENVIRONMENT_FILE) $(GIT_HASH_FILE)
+	! test -f eos_raspberrypi.img || rm eos_raspberrypi.img
+	./build/raspberry-pi/make-initialized-image.sh
+
+# For creating os images. DO NOT USE
+install: all
+	mkdir -p $(DESTDIR)/usr/bin
+	cp backend/target/$(ARCH)-unknown-linux-gnu/release/embassy-init $(DESTDIR)/usr/bin/
+	cp backend/target/$(ARCH)-unknown-linux-gnu/release/embassyd $(DESTDIR)/usr/bin/
+	cp backend/target/$(ARCH)-unknown-linux-gnu/release/embassy-cli $(DESTDIR)/usr/bin/
+	cp backend/target/$(ARCH)-unknown-linux-gnu/release/avahi-alias $(DESTDIR)/usr/bin/
+	
+	mkdir -p $(DESTDIR)/usr/lib
+	rm -rf $(DESTDIR)/usr/lib/embassy
+	cp -r build/lib $(DESTDIR)/usr/lib/embassy
+
+	cp ENVIRONMENT.txt $(DESTDIR)/usr/lib/embassy/
+	cp GIT_HASH.txt $(DESTDIR)/usr/lib/embassy/
+	cp VERSION.txt $(DESTDIR)/usr/lib/embassy/
+
+	mkdir -p $(DESTDIR)/usr/lib/embassy/container
+	cp libs/target/aarch64-unknown-linux-musl/release/embassy_container_init $(DESTDIR)/usr/lib/embassy/container/embassy_container_init.arm64
+	cp libs/target/x86_64-unknown-linux-musl/release/embassy_container_init $(DESTDIR)/usr/lib/embassy/container/embassy_container_init.amd64
+
+	mkdir -p $(DESTDIR)/usr/lib/embassy/system-images
+	cp system-images/compat/docker-images/aarch64.tar $(DESTDIR)/usr/lib/embassy/system-images/compat.tar
+	cp system-images/utils/docker-images/$(ARCH).tar $(DESTDIR)/usr/lib/embassy/system-images/utils.tar
+	cp system-images/binfmt/docker-images/$(ARCH).tar $(DESTDIR)/usr/lib/embassy/system-images/binfmt.tar
+
+	mkdir -p $(DESTDIR)/var/www/html
+	cp -r frontend/dist/diagnostic-ui $(DESTDIR)/var/www/html/diagnostic
+	cp -r frontend/dist/setup-wizard $(DESTDIR)/var/www/html/setup
+	cp -r frontend/dist/install-wizard $(DESTDIR)/var/www/html/install
+	cp -r frontend/dist/ui $(DESTDIR)/var/www/html/main
+	cp index.html $(DESTDIR)/var/www/html/
+
+system-images/compat/docker-images/aarch64.tar: $(COMPAT_SRC)
+	cd system-images/compat && make
+
+system-images/utils/docker-images/aarch64.tar system-images/utils/docker-images/x86_64.tar: $(UTILS_SRC)
+	cd system-images/utils && make
+
+system-images/binfmt/docker-images/aarch64.tar system-images/binfmt/docker-images/x86_64.tar: $(BINFMT_SRC)
+	cd system-images/binfmt && make
 
 raspios.img:
-	wget https://downloads.raspberrypi.org/raspios_lite_arm64/images/raspios_lite_arm64-2022-01-28/2022-01-28-raspios-bullseye-arm64-lite.zip
+	wget --continue https://downloads.raspberrypi.org/raspios_lite_arm64/images/raspios_lite_arm64-2022-01-28/2022-01-28-raspios-bullseye-arm64-lite.zip
 	unzip 2022-01-28-raspios-bullseye-arm64-lite.zip
 	mv 2022-01-28-raspios-bullseye-arm64-lite.img raspios.img
 
-product_key.txt:
-	$(shell which echo) -n "X" > product_key.txt
-	cat /dev/urandom | base32 | head -c11 | tr '[:upper:]' '[:lower:]' >> product_key.txt
-	if [ "$(KEY)" != "" ]; then $(shell which echo) -n "$(KEY)" > product_key.txt; fi
-	echo >> product_key.txt
+snapshots: libs/snapshot_creator/Cargo.toml
+	cd libs/  && ./build-v8-snapshot.sh
+	cd libs/  && ./build-arm-v8-snapshot.sh
 
-$(EMBASSY_BINS): $(BACKEND_SRC)
-	cd backend && ./build-prod.sh
+$(EMBASSY_BINS): $(BACKEND_SRC) $(ENVIRONMENT_FILE) $(GIT_HASH_FILE) frontend/patchdb-ui-seed.json
+	cd backend && ARCH=$(ARCH) ./build-prod.sh
+	touch $(EMBASSY_BINS)
 
 frontend/node_modules: frontend/package.json
 	npm --prefix frontend ci
 
-$(EMBASSY_UIS): $(FRONTEND_SRC) frontend/node_modules patch-db/client patch-db/client/dist frontend/config.json
-	npm --prefix frontend run build:all
+frontend/dist/ui: $(FRONTEND_UI_SRC) $(FRONTEND_SHARED_SRC) $(ENVIRONMENT_FILE)
+	npm --prefix frontend run build:ui
 
-frontend/config.json: .git/HEAD $(GIT_REFS)
+frontend/dist/setup-wizard: $(FRONTEND_SETUP_WIZARD_SRC) $(FRONTEND_SHARED_SRC) $(ENVIRONMENT_FILE)
+	npm --prefix frontend run build:setup
+
+frontend/dist/diagnostic-ui: $(FRONTEND_DIAGNOSTIC_UI_SRC) $(FRONTEND_SHARED_SRC) $(ENVIRONMENT_FILE)
+	npm --prefix frontend run build:dui
+
+frontend/dist/install-wizard: $(FRONTEND_INSTALL_WIZARD_SRC) $(FRONTEND_SHARED_SRC) $(ENVIRONMENT_FILE)
+	npm --prefix frontend run build:install-wiz
+
+frontend/config.json: $(GIT_HASH_FILE) frontend/config-sample.json
 	jq '.useMocks = false' frontend/config-sample.json > frontend/config.json
+	jq '.packageArch = "$(ARCH)"' frontend/config.json > frontend/config.json.tmp
+	jq '.osArch = "$(OS_ARCH)"' frontend/config.json.tmp > frontend/config.json
+	rm frontend/config.json.tmp
 	npm --prefix frontend run-script build-config
 
+frontend/patchdb-ui-seed.json: frontend/package.json
+	jq '."ack-welcome" = "$(shell yq '.version' frontend/package.json)"' frontend/patchdb-ui-seed.json > ui-seed.tmp
+	mv ui-seed.tmp frontend/patchdb-ui-seed.json
+
 patch-db/client/node_modules: patch-db/client/package.json
-	npm --prefix patch-db/client install
+	npm --prefix patch-db/client ci
 
 patch-db/client/dist: $(PATCH_DB_CLIENT_SRC) patch-db/client/node_modules
 	! test -d patch-db/client/dist || rm -rf patch-db/client/dist
-	npm --prefix patch-db/client run build
+	npm --prefix frontend run build:deps
+
+# used by github actions
+backend-$(ARCH).tar: $(EMBASSY_BINS)
+	tar -cvf $@ $^
 
 # this is a convenience step to build all frontends - it is not referenced elsewhere in this file
-frontend: frontend/node_modules $(EMBASSY_UIS) 
+frontends: $(EMBASSY_UIS) 
+
+# this is a convenience step to build the UI
+ui: frontend/dist/ui
+
+# used by github actions
+backend: $(EMBASSY_BINS)
+
+cargo-deps/aarch64-unknown-linux-gnu/release/nc-broadcast:
+	./build-cargo-dep.sh nc-broadcast

README.md

@@ -1,20 +1,21 @@
-# EmbassyOS
-[![Version](https://img.shields.io/github/v/tag/Start9Labs/embassy-os?color=success)](https://github.com/Start9Labs/embassy-os/releases)
+# embassyOS
+[![version](https://img.shields.io/github/v/tag/Start9Labs/embassy-os?color=success)](https://github.com/Start9Labs/embassy-os/releases)
+[![build](https://github.com/Start9Labs/embassy-os/actions/workflows/product.yaml/badge.svg)](https://github.com/Start9Labs/embassy-os/actions/workflows/product.yaml)
 [![community](https://img.shields.io/badge/community-matrix-yellow)](https://matrix.to/#/#community:matrix.start9labs.com)
 [![community](https://img.shields.io/badge/community-telegram-informational)](https://t.me/start9_labs)
-[![support](https://img.shields.io/badge/support-docs-important)](https://docs.start9labs.com)
+[![support](https://img.shields.io/badge/support-docs-important)](https://docs.start9.com)
 [![developer](https://img.shields.io/badge/developer-matrix-blueviolet)](https://matrix.to/#/#community-dev:matrix.start9labs.com)
-[![website](https://img.shields.io/website?down_color=lightgrey&down_message=offline&up_color=green&up_message=online&url=https%3A%2F%2Fstart9labs.com)](https://start9labs.com)
+[![website](https://img.shields.io/website?down_color=lightgrey&down_message=offline&up_color=green&up_message=online&url=https%3A%2F%2Fstart9.com)](https://start9.com)
 
 [![mastodon](https://img.shields.io/mastodon/follow/000000001?domain=https%3A%2F%2Fmastodon.start9labs.com&label=Follow&style=social)](http://mastodon.start9labs.com)
 [![twitter](https://img.shields.io/twitter/follow/start9labs?label=Follow)](https://twitter.com/start9labs)
 
 ### _Welcome to the era of Sovereign Computing_ ###
 
-EmbassyOS is a browser-based, graphical operating system for a personal server. EmbassyOS facilitates the discovery, installation, network configuration, service configuration, data backup, dependency management, and health monitoring of self-hosted software services. It is the most advanced, secure, reliable, and user friendly personal server OS in the world.
+embassyOS is a browser-based, graphical operating system for a personal server. embassyOS facilitates the discovery, installation, network configuration, service configuration, data backup, dependency management, and health monitoring of self-hosted software services. It is the most advanced, secure, reliable, and user friendly personal server OS in the world.
 
-## Running EmbassyOS
-There are multiple ways to get your hands on EmbassyOS.
+## Running embassyOS
+There are multiple ways to get your hands on embassyOS.
 
 ### :moneybag: Buy an Embassy
 This is the most convenient option. Simply [buy an Embassy](https://start9.com) from Start9 and plug it in. Depending on where you live, shipping costs and import duties will vary.
@@ -28,22 +29,23 @@ While not as convenient as buying an Embassy, this option is easier than you mig
 
 To pursue this option, follow this [guide](https://start9.com/latest/diy).
 
-### :hammer_and_wrench: Build EmbassyOS from Source
+### :hammer_and_wrench: Build embassyOS from Source
 
-EmbassyOS can be built from source, for personal use, for free.
+embassyOS can be built from source, for personal use, for free.
 A detailed guide for doing so can be found [here](https://github.com/Start9Labs/embassy-os/blob/master/build/README.md).
 
 ## :heart: Contributing
-There are multiple ways to contribute: work directly on EmbassyOS, package a service for the marketplace, or help with documentation and guides. To learn more about contributing, see [here](https://github.com/Start9Labs/embassy-os/blob/master/CONTRIBUTING.md).
+There are multiple ways to contribute: work directly on embassyOS, package a service for the marketplace, or help with documentation and guides. To learn more about contributing, see [here](https://github.com/Start9Labs/embassy-os/blob/master/CONTRIBUTING.md).
 
 ## UI Screenshots
 <p align="center">
-<img src="assets/EmbassyOS.png" alt="EmbassyOS" width="65%">
+<img src="assets/embassyOS.png" alt="embassyOS" width="85%">
 </p>
 <p align="center">
-<img src="assets/eos-services.png" alt="Embassy Services" width="45%">
-<img src="assets/eos-preferences.png" alt="Embassy Preferences" width="45%">
+<img src="assets/eOS-preferences.png" alt="Embassy Preferences" width="49%">
+<img src="assets/eOS-ghost.png" alt="Embassy Ghost Service" width="49%">
 </p>
 <p align="center">
-<img src="assets/eos-bitcoind-health-check.png" alt="Embassy Bitcoin Health Checks" width="45%"> <img src="assets/eos-logs.png" alt="Embassy Logs" width="45%">
+<img src="assets/eOS-synapse-health-check.png" alt="Embassy Synapse Health Checks" width="49%">
+<img src="assets/eOS-sideload.png" alt="Embassy Sideload Service" width="49%">
 </p>

backend/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Aiden McClelland <me@drbonez.dev>"]
 description = "The core of the Start9 Embassy Operating System"
 documentation = "https://docs.rs/embassy-os"
-edition = "2018"
+edition = "2021"
 keywords = [
   "self-hosted",
   "raspberry-pi",
@@ -14,7 +14,7 @@ keywords = [
 name = "embassy-os"
 readme = "README.md"
 repository = "https://github.com/Start9Labs/embassy-os"
-version = "0.3.0"
+version = "0.3.3"
 
 [lib]
 name = "embassy"
@@ -36,103 +36,127 @@ path = "src/bin/embassy-sdk.rs"
 name = "embassy-cli"
 path = "src/bin/embassy-cli.rs"
 
+[[bin]]
+name = "avahi-alias"
+path = "src/bin/avahi-alias.rs"
+
 [features]
 avahi = ["avahi-sys"]
-beta = []
-default = ["avahi", "sound", "metal"]
-metal = []
-sound = []
+default = ["avahi", "js_engine"]
+dev = []
 unstable = ["patch-db/unstable"]
 
 [dependencies]
 aes = { version = "0.7.5", features = ["ctr"] }
-async-trait = "0.1.51"
+async-stream = "0.3.3"
+async-trait = "0.1.56"
 avahi-sys = { git = "https://github.com/Start9Labs/avahi-sys", version = "0.10.0", branch = "feature/dynamic-linking", features = [
   "dynamic",
 ], optional = true }
 base32 = "0.4.0"
+base64 = "0.13.0"
+base64ct = "1.5.1"
 basic-cookies = "0.1.4"
-bollard = "0.11.0"
+bollard = "0.13.0"
+bytes = "1"
 chrono = { version = "0.4.19", features = ["serde"] }
-clap = "2.33"
-color-eyre = "0.5"
-cookie_store = "0.15.0"
-digest = "0.9.0"
+clap = "3.2.8"
+color-eyre = "0.6.1"
+cookie_store = "0.16.1"
+current_platform = "0.2.0"
+digest = "0.10.3"
+digest-old = { package = "digest", version = "0.9.0" }
 divrem = "1.0.0"
+ed25519 = { version = "1.5.2", features = ["pkcs8", "pem", "alloc"] }
 ed25519-dalek = { version = "1.0.1", features = ["serde"] }
-emver = { version = "0.1.6", features = ["serde"] }
-fd-lock-rs = "0.1.3"
-futures = "0.3.17"
+emver = { version = "0.1.7", git = "https://github.com/Start9Labs/emver-rs.git", features = [
+  "serde",
+] }
+fd-lock-rs = "0.1.4"
+futures = "0.3.21"
 git-version = "0.3.5"
+helpers = { path = "../libs/helpers" }
+embassy_container_init = { path = "../libs/embassy_container_init" }
 hex = "0.4.3"
-hmac = "0.11.0"
-http = "0.2.5"
-hyper = "0.14.13"
-hyper-ws-listener = { git = "https://github.com/Start9Labs/hyper-ws-listener.git", branch = "main" }
-imbl = "1.0.1"
-indexmap = { version = "1.7.0", features = ["serde"] }
+hmac = "0.12.1"
+http = "0.2.8"
+hyper = { version = "0.14.20", features = ["full"] }
+hyper-ws-listener = "0.2.0"
+imbl = "2.0.0"
+indexmap = { version = "1.9.1", features = ["serde"] }
 isocountry = "0.3.2"
-itertools = "0.10.1"
+itertools = "0.10.3"
+josekit = "0.8.1"
+js_engine = { path = '../libs/js_engine', optional = true }
 jsonpath_lib = "0.3.0"
-lazy_static = "1.4"
-libc = "0.2.103"
-log = "0.4.14"
-nix = "0.23.0"
-nom = "7.0.0"
+lazy_static = "1.4.0"
+libc = "0.2.126"
+log = "0.4.17"
+mbrman = "0.5.0"
+models = { version = "*", path = "../libs/models" }
+nix = "0.25.0"
+nom = "7.1.1"
 num = "0.4.0"
-num_enum = "0.5.4"
+num_enum = "0.5.7"
 openssh-keys = "0.5.0"
-openssl = { version = "0.10.36", features = ["vendored"] }
+openssl = { version = "0.10.41", features = ["vendored"] }
 patch-db = { version = "*", path = "../patch-db/patch-db", features = [
   "trace",
 ] }
-pbkdf2 = "0.9.0"
-pin-project = "1.0.8"
-platforms = "1.1.0"
-prettytable-rs = "0.8.0"
+pbkdf2 = "0.11.0"
+pin-project = "1.0.11"
+pkcs8 = { version = "0.9.0", features = ["std"] }
+prettytable-rs = "0.9.0"
 proptest = "1.0.0"
 proptest-derive = "0.3.0"
-rand = "0.7.3"
-regex = "1.5.4"
-reqwest = { version = "0.11.4", features = ["stream", "json", "socks"] }
-reqwest_cookie_store = "0.2.0"
-rpassword = "5.0.1"
-rpc-toolkit = { version = "*", path = "../rpc-toolkit/rpc-toolkit" }
-rust-argon2 = "0.8.3"
+rand = { version = "0.8.5", features = ["std"] }
+rand-old = { package = "rand", version = "0.7.3" }
+regex = "1.6.0"
+reqwest = { version = "0.11.11", features = ["stream", "json", "socks"] }
+reqwest_cookie_store = "0.4.0"
+rpassword = "7.0.0"
+rpc-toolkit = "0.2.2"
+rust-argon2 = "1.0.0"
 scopeguard = "1.1" # because avahi-sys fucks your shit up
-serde = { version = "1.0.130", features = ["derive", "rc"] }
+serde = { version = "1.0.139", features = ["derive", "rc"] }
 serde_cbor = { package = "ciborium", version = "0.2.0" }
-serde_json = "1.0.68"
-serde_toml = { package = "toml", version = "0.5.8" }
-serde_yaml = "0.8.21"
-sha2 = "0.9.8"
-simple-logging = "2.0"
-sqlx = { version = "0.5", features = [
+serde_json = "1.0.82"
+serde_toml = { package = "toml", version = "0.5.9" }
+serde_with = { version = "2.0.1", features = ["macros", "json"] }
+serde_yaml = "0.9.11"
+sha2 = "0.10.2"
+sha2-old = { package = "sha2", version = "0.9.9" }
+simple-logging = "2.0.2"
+sqlx = { version = "0.6.0", features = [
   "chrono",
   "offline",
   "runtime-tokio-rustls",
-  "sqlite",
+  "postgres",
 ] }
-stderrlog = "0.5.1"
-tar = "0.4.37"
-thiserror = "1.0.29"
-tokio = { version = "1.15.0", features = ["full"] }
-tokio-compat-02 = "0.2.0"
-tokio-stream = { version = "0.1.7", features = ["io-util", "sync"] }
+stderrlog = "0.5.3"
+tar = "0.4.38"
+thiserror = "1.0.31"
+tokio = { version = "1.21.2", features = ["full"] }
+tokio-stream = { version = "0.1.9", features = ["io-util", "sync"] }
 tokio-tar = { git = "https://github.com/dr-bonez/tokio-tar.git" }
-tokio-tungstenite = "0.14.0"
-tokio-util = { version = "0.6.8", features = ["io"] }
-torut = "0.2.0"
-tracing = "0.1"
-tracing-error = "0.1"
-tracing-futures = "0.2"
-tracing-subscriber = "0.2"
-typed-builder = "0.9.1"
+tokio-tungstenite = { version = "0.17.1", features = ["native-tls"] }
+tokio-rustls = "0.23.4"
+tokio-util = { version = "0.7.3", features = ["io"] }
+torut = "0.2.1"
+tracing = "0.1.35"
+tracing-error = "0.2.0"
+tracing-futures = "0.2.5"
+tracing-subscriber = { version = "0.3.14", features = ["env-filter"] }
+trust-dns-server = "0.22.0"
+typed-builder = "0.10.0"
 url = { version = "2.2.2", features = ["serde"] }
+uuid = { version = "1.1.2", features = ["v4"] }
 
-[dependencies.serde_with]
-features = ["macros", "json"]
-version = "1.10.0"
+[profile.test]
+opt-level = 3
 
 [profile.dev.package.backtrace]
 opt-level = 3
+
+[profile.dev.package.sqlx-macros]
+opt-level = 3

backend/README.md

@@ -1,4 +1,4 @@
-# EmbassyOS Backend
+# embassyOS Backend
 
 - Requirements:
   - [Install Rust](https://rustup.rs)
@@ -6,18 +6,22 @@
   - [Docker](https://docs.docker.com/get-docker/)
   - [Rust ARM64 Build Container](https://github.com/Start9Labs/rust-arm-builder)
 - Scripts (run withing the `./backend` directory)
-  - `build-prod.sh` - compiles a release build of the artifacts for running on ARM64
-  - `build-dev.sh` - compiles a development build of the artifacts for running on ARM64
+  - `build-prod.sh` - compiles a release build of the artifacts for running on
+    ARM64
 - A Linux computer or VM
 
 ## Structure
 
-The EmbassyOS backend is broken up into 4 different binaries:
+The embassyOS backend is broken up into 4 different binaries:
 
-- embassyd: This is the main workhorse of EmbassyOS - any new functionality you want will likely go here
-- embassy-init: This is the component responsible for allowing you to set up your device, and handles system initialization on startup
-- embassy-cli: This is a CLI tool that will allow you to issue commands to embassyd and control it similarly to the UI
-- embassy-sdk: This is a CLI tool that aids in building and packaging services you wish to deploy to the Embassy
+- embassyd: This is the main workhorse of embassyOS - any new functionality you
+  want will likely go here
+- embassy-init: This is the component responsible for allowing you to set up
+  your device, and handles system initialization on startup
+- embassy-cli: This is a CLI tool that will allow you to issue commands to
+  embassyd and control it similarly to the UI
+- embassy-sdk: This is a CLI tool that aids in building and packaging services
+  you wish to deploy to the Embassy
 
 Finally there is a library `embassy` that supports all four of these tools.
 
@@ -25,11 +29,14 @@ See [here](/backend/Cargo.toml) for details.
 
 ## Building
 
-You can build the entire operating system image using `make` from the root of the EmbassyOS project. This will subsequently invoke the build scripts above to actually create the requisite binaries and put them onto the final operating system image.
+You can build the entire operating system image using `make` from the root of
+the embassyOS project. This will subsequently invoke the build scripts above to
+actually create the requisite binaries and put them onto the final operating
+system image.
 
 ## Questions
 
-If you have questions about how various pieces of the backend system work. Open an issue and tag the following people
+If you have questions about how various pieces of the backend system work. Open
+an issue and tag the following people
 
 - dr-bonez
-- ProofOfKeags

backend/build-dev.sh

@@ -8,9 +8,17 @@ if [ "$0" != "./build-dev.sh" ]; then
 	exit 1
 fi
 
-alias 'rust-arm64-builder'='docker run --rm -it -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$(pwd)":/home/rust/src start9/rust-arm-cross:aarch64'
+USE_TTY=
+if tty -s; then
+	USE_TTY="-it"
+fi
+
+alias 'rust-arm64-builder'='docker run $USE_TTY --rm -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$(pwd)":/home/rust/src start9/rust-arm-cross:aarch64'
 
 cd ..
-rust-arm64-builder sh -c "(cd backend && cargo build)"
+rust-arm64-builder sh -c "(cd backend && cargo build --locked)"
 cd backend
+
+sudo chown -R $USER target
+sudo chown -R $USER ~/.cargo
 #rust-arm64-builder aarch64-linux-gnu-strip target/aarch64-unknown-linux-gnu/release/embassyd

backend/build-portable-dev.sh

@@ -8,8 +8,16 @@ if [ "$0" != "./build-portable-dev.sh" ]; then
 	exit 1
 fi
 
-alias 'rust-musl-builder'='docker run --rm -it -v "$HOME"/.cargo/registry:/root/.cargo/registry -v "$(pwd)":/home/rust/src start9/rust-musl-cross:x86_64-musl'
+USE_TTY=
+if tty -s; then
+	USE_TTY="-it"
+fi
+
+alias 'rust-musl-builder'='docker run $USE_TTY --rm -v "$HOME"/.cargo/registry:/root/.cargo/registry -v "$(pwd)":/home/rust/src start9/rust-musl-cross:x86_64-musl'
 
 cd ..
-rust-musl-builder sh -c "(cd backend && cargo +beta build --target=x86_64-unknown-linux-musl --no-default-features)"
+rust-musl-builder sh -c "(cd backend && cargo +beta build --target=x86_64-unknown-linux-musl --no-default-features --locked)"
 cd backend
+
+sudo chown -R $USER target
+sudo chown -R $USER ~/.cargo

backend/build-portable.sh

@@ -8,8 +8,16 @@ if [ "$0" != "./build-portable.sh" ]; then
 	exit 1
 fi
 
-alias 'rust-musl-builder'='docker run --rm -it -v "$HOME"/.cargo/registry:/root/.cargo/registry -v "$(pwd)":/home/rust/src start9/rust-musl-cross:x86_64-musl'
+USE_TTY=
+if tty -s; then
+	USE_TTY="-it"
+fi
+
+alias 'rust-musl-builder'='docker run $USE_TTY --rm -v "$HOME"/.cargo/registry:/root/.cargo/registry -v "$(pwd)":/home/rust/src start9/rust-musl-cross:x86_64-musl'
 
 cd ..
-rust-musl-builder sh -c "(cd backend && cargo +beta build --release --target=x86_64-unknown-linux-musl --no-default-features)"
+rust-musl-builder sh -c "(cd backend && cargo +beta build --release --target=x86_64-unknown-linux-musl --no-default-features --locked)"
 cd backend
+
+sudo chown -R $USER target
+sudo chown -R $USER ~/.cargo

backend/build-prod.sh

@@ -3,26 +3,69 @@
 set -e
 shopt -s expand_aliases
 
+if [ -z "$ARCH" ]; then
+	ARCH=$(uname -m)
+fi
+
 if [ "$0" != "./build-prod.sh" ]; then
 	>&2 echo "Must be run from backend directory"
 	exit 1
 fi
 
-alias 'rust-arm64-builder'='docker run --rm -it -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$(pwd)":/home/rust/src start9/rust-arm-cross:aarch64'
+USE_TTY=
+if tty -s; then
+	USE_TTY="-it"
+fi
+
+alias 'rust-gnu-builder'='docker run $USE_TTY --rm -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$(pwd)":/home/rust/src -P start9/rust-arm-cross:aarch64'
+alias 'rust-musl-builder'='docker run $USE_TTY --rm -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$(pwd)":/home/rust/src -P messense/rust-musl-cross:$ARCH-musl'
 
 cd ..
+FLAGS=""
 if [[ "$ENVIRONMENT" =~ (^|-)unstable($|-) ]]; then
-	if [[ "$ENVIRONMENT" =~ (^|-)beta($|-) ]]; then
-		rust-arm64-builder sh -c "(cd backend && cargo build --release --features beta,unstable)"
-	else
-		rust-arm64-builder sh -c "(cd backend && cargo build --release --features unstable)"
-	fi
-else
-	if [[ "$ENVIRONMENT" =~ (^|-)beta($|-) ]]; then
-		rust-arm64-builder sh -c "(cd backend && cargo build --release --features beta)"
-	else
-		rust-arm64-builder sh -c "(cd backend && cargo build --release)"
-	fi
+	FLAGS="unstable,$FLAGS"
 fi
+if [[ "$ENVIRONMENT" =~ (^|-)dev($|-) ]]; then
+	FLAGS="dev,$FLAGS"
+fi
+
+set +e
+fail=
+if [[ "$FLAGS" = "" ]]; then
+	rust-gnu-builder sh -c "(git config --global --add safe.directory '*'; cd backend && cargo build --release --locked  --target=$ARCH-unknown-linux-gnu)"
+	if test $? -ne 0; then 
+		fail=true
+	fi
+	for ARCH in x86_64 aarch64
+	do
+		rust-musl-builder sh -c "(git config --global --add safe.directory '*'; cd libs && cargo build --release --locked --bin embassy_container_init )"
+		if test $? -ne 0; then 
+			fail=true
+		fi
+	done
+else
+	echo "FLAGS=$FLAGS"
+	rust-gnu-builder sh -c "(git config --global --add safe.directory '*'; cd backend && cargo build --release --features $FLAGS --locked --target=$ARCH-unknown-linux-gnu)"
+	if test $? -ne 0; then 
+		fail=true
+	fi
+	for ARCH in x86_64 aarch64
+	do
+		rust-musl-builder sh -c "(git config --global --add safe.directory '*'; cd libs && cargo build --release --features $FLAGS --locked --bin embassy_container_init)"
+		if test $? -ne 0; then 
+			fail=true
+		fi
+	done
+fi
+set -e
 cd backend
+
+sudo chown -R $USER target
+sudo chown -R $USER ~/.cargo
+sudo chown -R $USER ../libs/target
+
+if [ -n "$fail" ]; then
+	exit 1
+fi
+
 #rust-arm64-builder aarch64-linux-gnu-strip target/aarch64-unknown-linux-gnu/release/embassyd

backend/copy.sh

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-# Enter the backend directory, copy over the built EmbassyOS binaries and systemd services, edit the nginx config, then create the .ssh directory
-
-cp target/aarch64-unknown-linux-gnu/release/embassy-init /mnt/usr/local/bin
-cp target/aarch64-unknown-linux-gnu/release/embassyd /mnt/usr/local/bin
-cp target/aarch64-unknown-linux-gnu/release/embassy-cli /mnt/usr/local/bin
-cp *.service /mnt/etc/systemd/system/
-
-echo "application/wasm		wasm;" | sudo tee -a "/mnt/etc/nginx/mime.types"
-
-mkdir -p /mnt/root/.ssh

backend/embassy-init.service

@@ -1,14 +1,15 @@
 [Unit]
 Description=Embassy Init
-After=network.target ntp.service
-Requires=network.target
-Wants=avahi-daemon.service nginx.service tor.service ntp.service
+After=network-online.target
+Requires=network-online.target
+Wants=avahi-daemon.service tor.service
 
 [Service]
 Type=oneshot
-Environment=RUST_LOG=embassy_init=debug,embassy=debug
-ExecStart=/usr/local/bin/embassy-init
+Environment=RUST_LOG=embassy_init=debug,embassy=debug,js_engine=debug,patch_db=warn
+ExecStart=/usr/bin/embassy-init
 RemainAfterExit=true
+StandardOutput=append:/var/log/embassy-init.log
 
 [Install]
 WantedBy=embassyd.service

backend/embassyd.service

@@ -5,10 +5,13 @@ Requires=embassy-init.service
 
 [Service]
 Type=simple
-Environment=RUST_LOG=embassyd=debug,embassy=debug
-ExecStart=/usr/local/bin/embassyd
+Environment=RUST_LOG=embassyd=debug,embassy=debug,js_engine=debug,patch_db=warn
+ExecStart=/usr/bin/embassyd
 Restart=always
 RestartSec=3
+ManagedOOMPreference=avoid
+CPUAccounting=true
+CPUWeight=1000
 
 [Install]
 WantedBy=multi-user.target

backend/install-sdk.sh

@@ -8,4 +8,4 @@ if [ "$0" != "./install-sdk.sh" ]; then
 	exit 1
 fi
 
-cargo install --bin=embassy-sdk --path=. --no-default-features
+cargo install --bin=embassy-sdk --bin=embassy-cli --path=. --no-default-features --features=js_engine --locked

backend/migrations/20210629193146_Init.sql

@@ -1,45 +1,47 @@
 -- Add migration script here
-CREATE TABLE IF NOT EXISTS tor
-(
-    package     TEXT NOT NULL,
-    interface   TEXT NOT NULL,
-    key         BLOB NOT NULL CHECK (length(key) = 64),
+CREATE TABLE IF NOT EXISTS tor (
+    package TEXT NOT NULL,
+    interface TEXT NOT NULL,
+    key BYTEA NOT NULL CHECK (length(key) = 64),
     PRIMARY KEY (package, interface)
 );
-CREATE TABLE IF NOT EXISTS session
-(
-    id         TEXT NOT NULL PRIMARY KEY,
+
+CREATE TABLE IF NOT EXISTS session (
+    id TEXT NOT NULL PRIMARY KEY,
     logged_in TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
     logged_out TIMESTAMP,
     last_active TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
     user_agent TEXT,
-    metadata   TEXT NOT NULL DEFAULT 'null'
+    metadata TEXT NOT NULL DEFAULT 'null'
 );
-CREATE TABLE IF NOT EXISTS account
-(
-    id INTEGER PRIMARY KEY CHECK (id = 0),
+
+CREATE TABLE IF NOT EXISTS account (
+    id SERIAL PRIMARY KEY CHECK (id = 0),
     password TEXT NOT NULL,
-    tor_key BLOB NOT NULL CHECK (length(tor_key) = 64)
+    tor_key BYTEA NOT NULL CHECK (length(tor_key) = 64)
 );
-CREATE TABLE IF NOT EXISTS ssh_keys
-(
-    fingerprint     TEXT NOT NULL,
-    openssh_pubkey  TEXT NOT NULL,
-    created_at      TEXT NOT NULL,
+
+CREATE TABLE IF NOT EXISTS ssh_keys (
+    fingerprint TEXT NOT NULL,
+    openssh_pubkey TEXT NOT NULL,
+    created_at TEXT NOT NULL,
     PRIMARY KEY (fingerprint)
 );
-CREATE TABLE IF NOT EXISTS certificates
-(
-    id INTEGER PRIMARY KEY, -- Root = 0, Int = 1, Other = 2..
+
+CREATE TABLE IF NOT EXISTS certificates (
+    id SERIAL PRIMARY KEY,
+    -- Root = 0, Int = 1, Other = 2..
     priv_key_pem TEXT NOT NULL,
     certificate_pem TEXT NOT NULL,
     lookup_string TEXT UNIQUE,
     created_at TEXT,
     updated_at TEXT
 );
-CREATE TABLE IF NOT EXISTS notifications
-(
-    id INTEGER PRIMARY KEY,
+
+ALTER SEQUENCE certificates_id_seq START 2 RESTART 2;
+
+CREATE TABLE IF NOT EXISTS notifications (
+    id SERIAL PRIMARY KEY,
     package_id TEXT,
     created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
     code INTEGER NOT NULL,
@@ -48,9 +50,9 @@ CREATE TABLE IF NOT EXISTS notifications
     message TEXT NOT NULL,
     data TEXT
 );
-CREATE TABLE IF NOT EXISTS cifs_shares
-(
-    id INTEGER PRIMARY KEY,
+
+CREATE TABLE IF NOT EXISTS cifs_shares (
+    id SERIAL PRIMARY KEY,
     hostname TEXT NOT NULL,
     path TEXT NOT NULL,
     username TEXT NOT NULL,

backend/src/action.rs

@@ -1,76 +1,23 @@
 use std::collections::{BTreeMap, BTreeSet};
-use std::path::Path;
-use std::str::FromStr;
-use std::time::Duration;
 
 use clap::ArgMatches;
 use color_eyre::eyre::eyre;
 use indexmap::IndexSet;
-use patch_db::HasModel;
+pub use models::ActionId;
 use rpc_toolkit::command;
 use serde::{Deserialize, Serialize};
 use tracing::instrument;
 
-use self::docker::DockerAction;
 use crate::config::{Config, ConfigSpec};
 use crate::context::RpcContext;
-use crate::id::{Id, ImageId, InvalidId};
+use crate::id::ImageId;
+use crate::procedure::docker::DockerContainers;
+use crate::procedure::{PackageProcedure, ProcedureName};
 use crate::s9pk::manifest::PackageId;
 use crate::util::serde::{display_serializable, parse_stdin_deserializable, IoFormat};
 use crate::util::Version;
 use crate::volume::Volumes;
 use crate::{Error, ResultExt};
-
-pub mod docker;
-
-// TODO: create RPC endpoint that looks up the appropriate action and calls `execute`
-
-#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
-pub struct ActionId<S: AsRef<str> = String>(Id<S>);
-impl FromStr for ActionId {
-    type Err = InvalidId;
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        Ok(ActionId(Id::try_from(s.to_owned())?))
-    }
-}
-impl From<ActionId> for String {
-    fn from(value: ActionId) -> Self {
-        value.0.into()
-    }
-}
-impl<S: AsRef<str>> AsRef<ActionId<S>> for ActionId<S> {
-    fn as_ref(&self) -> &ActionId<S> {
-        self
-    }
-}
-impl<S: AsRef<str>> std::fmt::Display for ActionId<S> {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{}", &self.0)
-    }
-}
-impl<S: AsRef<str>> AsRef<str> for ActionId<S> {
-    fn as_ref(&self) -> &str {
-        self.0.as_ref()
-    }
-}
-impl<S: AsRef<str>> AsRef<Path> for ActionId<S> {
-    fn as_ref(&self) -> &Path {
-        self.0.as_ref().as_ref()
-    }
-}
-impl<'de, S> Deserialize<'de> for ActionId<S>
-where
-    S: AsRef<str>,
-    Id<S>: Deserialize<'de>,
-{
-    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
-    where
-        D: serde::de::Deserializer<'de>,
-    {
-        Ok(ActionId(Deserialize::deserialize(deserializer)?))
-    }
-}
-
 #[derive(Clone, Debug, Default, Deserialize, Serialize)]
 pub struct Actions(pub BTreeMap<ActionId, Action>);
 
@@ -103,16 +50,22 @@ pub struct Action {
     pub description: String,
     #[serde(default)]
     pub warning: Option<String>,
-    pub implementation: ActionImplementation,
+    pub implementation: PackageProcedure,
     pub allowed_statuses: IndexSet<DockerStatus>,
     #[serde(default)]
     pub input_spec: ConfigSpec,
 }
 impl Action {
     #[instrument]
-    pub fn validate(&self, volumes: &Volumes, image_ids: &BTreeSet<ImageId>) -> Result<(), Error> {
+    pub fn validate(
+        &self,
+        container: &Option<DockerContainers>,
+        eos_version: &Version,
+        volumes: &Volumes,
+        image_ids: &BTreeSet<ImageId>,
+    ) -> Result<(), Error> {
         self.implementation
-            .validate(volumes, image_ids, true)
+            .validate(container, eos_version, volumes, image_ids, true)
             .with_ctx(|_| {
                 (
                     crate::ErrorKind::ValidateS9pk,
@@ -141,10 +94,9 @@ impl Action {
                 ctx,
                 pkg_id,
                 pkg_version,
-                Some(&format!("{}Action", action_id)),
+                ProcedureName::Action(action_id.clone()),
                 volumes,
                 input,
-                true,
                 None,
             )
             .await?
@@ -152,77 +104,7 @@ impl Action {
     }
 }
 
-#[derive(Clone, Debug, Deserialize, Serialize, HasModel)]
-#[serde(rename_all = "kebab-case")]
-#[serde(tag = "type")]
-pub enum ActionImplementation {
-    Docker(DockerAction),
-}
-impl ActionImplementation {
-    #[instrument]
-    pub fn validate(
-        &self,
-        volumes: &Volumes,
-        image_ids: &BTreeSet<ImageId>,
-        expected_io: bool,
-    ) -> Result<(), color_eyre::eyre::Report> {
-        match self {
-            ActionImplementation::Docker(action) => {
-                action.validate(volumes, image_ids, expected_io)
-            }
-        }
-    }
-
-    #[instrument(skip(ctx, input))]
-    pub async fn execute<I: Serialize, O: for<'de> Deserialize<'de>>(
-        &self,
-        ctx: &RpcContext,
-        pkg_id: &PackageId,
-        pkg_version: &Version,
-        name: Option<&str>,
-        volumes: &Volumes,
-        input: Option<I>,
-        allow_inject: bool,
-        timeout: Option<Duration>,
-    ) -> Result<Result<O, (i32, String)>, Error> {
-        match self {
-            ActionImplementation::Docker(action) => {
-                action
-                    .execute(
-                        ctx,
-                        pkg_id,
-                        pkg_version,
-                        name,
-                        volumes,
-                        input,
-                        allow_inject,
-                        timeout,
-                    )
-                    .await
-            }
-        }
-    }
-    #[instrument(skip(ctx, input))]
-    pub async fn sandboxed<I: Serialize, O: for<'de> Deserialize<'de>>(
-        &self,
-        ctx: &RpcContext,
-        pkg_id: &PackageId,
-        pkg_version: &Version,
-        volumes: &Volumes,
-        input: Option<I>,
-        timeout: Option<Duration>,
-    ) -> Result<Result<O, (i32, String)>, Error> {
-        match self {
-            ActionImplementation::Docker(action) => {
-                action
-                    .sandboxed(ctx, pkg_id, pkg_version, volumes, input, timeout)
-                    .await
-            }
-        }
-    }
-}
-
-fn display_action_result(action_result: ActionResult, matches: &ArgMatches<'_>) {
+fn display_action_result(action_result: ActionResult, matches: &ArgMatches) {
     if matches.is_present("format") {
         return display_serializable(action_result, matches);
     }
@@ -257,9 +139,10 @@ pub async fn action(
         .await
         .with_kind(crate::ErrorKind::NotFound)?
         .manifest()
-        .get(&mut db, true)
+        .get(&mut db)
         .await?
         .to_owned();
+
     if let Some(action) = manifest.actions.0.get(&action_id) {
         action
             .execute(
@@ -278,13 +161,3 @@ pub async fn action(
         ))
     }
 }
-
-pub struct NoOutput;
-impl<'de> Deserialize<'de> for NoOutput {
-    fn deserialize<D>(_: D) -> Result<Self, D::Error>
-    where
-        D: serde::Deserializer<'de>,
-    {
-        Ok(NoOutput)
-    }
-}

backend/src/action/docker.rs

@@ -1,354 +0,0 @@
-use std::borrow::Cow;
-use std::collections::{BTreeMap, BTreeSet};
-use std::ffi::{OsStr, OsString};
-use std::net::Ipv4Addr;
-use std::path::PathBuf;
-use std::time::Duration;
-
-use bollard::container::RemoveContainerOptions;
-use futures::future::Either as EitherFuture;
-use nix::sys::signal;
-use nix::unistd::Pid;
-use serde::{Deserialize, Serialize};
-use serde_json::Value;
-use tracing::instrument;
-
-use crate::context::RpcContext;
-use crate::id::{Id, ImageId};
-use crate::s9pk::manifest::{PackageId, SYSTEM_PACKAGE_ID};
-use crate::util::serde::{Duration as SerdeDuration, IoFormat};
-use crate::util::Version;
-use crate::volume::{VolumeId, Volumes};
-use crate::{Error, ResultExt, HOST_IP};
-
-pub const NET_TLD: &str = "embassy";
-
-lazy_static::lazy_static! {
-    pub static ref SYSTEM_IMAGES: BTreeSet<ImageId> = {
-        let mut set = BTreeSet::new();
-
-        set.insert("compat".parse().unwrap());
-        set.insert("utils".parse().unwrap());
-
-        set
-    };
-}
-
-#[derive(Clone, Debug, Deserialize, Serialize)]
-#[serde(rename_all = "kebab-case")]
-pub struct DockerAction {
-    pub image: ImageId,
-    #[serde(default)]
-    pub system: bool,
-    pub entrypoint: String,
-    #[serde(default)]
-    pub args: Vec<String>,
-    #[serde(default)]
-    pub mounts: BTreeMap<VolumeId, PathBuf>,
-    #[serde(default)]
-    pub io_format: Option<IoFormat>,
-    #[serde(default)]
-    pub inject: bool,
-    #[serde(default)]
-    pub shm_size_mb: Option<usize>, // TODO: use postfix sizing? like 1k vs 1m vs 1g
-    #[serde(default)]
-    pub sigterm_timeout: Option<SerdeDuration>,
-}
-impl DockerAction {
-    pub fn validate(
-        &self,
-        volumes: &Volumes,
-        image_ids: &BTreeSet<ImageId>,
-        expected_io: bool,
-    ) -> Result<(), color_eyre::eyre::Report> {
-        for (volume, _) in &self.mounts {
-            if !volumes.contains_key(volume) && !matches!(&volume, &VolumeId::Backup) {
-                color_eyre::eyre::bail!("unknown volume: {}", volume);
-            }
-        }
-        if self.system {
-            if !SYSTEM_IMAGES.contains(&self.image) {
-                color_eyre::eyre::bail!("unknown system image: {}", self.image);
-            }
-        } else {
-            if !image_ids.contains(&self.image) {
-                color_eyre::eyre::bail!("image for {} not contained in package", self.image);
-            }
-        }
-        if expected_io && self.io_format.is_none() {
-            color_eyre::eyre::bail!("expected io-format");
-        }
-        Ok(())
-    }
-
-    #[instrument(skip(ctx, input))]
-    pub async fn execute<I: Serialize, O: for<'de> Deserialize<'de>>(
-        &self,
-        ctx: &RpcContext,
-        pkg_id: &PackageId,
-        pkg_version: &Version,
-        name: Option<&str>,
-        volumes: &Volumes,
-        input: Option<I>,
-        allow_inject: bool,
-        timeout: Option<Duration>,
-    ) -> Result<Result<O, (i32, String)>, Error> {
-        let mut cmd = tokio::process::Command::new("docker");
-        if self.inject && allow_inject {
-            cmd.arg("exec");
-        } else {
-            let container_name = Self::container_name(pkg_id, name);
-            cmd.arg("run")
-                .arg("--rm")
-                .arg("--network=start9")
-                .arg(format!("--add-host=embassy:{}", Ipv4Addr::from(HOST_IP)))
-                .arg("--name")
-                .arg(&container_name)
-                .arg(format!("--hostname={}", &container_name))
-                .arg("--no-healthcheck");
-            match ctx
-                .docker
-                .remove_container(
-                    &container_name,
-                    Some(RemoveContainerOptions {
-                        v: false,
-                        force: true,
-                        link: false,
-                    }),
-                )
-                .await
-            {
-                Ok(()) | Err(bollard::errors::Error::DockerResponseNotFoundError { .. }) => Ok(()),
-                Err(e) => Err(e),
-            }?;
-        }
-        cmd.args(
-            self.docker_args(ctx, pkg_id, pkg_version, volumes, allow_inject)
-                .await,
-        );
-        let input_buf = if let (Some(input), Some(format)) = (&input, &self.io_format) {
-            cmd.stdin(std::process::Stdio::piped());
-            Some(format.to_vec(input)?)
-        } else {
-            None
-        };
-        cmd.stdout(std::process::Stdio::piped());
-        cmd.stderr(std::process::Stdio::piped());
-        tracing::trace!(
-            "{}",
-            format!("{:?}", cmd)
-                .split(r#"" ""#)
-                .collect::<Vec<&str>>()
-                .join(" ")
-        );
-        let mut handle = cmd.spawn().with_kind(crate::ErrorKind::Docker)?;
-        let id = handle.id();
-        let timeout_fut = if let Some(timeout) = timeout {
-            EitherFuture::Right(async move {
-                tokio::time::sleep(timeout).await;
-
-                Ok(())
-            })
-        } else {
-            EitherFuture::Left(futures::future::pending::<Result<_, Error>>())
-        };
-        if let (Some(input), Some(mut stdin)) = (&input_buf, handle.stdin.take()) {
-            use tokio::io::AsyncWriteExt;
-            stdin
-                .write_all(input)
-                .await
-                .with_kind(crate::ErrorKind::Docker)?;
-            stdin.flush().await?;
-            stdin.shutdown().await?;
-            drop(stdin);
-        }
-        enum Race<T> {
-            Done(T),
-            TimedOut,
-        }
-        let res = tokio::select! {
-            res = handle.wait_with_output() => Race::Done(res.with_kind(crate::ErrorKind::Docker)?),
-            res = timeout_fut => {
-                res?;
-                Race::TimedOut
-            },
-        };
-        let res = match res {
-            Race::Done(x) => x,
-            Race::TimedOut => {
-                if let Some(id) = id {
-                    signal::kill(Pid::from_raw(id as i32), signal::SIGKILL)
-                        .with_kind(crate::ErrorKind::Docker)?;
-                }
-                return Ok(Err((143, "Timed out. Retrying soon...".to_owned())));
-            }
-        };
-        Ok(if res.status.success() || res.status.code() == Some(143) {
-            Ok(if let Some(format) = self.io_format {
-                match format.from_slice(&res.stdout) {
-                    Ok(a) => a,
-                    Err(e) => {
-                        tracing::warn!(
-                            "Failed to deserialize stdout from {}: {}, falling back to UTF-8 string.",
-                            format,
-                            e
-                        );
-                        serde_json::from_value(String::from_utf8(res.stdout)?.into())
-                            .with_kind(crate::ErrorKind::Deserialization)?
-                    }
-                }
-            } else if res.stdout.is_empty() {
-                serde_json::from_value(Value::Null).with_kind(crate::ErrorKind::Deserialization)?
-            } else {
-                serde_json::from_value(String::from_utf8(res.stdout)?.into())
-                    .with_kind(crate::ErrorKind::Deserialization)?
-            })
-        } else {
-            Err((
-                res.status.code().unwrap_or_default(),
-                String::from_utf8(res.stderr)?,
-            ))
-        })
-    }
-
-    #[instrument(skip(ctx, input))]
-    pub async fn sandboxed<I: Serialize, O: for<'de> Deserialize<'de>>(
-        &self,
-        ctx: &RpcContext,
-        pkg_id: &PackageId,
-        pkg_version: &Version,
-        volumes: &Volumes,
-        input: Option<I>,
-        timeout: Option<Duration>,
-    ) -> Result<Result<O, (i32, String)>, Error> {
-        let mut cmd = tokio::process::Command::new("docker");
-        cmd.arg("run").arg("--rm").arg("--network=none");
-        cmd.args(
-            self.docker_args(ctx, pkg_id, pkg_version, &volumes.to_readonly(), false)
-                .await,
-        );
-        let input_buf = if let (Some(input), Some(format)) = (&input, &self.io_format) {
-            cmd.stdin(std::process::Stdio::piped());
-            Some(format.to_vec(input)?)
-        } else {
-            None
-        };
-        cmd.stdout(std::process::Stdio::piped());
-        cmd.stderr(std::process::Stdio::piped());
-        let mut handle = cmd.spawn().with_kind(crate::ErrorKind::Docker)?;
-        if let (Some(input), Some(stdin)) = (&input_buf, &mut handle.stdin) {
-            use tokio::io::AsyncWriteExt;
-            stdin
-                .write_all(input)
-                .await
-                .with_kind(crate::ErrorKind::Docker)?;
-        }
-        let res = handle
-            .wait_with_output()
-            .await
-            .with_kind(crate::ErrorKind::Docker)?;
-        Ok(if res.status.success() || res.status.code() == Some(143) {
-            Ok(if let Some(format) = &self.io_format {
-                match format.from_slice(&res.stdout) {
-                    Ok(a) => a,
-                    Err(e) => {
-                        tracing::warn!(
-                            "Failed to deserialize stdout from {}: {}, falling back to UTF-8 string.",
-                            format,
-                            e
-                        );
-                        serde_json::from_value(String::from_utf8(res.stdout)?.into())
-                            .with_kind(crate::ErrorKind::Deserialization)?
-                    }
-                }
-            } else if res.stdout.is_empty() {
-                serde_json::from_value(Value::Null).with_kind(crate::ErrorKind::Deserialization)?
-            } else {
-                serde_json::from_value(String::from_utf8(res.stdout)?.into())
-                    .with_kind(crate::ErrorKind::Deserialization)?
-            })
-        } else {
-            Err((
-                res.status.code().unwrap_or_default(),
-                String::from_utf8(res.stderr)?,
-            ))
-        })
-    }
-
-    pub fn container_name(pkg_id: &PackageId, name: Option<&str>) -> String {
-        if let Some(name) = name {
-            format!("{}_{}.{}", pkg_id, name, NET_TLD)
-        } else {
-            format!("{}.{}", pkg_id, NET_TLD)
-        }
-    }
-
-    pub fn uncontainer_name(name: &str) -> Option<(PackageId<&str>, Option<&str>)> {
-        let (pre_tld, _) = name.split_once(".")?;
-        if pre_tld.contains('_') {
-            let (pkg, name) = name.split_once("_")?;
-            Some((Id::try_from(pkg).ok()?.into(), Some(name)))
-        } else {
-            Some((Id::try_from(pre_tld).ok()?.into(), None))
-        }
-    }
-
-    async fn docker_args(
-        &self,
-        ctx: &RpcContext,
-        pkg_id: &PackageId,
-        pkg_version: &Version,
-        volumes: &Volumes,
-        allow_inject: bool,
-    ) -> Vec<Cow<'_, OsStr>> {
-        let mut res = Vec::with_capacity(
-            (2 * self.mounts.len()) // --mount <MOUNT_ARG>
-                + (2 * self.shm_size_mb.is_some() as usize) // --shm-size <SHM_SIZE>
-                + 5 // --interactive --log-driver=journald --entrypoint <ENTRYPOINT> <IMAGE>
-                + self.args.len(), // [ARG...]
-        );
-        for (volume_id, dst) in &self.mounts {
-            let volume = if let Some(v) = volumes.get(volume_id) {
-                v
-            } else {
-                continue;
-            };
-            let src = volume.path_for(ctx, pkg_id, pkg_version, volume_id);
-            if let Err(e) = tokio::fs::metadata(&src).await {
-                tracing::warn!("{} not mounted to container: {}", src.display(), e);
-                continue;
-            }
-            res.push(OsStr::new("--mount").into());
-            res.push(
-                OsString::from(format!(
-                    "type=bind,src={},dst={}{}",
-                    src.display(),
-                    dst.display(),
-                    if volume.readonly() { ",readonly" } else { "" }
-                ))
-                .into(),
-            );
-        }
-        if let Some(shm_size_mb) = self.shm_size_mb {
-            res.push(OsStr::new("--shm-size").into());
-            res.push(OsString::from(format!("{}m", shm_size_mb)).into());
-        }
-        res.push(OsStr::new("--interactive").into());
-        if self.inject && allow_inject {
-            res.push(OsString::from(Self::container_name(pkg_id, None)).into());
-            res.push(OsStr::new(&self.entrypoint).into());
-        } else {
-            res.push(OsStr::new("--log-driver=journald").into());
-            res.push(OsStr::new("--entrypoint").into());
-            res.push(OsStr::new(&self.entrypoint).into());
-            if self.system {
-                res.push(OsString::from(self.image.for_package(SYSTEM_PACKAGE_ID, None)).into());
-            } else {
-                res.push(OsString::from(self.image.for_package(pkg_id, Some(pkg_version))).into());
-            }
-        }
-        res.extend(self.args.iter().map(|s| OsStr::new(s).into()));
-
-        res
-    }
-}

backend/src/auth.rs

@@ -4,29 +4,77 @@ use std::marker::PhantomData;
 use chrono::{DateTime, Utc};
 use clap::ArgMatches;
 use color_eyre::eyre::eyre;
+use josekit::jwk::Jwk;
+use patch_db::{DbHandle, LockReceipt};
 use rpc_toolkit::command;
 use rpc_toolkit::command_helpers::prelude::{RequestParts, ResponseParts};
 use rpc_toolkit::yajrc::RpcError;
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
-use sqlx::{Executor, Sqlite};
+use sqlx::{Executor, Postgres};
 use tracing::instrument;
 
 use crate::context::{CliContext, RpcContext};
 use crate::middleware::auth::{AsLogoutSessionId, HasLoggedOutSessions, HashSessionToken};
+use crate::middleware::encrypt::EncryptedWire;
 use crate::util::display_none;
 use crate::util::serde::{display_serializable, IoFormat};
 use crate::{ensure_code, Error, ResultExt};
+#[derive(Clone, Serialize, Deserialize)]
+#[serde(untagged)]
+pub enum PasswordType {
+    EncryptedWire(EncryptedWire),
+    String(String),
+}
+impl PasswordType {
+    pub fn decrypt(self, current_secret: impl AsRef<Jwk>) -> Result<String, Error> {
+        match self {
+            PasswordType::String(x) => Ok(x),
+            PasswordType::EncryptedWire(x) => x.decrypt(current_secret).ok_or_else(|| {
+                Error::new(
+                    color_eyre::eyre::eyre!("Couldn't decode password"),
+                    crate::ErrorKind::Unknown,
+                )
+            }),
+        }
+    }
+}
+impl Default for PasswordType {
+    fn default() -> Self {
+        PasswordType::String(String::default())
+    }
+}
+impl std::fmt::Debug for PasswordType {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "<REDACTED_PASSWORD>")?;
+        Ok(())
+    }
+}
 
-#[command(subcommands(login, logout, session))]
+impl std::str::FromStr for PasswordType {
+    type Err = String;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Ok(match serde_json::from_str(s) {
+            Ok(a) => a,
+            Err(_) => PasswordType::String(s.to_string()),
+        })
+    }
+}
+
+#[command(subcommands(login, logout, session, reset_password, get_pubkey))]
 pub fn auth() -> Result<(), Error> {
     Ok(())
 }
 
-pub fn parse_metadata(_: &str, _: &ArgMatches<'_>) -> Result<Value, Error> {
-    Ok(serde_json::json!({
+pub fn cli_metadata() -> Value {
+    serde_json::json!({
         "platforms": ["cli"],
-    }))
+    })
+}
+
+pub fn parse_metadata(_: &str, _: &ArgMatches) -> Result<Value, Error> {
+    Ok(cli_metadata())
 }
 
 #[test]
@@ -45,13 +93,13 @@ fn gen_pwd() {
 #[instrument(skip(ctx, password))]
 async fn cli_login(
     ctx: CliContext,
-    password: Option<String>,
+    password: Option<PasswordType>,
     metadata: Value,
 ) -> Result<(), RpcError> {
     let password = if let Some(password) = password {
-        password
+        password.decrypt(&ctx)?
     } else {
-        rpassword::prompt_password_stdout("Password: ")?
+        rpassword::prompt_password("Password: ")?
     };
 
     rpc_toolkit::command_helpers::call_remote(
@@ -82,7 +130,7 @@ pub fn check_password(hash: &str, password: &str) -> Result<(), Error> {
 
 pub async fn check_password_against_db<Ex>(secrets: &mut Ex, password: &str) -> Result<(), Error>
 where
-    for<'a> &'a mut Ex: Executor<'a, Database = Sqlite>,
+    for<'a> &'a mut Ex: Executor<'a, Database = Postgres>,
 {
     let pw_hash = sqlx::query!("SELECT password FROM account")
         .fetch_one(secrets)
@@ -102,15 +150,15 @@ pub async fn login(
     #[context] ctx: RpcContext,
     #[request] req: &RequestParts,
     #[response] res: &mut ResponseParts,
-    #[arg] password: Option<String>,
+    #[arg] password: Option<PasswordType>,
     #[arg(
         parse(parse_metadata),
-        default = "",
+        default = "cli_metadata",
         help = "RPC Only: This value cannot be overidden from the cli"
     )]
     metadata: Value,
 ) -> Result<(), Error> {
-    let password = password.unwrap_or_default();
+    let password = password.unwrap_or_default().decrypt(&ctx)?;
     let mut handle = ctx.secret_store.acquire().await?;
     check_password_against_db(&mut handle, &password).await?;
 
@@ -119,7 +167,7 @@ pub async fn login(
     let metadata = serde_json::to_string(&metadata).with_kind(crate::ErrorKind::Database)?;
     let hash_token_hashed = hash_token.hashed();
     sqlx::query!(
-        "INSERT INTO session (id, user_agent, metadata) VALUES (?, ?, ?)",
+        "INSERT INTO session (id, user_agent, metadata) VALUES ($1, $2, $3)",
         hash_token_hashed,
         user_agent,
         metadata,
@@ -168,7 +216,7 @@ pub async fn session() -> Result<(), Error> {
     Ok(())
 }
 
-fn display_sessions(arg: SessionList, matches: &ArgMatches<'_>) {
+fn display_sessions(arg: SessionList, matches: &ArgMatches) {
     use prettytable::*;
 
     if matches.is_present("format") {
@@ -198,7 +246,7 @@ fn display_sessions(arg: SessionList, matches: &ArgMatches<'_>) {
         }
         table.add_row(row);
     }
-    table.print_tty(false);
+    table.print_tty(false).unwrap();
 }
 
 #[command(display(display_sessions))]
@@ -234,7 +282,7 @@ pub async fn list(
     })
 }
 
-fn parse_comma_separated(arg: &str, _: &ArgMatches<'_>) -> Result<Vec<String>, RpcError> {
+fn parse_comma_separated(arg: &str, _: &ArgMatches) -> Result<Vec<String>, RpcError> {
     Ok(arg.split(",").map(|s| s.trim().to_owned()).collect())
 }
 
@@ -256,3 +304,121 @@ pub async fn kill(
     HasLoggedOutSessions::new(ids.into_iter().map(KillSessionId), &ctx).await?;
     Ok(())
 }
+
+#[instrument(skip(ctx, old_password, new_password))]
+async fn cli_reset_password(
+    ctx: CliContext,
+    old_password: Option<PasswordType>,
+    new_password: Option<PasswordType>,
+) -> Result<(), RpcError> {
+    let old_password = if let Some(old_password) = old_password {
+        old_password.decrypt(&ctx)?
+    } else {
+        rpassword::prompt_password("Current Password: ")?
+    };
+
+    let new_password = if let Some(new_password) = new_password {
+        new_password.decrypt(&ctx)?
+    } else {
+        let new_password = rpassword::prompt_password("New Password: ")?;
+        if new_password != rpassword::prompt_password("Confirm: ")? {
+            return Err(Error::new(
+                eyre!("Passwords do not match"),
+                crate::ErrorKind::IncorrectPassword,
+            )
+            .into());
+        }
+        new_password
+    };
+
+    rpc_toolkit::command_helpers::call_remote(
+        ctx,
+        "auth.reset-password",
+        serde_json::json!({ "old-password": old_password, "new-password": new_password }),
+        PhantomData::<()>,
+    )
+    .await?
+    .result?;
+
+    Ok(())
+}
+
+pub struct SetPasswordReceipt(LockReceipt<String, ()>);
+impl SetPasswordReceipt {
+    pub async fn new<Db: DbHandle>(db: &mut Db) -> Result<Self, Error> {
+        let mut locks = Vec::new();
+
+        let setup = Self::setup(&mut locks);
+        Ok(setup(&db.lock_all(locks).await?)?)
+    }
+
+    pub fn setup(
+        locks: &mut Vec<patch_db::LockTargetId>,
+    ) -> impl FnOnce(&patch_db::Verifier) -> Result<Self, Error> {
+        let password_hash = crate::db::DatabaseModel::new()
+            .server_info()
+            .password_hash()
+            .make_locker(patch_db::LockType::Write)
+            .add_to_keys(locks);
+        move |skeleton_key| Ok(Self(password_hash.verify(skeleton_key)?))
+    }
+}
+
+pub async fn set_password<Db: DbHandle, Ex>(
+    db: &mut Db,
+    receipt: &SetPasswordReceipt,
+    secrets: &mut Ex,
+    password: &str,
+) -> Result<(), Error>
+where
+    for<'a> &'a mut Ex: Executor<'a, Database = Postgres>,
+{
+    let password = argon2::hash_encoded(
+        password.as_bytes(),
+        &rand::random::<[u8; 16]>()[..],
+        &argon2::Config::default(),
+    )
+    .with_kind(crate::ErrorKind::PasswordHashGeneration)?;
+
+    sqlx::query!("UPDATE account SET password = $1", password,)
+        .execute(secrets)
+        .await?;
+
+    receipt.0.set(db, password).await?;
+
+    Ok(())
+}
+
+#[command(
+    rename = "reset-password",
+    custom_cli(cli_reset_password(async, context(CliContext))),
+    display(display_none)
+)]
+#[instrument(skip(ctx, old_password, new_password))]
+pub async fn reset_password(
+    #[context] ctx: RpcContext,
+    #[arg(rename = "old-password")] old_password: Option<PasswordType>,
+    #[arg(rename = "new-password")] new_password: Option<PasswordType>,
+) -> Result<(), Error> {
+    let old_password = old_password.unwrap_or_default().decrypt(&ctx)?;
+    let new_password = new_password.unwrap_or_default().decrypt(&ctx)?;
+
+    let mut secrets = ctx.secret_store.acquire().await?;
+    check_password_against_db(&mut secrets, &old_password).await?;
+
+    let mut db = ctx.db.handle();
+
+    let set_password_receipt = SetPasswordReceipt::new(&mut db).await?;
+
+    set_password(&mut db, &set_password_receipt, &mut secrets, &new_password).await?;
+
+    Ok(())
+}
+
+#[command(rename = "get-pubkey", display(display_none))]
+#[instrument(skip(ctx))]
+pub async fn get_pubkey(#[context] ctx: RpcContext) -> Result<Jwk, RpcError> {
+    let secret = ctx.as_ref().clone();
+    let pub_key = secret.to_public_key()?;
+    Ok(pub_key)
+}

backend/src/backup/backup_bulk.rs

@@ -1,11 +1,13 @@
-use std::collections::BTreeMap;
-use std::sync::Arc;
+use std::collections::{BTreeMap, BTreeSet};
+use std::path::PathBuf;
 
 use chrono::Utc;
+use clap::ArgMatches;
 use color_eyre::eyre::eyre;
+use helpers::AtomicFile;
 use openssl::pkey::{PKey, Private};
 use openssl::x509::X509;
-use patch_db::{DbHandle, LockType, PatchDbHandle, Revision};
+use patch_db::{DbHandle, LockType, PatchDbHandle};
 use rpc_toolkit::command;
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
@@ -18,17 +20,17 @@ use super::PackageBackupReport;
 use crate::auth::check_password_against_db;
 use crate::backup::{BackupReport, ServerBackupReport};
 use crate::context::RpcContext;
-use crate::db::util::WithRevision;
+use crate::db::model::BackupProgress;
 use crate::disk::mount::backup::BackupMountGuard;
 use crate::disk::mount::filesystem::ReadWrite;
 use crate::disk::mount::guard::TmpMountGuard;
 use crate::notifications::NotificationLevel;
 use crate::s9pk::manifest::PackageId;
 use crate::status::MainStatus;
+use crate::util::display_none;
 use crate::util::serde::IoFormat;
-use crate::util::{display_none, AtomicFile};
 use crate::version::VersionT;
-use crate::Error;
+use crate::{Error, ErrorKind, ResultExt};
 
 #[derive(Debug)]
 pub struct OsBackup {
@@ -112,35 +114,65 @@ impl Serialize for OsBackup {
     }
 }
 
+fn parse_comma_separated(arg: &str, _: &ArgMatches) -> Result<BTreeSet<PackageId>, Error> {
+    arg.split(',')
+        .map(|s| s.trim().parse().map_err(Error::from))
+        .collect()
+}
+
 #[command(rename = "create", display(display_none))]
 #[instrument(skip(ctx, old_password, password))]
 pub async fn backup_all(
     #[context] ctx: RpcContext,
     #[arg(rename = "target-id")] target_id: BackupTargetId,
-    #[arg(rename = "old-password", long = "old-password")] old_password: Option<String>,
-    #[arg] password: String,
-) -> Result<WithRevision<()>, Error> {
+    #[arg(rename = "old-password", long = "old-password")] old_password: Option<
+        crate::auth::PasswordType,
+    >,
+    #[arg(
+        rename = "package-ids",
+        long = "package-ids",
+        parse(parse_comma_separated)
+    )]
+    package_ids: Option<BTreeSet<PackageId>>,
+    #[arg] password: crate::auth::PasswordType,
+) -> Result<(), Error> {
     let mut db = ctx.db.handle();
+    let old_password_decrypted = old_password
+        .as_ref()
+        .unwrap_or(&password)
+        .clone()
+        .decrypt(&ctx)?;
+    let password = password.decrypt(&ctx)?;
     check_password_against_db(&mut ctx.secret_store.acquire().await?, &password).await?;
     let fs = target_id
         .load(&mut ctx.secret_store.acquire().await?)
         .await?;
     let mut backup_guard = BackupMountGuard::mount(
         TmpMountGuard::mount(&fs, ReadWrite).await?,
-        old_password.as_ref().unwrap_or(&password),
+        &old_password_decrypted,
     )
     .await?;
+    let all_packages = crate::db::DatabaseModel::new()
+        .package_data()
+        .get(&mut db)
+        .await?
+        .0
+        .keys()
+        .into_iter()
+        .cloned()
+        .collect();
+    let package_ids = package_ids.unwrap_or(all_packages);
     if old_password.is_some() {
         backup_guard.change_password(&password)?;
     }
-    let revision = assure_backing_up(&mut db).await?;
+    assure_backing_up(&mut db, &package_ids).await?;
     tokio::task::spawn(async move {
-        let backup_res = perform_backup(&ctx, &mut db, backup_guard).await;
-        let status_model = crate::db::DatabaseModel::new()
+        let backup_res = perform_backup(&ctx, &mut db, backup_guard, &package_ids).await;
+        let backup_progress = crate::db::DatabaseModel::new()
             .server_info()
             .status_info()
-            .backing_up();
-        status_model
+            .backup_progress();
+        backup_progress
             .clone()
             .lock(&mut db, LockType::Write)
             .await
@@ -207,36 +239,51 @@ pub async fn backup_all(
                     .expect("failed to send notification");
             }
         }
-        status_model
-            .put(&mut db, &false)
+        backup_progress
+            .delete(&mut db)
             .await
             .expect("failed to change server status");
     });
-    Ok(WithRevision {
-        response: (),
-        revision,
-    })
+    Ok(())
 }
 
-#[instrument(skip(db))]
-async fn assure_backing_up(db: &mut PatchDbHandle) -> Result<Option<Arc<Revision>>, Error> {
+#[instrument(skip(db, packages))]
+async fn assure_backing_up(
+    db: &mut PatchDbHandle,
+    packages: impl IntoIterator<Item = &PackageId>,
+) -> Result<(), Error> {
     let mut tx = db.begin().await?;
     let mut backing_up = crate::db::DatabaseModel::new()
         .server_info()
         .status_info()
-        .backing_up()
+        .backup_progress()
         .get_mut(&mut tx)
         .await?;
 
-    if *backing_up {
+    if backing_up
+        .iter()
+        .flat_map(|x| x.values())
+        .fold(false, |acc, x| {
+            if !x.complete {
+                return true;
+            }
+            acc
+        })
+    {
         return Err(Error::new(
             eyre!("Server is already backing up!"),
             crate::ErrorKind::InvalidRequest,
         ));
     }
-    *backing_up = true;
+    *backing_up = Some(
+        packages
+            .into_iter()
+            .map(|x| (x.clone(), BackupProgress { complete: false }))
+            .collect(),
+    );
     backing_up.save(&mut tx).await?;
-    Ok(tx.commit(None).await?)
+    tx.commit().await?;
+    Ok(())
 }
 
 #[instrument(skip(ctx, db, backup_guard))]
@@ -244,13 +291,16 @@ async fn perform_backup<Db: DbHandle>(
     ctx: &RpcContext,
     mut db: Db,
     mut backup_guard: BackupMountGuard<TmpMountGuard>,
+    package_ids: &BTreeSet<PackageId>,
 ) -> Result<BTreeMap<PackageId, PackageBackupReport>, Error> {
     let mut backup_report = BTreeMap::new();
 
     for package_id in crate::db::DatabaseModel::new()
         .package_data()
-        .keys(&mut db, false)
+        .keys(&mut db)
         .await?
+        .into_iter()
+        .filter(|id| package_ids.contains(id))
     {
         let mut tx = db.begin().await?; // for lock scope
         let installed_model = if let Some(installed_model) = crate::db::DatabaseModel::new()
@@ -267,10 +317,12 @@ async fn perform_backup<Db: DbHandle>(
         let main_status_model = installed_model.clone().status().main();
 
         main_status_model.lock(&mut tx, LockType::Write).await?;
-        let (started, health) = match main_status_model.get(&mut tx, true).await?.into_owned() {
-            MainStatus::Starting => (Some(Utc::now()), Default::default()),
+        let (started, health) = match main_status_model.get(&mut tx).await?.into_owned() {
+            MainStatus::Starting { .. } => (Some(Utc::now()), Default::default()),
             MainStatus::Running { started, health } => (Some(started), health.clone()),
-            MainStatus::Stopped | MainStatus::Stopping => (None, Default::default()),
+            MainStatus::Stopped | MainStatus::Stopping | MainStatus::Restarting => {
+                (None, Default::default())
+            }
             MainStatus::BackingUp { .. } => {
                 backup_report.insert(
                     package_id,
@@ -294,11 +346,7 @@ async fn perform_backup<Db: DbHandle>(
             .await?;
         tx.save().await?; // drop locks
 
-        let manifest = installed_model
-            .clone()
-            .manifest()
-            .get(&mut db, false)
-            .await?;
+        let manifest = installed_model.clone().manifest().get(&mut db).await?;
 
         ctx.managers
             .get(&(manifest.id.clone(), manifest.version.clone()))
@@ -318,6 +366,7 @@ async fn perform_backup<Db: DbHandle>(
             .backup
             .create(
                 ctx,
+                &mut tx,
                 &package_id,
                 &manifest.title,
                 &manifest.version,
@@ -341,7 +390,7 @@ async fn perform_backup<Db: DbHandle>(
             backup_guard
                 .metadata
                 .package_backups
-                .insert(package_id, pkg_meta);
+                .insert(package_id.clone(), pkg_meta);
         }
 
         main_status_model
@@ -353,6 +402,23 @@ async fn perform_backup<Db: DbHandle>(
                 },
             )
             .await?;
+
+        let mut backup_progress = crate::db::DatabaseModel::new()
+            .server_info()
+            .status_info()
+            .backup_progress()
+            .get_mut(&mut tx)
+            .await?;
+        if backup_progress.is_none() {
+            *backup_progress = Some(Default::default());
+        }
+        if let Some(mut backup_progress) = backup_progress
+            .as_mut()
+            .and_then(|bp| bp.get_mut(&package_id))
+        {
+            (*backup_progress).complete = true;
+        }
+        backup_progress.save(&mut tx).await?;
         tx.save().await?;
     }
 
@@ -361,7 +427,12 @@ async fn perform_backup<Db: DbHandle>(
         .await?;
 
     let (root_ca_key, root_ca_cert) = ctx.net_controller.ssl.export_root_ca().await?;
-    let mut os_backup_file = AtomicFile::new(backup_guard.as_ref().join("os-backup.cbor")).await?;
+    let mut os_backup_file = AtomicFile::new(
+        backup_guard.as_ref().join("os-backup.cbor"),
+        None::<PathBuf>,
+    )
+    .await
+    .with_kind(ErrorKind::Filesystem)?;
     os_backup_file
         .write_all(
             &IoFormat::Cbor.to_vec(&OsBackup {
@@ -370,13 +441,16 @@ async fn perform_backup<Db: DbHandle>(
                 root_ca_cert,
                 ui: crate::db::DatabaseModel::new()
                     .ui()
-                    .get(&mut db, true)
+                    .get(&mut db)
                     .await?
                     .into_owned(),
             })?,
         )
         .await?;
-    os_backup_file.save().await?;
+    os_backup_file
+        .save()
+        .await
+        .with_kind(ErrorKind::Filesystem)?;
 
     let timestamp = Some(Utc::now());
 
@@ -392,6 +466,5 @@ async fn perform_backup<Db: DbHandle>(
         .last_backup()
         .put(&mut db, &timestamp)
         .await?;
-
     Ok(backup_report)
 }

backend/src/backup/mod.rs

@@ -1,29 +1,31 @@
 use std::collections::{BTreeMap, BTreeSet};
-use std::path::Path;
+use std::path::{Path, PathBuf};
 
 use chrono::{DateTime, Utc};
 use color_eyre::eyre::eyre;
-use patch_db::{DbHandle, HasModel, LockType};
+use helpers::AtomicFile;
+use patch_db::{DbHandle, HasModel};
+use reqwest::Url;
 use rpc_toolkit::command;
 use serde::{Deserialize, Serialize};
-use sqlx::{Executor, Sqlite};
 use tokio::fs::File;
 use tokio::io::AsyncWriteExt;
 use tracing::instrument;
 
 use self::target::PackageBackupInfo;
-use crate::action::{ActionImplementation, NoOutput};
 use crate::context::RpcContext;
 use crate::dependencies::reconfigure_dependents_with_live_pointers;
 use crate::id::ImageId;
 use crate::install::PKG_ARCHIVE_DIR;
 use crate::net::interface::{InterfaceId, Interfaces};
+use crate::procedure::docker::DockerContainers;
+use crate::procedure::{NoOutput, PackageProcedure, ProcedureName};
 use crate::s9pk::manifest::PackageId;
 use crate::util::serde::IoFormat;
-use crate::util::{AtomicFile, Version};
+use crate::util::Version;
 use crate::version::{Current, VersionT};
 use crate::volume::{backup_dir, Volume, VolumeId, Volumes, BACKUP_DIR};
-use crate::{Error, ResultExt};
+use crate::{Error, ErrorKind, ResultExt};
 
 pub mod backup_bulk;
 pub mod restore;
@@ -60,28 +62,36 @@ pub fn package_backup() -> Result<(), Error> {
 struct BackupMetadata {
     pub timestamp: DateTime<Utc>,
     pub tor_keys: BTreeMap<InterfaceId, String>,
+    pub marketplace_url: Option<Url>,
 }
 
 #[derive(Clone, Debug, Deserialize, Serialize, HasModel)]
 pub struct BackupActions {
-    pub create: ActionImplementation,
-    pub restore: ActionImplementation,
+    pub create: PackageProcedure,
+    pub restore: PackageProcedure,
 }
 impl BackupActions {
-    pub fn validate(&self, volumes: &Volumes, image_ids: &BTreeSet<ImageId>) -> Result<(), Error> {
+    pub fn validate(
+        &self,
+        container: &Option<DockerContainers>,
+        eos_version: &Version,
+        volumes: &Volumes,
+        image_ids: &BTreeSet<ImageId>,
+    ) -> Result<(), Error> {
         self.create
-            .validate(volumes, image_ids, false)
+            .validate(container, eos_version, volumes, image_ids, false)
             .with_ctx(|_| (crate::ErrorKind::ValidateS9pk, "Backup Create"))?;
         self.restore
-            .validate(volumes, image_ids, false)
+            .validate(container, eos_version, volumes, image_ids, false)
             .with_ctx(|_| (crate::ErrorKind::ValidateS9pk, "Backup Restore"))?;
         Ok(())
     }
 
-    #[instrument(skip(ctx))]
-    pub async fn create(
+    #[instrument(skip(ctx, db))]
+    pub async fn create<Db: DbHandle>(
         &self,
         ctx: &RpcContext,
+        db: &mut Db,
         pkg_id: &PackageId,
         pkg_title: &str,
         pkg_version: &Version,
@@ -99,10 +109,9 @@ impl BackupActions {
                 ctx,
                 pkg_id,
                 pkg_version,
-                Some("CreateBackup"),
+                ProcedureName::CreateBackup,
                 &volumes,
                 None,
-                false,
                 None,
             )
             .await?
@@ -119,6 +128,18 @@ impl BackupActions {
                 )
             })
             .collect();
+        let marketplace_url = crate::db::DatabaseModel::new()
+            .package_data()
+            .idx_model(pkg_id)
+            .expect(db)
+            .await?
+            .installed()
+            .expect(db)
+            .await?
+            .marketplace_url()
+            .get(db)
+            .await?
+            .into_owned();
         let tmp_path = Path::new(BACKUP_DIR)
             .join(pkg_id)
             .join(format!("{}.s9pk", pkg_id));
@@ -129,7 +150,9 @@ impl BackupActions {
             .join(pkg_version.as_str())
             .join(format!("{}.s9pk", pkg_id));
         let mut infile = File::open(&s9pk_path).await?;
-        let mut outfile = AtomicFile::new(&tmp_path).await?;
+        let mut outfile = AtomicFile::new(&tmp_path, None::<PathBuf>)
+            .await
+            .with_kind(ErrorKind::Filesystem)?;
         tokio::io::copy(&mut infile, &mut *outfile)
             .await
             .with_ctx(|_| {
@@ -138,17 +161,20 @@ impl BackupActions {
                     format!("cp {} -> {}", s9pk_path.display(), tmp_path.display()),
                 )
             })?;
-        outfile.save().await?;
+        outfile.save().await.with_kind(ErrorKind::Filesystem)?;
         let timestamp = Utc::now();
         let metadata_path = Path::new(BACKUP_DIR).join(pkg_id).join("metadata.cbor");
-        let mut outfile = AtomicFile::new(&metadata_path).await?;
+        let mut outfile = AtomicFile::new(&metadata_path, None::<PathBuf>)
+            .await
+            .with_kind(ErrorKind::Filesystem)?;
         outfile
             .write_all(&IoFormat::Cbor.to_vec(&BackupMetadata {
                 timestamp,
                 tor_keys,
+                marketplace_url,
             })?)
             .await?;
-        outfile.save().await?;
+        outfile.save().await.with_kind(ErrorKind::Filesystem)?;
         Ok(PackageBackupInfo {
             os_version: Current::new().semver().into(),
             title: pkg_title.to_owned(),
@@ -157,20 +183,16 @@ impl BackupActions {
         })
     }
 
-    #[instrument(skip(ctx, db, secrets))]
-    pub async fn restore<Ex, Db: DbHandle>(
+    #[instrument(skip(ctx, db))]
+    pub async fn restore<Db: DbHandle>(
         &self,
         ctx: &RpcContext,
         db: &mut Db,
-        secrets: &mut Ex,
         pkg_id: &PackageId,
         pkg_version: &Version,
         interfaces: &Interfaces,
         volumes: &Volumes,
-    ) -> Result<(), Error>
-    where
-        for<'a> &'a mut Ex: Executor<'a, Database = Sqlite>,
-    {
+    ) -> Result<(), Error> {
         let mut volumes = volumes.clone();
         volumes.insert(VolumeId::Backup, Volume::Backup { readonly: true });
         self.restore
@@ -178,10 +200,9 @@ impl BackupActions {
                 ctx,
                 pkg_id,
                 pkg_version,
-                Some("RestoreBackup"),
+                ProcedureName::RestoreBackup,
                 &volumes,
                 None,
-                false,
                 None,
             )
             .await?
@@ -196,37 +217,16 @@ impl BackupActions {
                 )
             })?,
         )?;
-        for (iface, key) in metadata.tor_keys {
-            let key_vec = base32::decode(base32::Alphabet::RFC4648 { padding: true }, &key)
-                .ok_or_else(|| {
-                    Error::new(
-                        eyre!("invalid base32 string"),
-                        crate::ErrorKind::Deserialization,
-                    )
-                })?;
-            sqlx::query!(
-                "REPLACE INTO tor (package, interface, key) VALUES (?, ?, ?)",
-                **pkg_id,
-                *iface,
-                key_vec,
-            )
-            .execute(&mut *secrets)
-            .await?;
-        }
-        crate::db::DatabaseModel::new()
-            .package_data()
-            .lock(db, LockType::Write)
-            .await?;
-        crate::db::DatabaseModel::new()
+        let pde = crate::db::DatabaseModel::new()
             .package_data()
             .idx_model(pkg_id)
             .expect(db)
             .await?
             .installed()
             .expect(db)
-            .await?
-            .interface_addresses()
-            .put(db, &interfaces.install(&mut *secrets, pkg_id).await?)
+            .await?;
+        pde.marketplace_url()
+            .put(db, &metadata.marketplace_url)
             .await?;
 
         let entry = crate::db::DatabaseModel::new()
@@ -237,10 +237,11 @@ impl BackupActions {
             .installed()
             .expect(db)
             .await?
-            .get(db, true)
+            .get(db)
             .await?;
 
-        reconfigure_dependents_with_live_pointers(ctx, db, &entry).await?;
+        let receipts = crate::config::ConfigReceipts::new(db).await?;
+        reconfigure_dependents_with_live_pointers(ctx, db, &receipts, &entry).await?;
 
         Ok(())
     }

backend/src/backup/restore.rs

@@ -1,15 +1,15 @@
-use std::collections::BTreeMap;
 use std::path::Path;
 use std::sync::atomic::Ordering;
 use std::sync::Arc;
 use std::time::Duration;
+use std::{collections::BTreeMap, pin::Pin};
 
 use clap::ArgMatches;
 use color_eyre::eyre::eyre;
-use futures::future::BoxFuture;
-use futures::FutureExt;
+use futures::{future::BoxFuture, stream, Future};
+use futures::{FutureExt, StreamExt};
 use openssl::x509::X509;
-use patch_db::{DbHandle, PatchDbHandle, Revision};
+use patch_db::{DbHandle, PatchDbHandle};
 use rpc_toolkit::command;
 use tokio::fs::File;
 use tokio::task::JoinHandle;
@@ -18,69 +18,87 @@ use tracing::instrument;
 
 use super::target::BackupTargetId;
 use crate::backup::backup_bulk::OsBackup;
+use crate::backup::BackupMetadata;
+use crate::context::rpc::RpcContextConfig;
 use crate::context::{RpcContext, SetupContext};
 use crate::db::model::{PackageDataEntry, StaticFiles};
-use crate::db::util::WithRevision;
 use crate::disk::mount::backup::{BackupMountGuard, PackageBackupMountGuard};
-use crate::disk::mount::filesystem::ReadOnly;
+use crate::disk::mount::filesystem::ReadWrite;
 use crate::disk::mount::guard::TmpMountGuard;
+use crate::hostname::{get_hostname, Hostname};
+use crate::init::init;
 use crate::install::progress::InstallProgress;
 use crate::install::{download_install_s9pk, PKG_PUBLIC_DIR};
 use crate::net::ssl::SslManager;
+use crate::notifications::NotificationLevel;
 use crate::s9pk::manifest::{Manifest, PackageId};
 use crate::s9pk::reader::S9pkReader;
-use crate::setup::RecoveryStatus;
+use crate::setup::SetupStatus;
 use crate::util::display_none;
 use crate::util::io::dir_size;
 use crate::util::serde::IoFormat;
 use crate::volume::{backup_dir, BACKUP_DIR, PKG_VOLUME_DIR};
-use crate::{auth::check_password_against_db, notifications::NotificationLevel};
 use crate::{Error, ResultExt};
 
-fn parse_comma_separated(arg: &str, _: &ArgMatches<'_>) -> Result<Vec<PackageId>, Error> {
+fn parse_comma_separated(arg: &str, _: &ArgMatches) -> Result<Vec<PackageId>, Error> {
     arg.split(',')
         .map(|s| s.trim().parse().map_err(Error::from))
         .collect()
 }
 
 #[command(rename = "restore", display(display_none))]
-#[instrument(skip(ctx, old_password, password))]
+#[instrument(skip(ctx, password))]
 pub async fn restore_packages_rpc(
     #[context] ctx: RpcContext,
     #[arg(parse(parse_comma_separated))] ids: Vec<PackageId>,
     #[arg(rename = "target-id")] target_id: BackupTargetId,
-    #[arg(rename = "old-password", long = "old-password")] old_password: Option<String>,
     #[arg] password: String,
-) -> Result<WithRevision<()>, Error> {
+) -> Result<(), Error> {
     let mut db = ctx.db.handle();
-    check_password_against_db(&mut ctx.secret_store.acquire().await?, &password).await?;
     let fs = target_id
         .load(&mut ctx.secret_store.acquire().await?)
         .await?;
-    let mut backup_guard = BackupMountGuard::mount(
-        TmpMountGuard::mount(&fs, ReadOnly).await?,
-        old_password.as_ref().unwrap_or(&password),
-    )
-    .await?;
-    if old_password.is_some() {
-        backup_guard.change_password(&password)?;
-    }
+    let backup_guard =
+        BackupMountGuard::mount(TmpMountGuard::mount(&fs, ReadWrite).await?, &password).await?;
 
-    let (revision, backup_guard, tasks, _) =
-        restore_packages(&ctx, &mut db, backup_guard, ids).await?;
+    let (backup_guard, tasks, _) = restore_packages(&ctx, &mut db, backup_guard, ids).await?;
 
-    tokio::spawn(async {
-        futures::future::join_all(tasks).await;
+    tokio::spawn(async move {
+        stream::iter(tasks.into_iter().map(|x| (x, ctx.clone())))
+            .for_each_concurrent(5, |(res, ctx)| async move {
+                let mut db = ctx.db.handle();
+                match res.await {
+                    (Ok(_), _) => (),
+                    (Err(err), package_id) => {
+                        if let Err(err) = ctx
+                            .notification_manager
+                            .notify(
+                                &mut db,
+                                Some(package_id.clone()),
+                                NotificationLevel::Error,
+                                "Restoration Failure".to_string(),
+                                format!("Error restoring package {}: {}", package_id, err),
+                                (),
+                                None,
+                            )
+                            .await
+                        {
+                            tracing::error!("Failed to notify: {}", err);
+                            tracing::debug!("{:?}", err);
+                        };
+                        tracing::error!("Error restoring package {}: {}", package_id, err);
+                        tracing::debug!("{:?}", err);
+                    }
+                }
+            })
+            .await;
         if let Err(e) = backup_guard.unmount().await {
             tracing::error!("Error unmounting backup drive: {}", e);
             tracing::debug!("{:?}", e);
         }
     });
 
-    Ok(WithRevision {
-        response: (),
-        revision,
-    })
+    Ok(())
 }
 
 async fn approximate_progress(
@@ -108,7 +126,7 @@ async fn approximate_progress_loop(
             tracing::error!("Failed to approximate restore progress: {}", e);
             tracing::debug!("{:?}", e);
         } else {
-            *ctx.recovery_status.write().await = Some(Ok(starting_info.flatten()));
+            *ctx.setup_status.write().await = Some(Ok(starting_info.flatten()));
         }
         tokio::time::sleep(Duration::from_secs(1)).await;
     }
@@ -121,7 +139,7 @@ struct ProgressInfo {
     target_volume_size: BTreeMap<PackageId, u64>,
 }
 impl ProgressInfo {
-    fn flatten(&self) -> RecoveryStatus {
+    fn flatten(&self) -> SetupStatus {
         let mut total_bytes = 0;
         let mut bytes_transferred = 0;
 
@@ -144,8 +162,8 @@ impl ProgressInfo {
             bytes_transferred = total_bytes;
         }
 
-        RecoveryStatus {
-            total_bytes,
+        SetupStatus {
+            total_bytes: Some(total_bytes),
             bytes_transferred,
             complete: false,
         }
@@ -159,7 +177,7 @@ pub async fn recover_full_embassy(
     embassy_password: String,
     recovery_source: TmpMountGuard,
     recovery_password: Option<String>,
-) -> Result<(OnionAddressV3, X509, BoxFuture<'static, Result<(), Error>>), Error> {
+) -> Result<(Arc<String>, Hostname, OnionAddressV3, X509), Error> {
     let backup_guard = BackupMountGuard::mount(
         recovery_source,
         recovery_password.as_deref().unwrap_or_default(),
@@ -184,7 +202,7 @@ pub async fn recover_full_embassy(
     let key_vec = os_backup.tor_key.as_bytes().to_vec();
     let secret_store = ctx.secret_store().await?;
     sqlx::query!(
-        "REPLACE INTO account (id, password, tor_key) VALUES (?, ?, ?)",
+        "INSERT INTO account (id, password, tor_key) VALUES ($1, $2, $3) ON CONFLICT (id) DO UPDATE SET password = $2, tor_key = $3",
         0,
         password,
         key_vec,
@@ -200,67 +218,64 @@ pub async fn recover_full_embassy(
     .await?;
     secret_store.close().await;
 
+    let cfg = RpcContextConfig::load(ctx.config_path.clone()).await?;
+
+    init(&cfg).await?;
+
+    let rpc_ctx = RpcContext::init(ctx.config_path.clone(), disk_guid.clone()).await?;
+    let mut db = rpc_ctx.db.handle();
+
+    let receipts = crate::hostname::HostNameReceipt::new(&mut db).await?;
+    let hostname = get_hostname(&mut db, &receipts).await?;
+
+    drop(db);
+    let mut db = rpc_ctx.db.handle();
+
+    let ids = backup_guard
+        .metadata
+        .package_backups
+        .keys()
+        .cloned()
+        .collect();
+    let (backup_guard, tasks, progress_info) =
+        restore_packages(&rpc_ctx, &mut db, backup_guard, ids).await?;
+    let task_consumer_rpc_ctx = rpc_ctx.clone();
+    tokio::select! {
+        _ = async move {
+            stream::iter(tasks.into_iter().map(|x| (x,  task_consumer_rpc_ctx.clone())))
+                .for_each_concurrent(5, |(res, ctx)| async move {
+                    let mut db = ctx.db.handle();
+                    match res.await {
+                        (Ok(_), _) => (),
+                        (Err(err), package_id) => {
+                            if let Err(err) = ctx.notification_manager.notify(
+                                &mut db,
+                                Some(package_id.clone()),
+                                NotificationLevel::Error,
+                                "Restoration Failure".to_string(), format!("Error restoring package {}: {}", package_id,err), (), None).await{
+                                tracing::error!("Failed to notify: {}", err);
+                                tracing::debug!("{:?}", err);
+                                };
+                            tracing::error!("Error restoring package {}: {}", package_id, err);
+                            tracing::debug!("{:?}", err);
+                        },
+                    }
+                }).await;
+
+        } => {
+
+        },
+        _ = approximate_progress_loop(&ctx, &rpc_ctx, progress_info) => unreachable!(concat!(module_path!(), "::approximate_progress_loop should not terminate")),
+    }
+
+    backup_guard.unmount().await?;
+    rpc_ctx.shutdown().await?;
+
     Ok((
+        disk_guid,
+        hostname,
         os_backup.tor_key.public().get_onion_address(),
         os_backup.root_ca_cert,
-        async move {
-            let rpc_ctx = RpcContext::init(ctx.config_path.as_ref(), disk_guid).await?;
-            let mut db = rpc_ctx.db.handle();
-
-            let ids = backup_guard
-            .metadata
-            .package_backups
-            .keys()
-            .cloned()
-            .collect();
-            let (_, backup_guard, tasks, progress_info) = restore_packages(
-                &rpc_ctx,
-                &mut db,
-                backup_guard,
-                ids,
-            )
-            .await?;
-
-            tokio::select! {
-                res = futures::future::join_all(tasks) => {
-                    for res in res {
-                        match res.with_kind(crate::ErrorKind::Unknown) {
-                            Ok((Ok(_), _)) => (),
-                            Ok((Err(err), package_id)) => {
-                                if let Err(err) = rpc_ctx.notification_manager.notify(
-                                    &mut db,
-                                    Some(package_id.clone()),
-                                    NotificationLevel::Error, 
-                                    "Restoration Failure".to_string(), format!("Error restoring package {}: {}", package_id,err), (), None).await{
-                                    tracing::error!("Failed to notify: {}", err);
-                                    tracing::debug!("{:?}", err);
-                                    };
-                                tracing::error!("Error restoring package {}: {}", package_id, err);
-                                tracing::debug!("{:?}", err);
-                            },
-                            Err(e) => {
-                                if let Err(err) = rpc_ctx.notification_manager.notify(
-                                    &mut db,
-                                    None,
-                                    NotificationLevel::Error, 
-                                    "Restoration Failure".to_string(), format!("Error restoring ?: {}", e), (), None).await {
-
-                                    tracing::error!("Failed to notify: {}", err);
-                                    tracing::debug!("{:?}", err);
-                                }
-                                tracing::error!("Error restoring packages: {}", e);
-                                tracing::debug!("{:?}", e);
-                            },
-
-                        }
-                    }
-                },
-                _ = approximate_progress_loop(&ctx, &rpc_ctx, progress_info) => unreachable!(concat!(module_path!(), "::approximate_progress_loop should not terminate")),
-            }
-
-            backup_guard.unmount().await?;
-            rpc_ctx.shutdown().await
-        }.boxed()
     ))
 }
 
@@ -271,14 +286,13 @@ async fn restore_packages(
     ids: Vec<PackageId>,
 ) -> Result<
     (
-        Option<Arc<Revision>>,
         BackupMountGuard<TmpMountGuard>,
-        Vec<JoinHandle<(Result<(), Error>, PackageId)>>,
+        Vec<BoxFuture<'static, (Result<(), Error>, PackageId)>>,
         ProgressInfo,
     ),
     Error,
 > {
-    let (revision, guards) = assure_restoring(ctx, db, ids, &backup_guard).await?;
+    let guards = assure_restoring(ctx, db, ids, &backup_guard).await?;
 
     let mut progress_info = ProgressInfo::default();
 
@@ -292,7 +306,7 @@ async fn restore_packages(
             .insert(id.clone(), dir_size(backup_dir(&id)).await?);
         progress_info.target_volume_size.insert(id.clone(), 0);
         let package_id = id.clone();
-        tasks.push(tokio::spawn(
+        tasks.push(
             async move {
                 if let Err(e) = task.await {
                     tracing::error!("Error restoring package {}: {}", id, e);
@@ -302,11 +316,12 @@ async fn restore_packages(
                     Ok(())
                 }
             }
-            .map(|x| (x, package_id)),
-        ));
+            .map(|x| (x, package_id))
+            .boxed(),
+        );
     }
 
-    Ok((revision, backup_guard, tasks, progress_info))
+    Ok((backup_guard, tasks, progress_info))
 }
 
 #[instrument(skip(ctx, db, backup_guard))]
@@ -315,13 +330,7 @@ async fn assure_restoring(
     db: &mut PatchDbHandle,
     ids: Vec<PackageId>,
     backup_guard: &BackupMountGuard<TmpMountGuard>,
-) -> Result<
-    (
-        Option<Arc<Revision>>,
-        Vec<(Manifest, PackageBackupMountGuard)>,
-    ),
-    Error,
-> {
+) -> Result<Vec<(Manifest, PackageBackupMountGuard)>, Error> {
     let mut tx = db.begin().await?;
 
     let mut guards = Vec::with_capacity(ids.len());
@@ -381,7 +390,8 @@ async fn assure_restoring(
         guards.push((manifest, guard));
     }
 
-    Ok((tx.commit(None).await?, guards))
+    tx.commit().await?;
+    Ok(guards)
 }
 
 #[instrument(skip(ctx, guard))]
@@ -390,9 +400,37 @@ async fn restore_package<'a>(
     manifest: Manifest,
     guard: PackageBackupMountGuard,
 ) -> Result<(Arc<InstallProgress>, BoxFuture<'static, Result<(), Error>>), Error> {
+    let id = manifest.id.clone();
     let s9pk_path = Path::new(BACKUP_DIR)
         .join(&manifest.id)
-        .join(format!("{}.s9pk", manifest.id));
+        .join(format!("{}.s9pk", id));
+
+    let metadata_path = Path::new(BACKUP_DIR).join(&id).join("metadata.cbor");
+    let metadata: BackupMetadata =
+        IoFormat::Cbor.from_slice(&tokio::fs::read(&metadata_path).await.with_ctx(|_| {
+            (
+                crate::ErrorKind::Filesystem,
+                metadata_path.display().to_string(),
+            )
+        })?)?;
+    for (iface, key) in metadata.tor_keys {
+        let key_vec = base32::decode(base32::Alphabet::RFC4648 { padding: true }, &key)
+            .ok_or_else(|| {
+                Error::new(
+                    eyre!("invalid base32 string"),
+                    crate::ErrorKind::Deserialization,
+                )
+            })?;
+        sqlx::query!(
+                "INSERT INTO tor (package, interface, key) VALUES ($1, $2, $3) ON CONFLICT (package, interface) DO UPDATE SET key = $3",
+                *id,
+                *iface,
+                key_vec,
+            )
+            .execute(&ctx.secret_store)
+            .await?;
+    }
+
     let len = tokio::fs::metadata(&s9pk_path)
         .await
         .with_ctx(|_| {

backend/src/backup/target/cifs.rs

@@ -4,7 +4,7 @@ use color_eyre::eyre::eyre;
 use futures::TryStreamExt;
 use rpc_toolkit::command;
 use serde::{Deserialize, Serialize};
-use sqlx::{Executor, Sqlite};
+use sqlx::{Executor, Postgres};
 
 use super::{BackupTarget, BackupTargetId};
 use crate::context::RpcContext;
@@ -49,8 +49,8 @@ pub async fn add(
     let embassy_os = recovery_info(&guard).await?;
     guard.unmount().await?;
     let path_string = Path::new("/").join(&cifs.path).display().to_string();
-    let id: u32 = sqlx::query!(
-        "INSERT INTO cifs_shares (hostname, path, username, password) VALUES (?, ?, ?, ?) RETURNING id AS \"id: u32\"",
+    let id: i32 = sqlx::query!(
+        "INSERT INTO cifs_shares (hostname, path, username, password) VALUES ($1, $2, $3, $4) RETURNING id",
         cifs.hostname,
         path_string,
         cifs.username,
@@ -98,7 +98,7 @@ pub async fn update(
     guard.unmount().await?;
     let path_string = Path::new("/").join(&cifs.path).display().to_string();
     if sqlx::query!(
-        "UPDATE cifs_shares SET hostname = ?, path = ?, username = ?, password = ? WHERE id = ?",
+        "UPDATE cifs_shares SET hostname = $1, path = $2, username = $3, password = $4 WHERE id = $5",
         cifs.hostname,
         path_string,
         cifs.username,
@@ -137,7 +137,7 @@ pub async fn remove(#[context] ctx: RpcContext, #[arg] id: BackupTargetId) -> Re
             crate::ErrorKind::NotFound,
         ));
     };
-    if sqlx::query!("DELETE FROM cifs_shares WHERE id = ?", id)
+    if sqlx::query!("DELETE FROM cifs_shares WHERE id = $1", id)
         .execute(&ctx.secret_store)
         .await?
         .rows_affected()
@@ -151,12 +151,12 @@ pub async fn remove(#[context] ctx: RpcContext, #[arg] id: BackupTargetId) -> Re
     Ok(())
 }
 
-pub async fn load<Ex>(secrets: &mut Ex, id: u32) -> Result<Cifs, Error>
+pub async fn load<Ex>(secrets: &mut Ex, id: i32) -> Result<Cifs, Error>
 where
-    for<'a> &'a mut Ex: Executor<'a, Database = Sqlite>,
+    for<'a> &'a mut Ex: Executor<'a, Database = Postgres>,
 {
     let record = sqlx::query!(
-        "SELECT hostname, path, username, password FROM cifs_shares WHERE id = ?",
+        "SELECT hostname, path, username, password FROM cifs_shares WHERE id = $1",
         id
     )
     .fetch_one(secrets)
@@ -170,14 +170,13 @@ where
     })
 }
 
-pub async fn list<Ex>(secrets: &mut Ex) -> Result<Vec<(u32, CifsBackupTarget)>, Error>
+pub async fn list<Ex>(secrets: &mut Ex) -> Result<Vec<(i32, CifsBackupTarget)>, Error>
 where
-    for<'a> &'a mut Ex: Executor<'a, Database = Sqlite>,
+    for<'a> &'a mut Ex: Executor<'a, Database = Postgres>,
 {
-    let mut records = sqlx::query!(
-        "SELECT id AS \"id: u32\", hostname, path, username, password FROM cifs_shares"
-    )
-    .fetch_many(secrets);
+    let mut records =
+        sqlx::query!("SELECT id, hostname, path, username, password FROM cifs_shares")
+            .fetch_many(secrets);
 
     let mut cifs = Vec::new();
     while let Some(query_result) = records.try_next().await? {

backend/src/backup/target/mod.rs

@@ -6,11 +6,11 @@ use chrono::{DateTime, Utc};
 use clap::ArgMatches;
 use color_eyre::eyre::eyre;
 use digest::generic_array::GenericArray;
-use digest::Digest;
+use digest::OutputSizeUser;
 use rpc_toolkit::command;
 use serde::{Deserialize, Serialize};
 use sha2::Sha256;
-use sqlx::{Executor, Sqlite};
+use sqlx::{Executor, Postgres};
 use tracing::instrument;
 
 use self::cifs::CifsBackupTarget;
@@ -18,7 +18,7 @@ use crate::context::RpcContext;
 use crate::disk::mount::backup::BackupMountGuard;
 use crate::disk::mount::filesystem::block_dev::BlockDev;
 use crate::disk::mount::filesystem::cifs::Cifs;
-use crate::disk::mount::filesystem::{FileSystem, MountType, ReadOnly};
+use crate::disk::mount::filesystem::{FileSystem, MountType, ReadWrite};
 use crate::disk::mount::guard::TmpMountGuard;
 use crate::disk::util::PartitionInfo;
 use crate::s9pk::manifest::PackageId;
@@ -45,12 +45,12 @@ pub enum BackupTarget {
 #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)]
 pub enum BackupTargetId {
     Disk { logicalname: PathBuf },
-    Cifs { id: u32 },
+    Cifs { id: i32 },
 }
 impl BackupTargetId {
     pub async fn load<Ex>(self, secrets: &mut Ex) -> Result<BackupTargetFS, Error>
     where
-        for<'a> &'a mut Ex: Executor<'a, Database = Sqlite>,
+        for<'a> &'a mut Ex: Executor<'a, Database = Postgres>,
     {
         Ok(match self {
             BackupTargetId::Disk { logicalname } => {
@@ -119,7 +119,9 @@ impl FileSystem for BackupTargetFS {
             BackupTargetFS::Cifs(a) => a.mount(mountpoint, mount_type).await,
         }
     }
-    async fn source_hash(&self) -> Result<GenericArray<u8, <Sha256 as Digest>::OutputSize>, Error> {
+    async fn source_hash(
+        &self,
+    ) -> Result<GenericArray<u8, <Sha256 as OutputSizeUser>::OutputSize>, Error> {
         match self {
             BackupTargetFS::Disk(a) => a.source_hash().await,
             BackupTargetFS::Cifs(a) => a.source_hash().await,
@@ -132,16 +134,16 @@ pub fn target() -> Result<(), Error> {
     Ok(())
 }
 
-// TODO: incorporate reconnect into this response as well
 #[command(display(display_serializable))]
 pub async fn list(
     #[context] ctx: RpcContext,
 ) -> Result<BTreeMap<BackupTargetId, BackupTarget>, Error> {
     let mut sql_handle = ctx.secret_store.acquire().await?;
-    let (disks_res, cifs) =
-        tokio::try_join!(crate::disk::util::list(), cifs::list(&mut sql_handle),)?;
+    let (disks_res, cifs) = tokio::try_join!(
+        crate::disk::util::list(&ctx.os_partitions),
+        cifs::list(&mut sql_handle),
+    )?;
     Ok(disks_res
-        .disks
         .into_iter()
         .flat_map(|mut disk| {
             std::mem::take(&mut disk.partitions)
@@ -184,7 +186,7 @@ pub struct PackageBackupInfo {
     pub timestamp: DateTime<Utc>,
 }
 
-fn display_backup_info(info: BackupInfo, matches: &ArgMatches<'_>) {
+fn display_backup_info(info: BackupInfo, matches: &ArgMatches) {
     use prettytable::*;
 
     if matches.is_present("format") {
@@ -217,7 +219,7 @@ fn display_backup_info(info: BackupInfo, matches: &ArgMatches<'_>) {
         ];
         table.add_row(row);
     }
-    table.print_tty(false);
+    table.print_tty(false).unwrap();
 }
 
 #[command(display(display_backup_info))]
@@ -232,7 +234,7 @@ pub async fn info(
             &target_id
                 .load(&mut ctx.secret_store.acquire().await?)
                 .await?,
-            ReadOnly,
+            ReadWrite,
         )
         .await?,
         &password,

backend/src/bin/avahi-alias.rs

@@ -0,0 +1,163 @@
+use avahi_sys::{
+    self, avahi_client_errno, avahi_entry_group_add_service, avahi_entry_group_commit,
+    avahi_strerror, AvahiClient,
+};
+
+fn log_str_error(action: &str, e: i32) {
+    unsafe {
+        let e_str = avahi_strerror(e);
+        eprintln!(
+            "Could not {}: {:?}",
+            action,
+            std::ffi::CStr::from_ptr(e_str)
+        );
+    }
+}
+
+fn main() {
+    let aliases: Vec<_> = std::env::args().skip(1).collect();
+    unsafe {
+        let simple_poll = avahi_sys::avahi_simple_poll_new();
+        let poll = avahi_sys::avahi_simple_poll_get(simple_poll);
+        let mut box_err = Box::pin(0 as i32);
+        let err_c: *mut i32 = box_err.as_mut().get_mut();
+        let avahi_client = avahi_sys::avahi_client_new(
+            poll,
+            avahi_sys::AvahiClientFlags::AVAHI_CLIENT_NO_FAIL,
+            Some(client_callback),
+            std::ptr::null_mut(),
+            err_c,
+        );
+        if avahi_client == std::ptr::null_mut::<AvahiClient>() {
+            log_str_error("create Avahi client", *box_err);
+            panic!("Failed to create Avahi Client");
+        }
+        let group = avahi_sys::avahi_entry_group_new(
+            avahi_client,
+            Some(entry_group_callback),
+            std::ptr::null_mut(),
+        );
+        if group == std::ptr::null_mut() {
+            log_str_error("create Avahi entry group", avahi_client_errno(avahi_client));
+            panic!("Failed to create Avahi Entry Group");
+        }
+        let mut hostname_buf = vec![0];
+        let hostname_raw = avahi_sys::avahi_client_get_host_name_fqdn(avahi_client);
+        hostname_buf.extend_from_slice(std::ffi::CStr::from_ptr(hostname_raw).to_bytes_with_nul());
+        let buflen = hostname_buf.len();
+        debug_assert!(hostname_buf.ends_with(b".local\0"));
+        debug_assert!(!hostname_buf[..(buflen - 7)].contains(&b'.'));
+        // assume fixed length prefix on hostname due to local address
+        hostname_buf[0] = (buflen - 8) as u8; // set the prefix length to len - 8 (leading byte, .local, nul) for the main address
+        hostname_buf[buflen - 7] = 5; // set the prefix length to 5 for "local"
+        let mut res;
+        let http_tcp_cstr =
+            std::ffi::CString::new("_http._tcp").expect("Could not cast _http._tcp to c string");
+        res = avahi_entry_group_add_service(
+            group,
+            avahi_sys::AVAHI_IF_UNSPEC,
+            avahi_sys::AVAHI_PROTO_UNSPEC,
+            avahi_sys::AvahiPublishFlags_AVAHI_PUBLISH_USE_MULTICAST,
+            hostname_raw,
+            http_tcp_cstr.as_ptr(),
+            std::ptr::null(),
+            std::ptr::null(),
+            443,
+            // below is a secret final argument that the type signature of this function does not tell you that it
+            // needs. This is because the C lib function takes a variable number of final arguments indicating the
+            // desired TXT records to add to this service entry. The way it decides when to stop taking arguments
+            // from the stack and dereferencing them is when it finds a null pointer...because fuck you, that's why.
+            // The consequence of this is that forgetting this last argument will cause segfaults or other undefined
+            // behavior. Welcome back to the stone age motherfucker.
+            std::ptr::null::<libc::c_char>(),
+        );
+        if res < avahi_sys::AVAHI_OK {
+            log_str_error("add service to Avahi entry group", res);
+            panic!("Failed to load Avahi services");
+        }
+        eprintln!("Published {:?}", std::ffi::CStr::from_ptr(hostname_raw));
+        for alias in aliases {
+            let lan_address = alias + ".local";
+            let lan_address_ptr = std::ffi::CString::new(lan_address)
+                .expect("Could not cast lan address to c string");
+            res = avahi_sys::avahi_entry_group_add_record(
+                group,
+                avahi_sys::AVAHI_IF_UNSPEC,
+                avahi_sys::AVAHI_PROTO_UNSPEC,
+                avahi_sys::AvahiPublishFlags_AVAHI_PUBLISH_USE_MULTICAST
+                    | avahi_sys::AvahiPublishFlags_AVAHI_PUBLISH_ALLOW_MULTIPLE,
+                lan_address_ptr.as_ptr(),
+                avahi_sys::AVAHI_DNS_CLASS_IN as u16,
+                avahi_sys::AVAHI_DNS_TYPE_CNAME as u16,
+                avahi_sys::AVAHI_DEFAULT_TTL,
+                hostname_buf.as_ptr().cast(),
+                hostname_buf.len(),
+            );
+            if res < avahi_sys::AVAHI_OK {
+                log_str_error("add CNAME record to Avahi entry group", res);
+                panic!("Failed to load Avahi services");
+            }
+            eprintln!("Published {:?}", lan_address_ptr);
+        }
+        let commit_err = avahi_entry_group_commit(group);
+        if commit_err < avahi_sys::AVAHI_OK {
+            log_str_error("reset Avahi entry group", commit_err);
+            panic!("Failed to load Avahi services: reset");
+        }
+    }
+    std::thread::park()
+}
+
+unsafe extern "C" fn entry_group_callback(
+    _group: *mut avahi_sys::AvahiEntryGroup,
+    state: avahi_sys::AvahiEntryGroupState,
+    _userdata: *mut core::ffi::c_void,
+) {
+    match state {
+        avahi_sys::AvahiEntryGroupState_AVAHI_ENTRY_GROUP_FAILURE => {
+            eprintln!("AvahiCallback: EntryGroupState = AVAHI_ENTRY_GROUP_FAILURE");
+        }
+        avahi_sys::AvahiEntryGroupState_AVAHI_ENTRY_GROUP_COLLISION => {
+            eprintln!("AvahiCallback: EntryGroupState = AVAHI_ENTRY_GROUP_COLLISION");
+        }
+        avahi_sys::AvahiEntryGroupState_AVAHI_ENTRY_GROUP_UNCOMMITED => {
+            eprintln!("AvahiCallback: EntryGroupState = AVAHI_ENTRY_GROUP_UNCOMMITED");
+        }
+        avahi_sys::AvahiEntryGroupState_AVAHI_ENTRY_GROUP_ESTABLISHED => {
+            eprintln!("AvahiCallback: EntryGroupState = AVAHI_ENTRY_GROUP_ESTABLISHED");
+        }
+        avahi_sys::AvahiEntryGroupState_AVAHI_ENTRY_GROUP_REGISTERING => {
+            eprintln!("AvahiCallback: EntryGroupState = AVAHI_ENTRY_GROUP_REGISTERING");
+        }
+        other => {
+            eprintln!("AvahiCallback: EntryGroupState = {}", other);
+        }
+    }
+}
+
+unsafe extern "C" fn client_callback(
+    _group: *mut avahi_sys::AvahiClient,
+    state: avahi_sys::AvahiClientState,
+    _userdata: *mut core::ffi::c_void,
+) {
+    match state {
+        avahi_sys::AvahiClientState_AVAHI_CLIENT_FAILURE => {
+            eprintln!("AvahiCallback: ClientState = AVAHI_CLIENT_FAILURE");
+        }
+        avahi_sys::AvahiClientState_AVAHI_CLIENT_S_RUNNING => {
+            eprintln!("AvahiCallback: ClientState = AVAHI_CLIENT_S_RUNNING");
+        }
+        avahi_sys::AvahiClientState_AVAHI_CLIENT_CONNECTING => {
+            eprintln!("AvahiCallback: ClientState = AVAHI_CLIENT_CONNECTING");
+        }
+        avahi_sys::AvahiClientState_AVAHI_CLIENT_S_COLLISION => {
+            eprintln!("AvahiCallback: ClientState = AVAHI_CLIENT_S_COLLISION");
+        }
+        avahi_sys::AvahiClientState_AVAHI_CLIENT_S_REGISTERING => {
+            eprintln!("AvahiCallback: ClientState = AVAHI_CLIENT_S_REGISTERING");
+        }
+        other => {
+            eprintln!("AvahiCallback: ClientState = {}", other);
+        }
+    }
+}

backend/src/bin/embassy-cli.rs

@@ -7,20 +7,24 @@ use rpc_toolkit::run_cli;
 use rpc_toolkit::yajrc::RpcError;
 use serde_json::Value;
 
+lazy_static::lazy_static! {
+    static ref VERSION_STRING: String = Current::new().semver().to_string();
+}
+
 fn inner_main() -> Result<(), Error> {
     run_cli!({
         command: embassy::main_api,
         app: app => app
             .name("Embassy CLI")
-            .version(Current::new().semver().to_string().as_str())
+            .version(&**VERSION_STRING)
             .arg(
                 clap::Arg::with_name("config")
-                    .short("c")
+                    .short('c')
                     .long("config")
                     .takes_value(true),
             )
-            .arg(Arg::with_name("host").long("host").short("h").takes_value(true))
-            .arg(Arg::with_name("proxy").long("proxy").short("p").takes_value(true)),
+            .arg(Arg::with_name("host").long("host").short('h').takes_value(true))
+            .arg(Arg::with_name("proxy").long("proxy").short('p').takes_value(true)),
         context: matches => {
             EmbassyLogger::init();
             CliContext::init(matches)?

backend/src/bin/embassy-init.rs

@@ -1,89 +1,142 @@
-use std::path::Path;
+use std::path::{Path, PathBuf};
 use std::sync::Arc;
 use std::time::Duration;
 
 use embassy::context::rpc::RpcContextConfig;
-use embassy::context::{DiagnosticContext, SetupContext};
+use embassy::context::{DiagnosticContext, InstallContext, SetupContext};
+use embassy::disk::fsck::RepairStrategy;
 use embassy::disk::main::DEFAULT_PASSWORD;
-use embassy::hostname::get_product_key;
-use embassy::middleware::cors::cors;
-use embassy::middleware::diagnostic::diagnostic;
-use embassy::middleware::encrypt::encrypt;
+use embassy::disk::REPAIR_DISK_PATH;
+use embassy::init::STANDBY_MODE_PATH;
+use embassy::net::embassy_service_http_server::EmbassyServiceHTTPServer;
 #[cfg(feature = "avahi")]
 use embassy::net::mdns::MdnsController;
+use embassy::net::net_utils::ResourceFqdn;
+use embassy::net::static_server::{
+    diag_ui_file_router, install_ui_file_router, setup_ui_file_router,
+};
 use embassy::shutdown::Shutdown;
 use embassy::sound::CHIME;
 use embassy::util::logger::EmbassyLogger;
 use embassy::util::Invoke;
-use embassy::{Error, ResultExt};
-use http::StatusCode;
-use rpc_toolkit::rpc_server;
+use embassy::{Error, ErrorKind, ResultExt};
 use tokio::process::Command;
 use tracing::instrument;
 
-fn status_fn(_: i32) -> StatusCode {
-    StatusCode::OK
-}
-
 #[instrument]
-async fn setup_or_init(cfg_path: Option<&str>) -> Result<(), Error> {
-    if tokio::fs::metadata("/embassy-os/disk.guid").await.is_err() {
+async fn setup_or_init(cfg_path: Option<PathBuf>) -> Result<(), Error> {
+    if tokio::fs::metadata("/cdrom").await.is_ok() {
         #[cfg(feature = "avahi")]
-        let _mdns = MdnsController::init();
-        tokio::fs::write(
-            "/etc/nginx/sites-available/default",
-            include_str!("../nginx/setup-wizard.conf"),
-        )
-        .await
-        .with_ctx(|_| {
-            (
-                embassy::ErrorKind::Filesystem,
-                "/etc/nginx/sites-available/default",
-            )
-        })?;
-        Command::new("systemctl")
-            .arg("reload")
-            .arg("nginx")
-            .invoke(embassy::ErrorKind::Nginx)
+        let _mdns = MdnsController::init().await?;
+
+        let ctx = InstallContext::init(cfg_path).await?;
+
+        let embassy_ip_fqdn: ResourceFqdn = ResourceFqdn::IpAddr;
+        let embassy_fqdn: ResourceFqdn = "embassy.local".parse()?;
+
+        let localhost_fqdn = ResourceFqdn::LocalHost;
+
+        let install_ui_handler = install_ui_file_router(ctx.clone()).await?;
+
+        let mut install_http_server =
+            EmbassyServiceHTTPServer::new([0, 0, 0, 0].into(), 80, None).await?;
+        install_http_server
+            .add_svc_handler_mapping(embassy_ip_fqdn, install_ui_handler.clone())
             .await?;
-        let ctx = SetupContext::init(cfg_path).await?;
-        let keysource_ctx = ctx.clone();
-        let keysource = move || {
-            let ctx = keysource_ctx.clone();
-            async move { ctx.product_key().await }
-        };
-        let encrypt = encrypt(keysource);
+        install_http_server
+            .add_svc_handler_mapping(embassy_fqdn, install_ui_handler.clone())
+            .await?;
+
+        install_http_server
+            .add_svc_handler_mapping(localhost_fqdn, install_ui_handler.clone())
+            .await?;
+
         tokio::time::sleep(Duration::from_secs(1)).await; // let the record state that I hate this
         CHIME.play().await?;
-        rpc_server!({
-            command: embassy::setup_api,
-            context: ctx.clone(),
-            status: status_fn,
-            middleware: [
-                cors,
-                encrypt,
-            ]
-        })
-        .with_graceful_shutdown({
-            let mut shutdown = ctx.shutdown.subscribe();
-            async move {
-                shutdown.recv().await.expect("context dropped");
-            }
-        })
+
+        ctx.shutdown
+            .subscribe()
+            .recv()
+            .await
+            .expect("context dropped");
+        install_http_server.shutdown.send(()).unwrap();
+        Command::new("reboot")
+            .invoke(embassy::ErrorKind::Unknown)
+            .await?;
+    } else if tokio::fs::metadata("/media/embassy/config/disk.guid")
         .await
-        .with_kind(embassy::ErrorKind::Network)?;
+        .is_err()
+    {
+        #[cfg(feature = "avahi")]
+        let _mdns = MdnsController::init().await?;
+
+        let ctx = SetupContext::init(cfg_path).await?;
+
+        let embassy_ip_fqdn: ResourceFqdn = ResourceFqdn::IpAddr;
+        let embassy_fqdn: ResourceFqdn = "embassy.local".parse()?;
+        let localhost_fqdn = ResourceFqdn::LocalHost;
+
+        let setup_ui_handler = setup_ui_file_router(ctx.clone()).await?;
+
+        let mut setup_http_server =
+            EmbassyServiceHTTPServer::new([0, 0, 0, 0].into(), 80, None).await?;
+        setup_http_server
+            .add_svc_handler_mapping(embassy_ip_fqdn, setup_ui_handler.clone())
+            .await?;
+        setup_http_server
+            .add_svc_handler_mapping(embassy_fqdn, setup_ui_handler.clone())
+            .await?;
+
+        setup_http_server
+            .add_svc_handler_mapping(localhost_fqdn, setup_ui_handler.clone())
+            .await?;
+
+        tokio::time::sleep(Duration::from_secs(1)).await; // let the record state that I hate this
+        CHIME.play().await?;
+        ctx.shutdown
+            .subscribe()
+            .recv()
+            .await
+            .expect("context dropped");
+        setup_http_server.shutdown.send(()).unwrap();
+        tokio::task::yield_now().await;
+        if let Err(e) = Command::new("killall")
+            .arg("firefox-esr")
+            .invoke(ErrorKind::NotFound)
+            .await
+        {
+            tracing::error!("Failed to kill kiosk: {}", e);
+            tracing::debug!("{:?}", e);
+        }
     } else {
         let cfg = RpcContextConfig::load(cfg_path).await?;
-        embassy::disk::main::import(
-            tokio::fs::read_to_string("/embassy-os/disk.guid") // unique identifier for volume group - keeps track of the disk that goes with your embassy
-                .await?
-                .trim(),
+        let guid_string = tokio::fs::read_to_string("/media/embassy/config/disk.guid") // unique identifier for volume group - keeps track of the disk that goes with your embassy
+            .await?;
+        let guid = guid_string.trim();
+        let requires_reboot = embassy::disk::main::import(
+            guid,
             cfg.datadir(),
+            if tokio::fs::metadata(REPAIR_DISK_PATH).await.is_ok() {
+                RepairStrategy::Aggressive
+            } else {
+                RepairStrategy::Preen
+            },
             DEFAULT_PASSWORD,
         )
         .await?;
+        if tokio::fs::metadata(REPAIR_DISK_PATH).await.is_ok() {
+            tokio::fs::remove_file(REPAIR_DISK_PATH)
+                .await
+                .with_ctx(|_| (embassy::ErrorKind::Filesystem, REPAIR_DISK_PATH))?;
+        }
+        if requires_reboot.0 {
+            embassy::disk::main::export(guid, cfg.datadir()).await?;
+            Command::new("reboot")
+                .invoke(embassy::ErrorKind::Unknown)
+                .await?;
+        }
         tracing::info!("Loaded Disk");
-        embassy::init::init(&cfg, &get_product_key().await?).await?;
+        embassy::init::init(&cfg).await?;
     }
 
     Ok(())
@@ -108,39 +161,34 @@ async fn run_script_if_exists<P: AsRef<Path>>(path: P) {
 }
 
 #[instrument]
-async fn inner_main(cfg_path: Option<&str>) -> Result<Option<Shutdown>, Error> {
+async fn inner_main(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Error> {
+    if tokio::fs::metadata(STANDBY_MODE_PATH).await.is_ok() {
+        tokio::fs::remove_file(STANDBY_MODE_PATH).await?;
+        Command::new("sync").invoke(ErrorKind::Filesystem).await?;
+        embassy::sound::SHUTDOWN.play().await?;
+        futures::future::pending::<()>().await;
+    }
+
     embassy::sound::BEP.play().await?;
 
-    run_script_if_exists("/embassy-os/preinit.sh").await;
+    run_script_if_exists("/media/embassy/config/preinit.sh").await;
 
-    let res = if let Err(e) = setup_or_init(cfg_path).await {
-        async {
+    let res = if let Err(e) = setup_or_init(cfg_path.clone()).await {
+        async move {
             tracing::error!("{}", e.source);
             tracing::debug!("{}", e.source);
             embassy::sound::BEETHOVEN.play().await?;
             #[cfg(feature = "avahi")]
-            let _mdns = MdnsController::init();
-            tokio::fs::write(
-                "/etc/nginx/sites-available/default",
-                include_str!("../nginx/diagnostic-ui.conf"),
-            )
-            .await
-            .with_ctx(|_| {
-                (
-                    embassy::ErrorKind::Filesystem,
-                    "/etc/nginx/sites-available/default",
-                )
-            })?;
-            Command::new("systemctl")
-                .arg("reload")
-                .arg("nginx")
-                .invoke(embassy::ErrorKind::Nginx)
-                .await?;
+            let _mdns = MdnsController::init().await?;
+
             let ctx = DiagnosticContext::init(
                 cfg_path,
-                if tokio::fs::metadata("/embassy-os/disk.guid").await.is_ok() {
+                if tokio::fs::metadata("/media/embassy/config/disk.guid")
+                    .await
+                    .is_ok()
+                {
                     Some(Arc::new(
-                        tokio::fs::read_to_string("/embassy-os/disk.guid") // unique identifier for volume group - keeps track of the disk that goes with your embassy
+                        tokio::fs::read_to_string("/media/embassy/config/disk.guid") // unique identifier for volume group - keeps track of the disk that goes with your embassy
                             .await?
                             .trim()
                             .to_owned(),
@@ -151,47 +199,46 @@ async fn inner_main(cfg_path: Option<&str>) -> Result<Option<Shutdown>, Error> {
                 e,
             )
             .await?;
-            let mut shutdown_recv = ctx.shutdown.subscribe();
-            rpc_server!({
-                command: embassy::diagnostic_api,
-                context: ctx.clone(),
-                status: status_fn,
-                middleware: [
-                    cors,
-                    diagnostic,
-                ]
-            })
-            .with_graceful_shutdown({
-                let mut shutdown = ctx.shutdown.subscribe();
-                async move {
-                    shutdown.recv().await.expect("context dropped");
-                }
-            })
-            .await
-            .with_kind(embassy::ErrorKind::Network)?;
 
-            Ok::<_, Error>(
-                shutdown_recv
-                    .recv()
-                    .await
-                    .with_kind(embassy::ErrorKind::Network)?,
-            )
+            let embassy_ip_fqdn: ResourceFqdn = ResourceFqdn::IpAddr;
+            let embassy_fqdn: ResourceFqdn = "embassy.local".parse()?;
+
+            let localhost_fqdn = ResourceFqdn::LocalHost;
+
+            let diag_ui_handler = diag_ui_file_router(ctx.clone()).await?;
+
+            let mut diag_http_server =
+                EmbassyServiceHTTPServer::new([0, 0, 0, 0].into(), 80, None).await?;
+            diag_http_server
+                .add_svc_handler_mapping(embassy_ip_fqdn, diag_ui_handler.clone())
+                .await?;
+            diag_http_server
+                .add_svc_handler_mapping(embassy_fqdn, diag_ui_handler.clone())
+                .await?;
+
+            diag_http_server
+                .add_svc_handler_mapping(localhost_fqdn, diag_ui_handler.clone())
+                .await?;
+
+            let shutdown = ctx.shutdown.subscribe().recv().await.unwrap();
+            diag_http_server.shutdown.send(()).unwrap();
+            Ok(shutdown)
         }
         .await
     } else {
         Ok(None)
     };
 
-    run_script_if_exists("/embassy-os/postinit.sh").await;
+    run_script_if_exists("/media/embassy/config/postinit.sh").await;
 
     res
 }
 
 fn main() {
-    let matches = clap::App::new("embassyd")
+    let matches = clap::App::new("embassy-init")
         .arg(
             clap::Arg::with_name("config")
-                .short("c")
+                .short('c')
                 .long("config")
                 .takes_value(true),
         )
@@ -199,7 +246,7 @@ fn main() {
 
     EmbassyLogger::init();
 
-    let cfg_path = matches.value_of("config");
+    let cfg_path = matches.value_of("config").map(|p| Path::new(p).to_owned());
     let res = {
         let rt = tokio::runtime::Builder::new_multi_thread()
             .enable_all()

backend/src/bin/embassy-sdk.rs

@@ -6,21 +6,25 @@ use rpc_toolkit::run_cli;
 use rpc_toolkit::yajrc::RpcError;
 use serde_json::Value;
 
+lazy_static::lazy_static! {
+    static ref VERSION_STRING: String = Current::new().semver().to_string();
+}
+
 fn inner_main() -> Result<(), Error> {
     run_cli!({
         command: embassy::portable_api,
         app: app => app
             .name("Embassy SDK")
-            .version(Current::new().semver().to_string().as_str())
+            .version(&**VERSION_STRING)
             .arg(
                 clap::Arg::with_name("config")
-                    .short("c")
+                    .short('c')
                     .long("config")
                     .takes_value(true),
             ),
         context: matches => {
             if let Err(_) = std::env::var("RUST_LOG") {
-                std::env::set_var("RUST_LOG", "embassy=warn");
+                std::env::set_var("RUST_LOG", "embassy=warn,js_engine=warn");
             }
             EmbassyLogger::init();
             SdkContext::init(matches)?

backend/src/bin/embassyd.rs

@@ -1,56 +1,41 @@
+use std::path::{Path, PathBuf};
 use std::sync::Arc;
 use std::time::Duration;
 
 use color_eyre::eyre::eyre;
 use embassy::context::{DiagnosticContext, RpcContext};
-use embassy::core::rpc_continuations::RequestGuid;
-use embassy::db::subscribe;
-use embassy::middleware::auth::auth;
-use embassy::middleware::cors::cors;
-use embassy::middleware::diagnostic::diagnostic;
+use embassy::net::embassy_service_http_server::EmbassyServiceHTTPServer;
 #[cfg(feature = "avahi")]
 use embassy::net::mdns::MdnsController;
+use embassy::net::net_controller::NetController;
+use embassy::net::net_utils::ResourceFqdn;
+use embassy::net::static_server::diag_ui_file_router;
 use embassy::net::tor::tor_health_check;
 use embassy::shutdown::Shutdown;
 use embassy::system::launch_metrics_task;
+use embassy::util::daemon;
 use embassy::util::logger::EmbassyLogger;
-use embassy::util::{daemon, Invoke};
-use embassy::{static_server, Error, ErrorKind, ResultExt};
+use embassy::{Error, ErrorKind, ResultExt};
 use futures::{FutureExt, TryFutureExt};
 use reqwest::{Client, Proxy};
-use rpc_toolkit::hyper::{Body, Response, Server, StatusCode};
-use rpc_toolkit::rpc_server;
-use tokio::process::Command;
 use tokio::signal::unix::signal;
 use tracing::instrument;
 
-fn status_fn(_: i32) -> StatusCode {
-    StatusCode::OK
-}
-
-fn err_to_500(e: Error) -> Response<Body> {
-    tracing::error!("{}", e);
-    tracing::debug!("{:?}", e);
-    Response::builder()
-        .status(StatusCode::INTERNAL_SERVER_ERROR)
-        .body(Body::empty())
-        .unwrap()
-}
-
 #[instrument]
-async fn inner_main(cfg_path: Option<&str>) -> Result<Option<Shutdown>, Error> {
+async fn inner_main(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Error> {
     let (rpc_ctx, shutdown) = {
-        embassy::hostname::sync_hostname().await?;
         let rpc_ctx = RpcContext::init(
             cfg_path,
             Arc::new(
-                tokio::fs::read_to_string("/embassy-os/disk.guid") // unique identifier for volume group - keeps track of the disk that goes with your embassy
+                tokio::fs::read_to_string("/media/embassy/config/disk.guid") // unique identifier for volume group - keeps track of the disk that goes with your embassy
                     .await?
                     .trim()
                     .to_owned(),
             ),
         )
         .await?;
+        NetController::setup_embassy_ui(rpc_ctx.clone()).await?;
+
         let mut shutdown_recv = rpc_ctx.shutdown.subscribe();
 
         let sig_handler_ctx = rpc_ctx.clone();
@@ -66,7 +51,7 @@ async fn inner_main(cfg_path: Option<&str>) -> Result<Option<Shutdown>, Error> {
                 .map(|s| {
                     async move {
                         signal(*s)
-                            .expect(&format!("register {:?} handler", s))
+                            .unwrap_or_else(|_| panic!("register {:?} handler", s))
                             .recv()
                             .await
                     }
@@ -81,24 +66,11 @@ async fn inner_main(cfg_path: Option<&str>) -> Result<Option<Shutdown>, Error> {
                 .expect("send shutdown signal");
         });
 
-        rpc_ctx.set_nginx_conf(&mut rpc_ctx.db.handle()).await?;
-        let auth = auth(rpc_ctx.clone());
-        let ctx = rpc_ctx.clone();
-        let server = rpc_server!({
-            command: embassy::main_api,
-            context: ctx,
-            status: status_fn,
-            middleware: [
-                cors,
-                auth,
-            ]
-        })
-        .with_graceful_shutdown({
-            let mut shutdown = rpc_ctx.shutdown.subscribe();
-            async move {
-                shutdown.recv().await.expect("context dropped");
-            }
-        });
+        {
+            let mut db = rpc_ctx.db.handle();
+            let receipts = embassy::context::rpc::RpcSetHostNameReceipts::new(&mut db).await?;
+            embassy::hostname::sync_hostname(&mut db, &receipts.hostname_receipts).await?;
+        }
 
         let metrics_ctx = rpc_ctx.clone();
         let metrics_task = tokio::spawn(async move {
@@ -108,105 +80,6 @@ async fn inner_main(cfg_path: Option<&str>) -> Result<Option<Shutdown>, Error> {
             .await
         });
 
-        let rev_cache_ctx = rpc_ctx.clone();
-        let revision_cache_task = tokio::spawn(async move {
-            let mut sub = rev_cache_ctx.db.subscribe();
-            let mut shutdown = rev_cache_ctx.shutdown.subscribe();
-            loop {
-                let rev = match tokio::select! {
-                    a = sub.recv() => a,
-                    _ = shutdown.recv() => break,
-                } {
-                    Ok(a) => a,
-                    Err(_) => {
-                        rev_cache_ctx.revision_cache.write().await.truncate(0);
-                        continue;
-                    }
-                }; // TODO: handle falling behind
-                let mut cache = rev_cache_ctx.revision_cache.write().await;
-                cache.push_back(rev);
-                if cache.len() > rev_cache_ctx.revision_cache_size {
-                    cache.pop_front();
-                }
-            }
-        });
-
-        let ws_ctx = rpc_ctx.clone();
-        let ws_server = {
-            let builder = Server::bind(&ws_ctx.bind_ws);
-
-            let make_svc = ::rpc_toolkit::hyper::service::make_service_fn(move |_| {
-                let ctx = ws_ctx.clone();
-                async move {
-                    Ok::<_, ::rpc_toolkit::hyper::Error>(::rpc_toolkit::hyper::service::service_fn(
-                        move |req| {
-                            let ctx = ctx.clone();
-                            async move {
-                                tracing::debug!("Request to {}", req.uri().path());
-                                match req.uri().path() {
-                                    "/ws/db" => {
-                                        Ok(subscribe(ctx, req).await.unwrap_or_else(err_to_500))
-                                    }
-                                    path if path.starts_with("/rest/rpc/") => {
-                                        match RequestGuid::from(
-                                            path.strip_prefix("/rest/rpc/").unwrap(),
-                                        ) {
-                                            None => {
-                                                tracing::debug!("No Guid Path");
-                                                Response::builder()
-                                                    .status(StatusCode::BAD_REQUEST)
-                                                    .body(Body::empty())
-                                            }
-                                            Some(guid) => {
-                                                match ctx
-                                                    .rpc_stream_continuations
-                                                    .lock()
-                                                    .await
-                                                    .remove(&guid)
-                                                {
-                                                    None => Response::builder()
-                                                        .status(StatusCode::NOT_FOUND)
-                                                        .body(Body::empty()),
-                                                    Some(cont) => match (cont.handler)(req).await {
-                                                        Ok(r) => Ok(r),
-                                                        Err(e) => Response::builder()
-                                                            .status(
-                                                                StatusCode::INTERNAL_SERVER_ERROR,
-                                                            )
-                                                            .body(Body::from(format!("{}", e))),
-                                                    },
-                                                }
-                                            }
-                                        }
-                                    }
-                                    _ => Response::builder()
-                                        .status(StatusCode::NOT_FOUND)
-                                        .body(Body::empty()),
-                                }
-                            }
-                        },
-                    ))
-                }
-            });
-            builder.serve(make_svc)
-        }
-        .with_graceful_shutdown({
-            let mut shutdown = rpc_ctx.shutdown.subscribe();
-            async move {
-                shutdown.recv().await.expect("context dropped");
-            }
-        });
-
-        let file_server_ctx = rpc_ctx.clone();
-        let file_server = {
-            static_server::init(file_server_ctx, {
-                let mut shutdown = rpc_ctx.shutdown.subscribe();
-                async move {
-                    shutdown.recv().await.expect("context dropped");
-                }
-            })
-        };
-
         let tor_health_ctx = rpc_ctx.clone();
         let tor_client = Client::builder()
             .proxy(
@@ -232,27 +105,12 @@ async fn inner_main(cfg_path: Option<&str>) -> Result<Option<Shutdown>, Error> {
         embassy::sound::CHIME.play().await?;
 
         futures::try_join!(
-            server
-                .map_err(|e| Error::new(e, ErrorKind::Network))
-                .map_ok(|_| tracing::debug!("RPC Server Shutdown")),
             metrics_task
                 .map_err(|e| Error::new(
                     eyre!("{}", e).wrap_err("Metrics daemon panicked!"),
                     ErrorKind::Unknown
                 ))
                 .map_ok(|_| tracing::debug!("Metrics daemon Shutdown")),
-            revision_cache_task
-                .map_err(|e| Error::new(
-                    eyre!("{}", e).wrap_err("Revision Cache daemon panicked!"),
-                    ErrorKind::Unknown
-                ))
-                .map_ok(|_| tracing::debug!("Revision Cache daemon Shutdown")),
-            ws_server
-                .map_err(|e| Error::new(e, ErrorKind::Network))
-                .map_ok(|_| tracing::debug!("WebSocket Server Shutdown")),
-            file_server
-                .map_err(|e| Error::new(e, ErrorKind::Network))
-                .map_ok(|_| tracing::debug!("Static File Server Shutdown")),
             tor_health_daemon
                 .map_err(|e| Error::new(
                     e.wrap_err("Tor Health daemon panicked!"),
@@ -283,7 +141,7 @@ fn main() {
     let matches = clap::App::new("embassyd")
         .arg(
             clap::Arg::with_name("config")
-                .short("c")
+                .short('c')
                 .long("config")
                 .takes_value(true),
         )
@@ -291,7 +149,7 @@ fn main() {
 
     EmbassyLogger::init();
 
-    let cfg_path = matches.value_of("config");
+    let cfg_path = matches.value_of("config").map(|p| Path::new(p).to_owned());
 
     let res = {
         let rt = tokio::runtime::Builder::new_multi_thread()
@@ -299,7 +157,7 @@ fn main() {
             .build()
             .expect("failed to initialize runtime");
         rt.block_on(async {
-            match inner_main(cfg_path).await {
+            match inner_main(cfg_path.clone()).await {
                 Ok(a) => Ok(a),
                 Err(e) => {
                     (|| async {
@@ -307,28 +165,15 @@ fn main() {
                         tracing::debug!("{:?}", e.source);
                         embassy::sound::BEETHOVEN.play().await?;
                         #[cfg(feature = "avahi")]
-                        let _mdns = MdnsController::init();
-                        tokio::fs::write(
-                            "/etc/nginx/sites-available/default",
-                            include_str!("../nginx/diagnostic-ui.conf"),
-                        )
-                        .await
-                        .with_ctx(|_| {
-                            (
-                                embassy::ErrorKind::Filesystem,
-                                "/etc/nginx/sites-available/default",
-                            )
-                        })?;
-                        Command::new("systemctl")
-                            .arg("reload")
-                            .arg("nginx")
-                            .invoke(embassy::ErrorKind::Nginx)
-                            .await?;
+                        let _mdns = MdnsController::init().await?;
                         let ctx = DiagnosticContext::init(
                             cfg_path,
-                            if tokio::fs::metadata("/embassy-os/disk.guid").await.is_ok() {
+                            if tokio::fs::metadata("/media/embassy/config/disk.guid")
+                                .await
+                                .is_ok()
+                            {
                                 Some(Arc::new(
-                                    tokio::fs::read_to_string("/embassy-os/disk.guid") // unique identifier for volume group - keeps track of the disk that goes with your embassy
+                                    tokio::fs::read_to_string("/media/embassy/config/disk.guid") // unique identifier for volume group - keeps track of the disk that goes with your embassy
                                         .await?
                                         .trim()
                                         .to_owned(),
@@ -339,24 +184,24 @@ fn main() {
                             e,
                         )
                         .await?;
-                        rpc_server!({
-                            command: embassy::diagnostic_api,
-                            context: ctx.clone(),
-                            status: status_fn,
-                            middleware: [
-                                cors,
-                                diagnostic,
-                            ]
-                        })
-                        .with_graceful_shutdown({
-                            let mut shutdown = ctx.shutdown.subscribe();
-                            async move {
-                                shutdown.recv().await.expect("context dropped");
-                            }
-                        })
-                        .await
-                        .with_kind(embassy::ErrorKind::Network)?;
-                        Ok::<_, Error>(None)
+
+                        let embassy_ip_fqdn: ResourceFqdn = ResourceFqdn::IpAddr;
+                        let embassy_fqdn: ResourceFqdn = "embassy.local".parse()?;
+
+                        let diag_ui_handler = diag_ui_file_router(ctx.clone()).await?;
+
+                        let mut diag_http_server =
+                            EmbassyServiceHTTPServer::new([0, 0, 0, 0].into(), 80, None).await?;
+                        diag_http_server
+                            .add_svc_handler_mapping(embassy_ip_fqdn, diag_ui_handler.clone())
+                            .await?;
+                        diag_http_server
+                            .add_svc_handler_mapping(embassy_fqdn, diag_ui_handler)
+                            .await?;
+
+                        let mut shutdown = ctx.shutdown.subscribe();
+
+                        shutdown.recv().await.with_kind(crate::ErrorKind::Unknown)
                     })()
                     .await
                 }

backend/src/config/action.rs

@@ -7,10 +7,11 @@ use serde::{Deserialize, Serialize};
 use tracing::instrument;
 
 use super::{Config, ConfigSpec};
-use crate::action::ActionImplementation;
 use crate::context::RpcContext;
 use crate::dependencies::Dependencies;
 use crate::id::ImageId;
+use crate::procedure::docker::DockerContainers;
+use crate::procedure::{PackageProcedure, ProcedureName};
 use crate::s9pk::manifest::PackageId;
 use crate::status::health_check::HealthCheckId;
 use crate::util::Version;
@@ -26,17 +27,23 @@ pub struct ConfigRes {
 
 #[derive(Clone, Debug, Deserialize, Serialize, HasModel)]
 pub struct ConfigActions {
-    pub get: ActionImplementation,
-    pub set: ActionImplementation,
+    pub get: PackageProcedure,
+    pub set: PackageProcedure,
 }
 impl ConfigActions {
     #[instrument]
-    pub fn validate(&self, volumes: &Volumes, image_ids: &BTreeSet<ImageId>) -> Result<(), Error> {
+    pub fn validate(
+        &self,
+        container: &Option<DockerContainers>,
+        eos_version: &Version,
+        volumes: &Volumes,
+        image_ids: &BTreeSet<ImageId>,
+    ) -> Result<(), Error> {
         self.get
-            .validate(volumes, image_ids, true)
+            .validate(container, eos_version, volumes, image_ids, true)
             .with_ctx(|_| (crate::ErrorKind::ValidateS9pk, "Config Get"))?;
         self.set
-            .validate(volumes, image_ids, true)
+            .validate(container, eos_version, volumes, image_ids, true)
             .with_ctx(|_| (crate::ErrorKind::ValidateS9pk, "Config Set"))?;
         Ok(())
     }
@@ -53,10 +60,9 @@ impl ConfigActions {
                 ctx,
                 pkg_id,
                 pkg_version,
-                Some("GetConfig"),
+                ProcedureName::GetConfig,
                 volumes,
                 None::<()>,
-                false,
                 None,
             )
             .await
@@ -81,10 +87,9 @@ impl ConfigActions {
                 ctx,
                 pkg_id,
                 pkg_version,
-                Some("SetConfig"),
+                ProcedureName::SetConfig,
                 volumes,
                 Some(input),
-                false,
                 None,
             )
             .await
@@ -107,6 +112,7 @@ impl ConfigActions {
 #[derive(Debug, Deserialize, Serialize)]
 #[serde(rename_all = "kebab-case")]
 pub struct SetResult {
+    #[serde(default)]
     #[serde(deserialize_with = "crate::util::serde::deserialize_from_str_opt")]
     #[serde(serialize_with = "crate::util::serde::serialize_display_opt")]
     pub signal: Option<Signal>,

backend/src/config/mod.rs

@@ -6,7 +6,7 @@ use color_eyre::eyre::eyre;
 use futures::future::{BoxFuture, FutureExt};
 use indexmap::IndexSet;
 use itertools::Itertools;
-use patch_db::{DbHandle, LockType};
+use patch_db::{DbHandle, LockReceipt, LockTarget, LockTargetId, LockType, Verifier};
 use rand::SeedableRng;
 use regex::Regex;
 use rpc_toolkit::command;
@@ -14,17 +14,18 @@ use serde_json::Value;
 use tracing::instrument;
 
 use crate::context::RpcContext;
-use crate::db::model::CurrentDependencyInfo;
-use crate::db::util::WithRevision;
+use crate::db::model::{CurrentDependencies, CurrentDependencyInfo, CurrentDependents};
 use crate::dependencies::{
     add_dependent_to_current_dependents_lists, break_transitive, heal_all_dependents_transitive,
-    BreakageRes, DependencyError, DependencyErrors, TaggedDependencyError,
+    BreakTransitiveReceipts, BreakageRes, Dependencies, DependencyConfig, DependencyError,
+    DependencyErrors, DependencyReceipt, TaggedDependencyError, TryHealReceipts,
 };
-use crate::install::cleanup::remove_from_current_dependents_lists;
+use crate::install::cleanup::{remove_from_current_dependents_lists, UpdateDependencyReceipts};
+use crate::procedure::docker::DockerContainers;
 use crate::s9pk::manifest::{Manifest, PackageId};
 use crate::util::display_none;
 use crate::util::serde::{display_serializable, parse_stdin_deserializable, IoFormat};
-use crate::{Error, ResultExt as _};
+use crate::Error;
 
 pub mod action;
 pub mod spec;
@@ -33,8 +34,8 @@ pub mod util;
 pub use spec::{ConfigSpec, Defaultable};
 use util::NumRange;
 
-use self::action::ConfigRes;
-use self::spec::{PackagePointerSpec, ValueSpecPointer};
+use self::action::{ConfigActions, ConfigRes};
+use self::spec::{ConfigPointerReceipts, PackagePointerSpec, ValueSpecPointer};
 
 pub type Config = serde_json::Map<String, Value>;
 pub trait TypeOf {
@@ -163,6 +164,55 @@ pub fn config(#[arg] id: PackageId) -> Result<PackageId, Error> {
     Ok(id)
 }
 
+pub struct ConfigGetReceipts {
+    manifest_volumes: LockReceipt<crate::volume::Volumes, ()>,
+    manifest_version: LockReceipt<crate::util::Version, ()>,
+    manifest_config: LockReceipt<Option<ConfigActions>, ()>,
+}
+
+impl ConfigGetReceipts {
+    pub async fn new<'a>(db: &'a mut impl DbHandle, id: &PackageId) -> Result<Self, Error> {
+        let mut locks = Vec::new();
+
+        let setup = Self::setup(&mut locks, id);
+        Ok(setup(&db.lock_all(locks).await?)?)
+    }
+
+    pub fn setup(
+        locks: &mut Vec<LockTargetId>,
+        id: &PackageId,
+    ) -> impl FnOnce(&Verifier) -> Result<Self, Error> {
+        let manifest_version = crate::db::DatabaseModel::new()
+            .package_data()
+            .idx_model(id)
+            .and_then(|x| x.installed())
+            .map(|x| x.manifest().version())
+            .make_locker(LockType::Write)
+            .add_to_keys(locks);
+        let manifest_volumes = crate::db::DatabaseModel::new()
+            .package_data()
+            .idx_model(id)
+            .and_then(|x| x.installed())
+            .map(|x| x.manifest().volumes())
+            .make_locker(LockType::Write)
+            .add_to_keys(locks);
+        let manifest_config = crate::db::DatabaseModel::new()
+            .package_data()
+            .idx_model(id)
+            .and_then(|x| x.installed())
+            .map(|x| x.manifest().config())
+            .make_locker(LockType::Write)
+            .add_to_keys(locks);
+        move |skeleton_key| {
+            Ok(Self {
+                manifest_volumes: manifest_volumes.verify(skeleton_key)?,
+                manifest_version: manifest_version.verify(skeleton_key)?,
+                manifest_config: manifest_config.verify(skeleton_key)?,
+            })
+        }
+    }
+}
+
 #[command(display(display_serializable))]
 #[instrument(skip(ctx))]
 pub async fn get(
@@ -173,34 +223,22 @@ pub async fn get(
     format: Option<IoFormat>,
 ) -> Result<ConfigRes, Error> {
     let mut db = ctx.db.handle();
-    let pkg_model = crate::db::DatabaseModel::new()
-        .package_data()
-        .idx_model(&id)
-        .and_then(|m| m.installed())
-        .expect(&mut db)
-        .await
-        .with_kind(crate::ErrorKind::NotFound)?;
-    let action = pkg_model
-        .clone()
-        .manifest()
-        .config()
-        .get(&mut db, true)
+    let receipts = ConfigGetReceipts::new(&mut db, &id).await?;
+    let action = receipts
+        .manifest_config
+        .get(&mut db)
         .await?
-        .to_owned()
         .ok_or_else(|| Error::new(eyre!("{} has no config", id), crate::ErrorKind::NotFound))?;
-    let version = pkg_model
-        .clone()
-        .manifest()
-        .version()
-        .get(&mut db, true)
-        .await?;
-    let volumes = pkg_model.manifest().volumes().get(&mut db, true).await?;
-    action.get(&ctx, &id, &*version, &*volumes).await
+
+    let volumes = receipts.manifest_volumes.get(&mut db).await?;
+    let version = receipts.manifest_version.get(&mut db).await?;
+    action.get(&ctx, &id, &version, &volumes).await
 }
 
 #[command(
     subcommands(self(set_impl(async, context(RpcContext))), set_dry),
-    display(display_none)
+    display(display_none),
+    metadata(sync_db = true)
 )]
 #[instrument]
 pub fn set(
@@ -210,25 +248,180 @@ pub fn set(
     format: Option<IoFormat>,
     #[arg(long = "timeout")] timeout: Option<crate::util::serde::Duration>,
     #[arg(stdin, parse(parse_stdin_deserializable))] config: Option<Config>,
-    #[arg(rename = "expire-id", long = "expire-id")] expire_id: Option<String>,
-) -> Result<(PackageId, Option<Config>, Option<Duration>, Option<String>), Error> {
-    Ok((id, config, timeout.map(|d| *d), expire_id))
+) -> Result<(PackageId, Option<Config>, Option<Duration>), Error> {
+    Ok((id, config, timeout.map(|d| *d)))
+}
+
+/// So, the new locking finds all the possible locks and lifts them up into a bundle of locks.
+/// Then this bundle will be passed down into the functions that will need to touch the db, and
+/// instead of doing the locks down in the system, we have already done the locks and can
+/// do the operation on the db.
+/// An UnlockedLock has two types, the type of setting and getting from the db, and the second type
+/// is the keys that we need to insert on getting/setting because we have included wild cards into the paths.
+pub struct ConfigReceipts {
+    pub dependency_receipt: DependencyReceipt,
+    pub config_receipts: ConfigPointerReceipts,
+    pub update_dependency_receipts: UpdateDependencyReceipts,
+    pub try_heal_receipts: TryHealReceipts,
+    pub break_transitive_receipts: BreakTransitiveReceipts,
+    configured: LockReceipt<bool, String>,
+    config_actions: LockReceipt<ConfigActions, String>,
+    dependencies: LockReceipt<Dependencies, String>,
+    volumes: LockReceipt<crate::volume::Volumes, String>,
+    version: LockReceipt<crate::util::Version, String>,
+    manifest: LockReceipt<Manifest, String>,
+    system_pointers: LockReceipt<Vec<spec::SystemPointerSpec>, String>,
+    pub current_dependents: LockReceipt<CurrentDependents, String>,
+    pub current_dependencies: LockReceipt<CurrentDependencies, String>,
+    dependency_errors: LockReceipt<DependencyErrors, String>,
+    manifest_dependencies_config: LockReceipt<DependencyConfig, (String, String)>,
+    docker_containers: LockReceipt<DockerContainers, String>,
+}
+
+impl ConfigReceipts {
+    pub async fn new<'a>(db: &'a mut impl DbHandle) -> Result<Self, Error> {
+        let mut locks = Vec::new();
+
+        let setup = Self::setup(&mut locks);
+        Ok(setup(&db.lock_all(locks).await?)?)
+    }
+
+    pub fn setup(locks: &mut Vec<LockTargetId>) -> impl FnOnce(&Verifier) -> Result<Self, Error> {
+        let dependency_receipt = DependencyReceipt::setup(locks);
+        let config_receipts = ConfigPointerReceipts::setup(locks);
+        let update_dependency_receipts = UpdateDependencyReceipts::setup(locks);
+        let break_transitive_receipts = BreakTransitiveReceipts::setup(locks);
+        let try_heal_receipts = TryHealReceipts::setup(locks);
+
+        let configured: LockTarget<bool, String> = crate::db::DatabaseModel::new()
+            .package_data()
+            .star()
+            .installed()
+            .map(|x| x.status().configured())
+            .make_locker(LockType::Write)
+            .add_to_keys(locks);
+
+        let config_actions = crate::db::DatabaseModel::new()
+            .package_data()
+            .star()
+            .installed()
+            .and_then(|x| x.manifest().config())
+            .make_locker(LockType::Read)
+            .add_to_keys(locks);
+
+        let dependencies = crate::db::DatabaseModel::new()
+            .package_data()
+            .star()
+            .installed()
+            .map(|x| x.manifest().dependencies())
+            .make_locker(LockType::Read)
+            .add_to_keys(locks);
+
+        let volumes = crate::db::DatabaseModel::new()
+            .package_data()
+            .star()
+            .installed()
+            .map(|x| x.manifest().volumes())
+            .make_locker(LockType::Read)
+            .add_to_keys(locks);
+
+        let version = crate::db::DatabaseModel::new()
+            .package_data()
+            .star()
+            .installed()
+            .map(|x| x.manifest().version())
+            .make_locker(LockType::Read)
+            .add_to_keys(locks);
+
+        let manifest = crate::db::DatabaseModel::new()
+            .package_data()
+            .star()
+            .installed()
+            .map(|x| x.manifest())
+            .make_locker(LockType::Read)
+            .add_to_keys(locks);
+
+        let system_pointers = crate::db::DatabaseModel::new()
+            .package_data()
+            .star()
+            .installed()
+            .map(|x| x.system_pointers())
+            .make_locker(LockType::Write)
+            .add_to_keys(locks);
+
+        let current_dependents = crate::db::DatabaseModel::new()
+            .package_data()
+            .star()
+            .installed()
+            .map(|x| x.current_dependents())
+            .make_locker(LockType::Write)
+            .add_to_keys(locks);
+
+        let current_dependencies = crate::db::DatabaseModel::new()
+            .package_data()
+            .star()
+            .installed()
+            .map(|x| x.current_dependencies())
+            .make_locker(LockType::Write)
+            .add_to_keys(locks);
+
+        let dependency_errors = crate::db::DatabaseModel::new()
+            .package_data()
+            .star()
+            .installed()
+            .map(|x| x.status().dependency_errors())
+            .make_locker(LockType::Write)
+            .add_to_keys(locks);
+
+        let manifest_dependencies_config = crate::db::DatabaseModel::new()
+            .package_data()
+            .star()
+            .installed()
+            .and_then(|x| x.manifest().dependencies().star().config())
+            .make_locker(LockType::Write)
+            .add_to_keys(locks);
+        let docker_containers = crate::db::DatabaseModel::new()
+            .package_data()
+            .star()
+            .installed()
+            .and_then(|x| x.manifest().containers())
+            .make_locker(LockType::Write)
+            .add_to_keys(locks);
+
+        move |skeleton_key| {
+            Ok(Self {
+                dependency_receipt: dependency_receipt(skeleton_key)?,
+                config_receipts: config_receipts(skeleton_key)?,
+                try_heal_receipts: try_heal_receipts(skeleton_key)?,
+                break_transitive_receipts: break_transitive_receipts(skeleton_key)?,
+                update_dependency_receipts: update_dependency_receipts(skeleton_key)?,
+                configured: configured.verify(skeleton_key)?,
+                config_actions: config_actions.verify(skeleton_key)?,
+                dependencies: dependencies.verify(skeleton_key)?,
+                volumes: volumes.verify(skeleton_key)?,
+                version: version.verify(skeleton_key)?,
+                manifest: manifest.verify(skeleton_key)?,
+                system_pointers: system_pointers.verify(skeleton_key)?,
+                current_dependents: current_dependents.verify(skeleton_key)?,
+                current_dependencies: current_dependencies.verify(skeleton_key)?,
+                dependency_errors: dependency_errors.verify(skeleton_key)?,
+                manifest_dependencies_config: manifest_dependencies_config.verify(skeleton_key)?,
+                docker_containers: docker_containers.verify(skeleton_key)?,
+            })
+        }
+    }
 }
 
 #[command(rename = "dry", display(display_serializable))]
 #[instrument(skip(ctx))]
 pub async fn set_dry(
     #[context] ctx: RpcContext,
-    #[parent_data] (id, config, timeout, _): (
-        PackageId,
-        Option<Config>,
-        Option<Duration>,
-        Option<String>,
-    ),
+    #[parent_data] (id, config, timeout): (PackageId, Option<Config>, Option<Duration>),
 ) -> Result<BreakageRes, Error> {
     let mut db = ctx.db.handle();
     let mut tx = db.begin().await?;
     let mut breakages = BTreeMap::new();
+    let locks = ConfigReceipts::new(&mut tx).await?;
     configure(
         &ctx,
         &mut tx,
@@ -238,20 +431,11 @@ pub async fn set_dry(
         true,
         &mut BTreeMap::new(),
         &mut breakages,
+        &locks,
     )
     .await?;
-    crate::db::DatabaseModel::new()
-        .package_data()
-        .idx_model(&id)
-        .expect(&mut tx)
-        .await?
-        .installed()
-        .expect(&mut tx)
-        .await?
-        .status()
-        .configured()
-        .put(&mut tx, &true)
-        .await?;
+
+    locks.configured.set(&mut tx, true, &id).await?;
     tx.abort().await?;
     Ok(BreakageRes(breakages))
 }
@@ -259,11 +443,12 @@ pub async fn set_dry(
 #[instrument(skip(ctx))]
 pub async fn set_impl(
     ctx: RpcContext,
-    (id, config, timeout, expire_id): (PackageId, Option<Config>, Option<Duration>, Option<String>),
-) -> Result<WithRevision<()>, Error> {
+    (id, config, timeout): (PackageId, Option<Config>, Option<Duration>),
+) -> Result<(), Error> {
     let mut db = ctx.db.handle();
     let mut tx = db.begin().await?;
     let mut breakages = BTreeMap::new();
+    let locks = ConfigReceipts::new(&mut tx).await?;
     configure(
         &ctx,
         &mut tx,
@@ -273,42 +458,34 @@ pub async fn set_impl(
         false,
         &mut BTreeMap::new(),
         &mut breakages,
+        &locks,
     )
     .await?;
-    Ok(WithRevision {
-        response: (),
-        revision: tx.commit(expire_id).await?,
-    })
+    tx.commit().await?;
+    Ok(())
 }
 
-#[instrument(skip(ctx, db))]
-pub async fn configure<Db: DbHandle>(
+#[instrument(skip(ctx, db, receipts))]
+pub async fn configure<'a, Db: DbHandle>(
     ctx: &RpcContext,
-    db: &mut Db,
+    db: &'a mut Db,
     id: &PackageId,
     config: Option<Config>,
     timeout: &Option<Duration>,
     dry_run: bool,
     overrides: &mut BTreeMap<PackageId, Config>,
     breakages: &mut BTreeMap<PackageId, TaggedDependencyError>,
+    receipts: &ConfigReceipts,
 ) -> Result<(), Error> {
-    configure_rec(ctx, db, id, config, timeout, dry_run, overrides, breakages).await?;
-    crate::db::DatabaseModel::new()
-        .package_data()
-        .idx_model(&id)
-        .expect(db)
-        .await?
-        .installed()
-        .expect(db)
-        .await?
-        .status()
-        .configured()
-        .put(db, &true)
-        .await?;
+    configure_rec(
+        ctx, db, id, config, timeout, dry_run, overrides, breakages, receipts,
+    )
+    .await?;
+    receipts.configured.set(db, true, &id).await?;
     Ok(())
 }
 
-#[instrument(skip(ctx, db))]
+#[instrument(skip(ctx, db, receipts))]
 pub fn configure_rec<'a, Db: DbHandle>(
     ctx: &'a RpcContext,
     db: &'a mut Db,
@@ -318,48 +495,33 @@ pub fn configure_rec<'a, Db: DbHandle>(
     dry_run: bool,
     overrides: &'a mut BTreeMap<PackageId, Config>,
     breakages: &'a mut BTreeMap<PackageId, TaggedDependencyError>,
+    receipts: &'a ConfigReceipts,
 ) -> BoxFuture<'a, Result<(), Error>> {
     async move {
-        crate::db::DatabaseModel::new()
-            .package_data()
-            .lock(db, LockType::Write)
-            .await?;
         // fetch data from db
-        let pkg_model = crate::db::DatabaseModel::new()
-            .package_data()
-            .idx_model(id)
-            .and_then(|m| m.installed())
-            .expect(db)
-            .await
-            .with_kind(crate::ErrorKind::NotFound)?;
-        let action = pkg_model
-            .clone()
-            .manifest()
-            .config()
-            .get(db, true)
+        let action = receipts
+            .config_actions
+            .get(db, id)
             .await?
-            .to_owned()
-            .ok_or_else(|| Error::new(eyre!("{} has no config", id), crate::ErrorKind::NotFound))?;
-        let version = pkg_model.clone().manifest().version().get(db, true).await?;
-        let dependencies = pkg_model
-            .clone()
-            .manifest()
-            .dependencies()
-            .get(db, true)
-            .await?;
-        let volumes = pkg_model.clone().manifest().volumes().get(db, true).await?;
-        let is_needs_config = !*pkg_model
-            .clone()
-            .status()
-            .configured()
-            .get(db, true)
-            .await?;
+            .ok_or_else(not_found)?;
+        let dependencies = receipts
+            .dependencies
+            .get(db, id)
+            .await?
+            .ok_or_else(not_found)?;
+        let volumes = receipts.volumes.get(db, id).await?.ok_or_else(not_found)?;
+        let is_needs_config = !receipts
+            .configured
+            .get(db, id)
+            .await?
+            .ok_or_else(not_found)?;
+        let version = receipts.version.get(db, id).await?.ok_or_else(not_found)?;
 
         // get current config and current spec
         let ConfigRes {
             config: old_config,
             spec,
-        } = action.get(ctx, id, &*version, &*volumes).await?;
+        } = action.get(ctx, id, &version, &volumes).await?;
 
         // determine new config to use
         let mut config = if let Some(config) = config.or_else(|| old_config.clone()) {
@@ -368,45 +530,49 @@ pub fn configure_rec<'a, Db: DbHandle>(
             spec.gen(&mut rand::rngs::StdRng::from_entropy(), timeout)?
         };
 
-        let manifest = crate::db::DatabaseModel::new()
-            .package_data()
-            .idx_model(id)
-            .and_then(|m| m.installed())
-            .map::<_, Manifest>(|i| i.manifest())
-            .expect(db)
-            .await?
-            .get(db, true)
-            .await
-            .with_kind(crate::ErrorKind::NotFound)?;
+        let manifest = receipts.manifest.get(db, id).await?.ok_or_else(not_found)?;
 
-        spec.validate(&*manifest)?;
+        spec.validate(&manifest)?;
         spec.matches(&config)?; // check that new config matches spec
-        spec.update(ctx, db, &*manifest, &*overrides, &mut config)
-            .await?; // dereference pointers in the new config
+        spec.update(
+            ctx,
+            db,
+            &manifest,
+            &*overrides,
+            &mut config,
+            &receipts.config_receipts,
+        )
+        .await?; // dereference pointers in the new config
 
         // create backreferences to pointers
-        let mut sys = pkg_model.clone().system_pointers().get_mut(db).await?;
+        let mut sys = receipts
+            .system_pointers
+            .get(db, &id)
+            .await?
+            .ok_or_else(not_found)?;
         sys.truncate(0);
-        let mut current_dependencies: BTreeMap<PackageId, CurrentDependencyInfo> = dependencies
-            .0
-            .iter()
-            .filter_map(|(id, info)| {
-                if info.requirement.required() {
-                    Some((id.clone(), CurrentDependencyInfo::default()))
-                } else {
-                    None
-                }
-            })
-            .collect();
+        let mut current_dependencies: CurrentDependencies = CurrentDependencies(
+            dependencies
+                .0
+                .iter()
+                .filter_map(|(id, info)| {
+                    if info.requirement.required() {
+                        Some((id.clone(), CurrentDependencyInfo::default()))
+                    } else {
+                        None
+                    }
+                })
+                .collect(),
+        );
         for ptr in spec.pointers(&config)? {
             match ptr {
                 ValueSpecPointer::Package(pkg_ptr) => {
                     if let Some(current_dependency) =
-                        current_dependencies.get_mut(pkg_ptr.package_id())
+                        current_dependencies.0.get_mut(pkg_ptr.package_id())
                     {
                         current_dependency.pointers.push(pkg_ptr);
                     } else {
-                        current_dependencies.insert(
+                        current_dependencies.0.insert(
                             pkg_ptr.package_id().to_owned(),
                             CurrentDependencyInfo {
                                 pointers: vec![pkg_ptr],
@@ -418,20 +584,20 @@ pub fn configure_rec<'a, Db: DbHandle>(
                 ValueSpecPointer::System(s) => sys.push(s),
             }
         }
-        sys.save(db).await?;
+        receipts.system_pointers.set(db, sys, &id).await?;
 
         let signal = if !dry_run {
             // run config action
             let res = action
-                .set(ctx, id, &*version, &*dependencies, &*volumes, &config)
+                .set(ctx, id, &version, &dependencies, &volumes, &config)
                 .await?;
 
             // track dependencies with no pointers
             for (package_id, health_checks) in res.depends_on.into_iter() {
-                if let Some(current_dependency) = current_dependencies.get_mut(&package_id) {
+                if let Some(current_dependency) = current_dependencies.0.get_mut(&package_id) {
                     current_dependency.health_checks.extend(health_checks);
                 } else {
-                    current_dependencies.insert(
+                    current_dependencies.0.insert(
                         package_id,
                         CurrentDependencyInfo {
                             pointers: Vec::new(),
@@ -442,79 +608,114 @@ pub fn configure_rec<'a, Db: DbHandle>(
             }
 
             // track dependency health checks
-            current_dependencies = current_dependencies
-                .into_iter()
-                .filter(|(dep_id, _)| {
-                    if dep_id != id && !manifest.dependencies.0.contains_key(dep_id) {
-                        tracing::warn!("Illegal dependency specified: {}", dep_id);
-                        false
-                    } else {
-                        true
-                    }
-                })
-                .collect();
+            current_dependencies = current_dependencies.map(|x| {
+                x.into_iter()
+                    .filter(|(dep_id, _)| {
+                        if dep_id != id && !manifest.dependencies.0.contains_key(dep_id) {
+                            tracing::warn!("Illegal dependency specified: {}", dep_id);
+                            false
+                        } else {
+                            true
+                        }
+                    })
+                    .collect()
+            });
             res.signal
         } else {
             None
         };
 
         // update dependencies
-        let mut deps = pkg_model.clone().current_dependencies().get_mut(db).await?;
-        remove_from_current_dependents_lists(db, id, deps.keys()).await?; // remove previous
-        add_dependent_to_current_dependents_lists(db, id, &current_dependencies).await?; // add new
-        current_dependencies.remove(id);
-        *deps = current_dependencies.clone();
-        deps.save(db).await?;
-        let mut errs = pkg_model
-            .clone()
-            .status()
-            .dependency_errors()
-            .get_mut(db)
+        let prev_current_dependencies = receipts
+            .current_dependencies
+            .get(db, &id)
+            .await?
+            .unwrap_or_default();
+        remove_from_current_dependents_lists(
+            db,
+            id,
+            &prev_current_dependencies,
+            &receipts.current_dependents,
+        )
+        .await?; // remove previous
+        add_dependent_to_current_dependents_lists(
+            db,
+            id,
+            &current_dependencies,
+            &receipts.current_dependents,
+        )
+        .await?; // add new
+        current_dependencies.0.remove(id);
+        receipts
+            .current_dependencies
+            .set(db, current_dependencies.clone(), &id)
             .await?;
-        *errs = DependencyErrors::init(ctx, db, &*manifest, &current_dependencies).await?;
-        errs.save(db).await?;
+
+        let errs = receipts
+            .dependency_errors
+            .get(db, &id)
+            .await?
+            .ok_or_else(not_found)?;
+        tracing::warn!("Dependency Errors: {:?}", errs);
+        let errs = DependencyErrors::init(
+            ctx,
+            db,
+            &manifest,
+            &current_dependencies,
+            &receipts.dependency_receipt.try_heal,
+        )
+        .await?;
+        receipts.dependency_errors.set(db, errs, &id).await?;
 
         // cache current config for dependents
         overrides.insert(id.clone(), config.clone());
 
         // handle dependents
-        let dependents = pkg_model.clone().current_dependents().get(db, true).await?;
+        let dependents = receipts
+            .current_dependents
+            .get(db, id)
+            .await?
+            .ok_or_else(not_found)?;
         let prev = if is_needs_config { None } else { old_config }
             .map(Value::Object)
             .unwrap_or_default();
         let next = Value::Object(config.clone());
-        for (dependent, dep_info) in dependents.iter().filter(|(dep_id, _)| dep_id != &id) {
+        for (dependent, dep_info) in dependents.0.iter().filter(|(dep_id, _)| dep_id != &id) {
+            let dependent_container = receipts.docker_containers.get(db, &dependent).await?;
+            let dependent_container = &dependent_container;
             // check if config passes dependent check
-            let dependent_model = crate::db::DatabaseModel::new()
-                .package_data()
-                .idx_model(dependent)
-                .and_then(|pkg| pkg.installed())
-                .expect(db)
-                .await?;
-            if let Some(cfg) = &*dependent_model
-                .clone()
-                .manifest()
-                .dependencies()
-                .idx_model(id)
-                .expect(db)
-                .await?
-                .config()
-                .get(db, true)
+            if let Some(cfg) = receipts
+                .manifest_dependencies_config
+                .get(db, (&dependent, &id))
                 .await?
             {
-                let manifest = dependent_model.clone().manifest().get(db, true).await?;
+                let manifest = receipts
+                    .manifest
+                    .get(db, &dependent)
+                    .await?
+                    .ok_or_else(not_found)?;
                 if let Err(error) = cfg
                     .check(
                         ctx,
+                        dependent_container,
                         dependent,
                         &manifest.version,
                         &manifest.volumes,
+                        id,
                         &config,
                     )
                     .await?
                 {
                     let dep_err = DependencyError::ConfigUnsatisfied { error };
-                    break_transitive(db, dependent, id, dep_err, breakages).await?;
+                    break_transitive(
+                        db,
+                        dependent,
+                        id,
+                        dep_err,
+                        breakages,
+                        &receipts.break_transitive_receipts,
+                    )
+                    .await?;
                 }
 
                 // handle backreferences
@@ -523,6 +724,7 @@ pub fn configure_rec<'a, Db: DbHandle>(
                         if cfg_ptr.select(&next) != cfg_ptr.select(&prev) {
                             if let Err(e) = configure_rec(
                                 ctx, db, dependent, None, timeout, dry_run, overrides, breakages,
+                                receipts,
                             )
                             .await
                             {
@@ -535,6 +737,7 @@ pub fn configure_rec<'a, Db: DbHandle>(
                                             error: format!("{}", e),
                                         },
                                         breakages,
+                                        &receipts.break_transitive_receipts,
                                     )
                                     .await?;
                                 } else {
@@ -544,7 +747,7 @@ pub fn configure_rec<'a, Db: DbHandle>(
                         }
                     }
                 }
-                heal_all_dependents_transitive(ctx, db, id).await?;
+                heal_all_dependents_transitive(ctx, db, id, &receipts.dependency_receipt).await?;
             }
         }
 
@@ -568,3 +771,67 @@ pub fn configure_rec<'a, Db: DbHandle>(
     }
     .boxed()
 }
+#[instrument]
+pub fn not_found() -> Error {
+    Error::new(eyre!("Could not find"), crate::ErrorKind::Incoherent)
+}
+
+/// We want to have a double check that the paths are what we expect them to be.
+/// Found that earlier the paths where not what we expected them to be.
+#[tokio::test]
+async fn ensure_creation_of_config_paths_makes_sense() {
+    let mut fake = patch_db::test_utils::NoOpDb();
+    let config_locks = ConfigReceipts::new(&mut fake).await.unwrap();
+    assert_eq!(
+        &format!("{}", config_locks.configured.lock.glob),
+        "/package-data/*/installed/status/configured"
+    );
+    assert_eq!(
+        &format!("{}", config_locks.config_actions.lock.glob),
+        "/package-data/*/installed/manifest/config"
+    );
+    assert_eq!(
+        &format!("{}", config_locks.dependencies.lock.glob),
+        "/package-data/*/installed/manifest/dependencies"
+    );
+    assert_eq!(
+        &format!("{}", config_locks.volumes.lock.glob),
+        "/package-data/*/installed/manifest/volumes"
+    );
+    assert_eq!(
+        &format!("{}", config_locks.version.lock.glob),
+        "/package-data/*/installed/manifest/version"
+    );
+    assert_eq!(
+        &format!("{}", config_locks.volumes.lock.glob),
+        "/package-data/*/installed/manifest/volumes"
+    );
+    assert_eq!(
+        &format!("{}", config_locks.manifest.lock.glob),
+        "/package-data/*/installed/manifest"
+    );
+    assert_eq!(
+        &format!("{}", config_locks.manifest.lock.glob),
+        "/package-data/*/installed/manifest"
+    );
+    assert_eq!(
+        &format!("{}", config_locks.system_pointers.lock.glob),
+        "/package-data/*/installed/system-pointers"
+    );
+    assert_eq!(
+        &format!("{}", config_locks.current_dependents.lock.glob),
+        "/package-data/*/installed/current-dependents"
+    );
+    assert_eq!(
+        &format!("{}", config_locks.dependency_errors.lock.glob),
+        "/package-data/*/installed/status/dependency-errors"
+    );
+    assert_eq!(
+        &format!("{}", config_locks.manifest_dependencies_config.lock.glob),
+        "/package-data/*/installed/manifest/dependencies/*/config"
+    );
+    assert_eq!(
+        &format!("{}", config_locks.system_pointers.lock.glob),
+        "/package-data/*/installed/system-pointers"
+    );
+}

backend/src/config/spec.rs

@@ -12,12 +12,13 @@ use async_trait::async_trait;
 use indexmap::{IndexMap, IndexSet};
 use itertools::Itertools;
 use jsonpath_lib::Compiled as CompiledJsonPath;
-use patch_db::{DbHandle, OptionModel};
+use patch_db::{DbHandle, LockReceipt, LockType};
 use rand::{CryptoRng, Rng};
 use regex::Regex;
+use serde::de::{MapAccess, Visitor};
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
 use serde_json::{Number, Value};
-use sqlx::SqlitePool;
+use sqlx::PgPool;
 
 use super::util::{self, CharSet, NumRange, UniqueBy, STATIC_NULL};
 use super::{Config, MatchError, NoMatchWithPath, TimeoutError, TypeOf};
@@ -44,6 +45,7 @@ pub trait ValueSpec {
         manifest: &Manifest,
         config_overrides: &BTreeMap<PackageId, Config>,
         value: &mut Value,
+        receipts: &ConfigPointerReceipts,
     ) -> Result<(), ConfigurationError>;
     // returns all pointers that are live in the provided config
     fn pointers(&self, value: &Value) -> Result<BTreeSet<ValueSpecPointer>, NoMatchWithPath>;
@@ -160,9 +162,10 @@ where
         manifest: &Manifest,
         config_overrides: &BTreeMap<PackageId, Config>,
         value: &mut Value,
+        receipts: &ConfigPointerReceipts,
     ) -> Result<(), ConfigurationError> {
         self.inner
-            .update(ctx, db, manifest, config_overrides, value)
+            .update(ctx, db, manifest, config_overrides, value, receipts)
             .await
     }
     fn pointers(&self, value: &Value) -> Result<BTreeSet<ValueSpecPointer>, NoMatchWithPath> {
@@ -204,9 +207,10 @@ where
         manifest: &Manifest,
         config_overrides: &BTreeMap<PackageId, Config>,
         value: &mut Value,
+        receipts: &ConfigPointerReceipts,
     ) -> Result<(), ConfigurationError> {
         self.inner
-            .update(ctx, db, manifest, config_overrides, value)
+            .update(ctx, db, manifest, config_overrides, value, receipts)
             .await
     }
     fn pointers(&self, value: &Value) -> Result<BTreeSet<ValueSpecPointer>, NoMatchWithPath> {
@@ -281,9 +285,10 @@ where
         manifest: &Manifest,
         config_overrides: &BTreeMap<PackageId, Config>,
         value: &mut Value,
+        receipts: &ConfigPointerReceipts,
     ) -> Result<(), ConfigurationError> {
         self.inner
-            .update(ctx, db, manifest, config_overrides, value)
+            .update(ctx, db, manifest, config_overrides, value, receipts)
             .await
     }
     fn pointers(&self, value: &Value) -> Result<BTreeSet<ValueSpecPointer>, NoMatchWithPath> {
@@ -343,7 +348,7 @@ pub enum ValueSpecAny {
     Pointer(WithDescription<ValueSpecPointer>),
 }
 impl ValueSpecAny {
-    pub fn name<'a>(&'a self) -> &'a str {
+    pub fn name(&self) -> &'_ str {
         match self {
             ValueSpecAny::Boolean(b) => b.name.as_str(),
             ValueSpecAny::Enum(e) => e.name.as_str(),
@@ -395,16 +400,41 @@ impl ValueSpec for ValueSpecAny {
         manifest: &Manifest,
         config_overrides: &BTreeMap<PackageId, Config>,
         value: &mut Value,
+        receipts: &ConfigPointerReceipts,
     ) -> Result<(), ConfigurationError> {
         match self {
-            ValueSpecAny::Boolean(a) => a.update(ctx, db, manifest, config_overrides, value).await,
-            ValueSpecAny::Enum(a) => a.update(ctx, db, manifest, config_overrides, value).await,
-            ValueSpecAny::List(a) => a.update(ctx, db, manifest, config_overrides, value).await,
-            ValueSpecAny::Number(a) => a.update(ctx, db, manifest, config_overrides, value).await,
-            ValueSpecAny::Object(a) => a.update(ctx, db, manifest, config_overrides, value).await,
-            ValueSpecAny::String(a) => a.update(ctx, db, manifest, config_overrides, value).await,
-            ValueSpecAny::Union(a) => a.update(ctx, db, manifest, config_overrides, value).await,
-            ValueSpecAny::Pointer(a) => a.update(ctx, db, manifest, config_overrides, value).await,
+            ValueSpecAny::Boolean(a) => {
+                a.update(ctx, db, manifest, config_overrides, value, receipts)
+                    .await
+            }
+            ValueSpecAny::Enum(a) => {
+                a.update(ctx, db, manifest, config_overrides, value, receipts)
+                    .await
+            }
+            ValueSpecAny::List(a) => {
+                a.update(ctx, db, manifest, config_overrides, value, receipts)
+                    .await
+            }
+            ValueSpecAny::Number(a) => {
+                a.update(ctx, db, manifest, config_overrides, value, receipts)
+                    .await
+            }
+            ValueSpecAny::Object(a) => {
+                a.update(ctx, db, manifest, config_overrides, value, receipts)
+                    .await
+            }
+            ValueSpecAny::String(a) => {
+                a.update(ctx, db, manifest, config_overrides, value, receipts)
+                    .await
+            }
+            ValueSpecAny::Union(a) => {
+                a.update(ctx, db, manifest, config_overrides, value, receipts)
+                    .await
+            }
+            ValueSpecAny::Pointer(a) => {
+                a.update(ctx, db, manifest, config_overrides, value, receipts)
+                    .await
+            }
         }
     }
     fn pointers(&self, value: &Value) -> Result<BTreeSet<ValueSpecPointer>, NoMatchWithPath> {
@@ -489,6 +519,7 @@ impl ValueSpec for ValueSpecBoolean {
         _manifest: &Manifest,
         _config_overrides: &BTreeMap<PackageId, Config>,
         _value: &mut Value,
+        _receipts: &ConfigPointerReceipts,
     ) -> Result<(), ConfigurationError> {
         Ok(())
     }
@@ -578,6 +609,7 @@ impl ValueSpec for ValueSpecEnum {
         _manifest: &Manifest,
         _config_overrides: &BTreeMap<PackageId, Config>,
         _value: &mut Value,
+        _receipts: &ConfigPointerReceipts,
     ) -> Result<(), ConfigurationError> {
         Ok(())
     }
@@ -664,12 +696,13 @@ where
         manifest: &Manifest,
         config_overrides: &BTreeMap<PackageId, Config>,
         value: &mut Value,
+        receipts: &ConfigPointerReceipts,
     ) -> Result<(), ConfigurationError> {
         if let Value::Array(ref mut ls) = value {
             for (i, val) in ls.into_iter().enumerate() {
                 match self
                     .spec
-                    .update(ctx, db, manifest, config_overrides, val)
+                    .update(ctx, db, manifest, config_overrides, val, receipts)
                     .await
                 {
                     Err(ConfigurationError::NoMatch(e)) => {
@@ -771,13 +804,29 @@ impl ValueSpec for ValueSpecList {
         manifest: &Manifest,
         config_overrides: &BTreeMap<PackageId, Config>,
         value: &mut Value,
+        receipts: &ConfigPointerReceipts,
     ) -> Result<(), ConfigurationError> {
         match self {
-            ValueSpecList::Enum(a) => a.update(ctx, db, manifest, config_overrides, value).await,
-            ValueSpecList::Number(a) => a.update(ctx, db, manifest, config_overrides, value).await,
-            ValueSpecList::Object(a) => a.update(ctx, db, manifest, config_overrides, value).await,
-            ValueSpecList::String(a) => a.update(ctx, db, manifest, config_overrides, value).await,
-            ValueSpecList::Union(a) => a.update(ctx, db, manifest, config_overrides, value).await,
+            ValueSpecList::Enum(a) => {
+                a.update(ctx, db, manifest, config_overrides, value, receipts)
+                    .await
+            }
+            ValueSpecList::Number(a) => {
+                a.update(ctx, db, manifest, config_overrides, value, receipts)
+                    .await
+            }
+            ValueSpecList::Object(a) => {
+                a.update(ctx, db, manifest, config_overrides, value, receipts)
+                    .await
+            }
+            ValueSpecList::String(a) => {
+                a.update(ctx, db, manifest, config_overrides, value, receipts)
+                    .await
+            }
+            ValueSpecList::Union(a) => {
+                a.update(ctx, db, manifest, config_overrides, value, receipts)
+                    .await
+            }
         }
     }
     fn pointers(&self, value: &Value) -> Result<BTreeSet<ValueSpecPointer>, NoMatchWithPath> {
@@ -898,6 +947,7 @@ impl ValueSpec for ValueSpecNumber {
         _manifest: &Manifest,
         _config_overrides: &BTreeMap<PackageId, Config>,
         _value: &mut Value,
+        _receipts: &ConfigPointerReceipts,
     ) -> Result<(), ConfigurationError> {
         Ok(())
     }
@@ -961,10 +1011,11 @@ impl ValueSpec for ValueSpecObject {
         manifest: &Manifest,
         config_overrides: &BTreeMap<PackageId, Config>,
         value: &mut Value,
+        receipts: &ConfigPointerReceipts,
     ) -> Result<(), ConfigurationError> {
         if let Value::Object(o) = value {
             self.spec
-                .update(ctx, db, manifest, config_overrides, o)
+                .update(ctx, db, manifest, config_overrides, o, receipts)
                 .await
         } else {
             Err(ConfigurationError::NoMatch(NoMatchWithPath::new(
@@ -1063,16 +1114,20 @@ impl ConfigSpec {
         manifest: &Manifest,
         config_overrides: &BTreeMap<PackageId, Config>,
         cfg: &mut Config,
+        receipts: &ConfigPointerReceipts,
     ) -> Result<(), ConfigurationError> {
         for (k, vs) in self.0.iter() {
             match cfg.get_mut(k) {
                 None => {
                     let mut v = Value::Null;
-                    vs.update(ctx, db, manifest, config_overrides, &mut v)
+                    vs.update(ctx, db, manifest, config_overrides, &mut v, receipts)
                         .await?;
                     cfg.insert(k.clone(), v);
                 }
-                Some(v) => match vs.update(ctx, db, manifest, config_overrides, v).await {
+                Some(v) => match vs
+                    .update(ctx, db, manifest, config_overrides, v, receipts)
+                    .await
+                {
                     Err(ConfigurationError::NoMatch(e)) => {
                         Err(ConfigurationError::NoMatch(e.prepend(k.clone())))
                     }
@@ -1113,18 +1168,95 @@ pub struct Pattern {
     pub pattern_description: String,
 }
 
-#[derive(Clone, Debug, Serialize, Deserialize)]
+#[derive(Clone, Debug, Serialize)]
+#[serde(rename_all = "kebab-case")]
 pub struct ValueSpecString {
     #[serde(flatten)]
     pub pattern: Option<Pattern>,
-    #[serde(default)]
+    pub textarea: bool,
     pub copyable: bool,
-    #[serde(default)]
     pub masked: bool,
     #[serde(skip_serializing_if = "Option::is_none")]
-    #[serde(default)]
     pub placeholder: Option<String>,
 }
+impl<'de> Deserialize<'de> for ValueSpecString {
+    fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<ValueSpecString, D::Error> {
+        struct ValueSpecStringVisitor;
+        impl<'de> Visitor<'de> for ValueSpecStringVisitor {
+            type Value = ValueSpecString;
+            fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
+                formatter.write_str("struct ValueSpecString")
+            }
+            fn visit_map<V: MapAccess<'de>>(self, mut map: V) -> Result<ValueSpecString, V::Error> {
+                let mut pattern = None;
+                let mut pattern_description = None;
+                let mut textarea = false;
+                let mut copyable = false;
+                let mut masked = false;
+                let mut placeholder = None;
+                while let Some::<String>(key) = map.next_key()? {
+                    if &key == "pattern" {
+                        if pattern.is_some() {
+                            return Err(serde::de::Error::duplicate_field("pattern"));
+                        } else {
+                            pattern = Some(
+                                Regex::new(&map.next_value::<String>()?)
+                                    .map_err(serde::de::Error::custom)?,
+                            );
+                        }
+                    } else if &key == "pattern-description" {
+                        if pattern_description.is_some() {
+                            return Err(serde::de::Error::duplicate_field("pattern-description"));
+                        } else {
+                            pattern_description = Some(map.next_value()?);
+                        }
+                    } else if &key == "textarea" {
+                        textarea = map.next_value()?;
+                    } else if &key == "copyable" {
+                        copyable = map.next_value()?;
+                    } else if &key == "masked" {
+                        masked = map.next_value()?;
+                    } else if &key == "placeholder" {
+                        if placeholder.is_some() {
+                            return Err(serde::de::Error::duplicate_field("placeholder"));
+                        } else {
+                            placeholder = Some(map.next_value()?);
+                        }
+                    }
+                }
+                let regex = match (pattern, pattern_description) {
+                    (None, None) => None,
+                    (Some(p), Some(d)) => Some(Pattern {
+                        pattern: p,
+                        pattern_description: d,
+                    }),
+                    (Some(_), None) => {
+                        return Err(serde::de::Error::missing_field("pattern-description"));
+                    }
+                    (None, Some(_)) => {
+                        return Err(serde::de::Error::missing_field("pattern"));
+                    }
+                };
+                Ok(ValueSpecString {
+                    pattern: regex,
+                    textarea,
+                    copyable,
+                    masked,
+                    placeholder,
+                })
+            }
+        }
+        const FIELDS: &'static [&'static str] = &[
+            "pattern",
+            "pattern-description",
+            "textarea",
+            "copyable",
+            "masked",
+            "placeholder",
+        ];
+        deserializer.deserialize_struct("ValueSpecString", FIELDS, ValueSpecStringVisitor)
+    }
+}
 #[async_trait]
 impl ValueSpec for ValueSpecString {
     fn matches(&self, value: &Value) -> Result<(), NoMatchWithPath> {
@@ -1160,6 +1292,7 @@ impl ValueSpec for ValueSpecString {
         _manifest: &Manifest,
         _config_overrides: &BTreeMap<PackageId, Config>,
         _value: &mut Value,
+        _receipts: &ConfigPointerReceipts,
     ) -> Result<(), ConfigurationError> {
         Ok(())
     }
@@ -1192,10 +1325,7 @@ impl DefaultableWith for ValueSpecString {
                 let candidate = spec.gen(rng);
                 match (spec, &self.pattern) {
                     (DefaultString::Entropy(_), Some(pattern))
-                        if !pattern.pattern.is_match(&candidate) =>
-                    {
-                        ()
-                    }
+                        if !pattern.pattern.is_match(&candidate) => {}
                     _ => {
                         return Ok(Value::String(candidate));
                     }
@@ -1204,6 +1334,8 @@ impl DefaultableWith for ValueSpecString {
                     if &now.elapsed() > timeout {
                         return Err(TimeoutError);
                     }
+                } else {
+                    return Ok(Value::String(candidate));
                 }
             }
         } else {
@@ -1371,6 +1503,7 @@ impl ValueSpec for ValueSpecUnion {
         manifest: &Manifest,
         config_overrides: &BTreeMap<PackageId, Config>,
         value: &mut Value,
+        receipts: &ConfigPointerReceipts,
     ) -> Result<(), ConfigurationError> {
         if let Value::Object(o) = value {
             match o.get(&self.tag.id) {
@@ -1381,7 +1514,10 @@ impl ValueSpec for ValueSpecUnion {
                     None => Err(ConfigurationError::NoMatch(NoMatchWithPath::new(
                         MatchError::Union(tag.clone(), self.variants.keys().cloned().collect()),
                     ))),
-                    Some(spec) => spec.update(ctx, db, manifest, config_overrides, o).await,
+                    Some(spec) => {
+                        spec.update(ctx, db, manifest, config_overrides, o, receipts)
+                            .await
+                    }
                 },
                 Some(other) => Err(ConfigurationError::NoMatch(
                     NoMatchWithPath::new(MatchError::InvalidType("string", other.type_of()))
@@ -1513,13 +1649,16 @@ impl ValueSpec for ValueSpecPointer {
         manifest: &Manifest,
         config_overrides: &BTreeMap<PackageId, Config>,
         value: &mut Value,
+        receipts: &ConfigPointerReceipts,
     ) -> Result<(), ConfigurationError> {
         match self {
             ValueSpecPointer::Package(a) => {
-                a.update(ctx, db, manifest, config_overrides, value).await
+                a.update(ctx, db, manifest, config_overrides, value, receipts)
+                    .await
             }
             ValueSpecPointer::System(a) => {
-                a.update(ctx, db, manifest, config_overrides, value).await
+                a.update(ctx, db, manifest, config_overrides, value, receipts)
+                    .await
             }
         }
     }
@@ -1563,12 +1702,17 @@ impl PackagePointerSpec {
         db: &mut Db,
         manifest: &Manifest,
         config_overrides: &BTreeMap<PackageId, Config>,
+        receipts: &ConfigPointerReceipts,
     ) -> Result<Value, ConfigurationError> {
         match &self {
             PackagePointerSpec::TorKey(key) => key.deref(&manifest.id, &ctx.secret_store).await,
-            PackagePointerSpec::TorAddress(tor) => tor.deref(db).await,
-            PackagePointerSpec::LanAddress(lan) => lan.deref(db).await,
-            PackagePointerSpec::Config(cfg) => cfg.deref(ctx, db, config_overrides).await,
+            PackagePointerSpec::TorAddress(tor) => {
+                tor.deref(db, &receipts.interface_addresses_receipt).await
+            }
+            PackagePointerSpec::LanAddress(lan) => {
+                lan.deref(db, &receipts.interface_addresses_receipt).await
+            }
+            PackagePointerSpec::Config(cfg) => cfg.deref(ctx, db, config_overrides, receipts).await,
         }
     }
 }
@@ -1616,8 +1760,11 @@ impl ValueSpec for PackagePointerSpec {
         manifest: &Manifest,
         config_overrides: &BTreeMap<PackageId, Config>,
         value: &mut Value,
+        receipts: &ConfigPointerReceipts,
     ) -> Result<(), ConfigurationError> {
-        *value = self.deref(ctx, db, manifest, config_overrides).await?;
+        *value = self
+            .deref(ctx, db, manifest, config_overrides, receipts)
+            .await?;
         Ok(())
     }
     fn pointers(&self, _value: &Value) -> Result<BTreeSet<ValueSpecPointer>, NoMatchWithPath> {
@@ -1640,16 +1787,17 @@ pub struct TorAddressPointer {
     interface: InterfaceId,
 }
 impl TorAddressPointer {
-    async fn deref<Db: DbHandle>(&self, db: &mut Db) -> Result<Value, ConfigurationError> {
-        let addr = crate::db::DatabaseModel::new()
-            .package_data()
-            .idx_model(&self.package_id)
-            .and_then(|pde| pde.installed())
-            .and_then(|installed| installed.interface_addresses().idx_model(&self.interface))
-            .and_then(|addresses| addresses.tor_address())
-            .get(db, true)
+    async fn deref<Db: DbHandle>(
+        &self,
+        db: &mut Db,
+        receipt: &InterfaceAddressesReceipt,
+    ) -> Result<Value, ConfigurationError> {
+        let addr = receipt
+            .interface_addresses
+            .get(db, (&self.package_id, &self.interface))
             .await
-            .map_err(|e| ConfigurationError::SystemError(Error::from(e)))?;
+            .map_err(|e| ConfigurationError::SystemError(Error::from(e)))?
+            .and_then(|addresses| addresses.tor_address);
         Ok(addr.to_owned().map(Value::String).unwrap_or(Value::Null))
     }
 }
@@ -1664,6 +1812,39 @@ impl fmt::Display for TorAddressPointer {
     }
 }
 
+pub struct InterfaceAddressesReceipt {
+    interface_addresses: LockReceipt<crate::db::model::InterfaceAddresses, (String, String)>,
+}
+
+impl InterfaceAddressesReceipt {
+    pub async fn new<'a>(db: &'a mut impl DbHandle) -> Result<Self, Error> {
+        let mut locks = Vec::new();
+
+        let setup = Self::setup(&mut locks);
+        Ok(setup(&db.lock_all(locks).await?)?)
+    }
+
+    pub fn setup(
+        locks: &mut Vec<patch_db::LockTargetId>,
+    ) -> impl FnOnce(&patch_db::Verifier) -> Result<Self, Error> {
+        // let cleanup_receipts = CleanupFailedReceipts::setup(locks);
+
+        let interface_addresses = crate::db::DatabaseModel::new()
+            .package_data()
+            .star()
+            .installed()
+            .map(|x| x.interface_addresses().star())
+            .make_locker(LockType::Read)
+            .add_to_keys(locks);
+        move |skeleton_key| {
+            Ok(Self {
+                // cleanup_receipts: cleanup_receipts(skeleton_key)?,
+                interface_addresses: interface_addresses.verify(skeleton_key)?,
+            })
+        }
+    }
+}
+
 #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
 #[serde(rename_all = "kebab-case")]
 pub struct LanAddressPointer {
@@ -1672,28 +1853,81 @@ pub struct LanAddressPointer {
 }
 impl fmt::Display for LanAddressPointer {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
-            LanAddressPointer {
-                package_id,
-                interface,
-            } => write!(f, "{}: lan-address: {}", package_id, interface),
-        }
+        let LanAddressPointer {
+            package_id,
+            interface,
+        } = self;
+        write!(f, "{}: lan-address: {}", package_id, interface)
     }
 }
 impl LanAddressPointer {
-    async fn deref<Db: DbHandle>(&self, db: &mut Db) -> Result<Value, ConfigurationError> {
-        let addr = crate::db::DatabaseModel::new()
-            .package_data()
-            .idx_model(&self.package_id)
-            .and_then(|pde| pde.installed())
-            .and_then(|installed| installed.interface_addresses().idx_model(&self.interface))
-            .and_then(|addresses| addresses.lan_address())
-            .get(db, true)
+    async fn deref<Db: DbHandle>(
+        &self,
+        db: &mut Db,
+        receipts: &InterfaceAddressesReceipt,
+    ) -> Result<Value, ConfigurationError> {
+        let addr = receipts
+            .interface_addresses
+            .get(db, (&self.package_id, &self.interface))
             .await
-            .map_err(|e| ConfigurationError::SystemError(Error::from(e)))?;
+            .ok()
+            .flatten()
+            .and_then(|x| x.lan_address);
         Ok(addr.to_owned().map(Value::String).unwrap_or(Value::Null))
     }
 }
+
+pub struct ConfigPointerReceipts {
+    interface_addresses_receipt: InterfaceAddressesReceipt,
+    manifest_volumes: LockReceipt<crate::volume::Volumes, String>,
+    manifest_version: LockReceipt<crate::util::Version, String>,
+    config_actions: LockReceipt<super::action::ConfigActions, String>,
+}
+
+impl ConfigPointerReceipts {
+    pub async fn new<'a>(db: &'a mut impl DbHandle) -> Result<Self, Error> {
+        let mut locks = Vec::new();
+
+        let setup = Self::setup(&mut locks);
+        Ok(setup(&db.lock_all(locks).await?)?)
+    }
+
+    pub fn setup(
+        locks: &mut Vec<patch_db::LockTargetId>,
+    ) -> impl FnOnce(&patch_db::Verifier) -> Result<Self, Error> {
+        let interface_addresses_receipt = InterfaceAddressesReceipt::setup(locks);
+
+        let manifest_volumes = crate::db::DatabaseModel::new()
+            .package_data()
+            .star()
+            .installed()
+            .map(|x| x.manifest().volumes())
+            .make_locker(LockType::Read)
+            .add_to_keys(locks);
+        let manifest_version = crate::db::DatabaseModel::new()
+            .package_data()
+            .star()
+            .installed()
+            .map(|x| x.manifest().version())
+            .make_locker(LockType::Read)
+            .add_to_keys(locks);
+        let config_actions = crate::db::DatabaseModel::new()
+            .package_data()
+            .star()
+            .installed()
+            .and_then(|x| x.manifest().config())
+            .make_locker(LockType::Read)
+            .add_to_keys(locks);
+        move |skeleton_key| {
+            Ok(Self {
+                interface_addresses_receipt: interface_addresses_receipt(skeleton_key)?,
+                manifest_volumes: manifest_volumes.verify(skeleton_key)?,
+                config_actions: config_actions.verify(skeleton_key)?,
+                manifest_version: manifest_version.verify(skeleton_key)?,
+            })
+        }
+    }
+}
 #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
 #[serde(rename_all = "kebab-case")]
 pub struct ConfigPointer {
@@ -1710,40 +1944,22 @@ impl ConfigPointer {
         ctx: &RpcContext,
         db: &mut Db,
         config_overrides: &BTreeMap<PackageId, Config>,
+        receipts: &ConfigPointerReceipts,
     ) -> Result<Value, ConfigurationError> {
         if let Some(cfg) = config_overrides.get(&self.package_id) {
             Ok(self.select(&Value::Object(cfg.clone())))
         } else {
-            let manifest_model: OptionModel<Manifest> = crate::db::DatabaseModel::new()
-                .package_data()
-                .idx_model(&self.package_id)
-                .and_then(|pde| pde.installed())
-                .map(|installed| installed.manifest())
-                .into();
-            let version = manifest_model
-                .clone()
-                .map(|manifest| manifest.version())
-                .get(db, true)
-                .await
-                .map_err(|e| ConfigurationError::SystemError(Error::from(e)))?;
-            let cfg_actions = manifest_model
-                .clone()
-                .and_then(|manifest| manifest.config())
-                .get(db, true)
-                .await
-                .map_err(|e| ConfigurationError::SystemError(Error::from(e)))?;
-            let volumes = manifest_model
-                .map(|manifest| manifest.volumes())
-                .get(db, true)
-                .await
-                .map_err(|e| ConfigurationError::SystemError(Error::from(e)))?;
+            let id = &self.package_id;
+            let version = receipts.manifest_version.get(db, id).await.ok().flatten();
+            let cfg_actions = receipts.config_actions.get(db, id).await.ok().flatten();
+            let volumes = receipts.manifest_volumes.get(db, id).await.ok().flatten();
             if let (Some(version), Some(cfg_actions), Some(volumes)) =
-                (&*version, &*cfg_actions, &*volumes)
+                (&version, &cfg_actions, &volumes)
             {
                 let cfg_res = cfg_actions
-                    .get(&ctx, &self.package_id, version, volumes)
+                    .get(ctx, &self.package_id, version, volumes)
                     .await
-                    .map_err(|e| ConfigurationError::SystemError(Error::from(e)))?;
+                    .map_err(|e| ConfigurationError::SystemError(e))?;
                 if let Some(cfg) = cfg_res.config {
                     Ok(self.select(&Value::Object(cfg)))
                 } else {
@@ -1757,13 +1973,12 @@ impl ConfigPointer {
 }
 impl fmt::Display for ConfigPointer {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
-            ConfigPointer {
-                package_id,
-                selector,
-                ..
-            } => write!(f, "{}: config: {}", package_id, selector),
-        }
+        let ConfigPointer {
+            package_id,
+            selector,
+            ..
+        } = self;
+        write!(f, "{}: config: {}", package_id, selector)
     }
 }
 
@@ -1837,7 +2052,7 @@ impl TorKeyPointer {
     async fn deref(
         &self,
         source_package: &PackageId,
-        secrets: &SqlitePool,
+        secrets: &PgPool,
     ) -> Result<Value, ConfigurationError> {
         if &self.package_id != source_package {
             return Err(ConfigurationError::PermissionDenied(
@@ -1845,7 +2060,7 @@ impl TorKeyPointer {
             ));
         }
         let x = sqlx::query!(
-            "SELECT key FROM tor WHERE package = ? AND interface = ?",
+            "SELECT key FROM tor WHERE package = $1 AND interface = $2",
             *self.package_id,
             *self.interface
         )
@@ -1909,6 +2124,8 @@ impl ValueSpec for SystemPointerSpec {
         _manifest: &Manifest,
         _config_overrides: &BTreeMap<PackageId, Config>,
         value: &mut Value,
+
+        _receipts: &ConfigPointerReceipts,
     ) -> Result<(), ConfigurationError> {
         *value = self.deref(db).await?;
         Ok(())
@@ -1926,3 +2143,42 @@ impl ValueSpec for SystemPointerSpec {
         false
     }
 }
+
+#[test]
+fn invalid_regex_produces_error() {
+    assert!(
+        serde_yaml::from_reader::<_, ConfigSpec>(std::io::Cursor::new(include_bytes!(
+            "../../test/config-spec/lnd-invalid-regex.yaml"
+        )))
+        .is_err()
+    )
+}
+
+#[test]
+fn missing_pattern_description_produces_error() {
+    assert!(
+        serde_yaml::from_reader::<_, ConfigSpec>(std::io::Cursor::new(include_bytes!(
+            "../../test/config-spec/lnd-missing-pattern-description.yaml"
+        )))
+        .is_err()
+    )
+}
+
+#[test]
+fn missing_pattern_produces_error() {
+    assert!(
+        serde_yaml::from_reader::<_, ConfigSpec>(std::io::Cursor::new(include_bytes!(
+            "../../test/config-spec/lnd-missing-pattern.yaml"
+        )))
+        .is_err()
+    )
+}
+
+#[test]
+fn regex_control() {
+    let spec = serde_yaml::from_reader::<_, ConfigSpec>(std::io::Cursor::new(include_bytes!(
+        "../../test/config-spec/lnd-correct.yaml"
+    )))
+    .unwrap();
+    println!("{}", serde_json::to_string_pretty(&spec).unwrap());
+}

backend/src/config/util.rs

@@ -16,7 +16,7 @@ impl CharSet {
         self.0.iter().any(|r| r.0.contains(c))
     }
     pub fn gen<R: Rng>(&self, rng: &mut R) -> char {
-        let mut idx = rng.gen_range(0, self.1);
+        let mut idx = rng.gen_range(0..self.1);
         for r in &self.0 {
             if idx < r.1 {
                 return std::convert::TryFrom::try_from(
@@ -147,6 +147,44 @@ impl serde::ser::Serialize for CharSet {
     }
 }
 
+pub trait MergeWith {
+    fn merge_with(&mut self, other: &serde_json::Value);
+}
+
+impl MergeWith for serde_json::Value {
+    fn merge_with(&mut self, other: &serde_json::Value) {
+        use serde_json::Value::Object;
+        if let (Object(orig), Object(ref other)) = (self, other) {
+            for (key, val) in other.into_iter() {
+                match (orig.get_mut(key), val) {
+                    (Some(new_orig @ Object(_)), other @ Object(_)) => {
+                        new_orig.merge_with(other);
+                    }
+                    (None, _) => {
+                        orig.insert(key.clone(), val.clone());
+                    }
+                    _ => (),
+                }
+            }
+        }
+    }
+}
+
+#[test]
+fn merge_with_tests() {
+    use serde_json::json;
+
+    let mut a = json!(
+        {"a": 1, "c": {"d": "123"}, "i": [1,2,3], "j": {}, "k":[1,2,3], "l": "test"}
+    );
+    a.merge_with(
+        &json!({"a":"a", "b": "b", "c":{"d":"d", "e":"e"}, "f":{"g":"g"}, "h": [1,2,3], "i":"i", "j":[1,2,3], "k":{}}),
+    );
+    assert_eq!(
+        a,
+        json!({"a": 1, "c": {"d": "123", "e":"e"}, "b":"b", "f": {"g":"g"}, "h":[1,2,3], "i":[1,2,3], "j": {}, "k":[1,2,3], "l": "test"})
+    )
+}
 pub mod serde_regex {
     use regex::Regex;
     use serde::*;

backend/src/context/cli.rs

@@ -1,12 +1,13 @@
 use std::fs::File;
 use std::io::BufReader;
-use std::net::{Ipv4Addr, SocketAddr};
+use std::net::Ipv4Addr;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
 
 use clap::ArgMatches;
 use color_eyre::eyre::eyre;
 use cookie_store::CookieStore;
+use josekit::jwk::Jwk;
 use reqwest::Proxy;
 use reqwest_cookie_store::CookieStoreMutex;
 use rpc_toolkit::reqwest::{Client, Url};
@@ -15,14 +16,17 @@ use rpc_toolkit::Context;
 use serde::Deserialize;
 use tracing::instrument;
 
+use crate::util::config::{load_config_from_paths, local_config_path};
 use crate::ResultExt;
 
+use super::setup::CURRENT_SECRET;
+
 #[derive(Debug, Default, Deserialize)]
 #[serde(rename_all = "kebab-case")]
 pub struct CliContextConfig {
-    pub bind_rpc: Option<SocketAddr>,
     pub host: Option<Url>,
     #[serde(deserialize_with = "crate::util::serde::deserialize_from_str_opt")]
+    #[serde(default)]
     pub proxy: Option<Url>,
     pub cookie_path: Option<PathBuf>,
 }
@@ -38,6 +42,10 @@ pub struct CliContextSeed {
 impl Drop for CliContextSeed {
     fn drop(&mut self) {
         let tmp = format!("{}.tmp", self.cookie_path.display());
+        let parent_dir = self.cookie_path.parent().unwrap_or(Path::new("/"));
+        if !parent_dir.exists() {
+            std::fs::create_dir_all(&parent_dir).unwrap();
+        }
         let mut writer = fd_lock_rs::FdLock::lock(
             File::create(&tmp).unwrap(),
             fd_lock_rs::LockType::Exclusive,
@@ -60,26 +68,22 @@ impl CliContext {
     /// BLOCKING
     #[instrument(skip(matches))]
     pub fn init(matches: &ArgMatches) -> Result<Self, crate::Error> {
-        let cfg_path = Path::new(matches.value_of("config").unwrap_or(crate::CONFIG_PATH));
-        let base = if cfg_path.exists() {
-            serde_yaml::from_reader(
-                File::open(cfg_path)
-                    .with_ctx(|_| (crate::ErrorKind::Filesystem, cfg_path.display().to_string()))?,
-            )
-            .with_kind(crate::ErrorKind::Deserialization)?
-        } else {
-            CliContextConfig::default()
-        };
+        let local_config_path = local_config_path();
+        let base: CliContextConfig = load_config_from_paths(
+            matches
+                .values_of("config")
+                .into_iter()
+                .flatten()
+                .map(|p| Path::new(p))
+                .chain(local_config_path.as_deref().into_iter())
+                .chain(std::iter::once(Path::new(crate::util::config::CONFIG_PATH))),
+        )?;
         let mut url = if let Some(host) = matches.value_of("host") {
             host.parse()?
         } else if let Some(host) = base.host {
             host
         } else {
-            format!(
-                "http://{}",
-                base.bind_rpc.unwrap_or(([127, 0, 0, 1], 80).into())
-            )
-            .parse()?
+            format!("http://localhost").parse()?
         };
         let proxy = if let Some(proxy) = matches.value_of("proxy") {
             Some(proxy.parse()?)
@@ -88,7 +92,9 @@ impl CliContext {
         };
 
         let cookie_path = base.cookie_path.unwrap_or_else(|| {
-            cfg_path
+            local_config_path
+                .as_deref()
+                .unwrap_or_else(|| Path::new(crate::util::config::CONFIG_PATH))
                 .parent()
                 .unwrap_or(Path::new("/"))
                 .join(".cookies.json")
@@ -123,6 +129,11 @@ impl CliContext {
         })))
     }
 }
+impl AsRef<Jwk> for CliContext {
+    fn as_ref(&self) -> &Jwk {
+        &*CURRENT_SECRET
+    }
+}
 impl std::ops::Deref for CliContext {
     type Target = CliContextSeed;
     fn deref(&self) -> &Self::Target {
@@ -149,3 +160,13 @@ impl Context for CliContext {
         &self.0.client
     }
 }
+/// When we had an empty proxy the system wasn't working like it used to, which allowed empty proxy
+#[test]
+fn test_cli_proxy_empty() {
+    serde_yaml::from_str::<CliContextConfig>(
+        "
+        bind_rpc:
+    ",
+    )
+    .unwrap();
+}

backend/src/context/diagnostic.rs

@@ -1,4 +1,3 @@
-use std::net::{IpAddr, SocketAddr};
 use std::ops::Deref;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
@@ -6,38 +5,36 @@ use std::sync::Arc;
 use rpc_toolkit::yajrc::RpcError;
 use rpc_toolkit::Context;
 use serde::Deserialize;
-use tokio::fs::File;
 use tokio::sync::broadcast::Sender;
 use tracing::instrument;
-use url::Host;
 
 use crate::shutdown::Shutdown;
-use crate::util::io::from_toml_async_reader;
-use crate::util::AsyncFileExt;
-use crate::{Error, ResultExt};
+use crate::util::config::load_config_from_paths;
+use crate::Error;
 
 #[derive(Debug, Default, Deserialize)]
 #[serde(rename_all = "kebab-case")]
 pub struct DiagnosticContextConfig {
-    pub bind_rpc: Option<SocketAddr>,
     pub datadir: Option<PathBuf>,
 }
 impl DiagnosticContextConfig {
     #[instrument(skip(path))]
-    pub async fn load<P: AsRef<Path>>(path: Option<P>) -> Result<Self, Error> {
-        let cfg_path = path
-            .as_ref()
-            .map(|p| p.as_ref())
-            .unwrap_or(Path::new(crate::CONFIG_PATH));
-        if let Some(f) = File::maybe_open(cfg_path)
-            .await
-            .with_ctx(|_| (crate::ErrorKind::Filesystem, cfg_path.display().to_string()))?
-        {
-            from_toml_async_reader(f).await
-        } else {
-            Ok(Self::default())
-        }
+    pub async fn load<P: AsRef<Path> + Send + 'static>(path: Option<P>) -> Result<Self, Error> {
+        tokio::task::spawn_blocking(move || {
+            load_config_from_paths(
+                path.as_ref()
+                    .into_iter()
+                    .map(|p| p.as_ref())
+                    .chain(std::iter::once(Path::new(
+                        crate::util::config::DEVICE_CONFIG_PATH,
+                    )))
+                    .chain(std::iter::once(Path::new(crate::util::config::CONFIG_PATH))),
+            )
+        })
+        .await
+        .unwrap()
     }
+
     pub fn datadir(&self) -> &Path {
         self.datadir
             .as_deref()
@@ -46,7 +43,6 @@ impl DiagnosticContextConfig {
 }
 
 pub struct DiagnosticContextSeed {
-    pub bind_rpc: SocketAddr,
     pub datadir: PathBuf,
     pub shutdown: Sender<Option<Shutdown>>,
     pub error: Arc<RpcError>,
@@ -57,17 +53,19 @@ pub struct DiagnosticContextSeed {
 pub struct DiagnosticContext(Arc<DiagnosticContextSeed>);
 impl DiagnosticContext {
     #[instrument(skip(path))]
-    pub async fn init<P: AsRef<Path>>(
+    pub async fn init<P: AsRef<Path> + Send + 'static>(
         path: Option<P>,
         disk_guid: Option<Arc<String>>,
         error: Error,
     ) -> Result<Self, Error> {
+        tracing::error!("Error: {}: Starting diagnostic UI", error);
+        tracing::debug!("{:?}", error);
+
         let cfg = DiagnosticContextConfig::load(path).await?;
 
         let (shutdown, _) = tokio::sync::broadcast::channel(1);
 
         Ok(Self(Arc::new(DiagnosticContextSeed {
-            bind_rpc: cfg.bind_rpc.unwrap_or(([127, 0, 0, 1], 5959).into()),
             datadir: cfg.datadir().to_owned(),
             shutdown,
             disk_guid,
@@ -76,17 +74,7 @@ impl DiagnosticContext {
     }
 }
 
-impl Context for DiagnosticContext {
-    fn host(&self) -> Host<&str> {
-        match self.0.bind_rpc.ip() {
-            IpAddr::V4(a) => Host::Ipv4(a),
-            IpAddr::V6(a) => Host::Ipv6(a),
-        }
-    }
-    fn port(&self) -> u16 {
-        self.0.bind_rpc.port()
-    }
-}
+impl Context for DiagnosticContext {}
 impl Deref for DiagnosticContext {
     type Target = DiagnosticContextSeed;
     fn deref(&self) -> &Self::Target {

backend/src/context/install.rs

@@ -0,0 +1,58 @@
+use std::ops::Deref;
+use std::path::Path;
+use std::sync::Arc;
+
+use rpc_toolkit::Context;
+use serde::Deserialize;
+use tokio::sync::broadcast::Sender;
+use tracing::instrument;
+
+use crate::os_install::find_eth_iface;
+use crate::util::config::load_config_from_paths;
+use crate::Error;
+
+#[derive(Debug, Default, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct InstallContextConfig {}
+impl InstallContextConfig {
+    #[instrument(skip(path))]
+    pub async fn load<P: AsRef<Path> + Send + 'static>(path: Option<P>) -> Result<Self, Error> {
+        tokio::task::spawn_blocking(move || {
+            load_config_from_paths(
+                path.as_ref()
+                    .into_iter()
+                    .map(|p| p.as_ref())
+                    .chain(std::iter::once(Path::new(crate::util::config::CONFIG_PATH))),
+            )
+        })
+        .await
+        .unwrap()
+    }
+}
+
+pub struct InstallContextSeed {
+    pub ethernet_interface: String,
+    pub shutdown: Sender<()>,
+}
+
+#[derive(Clone)]
+pub struct InstallContext(Arc<InstallContextSeed>);
+impl InstallContext {
+    #[instrument(skip(path))]
+    pub async fn init<P: AsRef<Path> + Send + 'static>(path: Option<P>) -> Result<Self, Error> {
+        let _cfg = InstallContextConfig::load(path.as_ref().map(|p| p.as_ref().to_owned())).await?;
+        let (shutdown, _) = tokio::sync::broadcast::channel(1);
+        Ok(Self(Arc::new(InstallContextSeed {
+            ethernet_interface: find_eth_iface().await?,
+            shutdown,
+        })))
+    }
+}
+
+impl Context for InstallContext {}
+impl Deref for InstallContext {
+    type Target = InstallContextSeed;
+    fn deref(&self) -> &Self::Target {
+        &*self.0
+    }
+}

backend/src/context/mod.rs

@@ -1,11 +1,13 @@
 pub mod cli;
 pub mod diagnostic;
+pub mod install;
 pub mod rpc;
 pub mod sdk;
 pub mod setup;
 
 pub use cli::CliContext;
 pub use diagnostic::DiagnosticContext;
+pub use install::InstallContext;
 pub use rpc::RpcContext;
 pub use sdk::SdkContext;
 pub use setup::SetupContext;
@@ -35,3 +37,8 @@ impl From<SetupContext> for () {
         ()
     }
 }
+impl From<InstallContext> for () {
+    fn from(_: InstallContext) -> Self {
+        ()
+    }
+}

backend/src/context/rpc.rs

@@ -1,124 +1,130 @@
 use std::collections::{BTreeMap, VecDeque};
-use std::net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV4};
+use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4};
 use std::ops::Deref;
 use std::path::{Path, PathBuf};
-use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::Arc;
 use std::time::Duration;
 
 use bollard::Docker;
-use chrono::Utc;
-use color_eyre::eyre::eyre;
+use helpers::to_tmp_path;
+use josekit::jwk::Jwk;
 use patch_db::json_ptr::JsonPointer;
-use patch_db::{DbHandle, LockType, PatchDb, Revision};
+use patch_db::{DbHandle, LockReceipt, LockType, PatchDb, Revision};
 use reqwest::Url;
-use rpc_toolkit::url::Host;
 use rpc_toolkit::Context;
 use serde::Deserialize;
-use sqlx::sqlite::SqliteConnectOptions;
-use sqlx::SqlitePool;
-use tokio::fs::File;
-use tokio::process::Command;
+use sqlx::postgres::PgConnectOptions;
+use sqlx::PgPool;
 use tokio::sync::{broadcast, oneshot, Mutex, RwLock};
 use tracing::instrument;
 
-use crate::core::rpc_continuations::{RequestGuid, RpcContinuation};
+use crate::core::rpc_continuations::{RequestGuid, RestHandler, RpcContinuation};
 use crate::db::model::{Database, InstalledPackageDataEntry, PackageDataEntry};
-use crate::hostname::{derive_hostname, derive_id, get_product_key};
-use crate::install::cleanup::{cleanup_failed, uninstall};
+use crate::disk::OsPartitionInfo;
+use crate::hostname::HostNameReceipt;
+use crate::init::{init_postgres, pgloader};
+use crate::install::cleanup::{cleanup_failed, uninstall, CleanupFailedReceipts};
 use crate::manager::ManagerMap;
 use crate::middleware::auth::HashSessionToken;
+use crate::net::net_controller::NetController;
 use crate::net::tor::os_key;
 use crate::net::wifi::WpaCli;
-use crate::net::NetController;
 use crate::notifications::NotificationManager;
 use crate::setup::password_hash;
 use crate::shutdown::Shutdown;
 use crate::status::{MainStatus, Status};
-use crate::util::io::from_toml_async_reader;
-use crate::util::{AsyncFileExt, Invoke};
-use crate::{Error, ResultExt};
+use crate::util::config::load_config_from_paths;
+use crate::{Error, ErrorKind, ResultExt};
+
+use super::setup::CURRENT_SECRET;
 
 #[derive(Debug, Default, Deserialize)]
 #[serde(rename_all = "kebab-case")]
 pub struct RpcContextConfig {
+    pub wifi_interface: Option<String>,
+    pub ethernet_interface: String,
+    pub os_partitions: OsPartitionInfo,
+    pub migration_batch_rows: Option<usize>,
+    pub migration_prefetch_rows: Option<usize>,
     pub bind_rpc: Option<SocketAddr>,
-    pub bind_ws: Option<SocketAddr>,
-    pub bind_static: Option<SocketAddr>,
     pub tor_control: Option<SocketAddr>,
     pub tor_socks: Option<SocketAddr>,
+    pub dns_bind: Option<Vec<SocketAddr>>,
     pub revision_cache_size: Option<usize>,
     pub datadir: Option<PathBuf>,
     pub log_server: Option<Url>,
 }
 impl RpcContextConfig {
-    pub async fn load<P: AsRef<Path>>(path: Option<P>) -> Result<Self, Error> {
-        let cfg_path = path
-            .as_ref()
-            .map(|p| p.as_ref())
-            .unwrap_or(Path::new(crate::CONFIG_PATH));
-        if let Some(f) = File::maybe_open(cfg_path)
-            .await
-            .with_ctx(|_| (crate::ErrorKind::Filesystem, cfg_path.display().to_string()))?
-        {
-            from_toml_async_reader(f).await
-        } else {
-            Ok(Self::default())
-        }
+    pub async fn load<P: AsRef<Path> + Send + 'static>(path: Option<P>) -> Result<Self, Error> {
+        tokio::task::spawn_blocking(move || {
+            load_config_from_paths(
+                path.as_ref()
+                    .into_iter()
+                    .map(|p| p.as_ref())
+                    .chain(std::iter::once(Path::new(
+                        crate::util::config::DEVICE_CONFIG_PATH,
+                    )))
+                    .chain(std::iter::once(Path::new(crate::util::config::CONFIG_PATH))),
+            )
+        })
+        .await
+        .unwrap()
     }
     pub fn datadir(&self) -> &Path {
         self.datadir
             .as_deref()
             .unwrap_or_else(|| Path::new("/embassy-data"))
     }
-    pub async fn db(&self, secret_store: &SqlitePool, product_key: &str) -> Result<PatchDb, Error> {
-        let sid = derive_id(product_key);
-        let hostname = derive_hostname(&sid);
+    pub async fn db(&self, secret_store: &PgPool) -> Result<PatchDb, Error> {
         let db_path = self.datadir().join("main").join("embassy.db");
         let db = PatchDb::open(&db_path)
             .await
             .with_ctx(|_| (crate::ErrorKind::Filesystem, db_path.display().to_string()))?;
-        if !db.exists(&<JsonPointer>::default()).await? {
+        if !db.exists(&<JsonPointer>::default()).await {
             db.put(
                 &<JsonPointer>::default(),
                 &Database::init(
-                    sid,
-                    &hostname,
                     &os_key(&mut secret_store.acquire().await?).await?,
                     password_hash(&mut secret_store.acquire().await?).await?,
                 ),
-                None,
             )
             .await?;
         }
         Ok(db)
     }
     #[instrument]
-    pub async fn secret_store(&self) -> Result<SqlitePool, Error> {
-        let secret_store = SqlitePool::connect_with(
-            SqliteConnectOptions::new()
-                .filename(self.datadir().join("main").join("secrets.db"))
-                .create_if_missing(true)
-                .busy_timeout(Duration::from_secs(30)),
-        )
-        .await?;
+    pub async fn secret_store(&self) -> Result<PgPool, Error> {
+        init_postgres(self.datadir()).await?;
+        let secret_store =
+            PgPool::connect_with(PgConnectOptions::new().database("secrets").username("root"))
+                .await?;
         sqlx::migrate!()
             .run(&secret_store)
             .await
             .with_kind(crate::ErrorKind::Database)?;
+        let old_db_path = self.datadir().join("main/secrets.db");
+        if tokio::fs::metadata(&old_db_path).await.is_ok() {
+            pgloader(
+                &old_db_path,
+                self.migration_batch_rows.unwrap_or(25000),
+                self.migration_prefetch_rows.unwrap_or(100_000),
+            )
+            .await?;
+        }
         Ok(secret_store)
     }
 }
 
 pub struct RpcContextSeed {
     is_closed: AtomicBool,
-    pub bind_rpc: SocketAddr,
-    pub bind_ws: SocketAddr,
-    pub bind_static: SocketAddr,
+    pub os_partitions: OsPartitionInfo,
+    pub wifi_interface: Option<String>,
+    pub ethernet_interface: String,
     pub datadir: PathBuf,
     pub disk_guid: Arc<String>,
     pub db: PatchDb,
-    pub secret_store: SqlitePool,
+    pub secret_store: PgPool,
     pub docker: Docker,
     pub net_controller: NetController,
     pub managers: ManagerMap,
@@ -130,14 +136,84 @@ pub struct RpcContextSeed {
     pub notification_manager: NotificationManager,
     pub open_authed_websockets: Mutex<BTreeMap<HashSessionToken, Vec<oneshot::Sender<()>>>>,
     pub rpc_stream_continuations: Mutex<BTreeMap<RequestGuid, RpcContinuation>>,
-    pub wifi_manager: Arc<RwLock<WpaCli>>,
+    pub wifi_manager: Option<Arc<RwLock<WpaCli>>>,
+    pub current_secret: Arc<Jwk>,
+}
+
+pub struct RpcCleanReceipts {
+    cleanup_receipts: CleanupFailedReceipts,
+    packages: LockReceipt<crate::db::model::AllPackageData, ()>,
+    package: LockReceipt<crate::db::model::PackageDataEntry, String>,
+}
+
+impl RpcCleanReceipts {
+    pub async fn new<'a>(db: &'a mut impl DbHandle) -> Result<Self, Error> {
+        let mut locks = Vec::new();
+
+        let setup = Self::setup(&mut locks);
+        Ok(setup(&db.lock_all(locks).await?)?)
+    }
+
+    pub fn setup(
+        locks: &mut Vec<patch_db::LockTargetId>,
+    ) -> impl FnOnce(&patch_db::Verifier) -> Result<Self, Error> {
+        let cleanup_receipts = CleanupFailedReceipts::setup(locks);
+
+        let packages = crate::db::DatabaseModel::new()
+            .package_data()
+            .make_locker(LockType::Write)
+            .add_to_keys(locks);
+        let package = crate::db::DatabaseModel::new()
+            .package_data()
+            .star()
+            .make_locker(LockType::Write)
+            .add_to_keys(locks);
+        move |skeleton_key| {
+            Ok(Self {
+                cleanup_receipts: cleanup_receipts(skeleton_key)?,
+                packages: packages.verify(skeleton_key)?,
+                package: package.verify(skeleton_key)?,
+            })
+        }
+    }
+}
+
+pub struct RpcSetHostNameReceipts {
+    pub hostname_receipts: HostNameReceipt,
+    #[allow(dead_code)]
+    server_info: LockReceipt<crate::db::model::ServerInfo, ()>,
+}
+
+impl RpcSetHostNameReceipts {
+    pub async fn new(db: &'_ mut impl DbHandle) -> Result<Self, Error> {
+        let mut locks = Vec::new();
+
+        let setup = Self::setup(&mut locks);
+        Ok(setup(&db.lock_all(locks).await?)?)
+    }
+
+    pub fn setup(
+        locks: &mut Vec<patch_db::LockTargetId>,
+    ) -> impl FnOnce(&patch_db::Verifier) -> Result<Self, Error> {
+        let hostname_receipts = HostNameReceipt::setup(locks);
+        let server_info = crate::db::DatabaseModel::new()
+            .server_info()
+            .make_locker(LockType::Read)
+            .add_to_keys(locks);
+        move |skeleton_key| {
+            Ok(Self {
+                hostname_receipts: hostname_receipts(skeleton_key)?,
+                server_info: server_info.verify(skeleton_key)?,
+            })
+        }
+    }
 }
 
 #[derive(Clone)]
 pub struct RpcContext(Arc<RpcContextSeed>);
 impl RpcContext {
     #[instrument(skip(cfg_path))]
-    pub async fn init<P: AsRef<Path>>(
+    pub async fn init<P: AsRef<Path> + Send + 'static>(
         cfg_path: Option<P>,
         disk_guid: Arc<String>,
     ) -> Result<Self, Error> {
@@ -149,17 +225,23 @@ impl RpcContext {
         )));
         let (shutdown, _) = tokio::sync::broadcast::channel(1);
         let secret_store = base.secret_store().await?;
-        tracing::info!("Opened Sqlite DB");
-        let db = base.db(&secret_store, &get_product_key().await?).await?;
+        tracing::info!("Opened Pg DB");
+        let db = base.db(&secret_store).await?;
         tracing::info!("Opened PatchDB");
-        let docker = Docker::connect_with_unix_defaults()?;
+        let mut docker = Docker::connect_with_unix_defaults()?;
+        docker.set_timeout(Duration::from_secs(600));
         tracing::info!("Connected to Docker");
         let net_controller = NetController::init(
-            ([127, 0, 0, 1], 80).into(),
+            ([0, 0, 0, 0], 80).into(),
             crate::net::tor::os_key(&mut secret_store.acquire().await?).await?,
             base.tor_control
                 .unwrap_or(SocketAddr::from(([127, 0, 0, 1], 9051))),
+            base.dns_bind
+                .as_ref()
+                .map(|v| v.as_slice())
+                .unwrap_or(&[SocketAddr::from(([127, 0, 0, 1], 53))]),
             secret_store.clone(),
+            &mut db.handle(),
             None,
         )
         .await?;
@@ -170,10 +252,10 @@ impl RpcContext {
         tracing::info!("Initialized Notification Manager");
         let seed = Arc::new(RpcContextSeed {
             is_closed: AtomicBool::new(false),
-            bind_rpc: base.bind_rpc.unwrap_or(([127, 0, 0, 1], 5959).into()),
-            bind_ws: base.bind_ws.unwrap_or(([127, 0, 0, 1], 5960).into()),
-            bind_static: base.bind_static.unwrap_or(([127, 0, 0, 1], 5961).into()),
             datadir: base.datadir().to_path_buf(),
+            os_partitions: base.os_partitions,
+            wifi_interface: base.wifi_interface.clone(),
+            ethernet_interface: base.ethernet_interface,
             disk_guid,
             db,
             secret_store,
@@ -188,7 +270,19 @@ impl RpcContext {
             notification_manager,
             open_authed_websockets: Mutex::new(BTreeMap::new()),
             rpc_stream_continuations: Mutex::new(BTreeMap::new()),
-            wifi_manager: Arc::new(RwLock::new(WpaCli::init("wlan0".to_string()))),
+            wifi_manager: base
+                .wifi_interface
+                .map(|i| Arc::new(RwLock::new(WpaCli::init(i)))),
+            current_secret: Arc::new(
+                Jwk::generate_ec_key(josekit::jwk::alg::ec::EcCurve::P256).map_err(|e| {
+                    tracing::debug!("{:?}", e);
+                    tracing::error!("Couldn't generate ec key");
+                    Error::new(
+                        color_eyre::eyre::eyre!("Couldn't generate ec key"),
+                        crate::ErrorKind::Unknown,
+                    )
+                })?,
+            ),
         });
 
         let res = Self(seed);
@@ -204,68 +298,28 @@ impl RpcContext {
         tracing::info!("Initialized Package Managers");
         Ok(res)
     }
-    #[instrument(skip(self, db))]
-    pub async fn set_nginx_conf<Db: DbHandle>(&self, db: &mut Db) -> Result<(), Error> {
-        tokio::fs::write("/etc/nginx/sites-available/default", {
-            let info = crate::db::DatabaseModel::new()
-                .server_info()
-                .get(db, true)
-                .await?;
-            format!(
-                include_str!("../nginx/main-ui.conf.template"),
-                lan_hostname = info.lan_address.host_str().unwrap(),
-                tor_hostname = info.tor_address.host_str().unwrap(),
-            )
-        })
-        .await
-        .with_ctx(|_| {
-            (
-                crate::ErrorKind::Filesystem,
-                "/etc/nginx/sites-available/default",
-            )
-        })?;
-        Command::new("systemctl")
-            .arg("reload")
-            .arg("nginx")
-            .invoke(crate::ErrorKind::Nginx)
-            .await?;
-        Ok(())
-    }
+
     #[instrument(skip(self))]
     pub async fn shutdown(self) -> Result<(), Error> {
         self.managers.empty().await?;
         self.secret_store.close().await;
         self.is_closed.store(true, Ordering::SeqCst);
+        // TODO: shutdown http servers
         Ok(())
     }
+
     #[instrument(skip(self))]
     pub async fn cleanup(&self) -> Result<(), Error> {
         let mut db = self.db.handle();
-        crate::db::DatabaseModel::new()
-            .package_data()
-            .lock(&mut db, LockType::Write)
-            .await?;
-        for package_id in crate::db::DatabaseModel::new()
-            .package_data()
-            .keys(&mut db, true)
-            .await?
-        {
+        let receipts = RpcCleanReceipts::new(&mut db).await?;
+        for (package_id, package) in receipts.packages.get(&mut db).await?.0 {
             if let Err(e) = async {
-                let mut pde = crate::db::DatabaseModel::new()
-                    .package_data()
-                    .idx_model(&package_id)
-                    .get_mut(&mut db)
-                    .await?;
-                match pde.as_mut().ok_or_else(|| {
-                    Error::new(
-                        eyre!("Node does not exist: /package-data/{}", package_id),
-                        crate::ErrorKind::Database,
-                    )
-                })? {
+                match package {
                     PackageDataEntry::Installing { .. }
                     | PackageDataEntry::Restoring { .. }
                     | PackageDataEntry::Updating { .. } => {
-                        cleanup_failed(self, &mut db, &package_id).await?;
+                        cleanup_failed(self, &mut db, &package_id, &receipts.cleanup_receipts)
+                            .await?;
                     }
                     PackageDataEntry::Removing { .. } => {
                         uninstall(
@@ -277,30 +331,48 @@ impl RpcContext {
                         .await?;
                     }
                     PackageDataEntry::Installed {
-                        installed:
-                            InstalledPackageDataEntry {
-                                status: Status { main, .. },
-                                ..
-                            },
-                        ..
+                        installed,
+                        static_files,
+                        manifest,
                     } => {
-                        let new_main = match std::mem::replace(
-                            main,
-                            MainStatus::Stopped, /* placeholder */
-                        ) {
+                        for (volume_id, volume_info) in &*manifest.volumes {
+                            let tmp_path = to_tmp_path(volume_info.path_for(
+                                &self.datadir,
+                                &package_id,
+                                &manifest.version,
+                                &volume_id,
+                            ))
+                            .with_kind(ErrorKind::Filesystem)?;
+                            if tokio::fs::metadata(&tmp_path).await.is_ok() {
+                                tokio::fs::remove_dir_all(&tmp_path).await?;
+                            }
+                        }
+                        let status = installed.status;
+                        let main = match status.main {
                             MainStatus::BackingUp { started, .. } => {
                                 if let Some(_) = started {
-                                    MainStatus::Starting
+                                    MainStatus::Starting { restarting: false }
                                 } else {
                                     MainStatus::Stopped
                                 }
                             }
-                            MainStatus::Running { .. } => MainStatus::Starting,
-                            a => a,
+                            MainStatus::Running { .. } => {
+                                MainStatus::Starting { restarting: false }
+                            }
+                            a => a.clone(),
                         };
-                        *main = new_main;
-
-                        pde.save(&mut db).await?;
+                        let new_package = PackageDataEntry::Installed {
+                            installed: InstalledPackageDataEntry {
+                                status: Status { main, ..status },
+                                ..installed
+                            },
+                            static_files,
+                            manifest,
+                        };
+                        receipts
+                            .package
+                            .set(&mut db, new_package, &package_id)
+                            .await?;
                     }
                 }
                 Ok::<_, Error>(())
@@ -313,18 +385,65 @@ impl RpcContext {
         }
         Ok(())
     }
-}
-impl Context for RpcContext {
-    fn host(&self) -> Host<&str> {
-        match self.0.bind_rpc.ip() {
-            IpAddr::V4(a) => Host::Ipv4(a),
-            IpAddr::V6(a) => Host::Ipv6(a),
+
+    #[instrument(skip(self))]
+    pub async fn clean_continuations(&self) {
+        let mut continuations = self.rpc_stream_continuations.lock().await;
+        let mut to_remove = Vec::new();
+        for (guid, cont) in &*continuations {
+            if cont.is_timed_out() {
+                to_remove.push(guid.clone());
+            }
+        }
+        for guid in to_remove {
+            continuations.remove(&guid);
         }
     }
-    fn port(&self) -> u16 {
-        self.0.bind_rpc.port()
+
+    #[instrument(skip(self, handler))]
+    pub async fn add_continuation(&self, guid: RequestGuid, handler: RpcContinuation) {
+        self.clean_continuations().await;
+        self.rpc_stream_continuations
+            .lock()
+            .await
+            .insert(guid, handler);
+    }
+
+    pub async fn get_continuation_handler(&self, guid: &RequestGuid) -> Option<RestHandler> {
+        let mut continuations = self.rpc_stream_continuations.lock().await;
+        if let Some(cont) = continuations.remove(guid) {
+            cont.into_handler().await
+        } else {
+            None
+        }
+    }
+
+    pub async fn get_ws_continuation_handler(&self, guid: &RequestGuid) -> Option<RestHandler> {
+        let continuations = self.rpc_stream_continuations.lock().await;
+        if matches!(continuations.get(guid), Some(RpcContinuation::WebSocket(_))) {
+            drop(continuations);
+            self.get_continuation_handler(guid).await
+        } else {
+            None
+        }
+    }
+
+    pub async fn get_rest_continuation_handler(&self, guid: &RequestGuid) -> Option<RestHandler> {
+        let continuations = self.rpc_stream_continuations.lock().await;
+        if matches!(continuations.get(guid), Some(RpcContinuation::Rest(_))) {
+            drop(continuations);
+            self.get_continuation_handler(guid).await
+        } else {
+            None
+        }
     }
 }
+impl AsRef<Jwk> for RpcContext {
+    fn as_ref(&self) -> &Jwk {
+        &*CURRENT_SECRET
+    }
+}
+impl Context for RpcContext {}
 impl Deref for RpcContext {
     type Target = RpcContextSeed;
     fn deref(&self) -> &Self::Target {

backend/src/context/sdk.rs

@@ -1,5 +1,3 @@
-use std::fs::File;
-use std::io::Read;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
 
@@ -9,6 +7,7 @@ use rpc_toolkit::Context;
 use serde::Deserialize;
 use tracing::instrument;
 
+use crate::util::config::{load_config_from_paths, local_config_path};
 use crate::{Error, ResultExt};
 
 #[derive(Debug, Default, Deserialize)]
@@ -28,22 +27,24 @@ impl SdkContext {
     /// BLOCKING
     #[instrument(skip(matches))]
     pub fn init(matches: &ArgMatches) -> Result<Self, crate::Error> {
-        let cfg_path = Path::new(matches.value_of("config").unwrap_or(crate::CONFIG_PATH));
-        let base = if cfg_path.exists() {
-            serde_yaml::from_reader(
-                File::open(cfg_path)
-                    .with_ctx(|_| (crate::ErrorKind::Filesystem, cfg_path.display().to_string()))?,
-            )
-            .with_kind(crate::ErrorKind::Deserialization)?
-        } else {
-            SdkContextConfig::default()
-        };
+        let local_config_path = local_config_path();
+        let base: SdkContextConfig = load_config_from_paths(
+            matches
+                .values_of("config")
+                .into_iter()
+                .flatten()
+                .map(|p| Path::new(p))
+                .chain(local_config_path.as_deref().into_iter())
+                .chain(std::iter::once(Path::new(crate::util::config::CONFIG_PATH))),
+        )?;
         Ok(SdkContext(Arc::new(SdkContextSeed {
             developer_key_path: base.developer_key_path.unwrap_or_else(|| {
-                cfg_path
+                local_config_path
+                    .as_deref()
+                    .unwrap_or_else(|| Path::new(crate::util::config::CONFIG_PATH))
                     .parent()
                     .unwrap_or(Path::new("/"))
-                    .join(".developer_key")
+                    .join("developer.key.pem")
             }),
         })))
     }
@@ -53,9 +54,17 @@ impl SdkContext {
         if !self.developer_key_path.exists() {
             return Err(Error::new(eyre!("Developer Key does not exist! Please run `embassy-sdk init` before running this command."), crate::ErrorKind::Uninitialized));
         }
-        let mut keypair_buf = [0; ed25519_dalek::KEYPAIR_LENGTH];
-        File::open(&self.developer_key_path)?.read_exact(&mut keypair_buf)?;
-        Ok(ed25519_dalek::Keypair::from_bytes(&keypair_buf)?)
+        let pair = <ed25519::KeypairBytes as ed25519::pkcs8::DecodePrivateKey>::from_pkcs8_pem(
+            &std::fs::read_to_string(&self.developer_key_path)?,
+        )
+        .with_kind(crate::ErrorKind::Pem)?;
+        let secret = ed25519_dalek::SecretKey::from_bytes(&pair.secret_key[..])?;
+        let public = if let Some(public) = pair.public_key {
+            ed25519_dalek::PublicKey::from_bytes(&public[..])?
+        } else {
+            (&secret).into()
+        };
+        Ok(ed25519_dalek::Keypair { secret, public })
     }
 }
 impl std::ops::Deref for SdkContext {

backend/src/context/setup.rs

@@ -1,30 +1,35 @@
-use std::net::{IpAddr, SocketAddr};
 use std::ops::Deref;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
-use std::time::Duration;
 
+use josekit::jwk::Jwk;
 use patch_db::json_ptr::JsonPointer;
 use patch_db::PatchDb;
 use rpc_toolkit::yajrc::RpcError;
 use rpc_toolkit::Context;
 use serde::{Deserialize, Serialize};
-use sqlx::sqlite::SqliteConnectOptions;
-use sqlx::SqlitePool;
-use tokio::fs::File;
+use sqlx::postgres::PgConnectOptions;
+use sqlx::PgPool;
 use tokio::sync::broadcast::Sender;
 use tokio::sync::RwLock;
 use tracing::instrument;
-use url::Host;
 
 use crate::db::model::Database;
-use crate::hostname::{derive_hostname, derive_id, get_product_key};
+use crate::disk::OsPartitionInfo;
+use crate::init::{init_postgres, pgloader};
 use crate::net::tor::os_key;
-use crate::setup::{password_hash, RecoveryStatus};
-use crate::util::io::from_toml_async_reader;
-use crate::util::AsyncFileExt;
+use crate::setup::{password_hash, SetupStatus};
+use crate::util::config::load_config_from_paths;
 use crate::{Error, ResultExt};
 
+lazy_static::lazy_static! {
+    pub static ref CURRENT_SECRET: Jwk = Jwk::generate_ec_key(josekit::jwk::alg::ec::EcCurve::P256).unwrap_or_else(|e| {
+        tracing::debug!("{:?}", e);
+        tracing::error!("Couldn't generate ec key");
+        panic!("Couldn't generate ec key")
+    });
+}
+
 #[derive(Clone, Serialize, Deserialize)]
 #[serde(rename_all = "kebab-case")]
 pub struct SetupResult {
@@ -36,24 +41,27 @@ pub struct SetupResult {
 #[derive(Debug, Default, Deserialize)]
 #[serde(rename_all = "kebab-case")]
 pub struct SetupContextConfig {
-    pub bind_rpc: Option<SocketAddr>,
+    pub os_partitions: OsPartitionInfo,
+    pub migration_batch_rows: Option<usize>,
+    pub migration_prefetch_rows: Option<usize>,
     pub datadir: Option<PathBuf>,
 }
 impl SetupContextConfig {
     #[instrument(skip(path))]
-    pub async fn load<P: AsRef<Path>>(path: Option<P>) -> Result<Self, Error> {
-        let cfg_path = path
-            .as_ref()
-            .map(|p| p.as_ref())
-            .unwrap_or(Path::new(crate::CONFIG_PATH));
-        if let Some(f) = File::maybe_open(cfg_path)
-            .await
-            .with_ctx(|_| (crate::ErrorKind::Filesystem, cfg_path.display().to_string()))?
-        {
-            from_toml_async_reader(f).await
-        } else {
-            Ok(Self::default())
-        }
+    pub async fn load<P: AsRef<Path> + Send + 'static>(path: Option<P>) -> Result<Self, Error> {
+        tokio::task::spawn_blocking(move || {
+            load_config_from_paths(
+                path.as_ref()
+                    .into_iter()
+                    .map(|p| p.as_ref())
+                    .chain(std::iter::once(Path::new(
+                        crate::util::config::DEVICE_CONFIG_PATH,
+                    )))
+                    .chain(std::iter::once(Path::new(crate::util::config::CONFIG_PATH))),
+            )
+        })
+        .await
+        .unwrap()
     }
     pub fn datadir(&self) -> &Path {
         self.datadir
@@ -63,104 +71,87 @@ impl SetupContextConfig {
 }
 
 pub struct SetupContextSeed {
+    pub os_partitions: OsPartitionInfo,
     pub config_path: Option<PathBuf>,
-    pub bind_rpc: SocketAddr,
+    pub migration_batch_rows: usize,
+    pub migration_prefetch_rows: usize,
     pub shutdown: Sender<()>,
     pub datadir: PathBuf,
     pub selected_v2_drive: RwLock<Option<PathBuf>>,
     pub cached_product_key: RwLock<Option<Arc<String>>>,
-    pub recovery_status: RwLock<Option<Result<RecoveryStatus, RpcError>>>,
+    pub setup_status: RwLock<Option<Result<SetupStatus, RpcError>>>,
     pub setup_result: RwLock<Option<(Arc<String>, SetupResult)>>,
 }
 
+impl AsRef<Jwk> for SetupContextSeed {
+    fn as_ref(&self) -> &Jwk {
+        &*CURRENT_SECRET
+    }
+}
+
 #[derive(Clone)]
 pub struct SetupContext(Arc<SetupContextSeed>);
 impl SetupContext {
     #[instrument(skip(path))]
-    pub async fn init<P: AsRef<Path>>(path: Option<P>) -> Result<Self, Error> {
-        let cfg = SetupContextConfig::load(path.as_ref()).await?;
+    pub async fn init<P: AsRef<Path> + Send + 'static>(path: Option<P>) -> Result<Self, Error> {
+        let cfg = SetupContextConfig::load(path.as_ref().map(|p| p.as_ref().to_owned())).await?;
         let (shutdown, _) = tokio::sync::broadcast::channel(1);
         let datadir = cfg.datadir().to_owned();
         Ok(Self(Arc::new(SetupContextSeed {
+            os_partitions: cfg.os_partitions,
             config_path: path.as_ref().map(|p| p.as_ref().to_owned()),
-            bind_rpc: cfg.bind_rpc.unwrap_or(([127, 0, 0, 1], 5959).into()),
+            migration_batch_rows: cfg.migration_batch_rows.unwrap_or(25000),
+            migration_prefetch_rows: cfg.migration_prefetch_rows.unwrap_or(100_000),
             shutdown,
             datadir,
             selected_v2_drive: RwLock::new(None),
             cached_product_key: RwLock::new(None),
-            recovery_status: RwLock::new(None),
+            setup_status: RwLock::new(None),
             setup_result: RwLock::new(None),
         })))
     }
     #[instrument(skip(self))]
-    pub async fn db(&self, secret_store: &SqlitePool) -> Result<PatchDb, Error> {
+    pub async fn db(&self, secret_store: &PgPool) -> Result<PatchDb, Error> {
         let db_path = self.datadir.join("main").join("embassy.db");
         let db = PatchDb::open(&db_path)
             .await
             .with_ctx(|_| (crate::ErrorKind::Filesystem, db_path.display().to_string()))?;
-        if !db.exists(&<JsonPointer>::default()).await? {
-            let pkey = self.product_key().await?;
-            let sid = derive_id(&*pkey);
-            let hostname = derive_hostname(&sid);
+        if !db.exists(&<JsonPointer>::default()).await {
             db.put(
                 &<JsonPointer>::default(),
                 &Database::init(
-                    sid,
-                    &hostname,
                     &os_key(&mut secret_store.acquire().await?).await?,
                     password_hash(&mut secret_store.acquire().await?).await?,
                 ),
-                None,
             )
             .await?;
         }
         Ok(db)
     }
     #[instrument(skip(self))]
-    pub async fn secret_store(&self) -> Result<SqlitePool, Error> {
-        let secret_store = SqlitePool::connect_with(
-            SqliteConnectOptions::new()
-                .filename(self.datadir.join("main").join("secrets.db"))
-                .create_if_missing(true)
-                .busy_timeout(Duration::from_secs(30)),
-        )
-        .await?;
+    pub async fn secret_store(&self) -> Result<PgPool, Error> {
+        init_postgres(&self.datadir).await?;
+        let secret_store =
+            PgPool::connect_with(PgConnectOptions::new().database("secrets").username("root"))
+                .await?;
         sqlx::migrate!()
             .run(&secret_store)
             .await
             .with_kind(crate::ErrorKind::Database)?;
+        let old_db_path = self.datadir.join("main/secrets.db");
+        if tokio::fs::metadata(&old_db_path).await.is_ok() {
+            pgloader(
+                &old_db_path,
+                self.migration_batch_rows,
+                self.migration_prefetch_rows,
+            )
+            .await?;
+        }
         Ok(secret_store)
     }
-    #[instrument(skip(self))]
-    pub async fn product_key(&self) -> Result<Arc<String>, Error> {
-        Ok(
-            if let Some(k) = {
-                let guard = self.cached_product_key.read().await;
-                let res = guard.clone();
-                drop(guard);
-                res
-            } {
-                k
-            } else {
-                let k = Arc::new(get_product_key().await?);
-                *self.cached_product_key.write().await = Some(k.clone());
-                k
-            },
-        )
-    }
 }
 
-impl Context for SetupContext {
-    fn host(&self) -> Host<&str> {
-        match self.0.bind_rpc.ip() {
-            IpAddr::V4(a) => Host::Ipv4(a),
-            IpAddr::V6(a) => Host::Ipv6(a),
-        }
-    }
-    fn port(&self) -> u16 {
-        self.0.bind_rpc.port()
-    }
-}
+impl Context for SetupContext {}
 impl Deref for SetupContext {
     type Target = SetupContextSeed;
     fn deref(&self) -> &Self::Target {

backend/src/control.rs

@@ -1,61 +1,80 @@
 use std::collections::BTreeMap;
 
 use color_eyre::eyre::eyre;
-use patch_db::{DbHandle, LockType};
+use patch_db::{DbHandle, LockReceipt, LockType};
 use rpc_toolkit::command;
 use tracing::instrument;
 
 use crate::context::RpcContext;
-use crate::db::util::WithRevision;
 use crate::dependencies::{
     break_all_dependents_transitive, heal_all_dependents_transitive, BreakageRes, DependencyError,
-    TaggedDependencyError,
+    DependencyReceipt, TaggedDependencyError,
 };
 use crate::s9pk::manifest::PackageId;
 use crate::status::MainStatus;
 use crate::util::display_none;
 use crate::util::serde::display_serializable;
-use crate::{Error, ResultExt};
+use crate::Error;
 
-#[command(display(display_none))]
+#[derive(Clone)]
+pub struct StartReceipts {
+    dependency_receipt: DependencyReceipt,
+    status: LockReceipt<MainStatus, ()>,
+    version: LockReceipt<crate::util::Version, ()>,
+}
+
+impl StartReceipts {
+    pub async fn new(db: &mut impl DbHandle, id: &PackageId) -> Result<Self, Error> {
+        let mut locks = Vec::new();
+
+        let setup = Self::setup(&mut locks, id);
+        Ok(setup(&db.lock_all(locks).await?)?)
+    }
+
+    pub fn setup(
+        locks: &mut Vec<patch_db::LockTargetId>,
+        id: &PackageId,
+    ) -> impl FnOnce(&patch_db::Verifier) -> Result<Self, Error> {
+        let dependency_receipt = DependencyReceipt::setup(locks);
+        let status = crate::db::DatabaseModel::new()
+            .package_data()
+            .idx_model(id)
+            .and_then(|x| x.installed())
+            .map(|x| x.status().main())
+            .make_locker(LockType::Write)
+            .add_to_keys(locks);
+        let version = crate::db::DatabaseModel::new()
+            .package_data()
+            .idx_model(id)
+            .and_then(|x| x.installed())
+            .map(|x| x.manifest().version())
+            .make_locker(LockType::Read)
+            .add_to_keys(locks);
+        move |skeleton_key| {
+            Ok(Self {
+                dependency_receipt: dependency_receipt(skeleton_key)?,
+                status: status.verify(skeleton_key)?,
+                version: version.verify(skeleton_key)?,
+            })
+        }
+    }
+}
+
+#[command(display(display_none), metadata(sync_db = true))]
 #[instrument(skip(ctx))]
-pub async fn start(
-    #[context] ctx: RpcContext,
-    #[arg] id: PackageId,
-) -> Result<WithRevision<()>, Error> {
+pub async fn start(#[context] ctx: RpcContext, #[arg] id: PackageId) -> Result<(), Error> {
     let mut db = ctx.db.handle();
     let mut tx = db.begin().await?;
-    crate::db::DatabaseModel::new()
-        .package_data()
-        .lock(&mut tx, LockType::Write)
+    let receipts = StartReceipts::new(&mut tx, &id).await?;
+    let version = receipts.version.get(&mut tx).await?;
+    receipts
+        .status
+        .set(&mut tx, MainStatus::Starting { restarting: false })
         .await?;
-    let installed = crate::db::DatabaseModel::new()
-        .package_data()
-        .idx_model(&id)
-        .and_then(|pkg| pkg.installed())
-        .expect(&mut tx)
-        .await
-        .with_ctx(|_| {
-            (
-                crate::ErrorKind::NotFound,
-                format!("{} is not installed", id),
-            )
-        })?;
-    installed.lock(&mut tx, LockType::Read).await?;
-    let version = installed
-        .clone()
-        .manifest()
-        .version()
-        .get(&mut tx, true)
-        .await?
-        .to_owned();
-    let mut status = installed.status().main().get_mut(&mut tx).await?;
+    heal_all_dependents_transitive(&ctx, &mut tx, &id, &receipts.dependency_receipt).await?;
 
-    *status = MainStatus::Starting;
-    status.save(&mut tx).await?;
-    heal_all_dependents_transitive(&ctx, &mut tx, &id).await?;
-
-    let revision = tx.commit(None).await?;
+    tx.commit().await?;
+    drop(receipts);
 
     ctx.managers
         .get(&(id, version))
@@ -64,45 +83,72 @@ pub async fn start(
         .synchronize()
         .await;
 
-    Ok(WithRevision {
-        revision,
-        response: (),
-    })
+    Ok(())
+}
+#[derive(Clone)]
+pub struct StopReceipts {
+    breaks: crate::dependencies::BreakTransitiveReceipts,
+    status: LockReceipt<MainStatus, ()>,
+}
+
+impl StopReceipts {
+    pub async fn new<'a>(db: &'a mut impl DbHandle, id: &PackageId) -> Result<Self, Error> {
+        let mut locks = Vec::new();
+
+        let setup = Self::setup(&mut locks, id);
+        Ok(setup(&db.lock_all(locks).await?)?)
+    }
+
+    pub fn setup(
+        locks: &mut Vec<patch_db::LockTargetId>,
+        id: &PackageId,
+    ) -> impl FnOnce(&patch_db::Verifier) -> Result<Self, Error> {
+        let breaks = crate::dependencies::BreakTransitiveReceipts::setup(locks);
+        let status = crate::db::DatabaseModel::new()
+            .package_data()
+            .idx_model(id)
+            .and_then(|x| x.installed())
+            .map(|x| x.status().main())
+            .make_locker(LockType::Write)
+            .add_to_keys(locks);
+        move |skeleton_key| {
+            Ok(Self {
+                breaks: breaks(skeleton_key)?,
+                status: status.verify(skeleton_key)?,
+            })
+        }
+    }
 }
 
 #[instrument(skip(db))]
-async fn stop_common<Db: DbHandle>(
+pub async fn stop_common<Db: DbHandle>(
     db: &mut Db,
     id: &PackageId,
     breakages: &mut BTreeMap<PackageId, TaggedDependencyError>,
-) -> Result<(), Error> {
+) -> Result<MainStatus, Error> {
     let mut tx = db.begin().await?;
-    let mut status = crate::db::DatabaseModel::new()
-        .package_data()
-        .idx_model(&id)
-        .and_then(|pkg| pkg.installed())
-        .expect(&mut tx)
-        .await
-        .with_ctx(|_| {
-            (
-                crate::ErrorKind::NotFound,
-                format!("{} is not installed", id),
-            )
-        })?
-        .status()
-        .main()
-        .get_mut(&mut tx)
-        .await?;
+    let receipts = StopReceipts::new(&mut tx, id).await?;
+    let last_status = receipts.status.get(&mut tx).await?;
+    receipts.status.set(&mut tx, MainStatus::Stopping).await?;
 
-    *status = MainStatus::Stopping;
-    status.save(&mut tx).await?;
     tx.save().await?;
-    break_all_dependents_transitive(db, &id, DependencyError::NotRunning, breakages).await?;
+    break_all_dependents_transitive(
+        db,
+        id,
+        DependencyError::NotRunning,
+        breakages,
+        &receipts.breaks,
+    )
+    .await?;
 
-    Ok(())
+    Ok(last_status)
 }
 
-#[command(subcommands(self(stop_impl(async)), stop_dry), display(display_none))]
+#[command(
+    subcommands(self(stop_impl(async)), stop_dry),
+    display(display_none),
+    metadata(sync_db = true)
+)]
 pub fn stop(#[arg] id: PackageId) -> Result<PackageId, Error> {
     Ok(id)
 }
@@ -125,14 +171,38 @@ pub async fn stop_dry(
 }
 
 #[instrument(skip(ctx))]
-pub async fn stop_impl(ctx: RpcContext, id: PackageId) -> Result<WithRevision<()>, Error> {
+pub async fn stop_impl(ctx: RpcContext, id: PackageId) -> Result<MainStatus, Error> {
     let mut db = ctx.db.handle();
     let mut tx = db.begin().await?;
 
-    stop_common(&mut tx, &id, &mut BTreeMap::new()).await?;
+    let last_statuts = stop_common(&mut tx, &id, &mut BTreeMap::new()).await?;
 
-    Ok(WithRevision {
-        revision: tx.commit(None).await?,
-        response: (),
-    })
+    tx.commit().await?;
+
+    Ok(last_statuts)
+}
+
+#[command(display(display_none), metadata(sync_db = true))]
+pub async fn restart(#[context] ctx: RpcContext, #[arg] id: PackageId) -> Result<(), Error> {
+    let mut db = ctx.db.handle();
+    let mut tx = db.begin().await?;
+
+    let mut status = crate::db::DatabaseModel::new()
+        .package_data()
+        .idx_model(&id)
+        .and_then(|pde| pde.installed())
+        .map(|i| i.status().main())
+        .get_mut(&mut tx)
+        .await?;
+    if !matches!(&*status, Some(MainStatus::Running { .. })) {
+        return Err(Error::new(
+            eyre!("{} is not running", id),
+            crate::ErrorKind::InvalidRequest,
+        ));
+    }
+    *status = Some(MainStatus::Restarting);
+    status.save(&mut tx).await?;
+    tx.commit().await?;
+
+    Ok(())
 }

backend/src/core/rpc_continuations.rs

@@ -1,20 +1,27 @@
-use std::time::Instant;
+use std::sync::Arc;
+use std::time::Duration;
 
 use futures::future::BoxFuture;
-use http::{Request, Response};
-use hyper::Body;
+use futures::FutureExt;
+use helpers::TimedResource;
+use hyper::upgrade::Upgraded;
+use hyper::{Body, Error as HyperError, Request, Response};
 use rand::RngCore;
+use tokio::task::JoinError;
+use tokio_tungstenite::WebSocketStream;
+
+use crate::{Error, ResultExt};
 
 #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, serde::Serialize, serde::Deserialize)]
-pub struct RequestGuid<T: AsRef<str> = String>(T);
+pub struct RequestGuid<T: AsRef<str> = String>(Arc<T>);
 impl RequestGuid {
     pub fn new() -> Self {
         let mut buf = [0; 40];
         rand::thread_rng().fill_bytes(&mut buf);
-        RequestGuid(base32::encode(
+        RequestGuid(Arc::new(base32::encode(
             base32::Alphabet::RFC4648 { padding: false },
             &buf,
-        ))
+        )))
     }
 
     pub fn from(r: &str) -> Option<RequestGuid> {
@@ -26,7 +33,7 @@ impl RequestGuid {
                 return None;
             }
         }
-        Some(RequestGuid(r.to_owned()))
+        Some(RequestGuid(Arc::new(r.to_owned())))
     }
 }
 #[test]
@@ -39,15 +46,71 @@ fn parse_guid() {
 
 impl<T: AsRef<str>> std::fmt::Display for RequestGuid<T> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        self.0.as_ref().fmt(f)
+        (&*self.0).as_ref().fmt(f)
     }
 }
 
-pub struct RpcContinuation {
-    pub created_at: Instant,
-    pub handler: Box<
-        dyn FnOnce(Request<Body>) -> BoxFuture<'static, Result<Response<Body>, crate::Error>>
-            + Send
-            + Sync,
-    >,
+pub type RestHandler = Box<
+    dyn FnOnce(Request<Body>) -> BoxFuture<'static, Result<Response<Body>, crate::Error>> + Send,
+>;
+
+pub type WebSocketHandler = Box<
+    dyn FnOnce(
+            BoxFuture<'static, Result<Result<WebSocketStream<Upgraded>, HyperError>, JoinError>>,
+        ) -> BoxFuture<'static, Result<(), Error>>
+        + Send,
+>;
+
+pub enum RpcContinuation {
+    Rest(TimedResource<RestHandler>),
+    WebSocket(TimedResource<WebSocketHandler>),
+}
+impl RpcContinuation {
+    pub fn rest(handler: RestHandler, timeout: Duration) -> Self {
+        RpcContinuation::Rest(TimedResource::new(handler, timeout))
+    }
+    pub fn ws(handler: WebSocketHandler, timeout: Duration) -> Self {
+        RpcContinuation::WebSocket(TimedResource::new(handler, timeout))
+    }
+    pub fn is_timed_out(&self) -> bool {
+        match self {
+            RpcContinuation::Rest(a) => a.is_timed_out(),
+            RpcContinuation::WebSocket(a) => a.is_timed_out(),
+        }
+    }
+    pub async fn into_handler(self) -> Option<RestHandler> {
+        match self {
+            RpcContinuation::Rest(handler) => handler.get().await,
+            RpcContinuation::WebSocket(handler) => {
+                if let Some(handler) = handler.get().await {
+                    Some(Box::new(
+                        |req: Request<Body>| -> BoxFuture<'static, Result<Response<Body>, Error>> {
+                            async move {
+                                let (parts, body) = req.into_parts();
+                                let req = Request::from_parts(parts, body);
+                                let (res, ws_fut) = hyper_ws_listener::create_ws(req)
+                                    .with_kind(crate::ErrorKind::Network)?;
+                                if let Some(ws_fut) = ws_fut {
+                                    tokio::task::spawn(async move {
+                                        match handler(ws_fut.boxed()).await {
+                                            Ok(()) => (),
+                                            Err(e) => {
+                                                tracing::error!("WebSocket Closed: {}", e);
+                                                tracing::debug!("{:?}", e);
+                                            }
+                                        }
+                                    });
+                                }
+
+                                Ok(res)
+                            }
+                            .boxed()
+                        },
+                    ))
+                } else {
+                    None
+                }
+            }
+        }
+    }
 }

backend/src/db/mod.rs

@@ -1,92 +1,61 @@
 pub mod model;
 pub mod package;
-pub mod util;
 
-use std::borrow::Cow;
 use std::future::Future;
 use std::sync::Arc;
-use std::time::Duration;
 
-use color_eyre::eyre::eyre;
 use futures::{FutureExt, SinkExt, StreamExt};
 use patch_db::json_ptr::JsonPointer;
 use patch_db::{Dump, Revision};
 use rpc_toolkit::command;
 use rpc_toolkit::hyper::upgrade::Upgraded;
 use rpc_toolkit::hyper::{Body, Error as HyperError, Request, Response};
-use rpc_toolkit::yajrc::{GenericRpcMethod, RpcError, RpcResponse};
+use rpc_toolkit::yajrc::RpcError;
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
-use tokio::sync::{broadcast, oneshot};
+use tokio::sync::oneshot;
 use tokio::task::JoinError;
+use tokio_tungstenite::tungstenite::protocol::frame::coding::CloseCode;
+use tokio_tungstenite::tungstenite::protocol::CloseFrame;
 use tokio_tungstenite::tungstenite::Message;
 use tokio_tungstenite::WebSocketStream;
 use tracing::instrument;
 
 pub use self::model::DatabaseModel;
-use self::util::WithRevision;
 use crate::context::RpcContext;
 use crate::middleware::auth::{HasValidSession, HashSessionToken};
 use crate::util::serde::{display_serializable, IoFormat};
 use crate::{Error, ResultExt};
 
-#[instrument(skip(ctx, ws_fut))]
+#[instrument(skip(ctx, session, ws_fut))]
 async fn ws_handler<
     WSFut: Future<Output = Result<Result<WebSocketStream<Upgraded>, HyperError>, JoinError>>,
 >(
     ctx: RpcContext,
+    session: Option<(HasValidSession, HashSessionToken)>,
     ws_fut: WSFut,
 ) -> Result<(), Error> {
-    let (dump, sub) = ctx.db.dump_and_sub().await;
+    let (dump, sub) = ctx.db.dump_and_sub().await?;
     let mut stream = ws_fut
         .await
         .with_kind(crate::ErrorKind::Network)?
         .with_kind(crate::ErrorKind::Unknown)?;
 
-    let (has_valid_session, token) = loop {
-        if let Some(Message::Text(cookie)) = stream
-            .next()
+    if let Some((session, token)) = session {
+        let kill = subscribe_to_session_kill(&ctx, token).await;
+        send_dump(session, &mut stream, dump).await?;
+
+        deal_with_messages(session, kill, sub, stream).await?;
+    } else {
+        stream
+            .close(Some(CloseFrame {
+                code: CloseCode::Error,
+                reason: "UNAUTHORIZED".into(),
+            }))
             .await
-            .transpose()
-            .with_kind(crate::ErrorKind::Network)?
-        {
-            let cookie_str = serde_json::from_str::<Cow<str>>(&cookie)
-                .with_kind(crate::ErrorKind::Deserialization)?;
+            .with_kind(crate::ErrorKind::Network)?;
+    }
 
-            let id = basic_cookies::Cookie::parse(&cookie_str)
-                .with_kind(crate::ErrorKind::Authorization)?
-                .into_iter()
-                .find(|c| c.get_name() == "session")
-                .ok_or_else(|| {
-                    Error::new(eyre!("UNAUTHORIZED"), crate::ErrorKind::Authorization)
-                })?;
-            let authenticated_session = HashSessionToken::from_cookie(&id);
-            match HasValidSession::from_session(&authenticated_session, &ctx).await {
-                Err(e) => {
-                    stream
-                        .send(Message::Text(
-                            serde_json::to_string(
-                                &RpcResponse::<GenericRpcMethod<String>>::from_result(Err::<
-                                    _,
-                                    RpcError,
-                                >(
-                                    e.into()
-                                )),
-                            )
-                            .with_kind(crate::ErrorKind::Serialization)?,
-                        ))
-                        .await
-                        .with_kind(crate::ErrorKind::Network)?;
-                    return Ok(());
-                }
-                Ok(has_validation) => break (has_validation, authenticated_session),
-            }
-        }
-    };
-    let kill = subscribe_to_session_kill(&ctx, token).await;
-    send_dump(has_valid_session, &mut stream, dump).await?;
-
-    deal_with_messages(has_valid_session, kill, sub, stream).await?;
     Ok(())
 }
 
@@ -108,55 +77,39 @@ async fn subscribe_to_session_kill(
 async fn deal_with_messages(
     _has_valid_authentication: HasValidSession,
     mut kill: oneshot::Receiver<()>,
-    mut sub: broadcast::Receiver<Arc<Revision>>,
+    mut sub: patch_db::Subscriber,
     mut stream: WebSocketStream<Upgraded>,
 ) -> Result<(), Error> {
     loop {
         futures::select! {
             _ = (&mut kill).fuse() => {
                 tracing::info!("Closing WebSocket: Reason: Session Terminated");
+                stream
+                    .close(Some(CloseFrame {
+                        code: CloseCode::Error,
+                        reason: "UNAUTHORIZED".into(),
+                    }))
+                    .await
+                    .with_kind(crate::ErrorKind::Network)?;
                 return Ok(())
             }
             new_rev = sub.recv().fuse() => {
-                let rev = new_rev.with_kind(crate::ErrorKind::Database)?;
+                let rev = new_rev.expect("UNREACHABLE: patch-db is dropped");
                 stream
-                    .send(Message::Text(
-                        serde_json::to_string(
-                            &RpcResponse::<GenericRpcMethod<String>>::from_result(Ok::<_, RpcError>(
-                                serde_json::to_value(&rev).with_kind(crate::ErrorKind::Serialization)?,
-                            )),
-                        )
-                        .with_kind(crate::ErrorKind::Serialization)?,
-                    ))
+                    .send(Message::Text(serde_json::to_string(&rev).with_kind(crate::ErrorKind::Serialization)?))
                     .await
                     .with_kind(crate::ErrorKind::Network)?;
             }
             message = stream.next().fuse() => {
                 let message = message.transpose().with_kind(crate::ErrorKind::Network)?;
                 match message {
-                    Some(Message::Ping(a)) => {
-                        stream
-                            .send(Message::Pong(a))
-                            .await
-                            .with_kind(crate::ErrorKind::Network)?;
-                    }
-                    Some(Message::Close(frame)) => {
-                        if let Some(reason) = frame.as_ref() {
-                            tracing::info!("Closing WebSocket: Reason: {} {}", reason.code, reason.reason);
-                        } else {
-                            tracing::info!("Closing WebSocket: Reason: Unknown");
-                        }
+                    None => {
+                        tracing::info!("Closing WebSocket: Stream Finished");
                         return Ok(())
                     }
                     _ => (),
                 }
             }
-            _ = tokio::time::sleep(Duration::from_secs(10)).fuse() => {
-                stream
-                    .send(Message::Ping(Vec::new()))
-                    .await
-                    .with_kind(crate::ErrorKind::Network)?;
-            }
         }
     }
 }
@@ -168,13 +121,7 @@ async fn send_dump(
 ) -> Result<(), Error> {
     stream
         .send(Message::Text(
-            serde_json::to_string(&RpcResponse::<GenericRpcMethod<String>>::from_result(Ok::<
-                _,
-                RpcError,
-            >(
-                serde_json::to_value(&dump).with_kind(crate::ErrorKind::Serialization)?,
-            )))
-            .with_kind(crate::ErrorKind::Serialization)?,
+            serde_json::to_string(&dump).with_kind(crate::ErrorKind::Serialization)?,
         ))
         .await
         .with_kind(crate::ErrorKind::Network)?;
@@ -183,11 +130,27 @@ async fn send_dump(
 
 pub async fn subscribe(ctx: RpcContext, req: Request<Body>) -> Result<Response<Body>, Error> {
     let (parts, body) = req.into_parts();
+    let session = match async {
+        let token = HashSessionToken::from_request_parts(&parts)?;
+        let session = HasValidSession::from_session(&token, &ctx).await?;
+        Ok::<_, Error>((session, token))
+    }
+    .await
+    {
+        Ok(a) => Some(a),
+        Err(e) => {
+            if e.kind != crate::ErrorKind::Authorization {
+                tracing::error!("Error Authenticating Websocket: {}", e);
+                tracing::debug!("{:?}", e);
+            }
+            None
+        }
+    };
     let req = Request::from_parts(parts, body);
     let (res, ws_fut) = hyper_ws_listener::create_ws(req).with_kind(crate::ErrorKind::Network)?;
     if let Some(ws_fut) = ws_fut {
         tokio::task::spawn(async move {
-            match ws_handler(ctx, ws_fut).await {
+            match ws_handler(ctx, session, ws_fut).await {
                 Ok(()) => (),
                 Err(e) => {
                     tracing::error!("WebSocket Closed: {}", e);
@@ -219,24 +182,11 @@ pub async fn revisions(
     #[allow(unused_variables)]
     #[arg(long = "format")]
     format: Option<IoFormat>,
-) -> Result<RevisionsRes, RpcError> {
-    let cache = ctx.revision_cache.read().await;
-    if cache
-        .front()
-        .map(|rev| rev.id <= since + 1)
-        .unwrap_or(false)
-    {
-        Ok(RevisionsRes::Revisions(
-            cache
-                .iter()
-                .skip_while(|rev| rev.id < since + 1)
-                .cloned()
-                .collect(),
-        ))
-    } else {
-        drop(cache);
-        Ok(RevisionsRes::Dump(ctx.db.dump().await))
-    }
+) -> Result<RevisionsRes, Error> {
+    Ok(match ctx.db.sync(since).await? {
+        Ok(revs) => RevisionsRes::Revisions(revs),
+        Err(dump) => RevisionsRes::Dump(dump),
+    })
 }
 
 #[command(display(display_serializable))]
@@ -245,8 +195,8 @@ pub async fn dump(
     #[allow(unused_variables)]
     #[arg(long = "format")]
     format: Option<IoFormat>,
-) -> Result<Dump, RpcError> {
-    Ok(ctx.db.dump().await)
+) -> Result<Dump, Error> {
+    Ok(ctx.db.dump().await?)
 }
 
 #[command(subcommands(ui))]
@@ -263,13 +213,11 @@ pub async fn ui(
     #[allow(unused_variables)]
     #[arg(long = "format")]
     format: Option<IoFormat>,
-) -> Result<WithRevision<()>, Error> {
+) -> Result<(), Error> {
     let ptr = "/ui"
         .parse::<JsonPointer>()
         .with_kind(crate::ErrorKind::Database)?
         + &pointer;
-    Ok(WithRevision {
-        response: (),
-        revision: ctx.db.put(&ptr, &value, None).await?,
-    })
+    ctx.db.put(&ptr, &value).await?;
+    Ok(())
 }

backend/src/db/model.rs

@@ -12,6 +12,7 @@ use serde_json::Value;
 use torut::onion::TorSecretKeyV3;
 
 use crate::config::spec::{PackagePointerSpec, SystemPointerSpec};
+use crate::hostname::{generate_hostname, generate_id};
 use crate::install::progress::InstallProgress;
 use crate::net::interface::InterfaceId;
 use crate::s9pk::manifest::{Manifest, ManifestModel, PackageId};
@@ -27,31 +28,28 @@ pub struct Database {
     pub server_info: ServerInfo,
     #[model]
     pub package_data: AllPackageData,
-    #[model]
-    pub recovered_packages: BTreeMap<PackageId, RecoveredPackageInfo>,
     pub ui: Value,
 }
 impl Database {
-    pub fn init(
-        id: String,
-        hostname: &str,
-        tor_key: &TorSecretKeyV3,
-        password_hash: String,
-    ) -> Self {
+    pub fn init(tor_key: &TorSecretKeyV3, password_hash: String) -> Self {
+        let id = generate_id();
+        let my_hostname = generate_hostname();
+        let lan_address = my_hostname.lan_address().parse().unwrap();
         // TODO
         Database {
             server_info: ServerInfo {
                 id,
                 version: Current::new().semver().into(),
+                hostname: Some(my_hostname.0),
                 last_backup: None,
                 last_wifi_region: None,
                 eos_version_compat: Current::new().compat().clone(),
-                lan_address: format!("https://{}.local", hostname).parse().unwrap(),
+                lan_address,
                 tor_address: format!("http://{}", tor_key.public().get_onion_address())
                     .parse()
                     .unwrap(),
                 status_info: ServerStatus {
-                    backing_up: false,
+                    backup_progress: None,
                     updated: false,
                     update_progress: None,
                 },
@@ -68,8 +66,8 @@ impl Database {
                 password_hash,
             },
             package_data: AllPackageData::default(),
-            recovered_packages: BTreeMap::new(),
-            ui: Value::Object(Default::default()),
+            ui: serde_json::from_str(include_str!("../../../frontend/patchdb-ui-seed.json"))
+                .unwrap(),
         }
     }
 }
@@ -83,6 +81,7 @@ impl DatabaseModel {
 #[serde(rename_all = "kebab-case")]
 pub struct ServerInfo {
     pub id: String,
+    pub hostname: Option<String>,
     pub version: Version,
     pub last_backup: Option<DateTime<Utc>>,
     /// Used in the wifi to determine the region to set the system to
@@ -99,10 +98,16 @@ pub struct ServerInfo {
     pub password_hash: String,
 }
 
+#[derive(Debug, Default, Deserialize, Serialize, HasModel)]
+pub struct BackupProgress {
+    pub complete: bool,
+}
+
 #[derive(Debug, Default, Deserialize, Serialize, HasModel)]
 #[serde(rename_all = "kebab-case")]
 pub struct ServerStatus {
-    pub backing_up: bool,
+    #[model]
+    pub backup_progress: Option<BTreeMap<PackageId, BackupProgress>>,
     pub updated: bool,
     #[model]
     pub update_progress: Option<UpdateProgress>,
@@ -232,6 +237,15 @@ impl PackageDataEntry {
             PackageDataEntry::Installed { manifest, .. } => manifest,
         }
     }
+    pub fn manifest_borrow(&self) -> &Manifest {
+        match self {
+            PackageDataEntry::Installing { manifest, .. } => manifest,
+            PackageDataEntry::Updating { manifest, .. } => manifest,
+            PackageDataEntry::Restoring { manifest, .. } => manifest,
+            PackageDataEntry::Removing { manifest, .. } => manifest,
+            PackageDataEntry::Installed { manifest, .. } => manifest,
+        }
+    }
 }
 impl PackageDataEntryModel {
     pub fn installed(self) -> OptionModel<InstalledPackageDataEntry> {
@@ -260,17 +274,66 @@ pub struct InstalledPackageDataEntry {
     #[model]
     pub manifest: Manifest,
     pub last_backup: Option<DateTime<Utc>>,
+    #[model]
     pub system_pointers: Vec<SystemPointerSpec>,
     #[model]
     pub dependency_info: BTreeMap<PackageId, StaticDependencyInfo>,
     #[model]
-    pub current_dependents: BTreeMap<PackageId, CurrentDependencyInfo>,
+    pub current_dependents: CurrentDependents,
     #[model]
-    pub current_dependencies: BTreeMap<PackageId, CurrentDependencyInfo>,
+    pub current_dependencies: CurrentDependencies,
     #[model]
     pub interface_addresses: InterfaceAddressMap,
 }
 
+#[derive(Debug, Clone, Deserialize, Serialize)]
+pub struct CurrentDependents(pub BTreeMap<PackageId, CurrentDependencyInfo>);
+impl CurrentDependents {
+    pub fn map(
+        mut self,
+        transform: impl Fn(
+            BTreeMap<PackageId, CurrentDependencyInfo>,
+        ) -> BTreeMap<PackageId, CurrentDependencyInfo>,
+    ) -> Self {
+        self.0 = transform(self.0);
+        self
+    }
+}
+impl Map for CurrentDependents {
+    type Key = PackageId;
+    type Value = CurrentDependencyInfo;
+    fn get(&self, key: &Self::Key) -> Option<&Self::Value> {
+        self.0.get(key)
+    }
+}
+impl HasModel for CurrentDependents {
+    type Model = MapModel<Self>;
+}
+
+#[derive(Debug, Clone, Default, Deserialize, Serialize)]
+pub struct CurrentDependencies(pub BTreeMap<PackageId, CurrentDependencyInfo>);
+impl CurrentDependencies {
+    pub fn map(
+        mut self,
+        transform: impl Fn(
+            BTreeMap<PackageId, CurrentDependencyInfo>,
+        ) -> BTreeMap<PackageId, CurrentDependencyInfo>,
+    ) -> Self {
+        self.0 = transform(self.0);
+        self
+    }
+}
+impl Map for CurrentDependencies {
+    type Key = PackageId;
+    type Value = CurrentDependencyInfo;
+    fn get(&self, key: &Self::Key) -> Option<&Self::Value> {
+        self.0.get(key)
+    }
+}
+impl HasModel for CurrentDependencies {
+    type Model = MapModel<Self>;
+}
+
 #[derive(Clone, Debug, Default, Deserialize, Serialize, HasModel)]
 #[serde(rename_all = "kebab-case")]
 pub struct StaticDependencyInfo {

backend/src/db/package.rs

@@ -1,25 +1,75 @@
-use patch_db::DbHandle;
+use patch_db::{DbHandle, LockReceipt, LockTargetId, LockType, Verifier};
 
 use crate::s9pk::manifest::{Manifest, PackageId};
 use crate::Error;
 
-pub async fn get_packages<Db: DbHandle>(db: &mut Db) -> Result<Vec<PackageId>, Error> {
-    let packages = crate::db::DatabaseModel::new()
-        .package_data()
-        .get(db, false)
-        .await?;
+pub struct PackageReceipts {
+    package_data: LockReceipt<super::model::AllPackageData, ()>,
+}
+
+impl PackageReceipts {
+    pub async fn new<'a>(db: &'a mut impl DbHandle) -> Result<Self, Error> {
+        let mut locks = Vec::new();
+
+        let setup = Self::setup(&mut locks);
+        Ok(setup(&db.lock_all(locks).await?)?)
+    }
+
+    pub fn setup(locks: &mut Vec<LockTargetId>) -> impl FnOnce(&Verifier) -> Result<Self, Error> {
+        let package_data = crate::db::DatabaseModel::new()
+            .package_data()
+            .make_locker(LockType::Read)
+            .add_to_keys(locks);
+        move |skeleton_key| {
+            Ok(Self {
+                package_data: package_data.verify(&skeleton_key)?,
+            })
+        }
+    }
+}
+
+pub async fn get_packages<Db: DbHandle>(
+    db: &mut Db,
+    receipts: &PackageReceipts,
+) -> Result<Vec<PackageId>, Error> {
+    let packages = receipts.package_data.get(db).await?;
     Ok(packages.0.keys().cloned().collect())
 }
 
+pub struct ManifestReceipts {
+    manifest: LockReceipt<Manifest, String>,
+}
+
+impl ManifestReceipts {
+    pub async fn new<'a>(db: &'a mut impl DbHandle, id: &PackageId) -> Result<Self, Error> {
+        let mut locks = Vec::new();
+
+        let setup = Self::setup(&mut locks, id);
+        Ok(setup(&db.lock_all(locks).await?)?)
+    }
+
+    pub fn setup(
+        locks: &mut Vec<LockTargetId>,
+        _id: &PackageId,
+    ) -> impl FnOnce(&Verifier) -> Result<Self, Error> {
+        let manifest = crate::db::DatabaseModel::new()
+            .package_data()
+            .star()
+            .manifest()
+            .make_locker(LockType::Read)
+            .add_to_keys(locks);
+        move |skeleton_key| {
+            Ok(Self {
+                manifest: manifest.verify(&skeleton_key)?,
+            })
+        }
+    }
+}
+
 pub async fn get_manifest<Db: DbHandle>(
     db: &mut Db,
     pkg: &PackageId,
+    receipts: &ManifestReceipts,
 ) -> Result<Option<Manifest>, Error> {
-    let mpde = crate::db::DatabaseModel::new()
-        .package_data()
-        .idx_model(pkg)
-        .get(db, false)
-        .await?
-        .into_owned();
-    Ok(mpde.map(|pde| pde.manifest()))
+    Ok(receipts.manifest.get(db, pkg).await?)
 }

backend/src/db/util.rs

@@ -1,10 +0,0 @@
-use std::sync::Arc;
-
-use patch_db::Revision;
-use serde::{Deserialize, Serialize};
-
-#[derive(Clone, Debug, Deserialize, Serialize)]
-pub struct WithRevision<T> {
-    pub response: T,
-    pub revision: Option<Arc<Revision>>,
-}

backend/src/dependencies.rs

@@ -6,19 +6,21 @@ use color_eyre::eyre::eyre;
 use emver::VersionRange;
 use futures::future::BoxFuture;
 use futures::FutureExt;
-use patch_db::{DbHandle, HasModel, LockType, Map, MapModel, PatchDbHandle};
+use patch_db::{
+    DbHandle, HasModel, LockReceipt, LockTargetId, LockType, Map, MapModel, PatchDbHandle, Verifier,
+};
 use rand::SeedableRng;
 use rpc_toolkit::command;
 use serde::{Deserialize, Serialize};
 use tracing::instrument;
 
-use crate::action::{ActionImplementation, NoOutput};
-use crate::config::action::ConfigRes;
+use crate::config::action::{ConfigActions, ConfigRes};
 use crate::config::spec::PackagePointerSpec;
-use crate::config::{Config, ConfigSpec};
+use crate::config::{not_found, Config, ConfigReceipts, ConfigSpec};
 use crate::context::RpcContext;
-use crate::db::model::{CurrentDependencyInfo, InstalledPackageDataEntry};
-use crate::error::ResultExt;
+use crate::db::model::{CurrentDependencies, CurrentDependents, InstalledPackageDataEntry};
+use crate::procedure::docker::DockerContainers;
+use crate::procedure::{NoOutput, PackageProcedure, ProcedureName};
 use crate::s9pk::manifest::{Manifest, PackageId};
 use crate::status::health_check::{HealthCheckId, HealthCheckResult};
 use crate::status::{MainStatus, Status};
@@ -55,6 +57,81 @@ pub enum DependencyError {
     Transitive, // { "type": "transitive" }
 }
 
+#[derive(Clone)]
+pub struct TryHealReceipts {
+    status: LockReceipt<Status, String>,
+    manifest: LockReceipt<Manifest, String>,
+    manifest_version: LockReceipt<Version, String>,
+    current_dependencies: LockReceipt<CurrentDependencies, String>,
+    dependency_errors: LockReceipt<DependencyErrors, String>,
+    docker_containers: LockReceipt<DockerContainers, String>,
+}
+
+impl TryHealReceipts {
+    pub async fn new<'a>(db: &'a mut impl DbHandle) -> Result<Self, Error> {
+        let mut locks = Vec::new();
+
+        let setup = Self::setup(&mut locks);
+        Ok(setup(&db.lock_all(locks).await?)?)
+    }
+
+    pub fn setup(locks: &mut Vec<LockTargetId>) -> impl FnOnce(&Verifier) -> Result<Self, Error> {
+        let manifest_version = crate::db::DatabaseModel::new()
+            .package_data()
+            .star()
+            .installed()
+            .map(|x| x.manifest().version())
+            .make_locker(LockType::Write)
+            .add_to_keys(locks);
+        let status = crate::db::DatabaseModel::new()
+            .package_data()
+            .star()
+            .installed()
+            .map(|x| x.status())
+            .make_locker(LockType::Write)
+            .add_to_keys(locks);
+        let manifest = crate::db::DatabaseModel::new()
+            .package_data()
+            .star()
+            .installed()
+            .map(|x| x.manifest())
+            .make_locker(LockType::Write)
+            .add_to_keys(locks);
+
+        let current_dependencies = crate::db::DatabaseModel::new()
+            .package_data()
+            .star()
+            .installed()
+            .map(|x| x.current_dependencies())
+            .make_locker(LockType::Write)
+            .add_to_keys(locks);
+        let dependency_errors = crate::db::DatabaseModel::new()
+            .package_data()
+            .star()
+            .installed()
+            .map(|x| x.status().dependency_errors())
+            .make_locker(LockType::Write)
+            .add_to_keys(locks);
+        let docker_containers = crate::db::DatabaseModel::new()
+            .package_data()
+            .star()
+            .installed()
+            .and_then(|x| x.manifest().containers())
+            .make_locker(LockType::Write)
+            .add_to_keys(locks);
+        move |skeleton_key| {
+            Ok(Self {
+                status: status.verify(skeleton_key)?,
+                manifest_version: manifest_version.verify(skeleton_key)?,
+                current_dependencies: current_dependencies.verify(skeleton_key)?,
+                manifest: manifest.verify(skeleton_key)?,
+                dependency_errors: dependency_errors.verify(skeleton_key)?,
+                docker_containers: docker_containers.verify(skeleton_key)?,
+            })
+        }
+    }
+}
+
 impl DependencyError {
     pub fn cmp_priority(&self, other: &DependencyError) -> std::cmp::Ordering {
         use std::cmp::Ordering::*;
@@ -114,7 +191,7 @@ impl DependencyError {
             (DependencyError::Transitive, _) => DependencyError::Transitive,
         }
     }
-    #[instrument(skip(ctx, db))]
+    #[instrument(skip(ctx, db, receipts))]
     pub fn try_heal<'a, Db: DbHandle>(
         self,
         ctx: &'a RpcContext,
@@ -123,42 +200,34 @@ impl DependencyError {
         dependency: &'a PackageId,
         mut dependency_config: Option<Config>,
         info: &'a DepInfo,
+        receipts: &'a TryHealReceipts,
     ) -> BoxFuture<'a, Result<Option<Self>, Error>> {
         async move {
+            let container = receipts.docker_containers.get(db, id).await?;
             Ok(match self {
                 DependencyError::NotInstalled => {
-                    if crate::db::DatabaseModel::new()
-                        .package_data()
-                        .idx_model(dependency)
-                        .and_then(|m| m.installed())
-                        .exists(db, true)
-                        .await?
-                    {
+                    if receipts.status.get(db, dependency).await?.is_some() {
                         DependencyError::IncorrectVersion {
                             expected: info.version.clone(),
                             received: Default::default(),
                         }
-                        .try_heal(ctx, db, id, dependency, dependency_config, info)
+                        .try_heal(ctx, db, id, dependency, dependency_config, info, receipts)
                         .await?
                     } else {
                         Some(DependencyError::NotInstalled)
                     }
                 }
                 DependencyError::IncorrectVersion { expected, .. } => {
-                    let version: Version = crate::db::DatabaseModel::new()
-                        .package_data()
-                        .idx_model(dependency)
-                        .and_then(|m| m.installed())
-                        .map(|m| m.manifest().version())
-                        .get(db, true)
+                    let version: Version = receipts
+                        .manifest_version
+                        .get(db, dependency)
                         .await?
-                        .into_owned()
                         .unwrap_or_default();
                     if version.satisfies(&expected) {
                         DependencyError::ConfigUnsatisfied {
                             error: String::new(),
                         }
-                        .try_heal(ctx, db, id, dependency, dependency_config, info)
+                        .try_heal(ctx, db, id, dependency, dependency_config, info, receipts)
                         .await?
                     } else {
                         Some(DependencyError::IncorrectVersion {
@@ -168,24 +237,14 @@ impl DependencyError {
                     }
                 }
                 DependencyError::ConfigUnsatisfied { .. } => {
-                    let dependent_manifest = crate::db::DatabaseModel::new()
-                        .package_data()
-                        .idx_model(id)
-                        .and_then(|m| m.installed())
-                        .map::<_, Manifest>(|m| m.manifest())
-                        .expect(db)
+                    let dependent_manifest =
+                        receipts.manifest.get(db, id).await?.ok_or_else(not_found)?;
+                    let dependency_manifest = receipts
+                        .manifest
+                        .get(db, dependency)
                         .await?
-                        .get(db, true)
-                        .await?;
-                    let dependency_manifest = crate::db::DatabaseModel::new()
-                        .package_data()
-                        .idx_model(dependency)
-                        .and_then(|m| m.installed())
-                        .map::<_, Manifest>(|m| m.manifest())
-                        .expect(db)
-                        .await?
-                        .get(db, true)
-                        .await?;
+                        .ok_or_else(not_found)?;
+
                     let dependency_config = if let Some(cfg) = dependency_config.take() {
                         cfg
                     } else if let Some(cfg_info) = &dependency_manifest.config {
@@ -206,9 +265,11 @@ impl DependencyError {
                         if let Err(error) = cfg_req
                             .check(
                                 ctx,
+                                &container,
                                 id,
                                 &dependent_manifest.version,
                                 &dependent_manifest.volumes,
+                                dependency,
                                 &dependency_config,
                             )
                             .await?
@@ -217,40 +278,39 @@ impl DependencyError {
                         }
                     }
                     DependencyError::NotRunning
-                        .try_heal(ctx, db, id, dependency, Some(dependency_config), info)
+                        .try_heal(
+                            ctx,
+                            db,
+                            id,
+                            dependency,
+                            Some(dependency_config),
+                            info,
+                            receipts,
+                        )
                         .await?
                 }
                 DependencyError::NotRunning => {
-                    let status = crate::db::DatabaseModel::new()
-                        .package_data()
-                        .idx_model(dependency)
-                        .and_then(|m| m.installed())
-                        .map::<_, Status>(|m| m.status())
-                        .expect(db)
+                    let status = receipts
+                        .status
+                        .get(db, dependency)
                         .await?
-                        .get(db, true)
-                        .await?;
+                        .ok_or_else(not_found)?;
                     if status.main.running() {
                         DependencyError::HealthChecksFailed {
                             failures: BTreeMap::new(),
                         }
-                        .try_heal(ctx, db, id, dependency, dependency_config, info)
+                        .try_heal(ctx, db, id, dependency, dependency_config, info, receipts)
                         .await?
                     } else {
                         Some(DependencyError::NotRunning)
                     }
                 }
                 DependencyError::HealthChecksFailed { .. } => {
-                    let status = crate::db::DatabaseModel::new()
-                        .package_data()
-                        .idx_model(dependency)
-                        .and_then(|m| m.installed())
-                        .map::<_, Status>(|m| m.status())
-                        .expect(db)
+                    let status = receipts
+                        .status
+                        .get(db, dependency)
                         .await?
-                        .get(db, true)
-                        .await?
-                        .into_owned();
+                        .ok_or_else(not_found)?;
                     match status.main {
                         MainStatus::BackingUp {
                             started: Some(_),
@@ -260,19 +320,14 @@ impl DependencyError {
                             let mut failures = BTreeMap::new();
                             for (check, res) in health {
                                 if !matches!(res, HealthCheckResult::Success)
-                                    && crate::db::DatabaseModel::new()
-                                        .package_data()
-                                        .idx_model(id)
-                                        .and_then(|m| m.installed())
-                                        .and_then::<_, CurrentDependencyInfo>(|m| {
-                                            m.current_dependencies().idx_model(dependency)
-                                        })
-                                        .get(db, true)
+                                    && receipts
+                                        .current_dependencies
+                                        .get(db, id)
                                         .await?
-                                        .into_owned()
-                                        .map(|i| i.health_checks)
-                                        .unwrap_or_default()
-                                        .contains(&check)
+                                        .ok_or_else(not_found)?
+                                        .get(dependency)
+                                        .map(|x| x.health_checks.contains(&check))
+                                        .unwrap_or(false)
                                 {
                                     failures.insert(check.clone(), res.clone());
                                 }
@@ -281,27 +336,39 @@ impl DependencyError {
                                 Some(DependencyError::HealthChecksFailed { failures })
                             } else {
                                 DependencyError::Transitive
-                                    .try_heal(ctx, db, id, dependency, dependency_config, info)
+                                    .try_heal(
+                                        ctx,
+                                        db,
+                                        id,
+                                        dependency,
+                                        dependency_config,
+                                        info,
+                                        receipts,
+                                    )
                                     .await?
                             }
                         }
-                        MainStatus::Starting => {
+                        MainStatus::Starting { .. } | MainStatus::Restarting => {
                             DependencyError::Transitive
-                                .try_heal(ctx, db, id, dependency, dependency_config, info)
+                                .try_heal(
+                                    ctx,
+                                    db,
+                                    id,
+                                    dependency,
+                                    dependency_config,
+                                    info,
+                                    receipts,
+                                )
                                 .await?
                         }
                         _ => return Ok(Some(DependencyError::NotRunning)),
                     }
                 }
                 DependencyError::Transitive => {
-                    if crate::db::DatabaseModel::new()
-                        .package_data()
-                        .idx_model(dependency)
-                        .and_then(|m| m.installed())
-                        .map::<_, DependencyErrors>(|m| m.status().dependency_errors())
-                        .get(db, true)
+                    if receipts
+                        .dependency_errors
+                        .get(db, dependency)
                         .await?
-                        .into_owned()
                         .unwrap_or_default()
                         .0
                         .is_empty()
@@ -406,6 +473,7 @@ impl DepInfo {
         dependency_id: &PackageId,
         dependency_config: Option<Config>, // fetch if none
         dependent_id: &PackageId,
+        receipts: &TryHealReceipts,
     ) -> Result<Result<(), DependencyError>, Error> {
         Ok(
             if let Some(err) = DependencyError::NotInstalled
@@ -416,6 +484,7 @@ impl DepInfo {
                     dependency_id,
                     dependency_config,
                     self,
+                    receipts,
                 )
                 .await?
             {
@@ -430,27 +499,31 @@ impl DepInfo {
 #[derive(Clone, Debug, Deserialize, Serialize, HasModel)]
 #[serde(rename_all = "kebab-case")]
 pub struct DependencyConfig {
-    check: ActionImplementation,
-    auto_configure: ActionImplementation,
+    check: PackageProcedure,
+    auto_configure: PackageProcedure,
 }
 impl DependencyConfig {
     pub async fn check(
         &self,
         ctx: &RpcContext,
+        container: &Option<DockerContainers>,
         dependent_id: &PackageId,
         dependent_version: &Version,
         dependent_volumes: &Volumes,
+        dependency_id: &PackageId,
         dependency_config: &Config,
     ) -> Result<Result<NoOutput, String>, Error> {
         Ok(self
             .check
             .sandboxed(
+                container,
                 ctx,
                 dependent_id,
                 dependent_version,
                 dependent_volumes,
                 Some(dependency_config),
                 None,
+                ProcedureName::Check(dependency_id.clone()),
             )
             .await?
             .map_err(|(_, e)| e))
@@ -458,6 +531,7 @@ impl DependencyConfig {
     pub async fn auto_configure(
         &self,
         ctx: &RpcContext,
+        container: &Option<DockerContainers>,
         dependent_id: &PackageId,
         dependent_version: &Version,
         dependent_volumes: &Volumes,
@@ -465,18 +539,113 @@ impl DependencyConfig {
     ) -> Result<Config, Error> {
         self.auto_configure
             .sandboxed(
+                container,
                 ctx,
                 dependent_id,
                 dependent_version,
                 dependent_volumes,
                 Some(old),
                 None,
+                ProcedureName::AutoConfig(dependent_id.clone()),
             )
             .await?
             .map_err(|e| Error::new(eyre!("{}", e.1), crate::ErrorKind::AutoConfigure))
     }
 }
 
+pub struct DependencyConfigReceipts {
+    config: ConfigReceipts,
+    dependencies: LockReceipt<Dependencies, ()>,
+    dependency_volumes: LockReceipt<Volumes, ()>,
+    dependency_version: LockReceipt<Version, ()>,
+    dependency_config_action: LockReceipt<ConfigActions, ()>,
+    package_volumes: LockReceipt<Volumes, ()>,
+    package_version: LockReceipt<Version, ()>,
+    docker_containers: LockReceipt<DockerContainers, String>,
+}
+
+impl DependencyConfigReceipts {
+    pub async fn new<'a>(
+        db: &'a mut impl DbHandle,
+        package_id: &PackageId,
+        dependency_id: &PackageId,
+    ) -> Result<Self, Error> {
+        let mut locks = Vec::new();
+
+        let setup = Self::setup(&mut locks, package_id, dependency_id);
+        Ok(setup(&db.lock_all(locks).await?)?)
+    }
+
+    pub fn setup(
+        locks: &mut Vec<LockTargetId>,
+        package_id: &PackageId,
+        dependency_id: &PackageId,
+    ) -> impl FnOnce(&Verifier) -> Result<Self, Error> {
+        let config = ConfigReceipts::setup(locks);
+        let dependencies = crate::db::DatabaseModel::new()
+            .package_data()
+            .idx_model(package_id)
+            .and_then(|x| x.installed())
+            .map(|x| x.manifest().dependencies())
+            .make_locker(LockType::Write)
+            .add_to_keys(locks);
+        let dependency_volumes = crate::db::DatabaseModel::new()
+            .package_data()
+            .idx_model(dependency_id)
+            .and_then(|x| x.installed())
+            .map(|x| x.manifest().volumes())
+            .make_locker(LockType::Write)
+            .add_to_keys(locks);
+        let dependency_version = crate::db::DatabaseModel::new()
+            .package_data()
+            .idx_model(dependency_id)
+            .and_then(|x| x.installed())
+            .map(|x| x.manifest().version())
+            .make_locker(LockType::Write)
+            .add_to_keys(locks);
+        let dependency_config_action = crate::db::DatabaseModel::new()
+            .package_data()
+            .idx_model(dependency_id)
+            .and_then(|x| x.installed())
+            .and_then(|x| x.manifest().config())
+            .make_locker(LockType::Write)
+            .add_to_keys(locks);
+        let package_volumes = crate::db::DatabaseModel::new()
+            .package_data()
+            .idx_model(package_id)
+            .and_then(|x| x.installed())
+            .map(|x| x.manifest().volumes())
+            .make_locker(LockType::Write)
+            .add_to_keys(locks);
+        let package_version = crate::db::DatabaseModel::new()
+            .package_data()
+            .idx_model(package_id)
+            .and_then(|x| x.installed())
+            .map(|x| x.manifest().version())
+            .make_locker(LockType::Write)
+            .add_to_keys(locks);
+        let docker_containers = crate::db::DatabaseModel::new()
+            .package_data()
+            .star()
+            .installed()
+            .and_then(|x| x.manifest().containers())
+            .make_locker(LockType::Write)
+            .add_to_keys(locks);
+        move |skeleton_key| {
+            Ok(Self {
+                config: config(skeleton_key)?,
+                dependencies: dependencies.verify(&skeleton_key)?,
+                dependency_volumes: dependency_volumes.verify(&skeleton_key)?,
+                dependency_version: dependency_version.verify(&skeleton_key)?,
+                dependency_config_action: dependency_config_action.verify(&skeleton_key)?,
+                package_volumes: package_volumes.verify(&skeleton_key)?,
+                package_version: package_version.verify(&skeleton_key)?,
+                docker_containers: docker_containers.verify(&skeleton_key)?,
+            })
+        }
+    }
+}
+
 #[command(
     subcommands(self(configure_impl(async)), configure_dry),
     display(display_none)
@@ -493,11 +662,14 @@ pub async fn configure_impl(
     (pkg_id, dep_id): (PackageId, PackageId),
 ) -> Result<(), Error> {
     let mut db = ctx.db.handle();
+    let receipts = DependencyConfigReceipts::new(&mut db, &pkg_id, &dep_id).await?;
     let ConfigDryRes {
         old_config: _,
         new_config,
         spec: _,
-    } = configure_logic(ctx.clone(), &mut db, (pkg_id, dep_id.clone())).await?;
+    } = configure_logic(ctx.clone(), &mut db, (pkg_id, dep_id.clone()), &receipts).await?;
+
+    let locks = &receipts.config;
     Ok(crate::config::configure(
         &ctx,
         &mut db,
@@ -507,6 +679,7 @@ pub async fn configure_impl(
         false,
         &mut BTreeMap::new(),
         &mut BTreeMap::new(),
+        locks,
     )
     .await?)
 }
@@ -526,67 +699,26 @@ pub async fn configure_dry(
     #[parent_data] (pkg_id, dependency_id): (PackageId, PackageId),
 ) -> Result<ConfigDryRes, Error> {
     let mut db = ctx.db.handle();
-    configure_logic(ctx, &mut db, (pkg_id, dependency_id)).await
+    let receipts = DependencyConfigReceipts::new(&mut db, &pkg_id, &dependency_id).await?;
+    configure_logic(ctx, &mut db, (pkg_id, dependency_id), &receipts).await
 }
 
 pub async fn configure_logic(
     ctx: RpcContext,
     db: &mut PatchDbHandle,
     (pkg_id, dependency_id): (PackageId, PackageId),
+    receipts: &DependencyConfigReceipts,
 ) -> Result<ConfigDryRes, Error> {
-    crate::db::DatabaseModel::new()
-        .package_data()
-        .lock(db, LockType::Read)
-        .await?;
-    let pkg_model = crate::db::DatabaseModel::new()
-        .package_data()
-        .idx_model(&pkg_id)
-        .and_then(|m| m.installed())
-        .expect(db)
-        .await
-        .with_kind(crate::ErrorKind::NotFound)?;
-    let pkg_version = pkg_model.clone().manifest().version().get(db, true).await?;
-    let pkg_volumes = pkg_model.clone().manifest().volumes().get(db, true).await?;
-    let dependency_model = crate::db::DatabaseModel::new()
-        .package_data()
-        .idx_model(&dependency_id)
-        .and_then(|m| m.installed())
-        .expect(db)
-        .await
-        .with_kind(crate::ErrorKind::NotFound)?;
-    let dependency_config_action = dependency_model
-        .clone()
-        .manifest()
-        .config()
-        .get(db, true)
-        .await?
-        .to_owned()
-        .ok_or_else(|| {
-            Error::new(
-                eyre!("{} has no config", dependency_id),
-                crate::ErrorKind::NotFound,
-            )
-        })?;
-    let dependency_version = dependency_model
-        .clone()
-        .manifest()
-        .version()
-        .get(db, true)
-        .await?;
-    let dependency_volumes = dependency_model
-        .clone()
-        .manifest()
-        .volumes()
-        .get(db, true)
-        .await?;
-    let dependencies = pkg_model
-        .clone()
-        .manifest()
-        .dependencies()
-        .get(db, true)
-        .await?;
+    let pkg_version = receipts.package_version.get(db).await?;
+    let pkg_volumes = receipts.package_volumes.get(db).await?;
+    let dependency_config_action = receipts.dependency_config_action.get(db).await?;
+    let dependency_version = receipts.dependency_version.get(db).await?;
+    let dependency_volumes = receipts.dependency_volumes.get(db).await?;
+    let dependencies = receipts.dependencies.get(db).await?;
+    let pkg_docker_container = receipts.docker_containers.get(db, &*pkg_id).await?;
 
     let dependency = dependencies
+        .0
         .get(&dependency_id)
         .ok_or_else(|| {
             Error::new(
@@ -617,8 +749,8 @@ pub async fn configure_logic(
         .get(
             &ctx,
             &dependency_id,
-            &*dependency_version,
-            &*dependency_volumes,
+            &dependency_version,
+            &dependency_volumes,
         )
         .await?;
 
@@ -634,12 +766,14 @@ pub async fn configure_logic(
     let new_config = dependency
         .auto_configure
         .sandboxed(
+            &pkg_docker_container,
             &ctx,
             &pkg_id,
             &pkg_version,
             &pkg_volumes,
             Some(&old_config),
             None,
+            ProcedureName::AutoConfig(dependency_id.clone()),
         )
         .await?
         .map_err(|e| Error::new(eyre!("{}", e.1), crate::ErrorKind::AutoConfigure))?;
@@ -650,29 +784,22 @@ pub async fn configure_logic(
         spec,
     })
 }
-
-#[instrument(skip(db, current_dependencies))]
-pub async fn add_dependent_to_current_dependents_lists<
-    'a,
-    Db: DbHandle,
-    I: IntoIterator<Item = (&'a PackageId, &'a CurrentDependencyInfo)>,
->(
+#[instrument(skip(db, current_dependencies, current_dependent_receipt))]
+pub async fn add_dependent_to_current_dependents_lists<'a, Db: DbHandle>(
     db: &mut Db,
     dependent_id: &PackageId,
-    current_dependencies: I,
+    current_dependencies: &CurrentDependencies,
+    current_dependent_receipt: &LockReceipt<CurrentDependents, String>,
 ) -> Result<(), Error> {
-    for (dependency, dep_info) in current_dependencies {
-        if let Some(dependency_model) = crate::db::DatabaseModel::new()
-            .package_data()
-            .idx_model(&dependency)
-            .and_then(|pkg| pkg.installed())
-            .check(db)
-            .await?
+    for (dependency, dep_info) in &current_dependencies.0 {
+        if let Some(mut dependency_dependents) =
+            current_dependent_receipt.get(db, dependency).await?
         {
-            dependency_model
-                .current_dependents()
-                .idx_model(dependent_id)
-                .put(db, &dep_info)
+            dependency_dependents
+                .0
+                .insert(dependent_id.clone(), dep_info.clone());
+            current_dependent_receipt
+                .set(db, dependency_dependents, dependency)
                 .await?;
         }
     }
@@ -696,10 +823,11 @@ impl DependencyErrors {
         ctx: &RpcContext,
         db: &mut Db,
         manifest: &Manifest,
-        current_dependencies: &BTreeMap<PackageId, CurrentDependencyInfo>,
+        current_dependencies: &CurrentDependencies,
+        receipts: &TryHealReceipts,
     ) -> Result<DependencyErrors, Error> {
         let mut res = BTreeMap::new();
-        for (dependency_id, info) in current_dependencies.keys().filter_map(|dependency_id| {
+        for (dependency_id, info) in current_dependencies.0.keys().filter_map(|dependency_id| {
             manifest
                 .dependencies
                 .0
@@ -707,7 +835,7 @@ impl DependencyErrors {
                 .map(|info| (dependency_id, info))
         }) {
             if let Err(e) = info
-                .satisfied(ctx, db, dependency_id, None, &manifest.id)
+                .satisfied(ctx, db, dependency_id, None, &manifest.id, receipts)
                 .await?
             {
                 res.insert(dependency_id.clone(), e);
@@ -735,49 +863,86 @@ pub async fn break_all_dependents_transitive<'a, Db: DbHandle>(
     id: &'a PackageId,
     error: DependencyError,
     breakages: &'a mut BTreeMap<PackageId, TaggedDependencyError>,
+    receipts: &'a BreakTransitiveReceipts,
 ) -> Result<(), Error> {
-    for dependent in crate::db::DatabaseModel::new()
-        .package_data()
-        .idx_model(id)
-        .and_then(|m| m.installed())
-        .expect(db)
+    for dependent in receipts
+        .current_dependents
+        .get(db, id)
         .await?
-        .current_dependents()
-        .keys(db, true)
-        .await?
-        .into_iter()
-        .filter(|dependent| id != dependent)
+        .iter()
+        .flat_map(|x| x.0.keys())
+        .filter(|dependent| id != *dependent)
     {
-        break_transitive(db, &dependent, id, error.clone(), breakages).await?;
+        break_transitive(db, dependent, id, error.clone(), breakages, receipts).await?;
     }
     Ok(())
 }
 
-#[instrument(skip(db))]
+#[derive(Clone)]
+pub struct BreakTransitiveReceipts {
+    pub dependency_receipt: DependencyReceipt,
+    dependency_errors: LockReceipt<DependencyErrors, String>,
+    current_dependents: LockReceipt<CurrentDependents, String>,
+}
+
+impl BreakTransitiveReceipts {
+    pub async fn new(db: &'_ mut impl DbHandle) -> Result<Self, Error> {
+        let mut locks = Vec::new();
+
+        let setup = Self::setup(&mut locks);
+        Ok(setup(&db.lock_all(locks).await?)?)
+    }
+
+    pub fn setup(locks: &mut Vec<LockTargetId>) -> impl FnOnce(&Verifier) -> Result<Self, Error> {
+        let dependency_receipt = DependencyReceipt::setup(locks);
+        let dependency_errors = crate::db::DatabaseModel::new()
+            .package_data()
+            .star()
+            .installed()
+            .map(|x| x.status().dependency_errors())
+            .make_locker(LockType::Write)
+            .add_to_keys(locks);
+        let current_dependents = crate::db::DatabaseModel::new()
+            .package_data()
+            .star()
+            .installed()
+            .map(|x| x.current_dependents())
+            .make_locker(LockType::Exist)
+            .add_to_keys(locks);
+        move |skeleton_key| {
+            Ok(Self {
+                dependency_receipt: dependency_receipt(skeleton_key)?,
+                dependency_errors: dependency_errors.verify(skeleton_key)?,
+                current_dependents: current_dependents.verify(skeleton_key)?,
+            })
+        }
+    }
+}
+
+#[instrument(skip(db, receipts))]
 pub fn break_transitive<'a, Db: DbHandle>(
     db: &'a mut Db,
     id: &'a PackageId,
     dependency: &'a PackageId,
     error: DependencyError,
     breakages: &'a mut BTreeMap<PackageId, TaggedDependencyError>,
+    receipts: &'a BreakTransitiveReceipts,
 ) -> BoxFuture<'a, Result<(), Error>> {
     async move {
         let mut tx = db.begin().await?;
-        let model = crate::db::DatabaseModel::new()
-            .package_data()
-            .idx_model(id)
-            .and_then(|m| m.installed())
-            .expect(&mut tx)
-            .await?;
-        let mut status = model.clone().status().get_mut(&mut tx).await?;
+        let mut dependency_errors = receipts
+            .dependency_errors
+            .get(&mut tx, id)
+            .await?
+            .ok_or_else(not_found)?;
 
-        let old = status.dependency_errors.0.remove(dependency);
+        let old = dependency_errors.0.remove(dependency);
         let newly_broken = if let Some(e) = &old {
             error.cmp_priority(&e) == Ordering::Greater
         } else {
             true
         };
-        status.dependency_errors.0.insert(
+        dependency_errors.0.insert(
             dependency.clone(),
             if let Some(old) = old {
                 old.merge_with(error.clone())
@@ -793,12 +958,25 @@ pub fn break_transitive<'a, Db: DbHandle>(
                     error: error.clone(),
                 },
             );
-            status.save(&mut tx).await?;
+            receipts
+                .dependency_errors
+                .set(&mut tx, dependency_errors, id)
+                .await?;
 
             tx.save().await?;
-            break_all_dependents_transitive(db, id, DependencyError::Transitive, breakages).await?;
+            break_all_dependents_transitive(
+                db,
+                id,
+                DependencyError::Transitive,
+                breakages,
+                receipts,
+            )
+            .await?;
         } else {
-            status.save(&mut tx).await?;
+            receipts
+                .dependency_errors
+                .set(&mut tx, dependency_errors, id)
+                .await?;
 
             tx.save().await?;
         }
@@ -808,68 +986,52 @@ pub fn break_transitive<'a, Db: DbHandle>(
     .boxed()
 }
 
-#[instrument(skip(ctx, db))]
+#[instrument(skip(ctx, db, locks))]
 pub async fn heal_all_dependents_transitive<'a, Db: DbHandle>(
     ctx: &'a RpcContext,
     db: &'a mut Db,
     id: &'a PackageId,
+    locks: &'a DependencyReceipt,
 ) -> Result<(), Error> {
-    for dependent in crate::db::DatabaseModel::new()
-        .package_data()
-        .idx_model(id)
-        .and_then(|m| m.installed())
-        .expect(db)
+    let dependents = locks
+        .current_dependents
+        .get(db, id)
         .await?
-        .current_dependents()
-        .keys(db, true)
-        .await?
-        .into_iter()
-        .filter(|dependent| id != dependent)
-    {
-        heal_transitive(ctx, db, &dependent, id).await?;
+        .ok_or_else(not_found)?;
+    for dependent in dependents.0.keys().filter(|dependent| id != *dependent) {
+        heal_transitive(ctx, db, dependent, id, locks).await?;
     }
     Ok(())
 }
 
-#[instrument(skip(ctx, db))]
+#[instrument(skip(ctx, db, receipts))]
 pub fn heal_transitive<'a, Db: DbHandle>(
     ctx: &'a RpcContext,
     db: &'a mut Db,
     id: &'a PackageId,
     dependency: &'a PackageId,
+    receipts: &'a DependencyReceipt,
 ) -> BoxFuture<'a, Result<(), Error>> {
     async move {
-        let mut tx = db.begin().await?;
-        let model = crate::db::DatabaseModel::new()
-            .package_data()
-            .idx_model(id)
-            .and_then(|m| m.installed())
-            .expect(&mut tx)
-            .await?;
-        let mut status = model.clone().status().get_mut(&mut tx).await?;
+        let mut status = receipts.status.get(db, id).await?.ok_or_else(not_found)?;
 
         let old = status.dependency_errors.0.remove(dependency);
 
         if let Some(old) = old {
-            let info = model
-                .manifest()
-                .dependencies()
-                .idx_model(dependency)
-                .expect(&mut tx)
+            let info = receipts
+                .dependency
+                .get(db, (id, dependency))
                 .await?
-                .get(&mut tx, true)
-                .await?;
+                .ok_or_else(not_found)?;
             if let Some(new) = old
-                .try_heal(ctx, &mut tx, id, dependency, None, &*info)
+                .try_heal(ctx, db, id, dependency, None, &info, &receipts.try_heal)
                 .await?
             {
                 status.dependency_errors.0.insert(dependency.clone(), new);
-                status.save(&mut tx).await?;
-                tx.save().await?;
+                receipts.status.set(db, status, id).await?;
             } else {
-                status.save(&mut tx).await?;
-                tx.save().await?;
-                heal_all_dependents_transitive(ctx, db, id).await?;
+                receipts.status.set(db, status, id).await?;
+                heal_all_dependents_transitive(ctx, db, id, receipts).await?;
             }
         }
 
@@ -881,11 +1043,12 @@ pub fn heal_transitive<'a, Db: DbHandle>(
 pub async fn reconfigure_dependents_with_live_pointers(
     ctx: &RpcContext,
     mut tx: impl DbHandle,
+    receipts: &ConfigReceipts,
     pde: &InstalledPackageDataEntry,
 ) -> Result<(), Error> {
     let dependents = &pde.current_dependents;
     let me = &pde.manifest.id;
-    for (dependent_id, dependency_info) in dependents {
+    for (dependent_id, dependency_info) in &dependents.0 {
         if dependency_info.pointers.iter().any(|ptr| match ptr {
             // dependency id matches the package being uninstalled
             PackagePointerSpec::TorAddress(ptr) => &ptr.package_id == me && dependent_id != me,
@@ -903,9 +1066,60 @@ pub async fn reconfigure_dependents_with_live_pointers(
                 false,
                 &mut BTreeMap::new(),
                 &mut BTreeMap::new(),
+                receipts,
             )
             .await?;
         }
     }
     Ok(())
 }
+
+#[derive(Clone)]
+pub struct DependencyReceipt {
+    pub try_heal: TryHealReceipts,
+    current_dependents: LockReceipt<CurrentDependents, String>,
+    status: LockReceipt<Status, String>,
+    dependency: LockReceipt<DepInfo, (String, String)>,
+}
+
+impl DependencyReceipt {
+    pub async fn new<'a>(db: &'a mut impl DbHandle) -> Result<Self, Error> {
+        let mut locks = Vec::new();
+
+        let setup = Self::setup(&mut locks);
+        Ok(setup(&db.lock_all(locks).await?)?)
+    }
+
+    pub fn setup(locks: &mut Vec<LockTargetId>) -> impl FnOnce(&Verifier) -> Result<Self, Error> {
+        let try_heal = TryHealReceipts::setup(locks);
+        let dependency = crate::db::DatabaseModel::new()
+            .package_data()
+            .star()
+            .installed()
+            .map(|x| x.manifest().dependencies().star())
+            .make_locker(LockType::Read)
+            .add_to_keys(locks);
+        let current_dependents = crate::db::DatabaseModel::new()
+            .package_data()
+            .star()
+            .installed()
+            .map(|x| x.current_dependents())
+            .make_locker(LockType::Write)
+            .add_to_keys(locks);
+        let status = crate::db::DatabaseModel::new()
+            .package_data()
+            .star()
+            .installed()
+            .map(|x| x.status())
+            .make_locker(LockType::Write)
+            .add_to_keys(locks);
+        move |skeleton_key| {
+            Ok(Self {
+                try_heal: try_heal(skeleton_key)?,
+                current_dependents: current_dependents.verify(skeleton_key)?,
+                status: status.verify(skeleton_key)?,
+                dependency: dependency.verify(skeleton_key)?,
+            })
+        }
+    }
+}

backend/src/developer/mod.rs

@@ -2,6 +2,7 @@ use std::fs::File;
 use std::io::Write;
 use std::path::Path;
 
+use ed25519::pkcs8::EncodePrivateKey;
 use ed25519_dalek::Keypair;
 use rpc_toolkit::command;
 use tracing::instrument;
@@ -20,10 +21,19 @@ pub fn init(#[context] ctx: SdkContext) -> Result<(), Error> {
                 .with_ctx(|_| (crate::ErrorKind::Filesystem, parent.display().to_string()))?;
         }
         tracing::info!("Generating new developer key...");
-        let keypair = Keypair::generate(&mut rand::thread_rng());
+        let keypair = Keypair::generate(&mut rand_old::thread_rng());
         tracing::info!("Writing key to {}", ctx.developer_key_path.display());
+        let keypair_bytes = ed25519::KeypairBytes {
+            secret_key: keypair.secret.to_bytes(),
+            public_key: Some(keypair.public.to_bytes()),
+        };
         let mut dev_key_file = File::create(&ctx.developer_key_path)?;
-        dev_key_file.write_all(&keypair.to_bytes())?;
+        dev_key_file.write_all(
+            keypair_bytes
+                .to_pkcs8_pem(base64ct::LineEnding::default())
+                .with_kind(crate::ErrorKind::Pem)?
+                .as_bytes(),
+        )?;
         dev_key_file.sync_all()?;
     }
     Ok(())

backend/src/diagnostic.rs

@@ -5,14 +5,15 @@ use rpc_toolkit::command;
 use rpc_toolkit::yajrc::RpcError;
 
 use crate::context::DiagnosticContext;
-use crate::logs::{display_logs, fetch_logs, LogResponse, LogSource};
+use crate::disk::repair;
+use crate::logs::{fetch_logs, LogResponse, LogSource};
 use crate::shutdown::Shutdown;
 use crate::util::display_none;
 use crate::Error;
 
 pub const SYSTEMD_UNIT: &'static str = "embassy-init";
 
-#[command(subcommands(error, logs, exit, restart, forget_disk))]
+#[command(subcommands(error, logs, exit, restart, forget_disk, disk))]
 pub fn diagnostic() -> Result<(), Error> {
     Ok(())
 }
@@ -22,19 +23,13 @@ pub fn error(#[context] ctx: DiagnosticContext) -> Result<Arc<RpcError>, Error>
     Ok(ctx.error.clone())
 }
 
-#[command(display(display_logs))]
+#[command(rpc_only)]
 pub async fn logs(
     #[arg] limit: Option<usize>,
     #[arg] cursor: Option<String>,
-    #[arg] before_flag: Option<bool>,
+    #[arg] before: bool,
 ) -> Result<LogResponse, Error> {
-    Ok(fetch_logs(
-        LogSource::Service(SYSTEMD_UNIT),
-        limit,
-        cursor,
-        before_flag.unwrap_or(false),
-    )
-    .await?)
+    Ok(fetch_logs(LogSource::Service(SYSTEMD_UNIT), limit, cursor, before).await?)
 }
 
 #[command(display(display_none))]
@@ -56,9 +51,14 @@ pub fn restart(#[context] ctx: DiagnosticContext) -> Result<(), Error> {
     Ok(())
 }
 
-#[command(rename = "forget-disk", display(display_none))]
+#[command(subcommands(forget_disk, repair))]
+pub fn disk() -> Result<(), Error> {
+    Ok(())
+}
+
+#[command(rename = "forget", display(display_none))]
 pub async fn forget_disk() -> Result<(), Error> {
-    let disk_guid = Path::new("/embassy-os/disk.guid");
+    let disk_guid = Path::new("/media/embassy/config/disk.guid");
     if tokio::fs::metadata(disk_guid).await.is_ok() {
         tokio::fs::remove_file(disk_guid).await?;
     }

backend/src/disk/fsck.rs

@@ -0,0 +1,120 @@
+use std::ffi::OsStr;
+use std::path::Path;
+
+use color_eyre::eyre::eyre;
+use futures::future::BoxFuture;
+use futures::FutureExt;
+use tokio::process::Command;
+use tracing::instrument;
+
+use crate::Error;
+
+#[derive(Debug, Clone, Copy)]
+#[must_use]
+pub struct RequiresReboot(pub bool);
+impl std::ops::BitOrAssign for RequiresReboot {
+    fn bitor_assign(&mut self, rhs: Self) {
+        self.0 |= rhs.0
+    }
+}
+
+#[derive(Debug, Clone, Copy)]
+pub enum RepairStrategy {
+    Preen,
+    Aggressive,
+}
+impl RepairStrategy {
+    pub async fn e2fsck(
+        &self,
+        logicalname: impl AsRef<Path> + std::fmt::Debug,
+    ) -> Result<RequiresReboot, Error> {
+        match self {
+            RepairStrategy::Preen => e2fsck_preen(logicalname).await,
+            RepairStrategy::Aggressive => e2fsck_aggressive(logicalname).await,
+        }
+    }
+}
+
+#[instrument]
+pub async fn e2fsck_preen(
+    logicalname: impl AsRef<Path> + std::fmt::Debug,
+) -> Result<RequiresReboot, Error> {
+    e2fsck_runner(Command::new("e2fsck").arg("-p"), logicalname).await
+}
+
+fn backup_existing_undo_file<'a>(path: &'a Path) -> BoxFuture<'a, Result<(), Error>> {
+    async move {
+        if tokio::fs::metadata(path).await.is_ok() {
+            let bak = path.with_extension(format!(
+                "{}.bak",
+                path.extension()
+                    .and_then(|s| s.to_str())
+                    .unwrap_or_default()
+            ));
+            backup_existing_undo_file(&bak).await?;
+            tokio::fs::rename(path, &bak).await?;
+        }
+        Ok(())
+    }
+    .boxed()
+}
+
+#[instrument]
+pub async fn e2fsck_aggressive(
+    logicalname: impl AsRef<Path> + std::fmt::Debug,
+) -> Result<RequiresReboot, Error> {
+    let undo_path = Path::new("/media/embassy/config")
+        .join(
+            logicalname
+                .as_ref()
+                .file_name()
+                .unwrap_or(OsStr::new("unknown")),
+        )
+        .with_extension("e2undo");
+    backup_existing_undo_file(&undo_path).await?;
+    e2fsck_runner(
+        Command::new("e2fsck").arg("-y").arg("-z").arg(undo_path),
+        logicalname,
+    )
+    .await
+}
+
+async fn e2fsck_runner(
+    e2fsck_cmd: &mut Command,
+    logicalname: impl AsRef<Path> + std::fmt::Debug,
+) -> Result<RequiresReboot, Error> {
+    let e2fsck_out = e2fsck_cmd.arg(logicalname.as_ref()).output().await?;
+    let e2fsck_stderr = String::from_utf8(e2fsck_out.stderr)?;
+    let code = e2fsck_out.status.code().ok_or_else(|| {
+        Error::new(
+            eyre!("e2fsck: process terminated by signal"),
+            crate::ErrorKind::DiskManagement,
+        )
+    })?;
+    if code & 4 != 0 {
+        tracing::error!(
+            "some filesystem errors NOT corrected on {}:\n{}",
+            logicalname.as_ref().display(),
+            e2fsck_stderr,
+        );
+    } else if code & 1 != 0 {
+        tracing::warn!(
+            "filesystem errors corrected on {}:\n{}",
+            logicalname.as_ref().display(),
+            e2fsck_stderr,
+        );
+    }
+    if code < 8 {
+        if code & 2 != 0 {
+            tracing::warn!("reboot required");
+            Ok(RequiresReboot(true))
+        } else {
+            Ok(RequiresReboot(false))
+        }
+    } else {
+        Err(Error::new(
+            eyre!("e2fsck: {}", e2fsck_stderr),
+            crate::ErrorKind::DiskManagement,
+        ))
+    }
+}

backend/src/disk/main.rs

@@ -5,12 +5,13 @@ use color_eyre::eyre::eyre;
 use tokio::process::Command;
 use tracing::instrument;
 
+use super::fsck::{RepairStrategy, RequiresReboot};
 use super::util::pvscan;
 use crate::disk::mount::filesystem::block_dev::mount;
 use crate::disk::mount::filesystem::ReadWrite;
 use crate::disk::mount::util::unmount;
 use crate::util::Invoke;
-use crate::{Error, ResultExt};
+use crate::{Error, ErrorKind, ResultExt};
 
 pub const PASSWORD_PATH: &'static str = "/etc/embassy/password";
 pub const DEFAULT_PASSWORD: &'static str = "password";
@@ -182,6 +183,7 @@ pub async fn unmount_all_fs<P: AsRef<Path>>(guid: &str, datadir: P) -> Result<()
 
 #[instrument(skip(datadir))]
 pub async fn export<P: AsRef<Path>>(guid: &str, datadir: P) -> Result<(), Error> {
+    Command::new("sync").invoke(ErrorKind::Filesystem).await?;
     unmount_all_fs(guid, datadir).await?;
     Command::new("vgchange")
         .arg("-an")
@@ -196,7 +198,12 @@ pub async fn export<P: AsRef<Path>>(guid: &str, datadir: P) -> Result<(), Error>
 }
 
 #[instrument(skip(datadir, password))]
-pub async fn import<P: AsRef<Path>>(guid: &str, datadir: P, password: &str) -> Result<(), Error> {
+pub async fn import<P: AsRef<Path>>(
+    guid: &str,
+    datadir: P,
+    repair: RepairStrategy,
+    password: &str,
+) -> Result<RequiresReboot, Error> {
     let scan = pvscan().await?;
     if scan
         .values()
@@ -244,8 +251,7 @@ pub async fn import<P: AsRef<Path>>(guid: &str, datadir: P, password: &str) -> R
         .arg(guid)
         .invoke(crate::ErrorKind::DiskManagement)
         .await?;
-    mount_all_fs(guid, datadir, password).await?;
-    Ok(())
+    mount_all_fs(guid, datadir, repair, password).await
 }
 
 #[instrument(skip(datadir, password))]
@@ -253,8 +259,9 @@ pub async fn mount_fs<P: AsRef<Path>>(
     guid: &str,
     datadir: P,
     name: &str,
+    repair: RepairStrategy,
     password: &str,
-) -> Result<(), Error> {
+) -> Result<RequiresReboot, Error> {
     tokio::fs::write(PASSWORD_PATH, password)
         .await
         .with_ctx(|_| (crate::ErrorKind::Filesystem, PASSWORD_PATH))?;
@@ -267,27 +274,26 @@ pub async fn mount_fs<P: AsRef<Path>>(
         .arg(format!("{}_{}", guid, name))
         .invoke(crate::ErrorKind::DiskManagement)
         .await?;
-    mount(
-        Path::new("/dev/mapper").join(format!("{}_{}", guid, name)),
-        datadir.as_ref().join(name),
-        ReadWrite,
-    )
-    .await?;
+    let mapper_path = Path::new("/dev/mapper").join(format!("{}_{}", guid, name));
+    let reboot = repair.e2fsck(&mapper_path).await?;
+    mount(&mapper_path, datadir.as_ref().join(name), ReadWrite).await?;
 
     tokio::fs::remove_file(PASSWORD_PATH)
         .await
         .with_ctx(|_| (crate::ErrorKind::Filesystem, PASSWORD_PATH))?;
 
-    Ok(())
+    Ok(reboot)
 }
 
 #[instrument(skip(datadir, password))]
 pub async fn mount_all_fs<P: AsRef<Path>>(
     guid: &str,
     datadir: P,
+    repair: RepairStrategy,
     password: &str,
-) -> Result<(), Error> {
-    mount_fs(guid, &datadir, "main", password).await?;
-    mount_fs(guid, &datadir, "package-data", password).await?;
-    Ok(())
+) -> Result<RequiresReboot, Error> {
+    let mut reboot = RequiresReboot(false);
+    reboot |= mount_fs(guid, &datadir, "main", repair, password).await?;
+    reboot |= mount_fs(guid, &datadir, "package-data", repair, password).await?;
+    Ok(reboot)
 }

backend/src/disk/mod.rs

@@ -1,23 +1,41 @@
+use std::path::{Path, PathBuf};
+
 use clap::ArgMatches;
 use rpc_toolkit::command;
+use serde::{Deserialize, Serialize};
 
-use self::util::DiskListResponse;
+use crate::context::RpcContext;
+use crate::disk::util::DiskInfo;
+use crate::util::display_none;
 use crate::util::serde::{display_serializable, IoFormat};
 use crate::Error;
 
+pub mod fsck;
 pub mod main;
 pub mod mount;
-pub mod quirks;
 pub mod util;
 
-pub const BOOT_RW_PATH: &'static str = "/media/boot-rw";
+pub const BOOT_RW_PATH: &str = "/media/boot-rw";
+pub const REPAIR_DISK_PATH: &str = "/media/embassy/config/repair-disk";
 
-#[command(subcommands(list))]
+#[derive(Debug, Default, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct OsPartitionInfo {
+    pub boot: PathBuf,
+    pub root: PathBuf,
+}
+impl OsPartitionInfo {
+    pub fn contains(&self, logicalname: impl AsRef<Path>) -> bool {
+        &*self.boot == logicalname.as_ref() || &*self.root == logicalname.as_ref()
+    }
+}
+
+#[command(subcommands(list, repair))]
 pub fn disk() -> Result<(), Error> {
     Ok(())
 }
 
-fn display_disk_info(info: DiskListResponse, matches: &ArgMatches<'_>) {
+fn display_disk_info(info: Vec<DiskInfo>, matches: &ArgMatches) {
     use prettytable::*;
 
     if matches.is_present("format") {
@@ -32,7 +50,7 @@ fn display_disk_info(info: DiskListResponse, matches: &ArgMatches<'_>) {
         "USED",
         "EMBASSY OS VERSION"
     ]);
-    for disk in info.disks {
+    for disk in info {
         let row = row![
             disk.logicalname.display(),
             "N/A",
@@ -68,14 +86,21 @@ fn display_disk_info(info: DiskListResponse, matches: &ArgMatches<'_>) {
             table.add_row(row);
         }
     }
-    table.print_tty(false);
+    table.print_tty(false).unwrap();
 }
 
 #[command(display(display_disk_info))]
 pub async fn list(
+    #[context] ctx: RpcContext,
     #[allow(unused_variables)]
     #[arg]
     format: Option<IoFormat>,
-) -> Result<DiskListResponse, Error> {
-    crate::disk::util::list().await
+) -> Result<Vec<DiskInfo>, Error> {
+    crate::disk::util::list(&ctx.os_partitions).await
+}
+
+#[command(display(display_none))]
+pub async fn repair() -> Result<(), Error> {
+    tokio::fs::write(REPAIR_DISK_PATH, b"").await?;
+    Ok(())
 }

backend/src/disk/mount/backup.rs

@@ -1,6 +1,7 @@
 use std::path::{Path, PathBuf};
 
 use color_eyre::eyre::eyre;
+use helpers::AtomicFile;
 use tokio::io::AsyncWriteExt;
 use tracing::instrument;
 
@@ -14,9 +15,9 @@ use crate::disk::util::EmbassyOsRecoveryInfo;
 use crate::middleware::encrypt::{decrypt_slice, encrypt_slice};
 use crate::s9pk::manifest::PackageId;
 use crate::util::serde::IoFormat;
-use crate::util::{AtomicFile, FileLock};
+use crate::util::FileLock;
 use crate::volume::BACKUP_DIR;
-use crate::{Error, ResultExt};
+use crate::{Error, ErrorKind, ResultExt};
 
 pub struct BackupMountGuard<G: GenericMountGuard> {
     backup_disk_mount_guard: Option<G>,
@@ -162,16 +163,20 @@ impl<G: GenericMountGuard> BackupMountGuard<G> {
     pub async fn save(&self) -> Result<(), Error> {
         let metadata_path = self.as_ref().join("metadata.cbor");
         let backup_disk_path = self.backup_disk_path();
-        let mut file = AtomicFile::new(&metadata_path).await?;
+        let mut file = AtomicFile::new(&metadata_path, None::<PathBuf>)
+            .await
+            .with_kind(ErrorKind::Filesystem)?;
         file.write_all(&IoFormat::Cbor.to_vec(&self.metadata)?)
             .await?;
-        file.save().await?;
+        file.save().await.with_kind(ErrorKind::Filesystem)?;
         let unencrypted_metadata_path =
             backup_disk_path.join("EmbassyBackups/unencrypted-metadata.cbor");
-        let mut file = AtomicFile::new(&unencrypted_metadata_path).await?;
+        let mut file = AtomicFile::new(&unencrypted_metadata_path, None::<PathBuf>)
+            .await
+            .with_kind(ErrorKind::Filesystem)?;
         file.write_all(&IoFormat::Cbor.to_vec(&self.unencrypted_metadata)?)
             .await?;
-        file.save().await?;
+        file.save().await.with_kind(ErrorKind::Filesystem)?;
         Ok(())
     }
 

backend/src/disk/mount/filesystem/bind.rs

@@ -0,0 +1,54 @@
+use std::os::unix::ffi::OsStrExt;
+use std::path::Path;
+
+use async_trait::async_trait;
+use digest::generic_array::GenericArray;
+use digest::{Digest, OutputSizeUser};
+use sha2::Sha256;
+
+use super::{FileSystem, MountType, ReadOnly};
+use crate::disk::mount::util::bind;
+use crate::{Error, ResultExt};
+
+pub struct Bind<SrcDir: AsRef<Path>> {
+    src_dir: SrcDir,
+}
+impl<SrcDir: AsRef<Path>> Bind<SrcDir> {
+    pub fn new(src_dir: SrcDir) -> Self {
+        Self { src_dir }
+    }
+}
+#[async_trait]
+impl<SrcDir: AsRef<Path> + Send + Sync> FileSystem for Bind<SrcDir> {
+    async fn mount<P: AsRef<Path> + Send + Sync>(
+        &self,
+        mountpoint: P,
+        mount_type: MountType,
+    ) -> Result<(), Error> {
+        bind(
+            self.src_dir.as_ref(),
+            mountpoint,
+            matches!(mount_type, ReadOnly),
+        )
+        .await
+    }
+    async fn source_hash(
+        &self,
+    ) -> Result<GenericArray<u8, <Sha256 as OutputSizeUser>::OutputSize>, Error> {
+        let mut sha = Sha256::new();
+        sha.update("Bind");
+        sha.update(
+            tokio::fs::canonicalize(self.src_dir.as_ref())
+                .await
+                .with_ctx(|_| {
+                    (
+                        crate::ErrorKind::Filesystem,
+                        self.src_dir.as_ref().display().to_string(),
+                    )
+                })?
+                .as_os_str()
+                .as_bytes(),
+        );
+        Ok(sha.finalize())
+    }
+}

backend/src/disk/mount/filesystem/block_dev.rs

@@ -3,7 +3,7 @@ use std::path::Path;
 
 use async_trait::async_trait;
 use digest::generic_array::GenericArray;
-use digest::Digest;
+use digest::{Digest, OutputSizeUser};
 use serde::{Deserialize, Serialize};
 use sha2::Sha256;
 
@@ -45,7 +45,9 @@ impl<LogicalName: AsRef<Path> + Send + Sync> FileSystem for BlockDev<LogicalName
     ) -> Result<(), Error> {
         mount(self.logicalname.as_ref(), mountpoint, mount_type).await
     }
-    async fn source_hash(&self) -> Result<GenericArray<u8, <Sha256 as Digest>::OutputSize>, Error> {
+    async fn source_hash(
+        &self,
+    ) -> Result<GenericArray<u8, <Sha256 as OutputSizeUser>::OutputSize>, Error> {
         let mut sha = Sha256::new();
         sha.update("BlockDev");
         sha.update(

backend/src/disk/mount/filesystem/cifs.rs

@@ -4,7 +4,7 @@ use std::path::{Path, PathBuf};
 
 use async_trait::async_trait;
 use digest::generic_array::GenericArray;
-use digest::Digest;
+use digest::{Digest, OutputSizeUser};
 use serde::{Deserialize, Serialize};
 use sha2::Sha256;
 use tokio::process::Command;
@@ -18,7 +18,7 @@ use crate::Error;
 async fn resolve_hostname(hostname: &str) -> Result<IpAddr, Error> {
     #[cfg(feature = "avahi")]
     if hostname.ends_with(".local") {
-        return Ok(crate::net::mdns::resolve_mdns(hostname).await?);
+        return Ok(IpAddr::V4(crate::net::mdns::resolve_mdns(hostname).await?));
     }
     Ok(String::from_utf8(
         Command::new("nmblookup")
@@ -93,7 +93,9 @@ impl FileSystem for Cifs {
         )
         .await
     }
-    async fn source_hash(&self) -> Result<GenericArray<u8, <Sha256 as Digest>::OutputSize>, Error> {
+    async fn source_hash(
+        &self,
+    ) -> Result<GenericArray<u8, <Sha256 as OutputSizeUser>::OutputSize>, Error> {
         let mut sha = Sha256::new();
         sha.update("Cifs");
         sha.update(self.hostname.as_bytes());

backend/src/disk/mount/filesystem/ecryptfs.rs

@@ -4,7 +4,7 @@ use std::path::Path;
 use async_trait::async_trait;
 use color_eyre::eyre::eyre;
 use digest::generic_array::GenericArray;
-use digest::Digest;
+use digest::{Digest, OutputSizeUser};
 use sha2::Sha256;
 use tokio::io::{AsyncReadExt, AsyncWriteExt};
 
@@ -63,7 +63,9 @@ impl<EncryptedDir: AsRef<Path> + Send + Sync, Key: AsRef<str> + Send + Sync> Fil
     ) -> Result<(), Error> {
         mount_ecryptfs(self.encrypted_dir.as_ref(), mountpoint, self.key.as_ref()).await
     }
-    async fn source_hash(&self) -> Result<GenericArray<u8, <Sha256 as Digest>::OutputSize>, Error> {
+    async fn source_hash(
+        &self,
+    ) -> Result<GenericArray<u8, <Sha256 as OutputSizeUser>::OutputSize>, Error> {
         let mut sha = Sha256::new();
         sha.update("EcryptFS");
         sha.update(

backend/src/disk/mount/filesystem/httpdirfs.rs

@@ -0,0 +1,52 @@
+use std::path::Path;
+
+use async_trait::async_trait;
+use digest::generic_array::GenericArray;
+use digest::{Digest, OutputSizeUser};
+use reqwest::Url;
+use serde::{Deserialize, Serialize};
+use sha2::Sha256;
+
+use super::{FileSystem, MountType};
+use crate::util::Invoke;
+use crate::Error;
+
+pub async fn mount_httpdirfs(url: &Url, mountpoint: impl AsRef<Path>) -> Result<(), Error> {
+    tokio::fs::create_dir_all(mountpoint.as_ref()).await?;
+    let mut cmd = tokio::process::Command::new("httpdirfs");
+    cmd.arg("--cache")
+        .arg("--single-file-mode")
+        .arg(url.as_str())
+        .arg(mountpoint.as_ref());
+    cmd.invoke(crate::ErrorKind::Filesystem).await?;
+    Ok(())
+}
+
+#[derive(Debug, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct HttpDirFS {
+    url: Url,
+}
+impl HttpDirFS {
+    pub fn new(url: Url) -> Self {
+        HttpDirFS { url }
+    }
+}
+#[async_trait]
+impl FileSystem for HttpDirFS {
+    async fn mount<P: AsRef<Path> + Send + Sync>(
+        &self,
+        mountpoint: P,
+        _mount_type: MountType,
+    ) -> Result<(), Error> {
+        mount_httpdirfs(&self.url, mountpoint).await
+    }
+    async fn source_hash(
+        &self,
+    ) -> Result<GenericArray<u8, <Sha256 as OutputSizeUser>::OutputSize>, Error> {
+        let mut sha = Sha256::new();
+        sha.update("HttpDirFS");
+        sha.update(self.url.as_str());
+        Ok(sha.finalize())
+    }
+}

backend/src/disk/mount/filesystem/label.rs

@@ -2,7 +2,7 @@ use std::path::Path;
 
 use async_trait::async_trait;
 use digest::generic_array::GenericArray;
-use digest::Digest;
+use digest::{Digest, OutputSizeUser};
 use sha2::Sha256;
 
 use super::{FileSystem, MountType, ReadOnly};
@@ -41,7 +41,9 @@ impl<S: AsRef<str> + Send + Sync> FileSystem for Label<S> {
     ) -> Result<(), Error> {
         mount_label(self.label.as_ref(), mountpoint, mount_type).await
     }
-    async fn source_hash(&self) -> Result<GenericArray<u8, <Sha256 as Digest>::OutputSize>, Error> {
+    async fn source_hash(
+        &self,
+    ) -> Result<GenericArray<u8, <Sha256 as OutputSizeUser>::OutputSize>, Error> {
         let mut sha = Sha256::new();
         sha.update("Label");
         sha.update(self.label.as_ref().as_bytes());

backend/src/disk/mount/filesystem/mod.rs

@@ -2,14 +2,16 @@ use std::path::Path;
 
 use async_trait::async_trait;
 use digest::generic_array::GenericArray;
-use digest::Digest;
+use digest::OutputSizeUser;
 use sha2::Sha256;
 
 use crate::Error;
 
+pub mod bind;
 pub mod block_dev;
 pub mod cifs;
 pub mod ecryptfs;
+pub mod httpdirfs;
 pub mod label;
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
@@ -27,5 +29,7 @@ pub trait FileSystem {
         mountpoint: P,
         mount_type: MountType,
     ) -> Result<(), Error>;
-    async fn source_hash(&self) -> Result<GenericArray<u8, <Sha256 as Digest>::OutputSize>, Error>;
+    async fn source_hash(
+        &self,
+    ) -> Result<GenericArray<u8, <Sha256 as OutputSizeUser>::OutputSize>, Error>;
 }

backend/src/disk/mount/guard.rs

@@ -11,7 +11,7 @@ use super::util::unmount;
 use crate::util::Invoke;
 use crate::Error;
 
-pub const TMP_MOUNTPOINT: &'static str = "/media/embassy-os/tmp";
+pub const TMP_MOUNTPOINT: &'static str = "/media/embassy/tmp";
 
 #[async_trait::async_trait]
 pub trait GenericMountGuard: AsRef<Path> + std::fmt::Debug + Send + Sync + 'static {

backend/src/disk/mount/util.rs

@@ -48,13 +48,15 @@ pub async fn unmount<P: AsRef<Path>>(mountpoint: P) -> Result<(), Error> {
         .arg(mountpoint.as_ref())
         .invoke(crate::ErrorKind::Filesystem)
         .await?;
-    tokio::fs::remove_dir_all(mountpoint.as_ref())
-        .await
-        .with_ctx(|_| {
-            (
-                crate::ErrorKind::Filesystem,
-                format!("rm {}", mountpoint.as_ref().display()),
-            )
-        })?;
+    match tokio::fs::remove_dir(mountpoint.as_ref()).await {
+        Err(e) if e.raw_os_error() == Some(39) => Ok(()), // directory not empty
+        a => a,
+    }
+    .with_ctx(|_| {
+        (
+            crate::ErrorKind::Filesystem,
+            format!("rm {}", mountpoint.as_ref().display()),
+        )
+    })?;
     Ok(())
 }

backend/src/disk/quirks.rs

@@ -1,161 +0,0 @@
-use std::num::ParseIntError;
-use std::path::Path;
-
-use color_eyre::eyre::eyre;
-use tokio::io::AsyncWriteExt;
-use tracing::instrument;
-
-use super::BOOT_RW_PATH;
-use crate::util::AtomicFile;
-use crate::Error;
-
-pub const QUIRK_PATH: &'static str = "/sys/module/usb_storage/parameters/quirks";
-
-pub const WHITELIST: [(VendorId, ProductId); 4] = [
-    (VendorId(0x1d6b), ProductId(0x0002)), // root hub usb2
-    (VendorId(0x1d6b), ProductId(0x0003)), // root hub usb3
-    (VendorId(0x2109), ProductId(0x3431)),
-    (VendorId(0x1058), ProductId(0x262f)), // western digital black HDD
-];
-
-#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
-pub struct VendorId(u16);
-impl std::str::FromStr for VendorId {
-    type Err = ParseIntError;
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        u16::from_str_radix(s.trim(), 16).map(VendorId)
-    }
-}
-impl std::fmt::Display for VendorId {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{:04x}", self.0)
-    }
-}
-
-#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
-pub struct ProductId(u16);
-impl std::str::FromStr for ProductId {
-    type Err = ParseIntError;
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        u16::from_str_radix(s.trim(), 16).map(ProductId)
-    }
-}
-impl std::fmt::Display for ProductId {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{:04x}", self.0)
-    }
-}
-
-#[derive(Clone, Debug)]
-pub struct Quirks(Vec<(VendorId, ProductId)>);
-impl Quirks {
-    pub fn add(&mut self, vendor: VendorId, product: ProductId) {
-        self.0.push((vendor, product));
-    }
-    pub fn contains(&self, vendor: VendorId, product: ProductId) -> bool {
-        self.0.contains(&(vendor, product))
-    }
-}
-impl std::fmt::Display for Quirks {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        let mut comma = false;
-        for (vendor, product) in &self.0 {
-            if comma {
-                write!(f, ",")?;
-            } else {
-                comma = true;
-            }
-            write!(f, "{}:{}:u", vendor, product)?;
-        }
-        Ok(())
-    }
-}
-impl std::str::FromStr for Quirks {
-    type Err = Error;
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        let s = s.trim();
-        let mut quirks = Vec::new();
-        for item in s.split(",") {
-            if let [vendor, product, "u"] = item.splitn(3, ":").collect::<Vec<_>>().as_slice() {
-                quirks.push((vendor.parse()?, product.parse()?));
-            } else {
-                return Err(Error::new(
-                    eyre!("Invalid quirk: `{}`", item),
-                    crate::ErrorKind::DiskManagement,
-                ));
-            }
-        }
-        Ok(Quirks(quirks))
-    }
-}
-
-#[instrument]
-pub async fn update_quirks(quirks: &mut Quirks) -> Result<Vec<String>, Error> {
-    let mut usb_devices = tokio::fs::read_dir("/sys/bus/usb/devices/").await?;
-    let mut to_reconnect = Vec::new();
-    while let Some(usb_device) = usb_devices.next_entry().await? {
-        if tokio::fs::metadata(usb_device.path().join("idVendor"))
-            .await
-            .is_err()
-        {
-            continue;
-        }
-        let vendor = tokio::fs::read_to_string(usb_device.path().join("idVendor"))
-            .await?
-            .parse()?;
-        let product = tokio::fs::read_to_string(usb_device.path().join("idProduct"))
-            .await?
-            .parse()?;
-        if WHITELIST.contains(&(vendor, product)) || quirks.contains(vendor, product) {
-            continue;
-        }
-        quirks.add(vendor, product);
-        {
-            // write quirks to sysfs
-            let mut quirk_file = tokio::fs::File::create(QUIRK_PATH).await?;
-            quirk_file.write_all(quirks.to_string().as_bytes()).await?;
-            quirk_file.sync_all().await?;
-            drop(quirk_file);
-        }
-
-        disconnect_usb(usb_device.path()).await?;
-        let (vendor_name, product_name) = tokio::try_join!(
-            tokio::fs::read_to_string(usb_device.path().join("manufacturer")),
-            tokio::fs::read_to_string(usb_device.path().join("product")),
-        )?;
-        to_reconnect.push(format!("{} {}", vendor_name, product_name));
-    }
-    Ok(to_reconnect)
-}
-
-#[instrument(skip(usb_device_path))]
-pub async fn disconnect_usb(usb_device_path: impl AsRef<Path>) -> Result<(), Error> {
-    let authorized_path = usb_device_path.as_ref().join("bConfigurationValue");
-    let mut authorized_file = tokio::fs::File::create(&authorized_path).await?;
-    authorized_file.write_all(b"0").await?;
-    authorized_file.sync_all().await?;
-    drop(authorized_file);
-    Ok(())
-}
-
-#[instrument]
-pub async fn fetch_quirks() -> Result<Quirks, Error> {
-    Ok(tokio::fs::read_to_string(QUIRK_PATH).await?.parse()?)
-}
-
-#[instrument]
-pub async fn save_quirks(quirks: &Quirks) -> Result<(), Error> {
-    let orig_path = Path::new(BOOT_RW_PATH).join("cmdline.txt.orig");
-    let target_path = Path::new(BOOT_RW_PATH).join("cmdline.txt");
-    if tokio::fs::metadata(&orig_path).await.is_err() {
-        tokio::fs::copy(&target_path, &orig_path).await?;
-    }
-    let cmdline = tokio::fs::read_to_string(&orig_path).await?;
-    let mut target = AtomicFile::new(&target_path).await?;
-    target
-        .write_all(format!("usb-storage.quirks={} {}", quirks, cmdline).as_bytes())
-        .await?;
-    target.save().await?;
-
-    Ok(())
-}

backend/src/disk/util.rs

@@ -12,26 +12,17 @@ use nom::sequence::{pair, preceded, terminated};
 use nom::IResult;
 use regex::Regex;
 use serde::{Deserialize, Serialize};
-use tokio::fs::File;
 use tokio::process::Command;
 use tracing::instrument;
 
 use super::mount::filesystem::block_dev::BlockDev;
 use super::mount::filesystem::ReadOnly;
 use super::mount::guard::TmpMountGuard;
-use super::quirks::{fetch_quirks, save_quirks, update_quirks};
-use crate::util::io::from_yaml_async_reader;
+use crate::disk::OsPartitionInfo;
 use crate::util::serde::IoFormat;
 use crate::util::{Invoke, Version};
 use crate::{Error, ResultExt as _};
 
-#[derive(Clone, Debug, Deserialize, Serialize)]
-#[serde(rename_all = "kebab-case")]
-pub struct DiskListResponse {
-    pub disks: Vec<DiskInfo>,
-    pub reconnect: Vec<String>,
-}
-
 #[derive(Clone, Debug, Deserialize, Serialize)]
 #[serde(rename_all = "kebab-case")]
 pub struct DiskInfo {
@@ -51,6 +42,7 @@ pub struct PartitionInfo {
     pub capacity: u64,
     pub used: Option<u64>,
     pub embassy_os: Option<EmbassyOsRecoveryInfo>,
+    pub guid: Option<String>,
 }
 
 #[derive(Clone, Debug, Default, Deserialize, Serialize)]
@@ -226,24 +218,16 @@ pub async fn recovery_info(
             )?,
         ));
     }
-    let version_path = mountpoint.as_ref().join("root/appmgr/version");
-    if tokio::fs::metadata(&version_path).await.is_ok() {
-        return Ok(Some(EmbassyOsRecoveryInfo {
-            version: from_yaml_async_reader(File::open(&version_path).await?).await?,
-            full: true,
-            password_hash: None,
-            wrapped_key: None,
-        }));
-    }
 
     Ok(None)
 }
 
 #[instrument]
-pub async fn list() -> Result<DiskListResponse, Error> {
-    let mut quirks = fetch_quirks().await?;
-    let reconnect = update_quirks(&mut quirks).await?;
-    save_quirks(&mut quirks).await?;
+pub async fn list(os: &OsPartitionInfo) -> Result<Vec<DiskInfo>, Error> {
+    struct DiskIndex {
+        parts: IndexSet<PathBuf>,
+        internal: bool,
+    }
     let disk_guids = pvscan().await?;
     let disks = tokio_stream::wrappers::ReadDirStream::new(
         tokio::fs::read_dir(DISK_PATH)
@@ -256,128 +240,160 @@ pub async fn list() -> Result<DiskListResponse, Error> {
             crate::ErrorKind::Filesystem,
         )
     })
-    .try_fold(BTreeMap::new(), |mut disks, dir_entry| async move {
-        if let Some(disk_path) = dir_entry.path().file_name().and_then(|s| s.to_str()) {
-            let (disk_path, part_path) = if let Some(end) = PARTITION_REGEX.find(disk_path) {
-                (
-                    disk_path.strip_suffix(end.as_str()).unwrap_or_default(),
-                    Some(disk_path),
-                )
-            } else {
-                (disk_path, None)
-            };
-            let disk_path = Path::new(DISK_PATH).join(disk_path);
-            let disk = tokio::fs::canonicalize(&disk_path).await.with_ctx(|_| {
-                (
-                    crate::ErrorKind::Filesystem,
-                    disk_path.display().to_string(),
-                )
-            })?;
-            if &*disk == Path::new("/dev/mmcblk0") {
-                return Ok(disks);
-            }
-            if !disks.contains_key(&disk) {
-                disks.insert(disk.clone(), IndexSet::new());
-            }
-            if let Some(part_path) = part_path {
-                let part_path = Path::new(DISK_PATH).join(part_path);
-                let part = tokio::fs::canonicalize(&part_path).await.with_ctx(|_| {
+    .try_fold(
+        BTreeMap::<PathBuf, DiskIndex>::new(),
+        |mut disks, dir_entry| async move {
+            if let Some(disk_path) = dir_entry.path().file_name().and_then(|s| s.to_str()) {
+                let (disk_path, part_path) = if let Some(end) = PARTITION_REGEX.find(disk_path) {
+                    (
+                        disk_path.strip_suffix(end.as_str()).unwrap_or_default(),
+                        Some(disk_path),
+                    )
+                } else {
+                    (disk_path, None)
+                };
+                let disk_path = Path::new(DISK_PATH).join(disk_path);
+                let disk = tokio::fs::canonicalize(&disk_path).await.with_ctx(|_| {
                     (
                         crate::ErrorKind::Filesystem,
-                        part_path.display().to_string(),
+                        disk_path.display().to_string(),
                     )
                 })?;
-                disks.get_mut(&disk).unwrap().insert(part);
+                let part = if let Some(part_path) = part_path {
+                    let part_path = Path::new(DISK_PATH).join(part_path);
+                    let part = tokio::fs::canonicalize(&part_path).await.with_ctx(|_| {
+                        (
+                            crate::ErrorKind::Filesystem,
+                            part_path.display().to_string(),
+                        )
+                    })?;
+                    Some(part)
+                } else {
+                    None
+                };
+                if !disks.contains_key(&disk) {
+                    disks.insert(
+                        disk.clone(),
+                        DiskIndex {
+                            parts: IndexSet::new(),
+                            internal: false,
+                        },
+                    );
+                }
+                if let Some(part) = part {
+                    if os.contains(&part) {
+                        disks.get_mut(&disk).unwrap().internal = true;
+                    } else {
+                        disks.get_mut(&disk).unwrap().parts.insert(part);
+                    }
+                }
             }
-        }
-        Ok(disks)
-    })
+            Ok(disks)
+        },
+    )
     .await?;
 
     let mut res = Vec::with_capacity(disks.len());
-    for (disk, parts) in disks {
-        let mut guid: Option<String> = None;
-        let mut partitions = Vec::with_capacity(parts.len());
-        let vendor = get_vendor(&disk)
-            .await
-            .map_err(|e| tracing::warn!("Could not get vendor of {}: {}", disk.display(), e.source))
-            .unwrap_or_default();
-        let model = get_model(&disk)
-            .await
-            .map_err(|e| tracing::warn!("Could not get model of {}: {}", disk.display(), e.source))
-            .unwrap_or_default();
-        let capacity = get_capacity(&disk)
-            .await
-            .map_err(|e| {
-                tracing::warn!("Could not get capacity of {}: {}", disk.display(), e.source)
-            })
-            .unwrap_or_default();
-        if let Some(g) = disk_guids.get(&disk) {
-            guid = g.clone();
-        } else {
-            for part in parts {
-                let mut embassy_os = None;
-                let label = get_label(&part).await?;
-                let capacity = get_capacity(&part)
-                    .await
-                    .map_err(|e| {
-                        tracing::warn!("Could not get capacity of {}: {}", part.display(), e.source)
-                    })
-                    .unwrap_or_default();
-                let mut used = None;
-
-                match TmpMountGuard::mount(&BlockDev::new(&part), ReadOnly).await {
-                    Err(e) => tracing::warn!("Could not collect usage information: {}", e.source),
-                    Ok(mount_guard) => {
-                        used = get_used(&mount_guard)
-                            .await
-                            .map_err(|e| {
-                                tracing::warn!(
-                                    "Could not get usage of {}: {}",
-                                    part.display(),
-                                    e.source
-                                )
-                            })
-                            .ok();
-                        if let Some(recovery_info) = match recovery_info(&mount_guard).await {
-                            Ok(a) => a,
-                            Err(e) => {
-                                tracing::error!(
-                                    "Error fetching unencrypted backup metadata: {}",
-                                    e
-                                );
-                                None
-                            }
-                        } {
-                            embassy_os = Some(recovery_info)
-                        }
-                        mount_guard.unmount().await?;
-                    }
+    for (disk, index) in disks {
+        if index.internal {
+            for part in index.parts {
+                let mut disk_info = disk_info(disk.clone()).await;
+                disk_info.logicalname = part;
+                if let Some(g) = disk_guids.get(&disk_info.logicalname) {
+                    disk_info.guid = g.clone();
+                } else {
+                    disk_info.partitions = vec![part_info(disk_info.logicalname.clone()).await];
                 }
-
-                partitions.push(PartitionInfo {
-                    logicalname: part,
-                    label,
-                    capacity,
-                    used,
-                    embassy_os,
-                });
+                res.push(disk_info);
             }
+        } else {
+            let mut disk_info = disk_info(disk).await;
+            disk_info.partitions = Vec::with_capacity(index.parts.len());
+            if let Some(g) = disk_guids.get(&disk_info.logicalname) {
+                disk_info.guid = g.clone();
+            } else {
+                for part in index.parts {
+                    let mut part_info = part_info(part).await;
+                    if let Some(g) = disk_guids.get(&part_info.logicalname) {
+                        part_info.guid = g.clone();
+                    }
+                    disk_info.partitions.push(part_info);
+                }
+            }
+            res.push(disk_info);
         }
-        res.push(DiskInfo {
-            logicalname: disk,
-            vendor,
-            model,
-            partitions,
-            capacity,
-            guid,
-        })
     }
 
-    Ok(DiskListResponse {
-        disks: res,
-        reconnect,
-    })
+    Ok(res)
+}
+
+async fn disk_info(disk: PathBuf) -> DiskInfo {
+    let vendor = get_vendor(&disk)
+        .await
+        .map_err(|e| tracing::warn!("Could not get vendor of {}: {}", disk.display(), e.source))
+        .unwrap_or_default();
+    let model = get_model(&disk)
+        .await
+        .map_err(|e| tracing::warn!("Could not get model of {}: {}", disk.display(), e.source))
+        .unwrap_or_default();
+    let capacity = get_capacity(&disk)
+        .await
+        .map_err(|e| tracing::warn!("Could not get capacity of {}: {}", disk.display(), e.source))
+        .unwrap_or_default();
+    DiskInfo {
+        logicalname: disk,
+        vendor,
+        model,
+        partitions: Vec::new(),
+        capacity,
+        guid: None,
+    }
+}
+
+async fn part_info(part: PathBuf) -> PartitionInfo {
+    let mut embassy_os = None;
+    let label = get_label(&part)
+        .await
+        .map_err(|e| tracing::warn!("Could not get label of {}: {}", part.display(), e.source))
+        .unwrap_or_default();
+    let capacity = get_capacity(&part)
+        .await
+        .map_err(|e| tracing::warn!("Could not get capacity of {}: {}", part.display(), e.source))
+        .unwrap_or_default();
+    let mut used = None;
+
+    match TmpMountGuard::mount(&BlockDev::new(&part), ReadOnly).await {
+        Err(e) => tracing::warn!("Could not collect usage information: {}", e.source),
+        Ok(mount_guard) => {
+            used = get_used(&mount_guard)
+                .await
+                .map_err(|e| {
+                    tracing::warn!("Could not get usage of {}: {}", part.display(), e.source)
+                })
+                .ok();
+            if let Some(recovery_info) = match recovery_info(&mount_guard).await {
+                Ok(a) => a,
+                Err(e) => {
+                    tracing::error!("Error fetching unencrypted backup metadata: {}", e);
+                    None
+                }
+            } {
+                embassy_os = Some(recovery_info)
+            }
+            if let Err(e) = mount_guard.unmount().await {
+                tracing::error!("Error unmounting partition {}: {}", part.display(), e);
+            }
+        }
+    }
+
+    PartitionInfo {
+        logicalname: part,
+        label,
+        capacity,
+        used,
+        embassy_os,
+        guid: None,
+    }
 }
 
 fn parse_pvscan_output(pvscan_output: &str) -> BTreeMap<PathBuf, Option<String>> {

backend/src/error.rs

@@ -1,253 +1,5 @@
-use std::fmt::Display;
-
 use color_eyre::eyre::eyre;
-use patch_db::Revision;
-use rpc_toolkit::yajrc::RpcError;
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum ErrorKind {
-    Unknown = 1,
-    Filesystem = 2,
-    Docker = 3,
-    ConfigSpecViolation = 4,
-    ConfigRulesViolation = 5,
-    NotFound = 6,
-    IncorrectPassword = 7,
-    VersionIncompatible = 8,
-    Network = 9,
-    Registry = 10,
-    Serialization = 11,
-    Deserialization = 12,
-    Utf8 = 13,
-    ParseVersion = 14,
-    IncorrectDisk = 15,
-    Nginx = 16,
-    Dependency = 17,
-    ParseS9pk = 18,
-    ParseUrl = 19,
-    DiskNotAvailable = 20,
-    BlockDevice = 21,
-    InvalidOnionAddress = 22,
-    Pack = 23,
-    ValidateS9pk = 24,
-    DiskCorrupted = 25,
-    Tor = 26,
-    ConfigGen = 27,
-    ParseNumber = 28,
-    Database = 29,
-    InvalidPackageId = 30,
-    InvalidSignature = 31,
-    Backup = 32,
-    Restore = 33,
-    Authorization = 34,
-    AutoConfigure = 35,
-    Action = 36,
-    RateLimited = 37,
-    InvalidRequest = 38,
-    MigrationFailed = 39,
-    Uninitialized = 40,
-    ParseNetAddress = 41,
-    ParseSshKey = 42,
-    SoundError = 43,
-    ParseTimestamp = 44,
-    ParseSysInfo = 45,
-    Wifi = 46,
-    Journald = 47,
-    DiskManagement = 48,
-    OpenSsl = 49,
-    PasswordHashGeneration = 50,
-    DiagnosticMode = 51,
-    ParseDbField = 52,
-    Duplicate = 53,
-    MultipleErrors = 54,
-    Incoherent = 55,
-    InvalidBackupTargetId = 56,
-    ProductKeyMismatch = 57,
-    LanPortConflict = 58,
-}
-impl ErrorKind {
-    pub fn as_str(&self) -> &'static str {
-        use ErrorKind::*;
-        match self {
-            Unknown => "Unknown Error",
-            Filesystem => "Filesystem I/O Error",
-            Docker => "Docker Error",
-            ConfigSpecViolation => "Config Spec Violation",
-            ConfigRulesViolation => "Config Rules Violation",
-            NotFound => "Not Found",
-            IncorrectPassword => "Incorrect Password",
-            VersionIncompatible => "Version Incompatible",
-            Network => "Network Error",
-            Registry => "Registry Error",
-            Serialization => "Serialization Error",
-            Deserialization => "Deserialization Error",
-            Utf8 => "UTF-8 Parse Error",
-            ParseVersion => "Version Parsing Error",
-            IncorrectDisk => "Incorrect Disk",
-            Nginx => "Nginx Error",
-            Dependency => "Dependency Error",
-            ParseS9pk => "S9PK Parsing Error",
-            ParseUrl => "URL Parsing Error",
-            DiskNotAvailable => "Disk Not Available",
-            BlockDevice => "Block Device Error",
-            InvalidOnionAddress => "Invalid Onion Address",
-            Pack => "Pack Error",
-            ValidateS9pk => "S9PK Validation Error",
-            DiskCorrupted => "Disk Corrupted", // Remove
-            Tor => "Tor Daemon Error",
-            ConfigGen => "Config Generation Error",
-            ParseNumber => "Number Parsing Error",
-            Database => "Database Error",
-            InvalidPackageId => "Invalid Package ID",
-            InvalidSignature => "Invalid Signature",
-            Backup => "Backup Error",
-            Restore => "Restore Error",
-            Authorization => "Unauthorized",
-            AutoConfigure => "Auto-Configure Error",
-            Action => "Action Failed",
-            RateLimited => "Rate Limited",
-            InvalidRequest => "Invalid Request",
-            MigrationFailed => "Migration Failed",
-            Uninitialized => "Uninitialized",
-            ParseNetAddress => "Net Address Parsing Error",
-            ParseSshKey => "SSH Key Parsing Error",
-            SoundError => "Sound Interface Error",
-            ParseTimestamp => "Timestamp Parsing Error",
-            ParseSysInfo => "System Info Parsing Error",
-            Wifi => "WiFi Internal Error",
-            Journald => "Journald Error",
-            DiskManagement => "Disk Management Error",
-            OpenSsl => "OpenSSL Internal Error",
-            PasswordHashGeneration => "Password Hash Generation Error",
-            DiagnosticMode => "Embassy is in Diagnostic Mode",
-            ParseDbField => "Database Field Parse Error",
-            Duplicate => "Duplication Error",
-            MultipleErrors => "Multiple Errors",
-            Incoherent => "Incoherent",
-            InvalidBackupTargetId => "Invalid Backup Target ID",
-            ProductKeyMismatch => "Incompatible Product Keys",
-            LanPortConflict => "Incompatible LAN port configuration",
-        }
-    }
-}
-impl Display for ErrorKind {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{}", self.as_str())
-    }
-}
-
-#[derive(Debug)]
-pub struct Error {
-    pub source: color_eyre::eyre::Error,
-    pub kind: ErrorKind,
-    pub revision: Option<Revision>,
-}
-impl Display for Error {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{}: {}", self.kind.as_str(), self.source)
-    }
-}
-impl Error {
-    pub fn new<E: Into<color_eyre::eyre::Error>>(source: E, kind: ErrorKind) -> Self {
-        Error {
-            source: source.into(),
-            kind,
-            revision: None,
-        }
-    }
-}
-impl From<std::io::Error> for Error {
-    fn from(e: std::io::Error) -> Self {
-        Error::new(e, ErrorKind::Filesystem)
-    }
-}
-impl From<std::str::Utf8Error> for Error {
-    fn from(e: std::str::Utf8Error) -> Self {
-        Error::new(e, ErrorKind::Utf8)
-    }
-}
-impl From<std::string::FromUtf8Error> for Error {
-    fn from(e: std::string::FromUtf8Error) -> Self {
-        Error::new(e, ErrorKind::Utf8)
-    }
-}
-impl From<emver::ParseError> for Error {
-    fn from(e: emver::ParseError) -> Self {
-        Error::new(e, ErrorKind::ParseVersion)
-    }
-}
-impl From<rpc_toolkit::url::ParseError> for Error {
-    fn from(e: rpc_toolkit::url::ParseError) -> Self {
-        Error::new(e, ErrorKind::ParseUrl)
-    }
-}
-impl From<std::num::ParseIntError> for Error {
-    fn from(e: std::num::ParseIntError) -> Self {
-        Error::new(e, ErrorKind::ParseNumber)
-    }
-}
-impl From<std::num::ParseFloatError> for Error {
-    fn from(e: std::num::ParseFloatError) -> Self {
-        Error::new(e, ErrorKind::ParseNumber)
-    }
-}
-impl From<patch_db::Error> for Error {
-    fn from(e: patch_db::Error) -> Self {
-        Error::new(e, ErrorKind::Database)
-    }
-}
-impl From<sqlx::Error> for Error {
-    fn from(e: sqlx::Error) -> Self {
-        Error::new(e, ErrorKind::Database)
-    }
-}
-impl From<ed25519_dalek::SignatureError> for Error {
-    fn from(e: ed25519_dalek::SignatureError) -> Self {
-        Error::new(e, ErrorKind::InvalidSignature)
-    }
-}
-impl From<bollard::errors::Error> for Error {
-    fn from(e: bollard::errors::Error) -> Self {
-        Error::new(e, ErrorKind::Docker)
-    }
-}
-impl From<torut::control::ConnError> for Error {
-    fn from(e: torut::control::ConnError) -> Self {
-        Error::new(eyre!("{:?}", e), ErrorKind::Tor)
-    }
-}
-impl From<std::net::AddrParseError> for Error {
-    fn from(e: std::net::AddrParseError) -> Self {
-        Error::new(e, ErrorKind::ParseNetAddress)
-    }
-}
-impl From<openssl::error::ErrorStack> for Error {
-    fn from(e: openssl::error::ErrorStack) -> Self {
-        Error::new(eyre!("OpenSSL ERROR:\n{}", e), ErrorKind::OpenSsl)
-    }
-}
-impl From<Error> for RpcError {
-    fn from(e: Error) -> Self {
-        let mut data_object = serde_json::Map::with_capacity(3);
-        data_object.insert("details".to_owned(), format!("{}", e.source).into());
-        data_object.insert("debug".to_owned(), format!("{:?}", e.source).into());
-        data_object.insert(
-            "revision".to_owned(),
-            match serde_json::to_value(&e.revision) {
-                Ok(a) => a,
-                Err(e) => {
-                    tracing::warn!("Error serializing revision for Error object: {}", e);
-                    serde_json::Value::Null
-                }
-            },
-        );
-        RpcError {
-            code: e.kind as i32,
-            message: e.kind.as_str().into(),
-            data: Some(data_object.into()),
-        }
-    }
-}
+pub use models::{Error, ErrorKind, ResultExt};
 
 #[derive(Debug, Default)]
 pub struct ErrorCollection(Vec<Error>);
@@ -298,46 +50,6 @@ impl std::fmt::Display for ErrorCollection {
     }
 }
 
-pub trait ResultExt<T, E>
-where
-    Self: Sized,
-{
-    fn with_kind(self, kind: ErrorKind) -> Result<T, Error>;
-    fn with_ctx<F: FnOnce(&E) -> (ErrorKind, D), D: Display + Send + Sync + 'static>(
-        self,
-        f: F,
-    ) -> Result<T, Error>;
-}
-impl<T, E> ResultExt<T, E> for Result<T, E>
-where
-    color_eyre::eyre::Error: From<E>,
-{
-    fn with_kind(self, kind: ErrorKind) -> Result<T, Error> {
-        self.map_err(|e| Error {
-            source: e.into(),
-            kind,
-            revision: None,
-        })
-    }
-
-    fn with_ctx<F: FnOnce(&E) -> (ErrorKind, D), D: Display + Send + Sync + 'static>(
-        self,
-        f: F,
-    ) -> Result<T, Error> {
-        self.map_err(|e| {
-            let (kind, ctx) = f(&e);
-            let source = color_eyre::eyre::Error::from(e);
-            let ctx = format!("{}: {}", ctx, source);
-            let source = source.wrap_err(ctx);
-            Error {
-                kind,
-                source: source.into(),
-                revision: None,
-            }
-        })
-    }
-}
-
 #[macro_export]
 macro_rules! ensure_code {
     ($x:expr, $c:expr, $fmt:expr $(, $arg:expr)*) => {

backend/src/fstab.template

@@ -0,0 +1,2 @@
+{boot}  /boot   vfat    defaults    0   2
+{root}  /       ext4    defaults    0   1

backend/src/hostname.rs

@@ -1,34 +1,84 @@
-use digest::Digest;
-use tokio::fs::File;
-use tokio::io::AsyncWriteExt;
+use patch_db::DbHandle;
+use rand::{thread_rng, Rng};
+use sqlx::Connection;
 use tokio::process::Command;
 use tracing::instrument;
 
+use crate::context::RpcContext;
 use crate::util::Invoke;
-use crate::{Error, ErrorKind, ResultExt};
+use crate::{Error, ErrorKind};
+#[derive(Clone, serde::Deserialize, serde::Serialize, Debug)]
+pub struct Hostname(pub String);
 
-pub const PRODUCT_KEY_PATH: &'static str = "/embassy-os/product_key.txt";
-
-#[instrument]
-pub async fn get_hostname() -> Result<String, Error> {
-    Ok(derive_hostname(&get_id().await?))
+lazy_static::lazy_static! {
+    static ref ADJECTIVES: Vec<String> = include_str!("./assets/adjectives.txt").lines().map(|x| x.to_string()).collect();
+    static ref NOUNS: Vec<String> = include_str!("./assets/nouns.txt").lines().map(|x| x.to_string()).collect();
+}
+impl AsRef<str> for Hostname {
+    fn as_ref(&self) -> &str {
+        &self.0
+    }
 }
 
-pub fn derive_hostname(id: &str) -> String {
-    format!("embassy-{}", id)
+impl Hostname {
+    pub fn lan_address(&self) -> String {
+        format!("https://{}.local", self.0)
+    }
+
+    pub fn local_domain_name(&self) -> String {
+        format!("{}.local", self.0)
+    }
+    pub fn no_dot_host_name(&self) -> String {
+        self.0.to_owned()
+    }
 }
 
-#[instrument]
-pub async fn get_current_hostname() -> Result<String, Error> {
-    let out = Command::new("hostname")
+pub async fn get_current_ip(eth: String) -> Result<String, Error> {
+    let cmd = format!(r"ifconfig {} | awk '/inet / {{print $2}}'", eth);
+
+    let out = Command::new("bash")
+        .arg("-c")
+        .arg(cmd)
         .invoke(ErrorKind::ParseSysInfo)
         .await?;
     let out_string = String::from_utf8(out)?;
     Ok(out_string.trim().to_owned())
 }
 
+pub async fn get_embassyd_tor_addr(rpc_ctx: RpcContext) -> Result<String, Error> {
+    let mut secrets_handle = rpc_ctx.secret_store.acquire().await?;
+
+    let mut secrets_tx = secrets_handle.begin().await?;
+
+    let tor_key = crate::net::tor::os_key(&mut secrets_tx).await?;
+
+    Ok(tor_key.public().get_onion_address().to_string())
+}
+
+pub fn generate_hostname() -> Hostname {
+    let mut rng = thread_rng();
+    let adjective = &ADJECTIVES[rng.gen_range(0..ADJECTIVES.len())];
+    let noun = &NOUNS[rng.gen_range(0..NOUNS.len())];
+    Hostname(format!("embassy-{adjective}-{noun}"))
+}
+
+pub fn generate_id() -> String {
+    let id = uuid::Uuid::new_v4();
+    id.to_string()
+}
+
 #[instrument]
-pub async fn set_hostname(hostname: &str) -> Result<(), Error> {
+pub async fn get_current_hostname() -> Result<Hostname, Error> {
+    let out = Command::new("hostname")
+        .invoke(ErrorKind::ParseSysInfo)
+        .await?;
+    let out_string = String::from_utf8(out)?;
+    Ok(Hostname(out_string.trim().to_owned()))
+}
+
+#[instrument]
+pub async fn set_hostname(hostname: &Hostname) -> Result<(), Error> {
+    let hostname: &String = &hostname.0;
     let _out = Command::new("hostnamectl")
         .arg("set-hostname")
         .arg(hostname)
@@ -37,38 +87,83 @@ pub async fn set_hostname(hostname: &str) -> Result<(), Error> {
     Ok(())
 }
 
-#[instrument]
-pub async fn get_product_key() -> Result<String, Error> {
-    let out = tokio::fs::read_to_string(PRODUCT_KEY_PATH)
-        .await
-        .with_ctx(|_| (crate::ErrorKind::Filesystem, PRODUCT_KEY_PATH))?;
-    Ok(out.trim().to_owned())
+#[instrument(skip(handle, receipts))]
+pub async fn get_id<Db: DbHandle>(
+    handle: &mut Db,
+    receipts: &HostNameReceipt,
+) -> Result<String, Error> {
+    let id = receipts.id.get(handle).await?;
+    Ok(id)
 }
 
-#[instrument]
-pub async fn set_product_key(key: &str) -> Result<(), Error> {
-    let mut pkey_file = File::create(PRODUCT_KEY_PATH).await?;
-    pkey_file.write_all(key.as_bytes()).await?;
+pub async fn get_hostname<Db: DbHandle>(
+    handle: &mut Db,
+    receipts: &HostNameReceipt,
+) -> Result<Hostname, Error> {
+    if let Ok(hostname) = receipts.hostname.get(handle).await {
+        if let Some(hostname) = hostname.to_owned() {
+            return Ok(Hostname(hostname));
+        }
+    }
+    let id = get_id(handle, receipts).await?;
+    if id.len() != 8 {
+        return Ok(generate_hostname());
+    }
+    return Ok(Hostname(format!("embassy-{}", id)));
+}
+
+pub async fn ensure_hostname_is_set<Db: DbHandle>(
+    handle: &mut Db,
+    receipts: &HostNameReceipt,
+) -> Result<(), Error> {
+    let hostname = get_hostname(handle, &receipts).await?;
+    receipts.hostname.set(handle, Some(hostname.0)).await?;
     Ok(())
 }
 
-pub fn derive_id(key: &str) -> String {
-    let mut hasher = sha2::Sha256::new();
-    hasher.update(key.as_bytes());
-    let res = hasher.finalize();
-    hex::encode(&res[0..4])
+#[derive(Clone)]
+pub struct HostNameReceipt {
+    hostname: patch_db::LockReceipt<Option<String>, ()>,
+    pub id: patch_db::LockReceipt<String, ()>,
 }
 
-#[instrument]
-pub async fn get_id() -> Result<String, Error> {
-    let key = get_product_key().await?;
-    Ok(derive_id(&key))
+impl HostNameReceipt {
+    pub async fn new<'a>(db: &'a mut impl DbHandle) -> Result<Self, Error> {
+        let mut locks = Vec::new();
+
+        let setup = Self::setup(&mut locks);
+        setup(&db.lock_all(locks).await?)
+    }
+
+    pub fn setup(
+        locks: &mut Vec<patch_db::LockTargetId>,
+    ) -> impl FnOnce(&patch_db::Verifier) -> Result<Self, Error> {
+        use patch_db::LockType;
+        let hostname = crate::db::DatabaseModel::new()
+            .server_info()
+            .hostname()
+            .make_locker(LockType::Write)
+            .add_to_keys(locks);
+        let id = crate::db::DatabaseModel::new()
+            .server_info()
+            .id()
+            .make_locker(LockType::Write)
+            .add_to_keys(locks);
+        move |skeleton_key| {
+            Ok(Self {
+                hostname: hostname.verify(skeleton_key)?,
+                id: id.verify(skeleton_key)?,
+            })
+        }
+    }
 }
 
-// cat /embassy-os/product_key.txt | shasum -a 256 | head -c 8 | awk '{print "embassy-"$1}' | xargs hostnamectl set-hostname && systemctl restart avahi-daemon
-#[instrument]
-pub async fn sync_hostname() -> Result<(), Error> {
-    set_hostname(&format!("embassy-{}", get_id().await?)).await?;
+#[instrument(skip(handle, receipts))]
+pub async fn sync_hostname<Db: DbHandle>(
+    handle: &mut Db,
+    receipts: &HostNameReceipt,
+) -> Result<(), Error> {
+    set_hostname(&get_hostname(handle, receipts).await?).await?;
     Command::new("systemctl")
         .arg("restart")
         .arg("avahi-daemon")

backend/src/id.rs

@@ -1,142 +1,10 @@
-use std::borrow::{Borrow, Cow};
 use std::fmt::Debug;
 use std::str::FromStr;
 
-use serde::{Deserialize, Deserializer, Serialize, Serializer};
+pub use models::{Id, IdUnchecked, InvalidId, SYSTEM_ID};
+use serde::{Deserialize, Deserializer, Serialize};
 
 use crate::util::Version;
-use crate::Error;
-
-pub const SYSTEM_ID: Id<&'static str> = Id("x_system");
-
-#[derive(Debug, thiserror::Error)]
-#[error("Invalid ID")]
-pub struct InvalidId;
-impl From<InvalidId> for Error {
-    fn from(err: InvalidId) -> Self {
-        Error::new(err, crate::error::ErrorKind::InvalidPackageId)
-    }
-}
-
-#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
-pub struct IdUnchecked<S: AsRef<str>>(pub S);
-impl<'de> Deserialize<'de> for IdUnchecked<Cow<'de, str>> {
-    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
-    where
-        D: Deserializer<'de>,
-    {
-        struct Visitor;
-        impl<'de> serde::de::Visitor<'de> for Visitor {
-            type Value = IdUnchecked<Cow<'de, str>>;
-            fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
-                write!(formatter, "a valid ID")
-            }
-            fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
-            where
-                E: serde::de::Error,
-            {
-                Ok(IdUnchecked(Cow::Owned(v.to_owned())))
-            }
-            fn visit_string<E>(self, v: String) -> Result<Self::Value, E>
-            where
-                E: serde::de::Error,
-            {
-                Ok(IdUnchecked(Cow::Owned(v)))
-            }
-            fn visit_borrowed_str<E>(self, v: &'de str) -> Result<Self::Value, E>
-            where
-                E: serde::de::Error,
-            {
-                Ok(IdUnchecked(Cow::Borrowed(v)))
-            }
-        }
-        deserializer.deserialize_any(Visitor)
-    }
-}
-impl<'de> Deserialize<'de> for IdUnchecked<String> {
-    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
-    where
-        D: Deserializer<'de>,
-    {
-        Ok(IdUnchecked(String::deserialize(deserializer)?))
-    }
-}
-impl<'de> Deserialize<'de> for IdUnchecked<&'de str> {
-    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
-    where
-        D: Deserializer<'de>,
-    {
-        Ok(IdUnchecked(<&'de str>::deserialize(deserializer)?))
-    }
-}
-
-#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
-pub struct Id<S: AsRef<str> = String>(S);
-impl<S: AsRef<str>> Id<S> {
-    pub fn try_from(value: S) -> Result<Self, InvalidId> {
-        if value
-            .as_ref()
-            .chars()
-            .all(|c| c.is_ascii_lowercase() || c == '-')
-        {
-            Ok(Id(value))
-        } else {
-            Err(InvalidId)
-        }
-    }
-}
-impl<'a> Id<&'a str> {
-    pub fn owned(&self) -> Id {
-        Id(self.0.to_owned())
-    }
-}
-impl From<Id> for String {
-    fn from(value: Id) -> Self {
-        value.0
-    }
-}
-impl<S: AsRef<str>> std::ops::Deref for Id<S> {
-    type Target = S;
-    fn deref(&self) -> &Self::Target {
-        &self.0
-    }
-}
-impl<S: AsRef<str>> std::fmt::Display for Id<S> {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{}", self.0.as_ref())
-    }
-}
-impl<S: AsRef<str>> AsRef<str> for Id<S> {
-    fn as_ref(&self) -> &str {
-        self.0.as_ref()
-    }
-}
-impl<S: AsRef<str>> Borrow<str> for Id<S> {
-    fn borrow(&self) -> &str {
-        self.0.as_ref()
-    }
-}
-impl<'de, S> Deserialize<'de> for Id<S>
-where
-    S: AsRef<str>,
-    IdUnchecked<S>: Deserialize<'de>,
-{
-    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
-    where
-        D: Deserializer<'de>,
-    {
-        let unchecked: IdUnchecked<S> = Deserialize::deserialize(deserializer)?;
-        Id::try_from(unchecked.0).map_err(serde::de::Error::custom)
-    }
-}
-impl<S: AsRef<str>> Serialize for Id<S> {
-    fn serialize<Ser>(&self, serializer: Ser) -> Result<Ser::Ok, Ser::Error>
-    where
-        Ser: Serializer,
-    {
-        serializer.serialize_str(self.as_ref())
-    }
-}
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
 pub struct ImageId<S: AsRef<str> = String>(Id<S>);

backend/src/init.rs

@@ -1,17 +1,226 @@
+use std::collections::HashMap;
+use std::path::Path;
+use std::process::Stdio;
+use std::time::Duration;
+
+use color_eyre::eyre::eyre;
+use helpers::NonDetachingJoinHandle;
+use patch_db::{DbHandle, LockReceipt, LockType};
+use sqlx::{Pool, Postgres};
 use tokio::process::Command;
 
 use crate::context::rpc::RpcContextConfig;
 use crate::db::model::ServerStatus;
-use crate::install::PKG_DOCKER_DIR;
+use crate::install::PKG_ARCHIVE_DIR;
+use crate::sound::CIRCLE_OF_5THS_SHORT;
 use crate::util::Invoke;
 use crate::Error;
 
-pub const SYSTEM_REBUILD_PATH: &str = "/embassy-os/system-rebuild";
+pub const SYSTEM_REBUILD_PATH: &str = "/media/embassy/config/system-rebuild";
+pub const STANDBY_MODE_PATH: &str = "/media/embassy/config/standby";
 
-pub async fn init(cfg: &RpcContextConfig, product_key: &str) -> Result<(), Error> {
-    let should_rebuild = tokio::fs::metadata(SYSTEM_REBUILD_PATH).await.is_ok();
+pub async fn check_time_is_synchronized() -> Result<bool, Error> {
+    Ok(String::from_utf8(
+        Command::new("timedatectl")
+            .arg("show")
+            .arg("-p")
+            .arg("NTPSynchronized")
+            .invoke(crate::ErrorKind::Unknown)
+            .await?,
+    )?
+    .trim()
+        == "NTPSynchronized=yes")
+}
+
+pub struct InitReceipts {
+    pub server_version: LockReceipt<crate::util::Version, ()>,
+    pub version_range: LockReceipt<emver::VersionRange, ()>,
+    pub last_wifi_region: LockReceipt<Option<isocountry::CountryCode>, ()>,
+    pub status_info: LockReceipt<ServerStatus, ()>,
+}
+impl InitReceipts {
+    pub async fn new(db: &mut impl DbHandle) -> Result<Self, Error> {
+        let mut locks = Vec::new();
+
+        let server_version = crate::db::DatabaseModel::new()
+            .server_info()
+            .version()
+            .make_locker(LockType::Write)
+            .add_to_keys(&mut locks);
+        let version_range = crate::db::DatabaseModel::new()
+            .server_info()
+            .eos_version_compat()
+            .make_locker(LockType::Write)
+            .add_to_keys(&mut locks);
+        let last_wifi_region = crate::db::DatabaseModel::new()
+            .server_info()
+            .last_wifi_region()
+            .make_locker(LockType::Write)
+            .add_to_keys(&mut locks);
+        let status_info = crate::db::DatabaseModel::new()
+            .server_info()
+            .status_info()
+            .into_model()
+            .make_locker(LockType::Write)
+            .add_to_keys(&mut locks);
+
+        let skeleton_key = db.lock_all(locks).await?;
+        Ok(Self {
+            server_version: server_version.verify(&skeleton_key)?,
+            version_range: version_range.verify(&skeleton_key)?,
+            status_info: status_info.verify(&skeleton_key)?,
+            last_wifi_region: last_wifi_region.verify(&skeleton_key)?,
+        })
+    }
+}
+
+pub async fn pgloader(
+    old_db_path: impl AsRef<Path>,
+    batch_rows: usize,
+    prefetch_rows: usize,
+) -> Result<(), Error> {
+    tokio::fs::write(
+        "/etc/embassy/migrate.load",
+        format!(
+            include_str!("migrate.load"),
+            sqlite_path = old_db_path.as_ref().display(),
+            batch_rows = batch_rows,
+            prefetch_rows = prefetch_rows
+        ),
+    )
+    .await?;
+    match tokio::fs::remove_dir_all("/tmp/pgloader").await {
+        Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
+        a => a,
+    }?;
+    tracing::info!("Running pgloader");
+    let out = Command::new("pgloader")
+        .arg("-v")
+        .arg("/etc/embassy/migrate.load")
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped())
+        .output()
+        .await?;
+    let stdout = String::from_utf8(out.stdout)?;
+    for line in stdout.lines() {
+        tracing::debug!("pgloader: {}", line);
+    }
+    let stderr = String::from_utf8(out.stderr)?;
+    for line in stderr.lines() {
+        tracing::debug!("pgloader err: {}", line);
+    }
+    tracing::debug!("pgloader exited with code {:?}", out.status);
+    if let Some(err) = stdout.lines().chain(stderr.lines()).find_map(|l| {
+        if l.split_ascii_whitespace()
+            .any(|word| word == "ERROR" || word == "FATAL")
+        {
+            Some(l)
+        } else {
+            None
+        }
+    }) {
+        return Err(Error::new(
+            eyre!("pgloader error: {}", err),
+            crate::ErrorKind::Database,
+        ));
+    }
+    tokio::fs::rename(
+        old_db_path.as_ref(),
+        old_db_path.as_ref().with_extension("bak"),
+    )
+    .await?;
+    Ok(())
+}
+
+// must be idempotent
+pub async fn init_postgres(datadir: impl AsRef<Path>) -> Result<(), Error> {
+    let db_dir = datadir.as_ref().join("main/postgresql");
+    let is_mountpoint = || async {
+        Ok::<_, Error>(
+            tokio::process::Command::new("mountpoint")
+                .arg("/var/lib/postgresql")
+                .stdout(std::process::Stdio::null())
+                .stderr(std::process::Stdio::null())
+                .status()
+                .await?
+                .success(),
+        )
+    };
+    let exists = tokio::fs::metadata(&db_dir).await.is_ok();
+    if !exists {
+        Command::new("cp")
+            .arg("-ra")
+            .arg("/var/lib/postgresql")
+            .arg(&db_dir)
+            .invoke(crate::ErrorKind::Filesystem)
+            .await?;
+    }
+    if !is_mountpoint().await? {
+        crate::disk::mount::util::bind(&db_dir, "/var/lib/postgresql", false).await?;
+    }
+    Command::new("chown")
+        .arg("-R")
+        .arg("postgres")
+        .arg("/var/lib/postgresql")
+        .invoke(crate::ErrorKind::Database)
+        .await?;
+    Command::new("systemctl")
+        .arg("start")
+        .arg("postgresql")
+        .invoke(crate::ErrorKind::Database)
+        .await?;
+    if !exists {
+        Command::new("sudo")
+            .arg("-u")
+            .arg("postgres")
+            .arg("createuser")
+            .arg("root")
+            .invoke(crate::ErrorKind::Database)
+            .await?;
+        Command::new("sudo")
+            .arg("-u")
+            .arg("postgres")
+            .arg("createdb")
+            .arg("secrets")
+            .arg("-O")
+            .arg("root")
+            .invoke(crate::ErrorKind::Database)
+            .await?;
+    }
+    Ok(())
+}
+
+pub struct InitResult {
+    pub secret_store: Pool<Postgres>,
+    pub db: patch_db::PatchDb,
+}
+
+pub async fn init(cfg: &RpcContextConfig) -> Result<InitResult, Error> {
     let secret_store = cfg.secret_store().await?;
-    let log_dir = cfg.datadir().join("main").join("logs");
+    let db = cfg.db(&secret_store).await?;
+    let mut handle = db.handle();
+    crate::db::DatabaseModel::new()
+        .server_info()
+        .lock(&mut handle, LockType::Write)
+        .await?;
+
+    let receipts = InitReceipts::new(&mut handle).await?;
+
+    let should_rebuild = tokio::fs::metadata(SYSTEM_REBUILD_PATH).await.is_ok()
+        || &*receipts.server_version.get(&mut handle).await? < &emver::Version::new(0, 3, 2, 0);
+
+    let song = if should_rebuild {
+        Some(NonDetachingJoinHandle::from(tokio::spawn(async {
+            loop {
+                CIRCLE_OF_5THS_SHORT.play().await.unwrap();
+                tokio::time::sleep(Duration::from_secs(10)).await;
+            }
+        })))
+    } else {
+        None
+    };
+
+    let log_dir = cfg.datadir().join("main/logs");
     if tokio::fs::metadata(&log_dir).await.is_err() {
         tokio::fs::create_dir_all(&log_dir).await?;
     }
@@ -33,7 +242,7 @@ pub async fn init(cfg: &RpcContextConfig, product_key: &str) -> Result<(), Error
             tokio::fs::remove_dir_all(&tmp_docker).await?;
         }
         Command::new("cp")
-            .arg("-r")
+            .arg("-ra")
             .arg("/var/lib/docker")
             .arg(&tmp_docker)
             .invoke(crate::ErrorKind::Filesystem)
@@ -58,53 +267,100 @@ pub async fn init(cfg: &RpcContextConfig, product_key: &str) -> Result<(), Error
     tracing::info!("Mounted Docker Data");
 
     if should_rebuild || !tmp_docker_exists {
+        tracing::info!("Creating Docker Network");
+        bollard::Docker::connect_with_unix_defaults()?
+            .create_network(bollard::network::CreateNetworkOptions {
+                name: "start9",
+                driver: "bridge",
+                ipam: bollard::models::Ipam {
+                    config: Some(vec![bollard::models::IpamConfig {
+                        subnet: Some("172.18.0.1/24".into()),
+                        ..Default::default()
+                    }]),
+                    ..Default::default()
+                },
+                options: {
+                    let mut m = HashMap::new();
+                    m.insert("com.docker.network.bridge.name", "br-start9");
+                    m
+                },
+                ..Default::default()
+            })
+            .await?;
+        tracing::info!("Created Docker Network");
+
         tracing::info!("Loading System Docker Images");
-        crate::install::load_images("/var/lib/embassy/system-images").await?;
+        crate::install::load_images("/usr/lib/embassy/system-images").await?;
         tracing::info!("Loaded System Docker Images");
 
         tracing::info!("Loading Package Docker Images");
-        crate::install::load_images(cfg.datadir().join(PKG_DOCKER_DIR)).await?;
+        crate::install::load_images(cfg.datadir().join(PKG_ARCHIVE_DIR)).await?;
         tracing::info!("Loaded Package Docker Images");
     }
 
-    crate::ssh::sync_keys_from_db(&secret_store, "/root/.ssh/authorized_keys").await?;
-    tracing::info!("Synced SSH Keys");
-    let db = cfg.db(&secret_store, product_key).await?;
-
-    let mut handle = db.handle();
-
-    crate::net::wifi::synchronize_wpa_supplicant_conf(
-        &cfg.datadir().join("main"),
-        &*crate::db::DatabaseModel::new()
-            .server_info()
-            .last_wifi_region()
-            .get(&mut handle, false)
-            .await
-            .map_err(|_e| {
-                Error::new(
-                    color_eyre::eyre::eyre!("Could not find the last wifi region"),
-                    crate::ErrorKind::NotFound,
-                )
-            })?,
-    )
-    .await?;
-    tracing::info!("Synchronized wpa_supplicant.conf");
-    let mut info = crate::db::DatabaseModel::new()
-        .server_info()
-        .get_mut(&mut handle)
+    tracing::info!("Enabling Docker QEMU Emulation");
+    Command::new("docker")
+        .arg("run")
+        .arg("--privileged")
+        .arg("--rm")
+        .arg("start9/x_system/binfmt")
+        .arg("--install")
+        .arg("all")
+        .invoke(crate::ErrorKind::Docker)
         .await?;
-    info.status_info = ServerStatus {
-        backing_up: false,
-        updated: false,
-        update_progress: None,
-    };
-    info.save(&mut handle).await?;
+    tracing::info!("Enabled Docker QEMU Emulation");
 
-    crate::version::init(&mut handle).await?;
+    crate::ssh::sync_keys_from_db(&secret_store, "/home/start9/.ssh/authorized_keys").await?;
+    tracing::info!("Synced SSH Keys");
 
-    if should_rebuild {
-        tokio::fs::remove_file(SYSTEM_REBUILD_PATH).await?;
+    if let Some(wifi_interface) = &cfg.wifi_interface {
+        crate::net::wifi::synchronize_wpa_supplicant_conf(
+            &cfg.datadir().join("main"),
+            wifi_interface,
+            &receipts.last_wifi_region.get(&mut handle).await?,
+        )
+        .await?;
+        tracing::info!("Synchronized WiFi");
+    }
+    receipts
+        .status_info
+        .set(
+            &mut handle,
+            ServerStatus {
+                updated: false,
+                update_progress: None,
+                backup_progress: None,
+            },
+        )
+        .await?;
+
+    let mut warn_time_not_synced = true;
+    for _ in 0..60 {
+        if check_time_is_synchronized().await? {
+            warn_time_not_synced = false;
+            break;
+        }
+        tokio::time::sleep(Duration::from_secs(1)).await;
+    }
+    if warn_time_not_synced {
+        tracing::warn!("Timed out waiting for system time to synchronize");
+    } else {
+        tracing::info!("Syncronized system clock");
     }
 
-    Ok(())
+    crate::version::init(&mut handle, &receipts).await?;
+
+    if should_rebuild {
+        match tokio::fs::remove_file(SYSTEM_REBUILD_PATH).await {
+            Ok(()) => Ok(()),
+            Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
+            Err(e) => Err(e),
+        }?;
+    }
+
+    drop(song);
+
+    tracing::info!("System initialized.");
+
+    Ok(InitResult { secret_store, db })
 }

backend/src/install/cleanup.rs

@@ -1,74 +1,98 @@
-use std::collections::{BTreeMap, HashMap};
+use std::collections::HashMap;
+use std::path::PathBuf;
+use std::sync::Arc;
 
-use bollard::image::ListImagesOptions;
-use color_eyre::eyre::eyre;
-use patch_db::{DbHandle, LockType, PatchDbHandle};
-use sqlx::{Executor, Sqlite};
+use bollard::image::{ListImagesOptions, RemoveImageOptions};
+use patch_db::{DbHandle, LockReceipt, LockTargetId, LockType, PatchDbHandle, Verifier};
+use sqlx::{Executor, Postgres};
 use tracing::instrument;
 
-use super::{PKG_ARCHIVE_DIR, PKG_DOCKER_DIR};
+use super::PKG_ARCHIVE_DIR;
+use crate::config::{not_found, ConfigReceipts};
 use crate::context::RpcContext;
-use crate::db::model::{CurrentDependencyInfo, InstalledPackageDataEntry, PackageDataEntry};
-use crate::dependencies::reconfigure_dependents_with_live_pointers;
+use crate::db::model::{
+    AllPackageData, CurrentDependencies, CurrentDependents, InstalledPackageDataEntry,
+    PackageDataEntry,
+};
+use crate::dependencies::{
+    reconfigure_dependents_with_live_pointers, DependencyErrors, TryHealReceipts,
+};
 use crate::error::ErrorCollection;
 use crate::s9pk::manifest::{Manifest, PackageId};
 use crate::util::{Apply, Version};
+use crate::volume::{asset_dir, script_dir};
 use crate::Error;
 
-#[instrument(skip(ctx, db, deps))]
-pub async fn update_dependency_errors_of_dependents<
-    'a,
-    Db: DbHandle,
-    I: IntoIterator<Item = &'a PackageId>,
->(
+pub struct UpdateDependencyReceipts {
+    try_heal: TryHealReceipts,
+    dependency_errors: LockReceipt<DependencyErrors, String>,
+    manifest: LockReceipt<Manifest, String>,
+}
+impl UpdateDependencyReceipts {
+    pub async fn new<'a>(db: &'a mut impl DbHandle) -> Result<Self, Error> {
+        let mut locks = Vec::new();
+
+        let setup = Self::setup(&mut locks);
+        Ok(setup(&db.lock_all(locks).await?)?)
+    }
+
+    pub fn setup(locks: &mut Vec<LockTargetId>) -> impl FnOnce(&Verifier) -> Result<Self, Error> {
+        let dependency_errors = crate::db::DatabaseModel::new()
+            .package_data()
+            .star()
+            .installed()
+            .map(|x| x.status().dependency_errors())
+            .make_locker(LockType::Write)
+            .add_to_keys(locks);
+        let manifest = crate::db::DatabaseModel::new()
+            .package_data()
+            .star()
+            .installed()
+            .map(|x| x.manifest())
+            .make_locker(LockType::Write)
+            .add_to_keys(locks);
+        let try_heal = TryHealReceipts::setup(locks);
+        move |skeleton_key| {
+            Ok(Self {
+                dependency_errors: dependency_errors.verify(skeleton_key)?,
+                manifest: manifest.verify(skeleton_key)?,
+                try_heal: try_heal(skeleton_key)?,
+            })
+        }
+    }
+}
+
+#[instrument(skip(ctx, db, deps, receipts))]
+pub async fn update_dependency_errors_of_dependents<'a, Db: DbHandle>(
     ctx: &RpcContext,
     db: &mut Db,
     id: &PackageId,
-    deps: I,
+    deps: &CurrentDependents,
+    receipts: &UpdateDependencyReceipts,
 ) -> Result<(), Error> {
-    for dep in deps {
-        if let Some(man) = &*crate::db::DatabaseModel::new()
-            .package_data()
-            .idx_model(&dep)
-            .and_then(|m| m.installed())
-            .map::<_, Manifest>(|m| m.manifest())
-            .get(db, true)
-            .await?
-        {
+    for dep in deps.0.keys() {
+        if let Some(man) = receipts.manifest.get(db, dep).await? {
             if let Err(e) = if let Some(info) = man.dependencies.0.get(id) {
-                info.satisfied(ctx, db, id, None, dep).await?
+                info.satisfied(ctx, db, id, None, dep, &receipts.try_heal)
+                    .await?
             } else {
                 Ok(())
             } {
-                let mut errs = crate::db::DatabaseModel::new()
-                    .package_data()
-                    .idx_model(&dep)
-                    .expect(db)
+                let mut errs = receipts
+                    .dependency_errors
+                    .get(db, dep)
                     .await?
-                    .installed()
-                    .expect(db)
-                    .await?
-                    .status()
-                    .dependency_errors()
-                    .get_mut(db)
-                    .await?;
+                    .ok_or_else(not_found)?;
                 errs.0.insert(id.clone(), e);
-                errs.save(db).await?;
+                receipts.dependency_errors.set(db, errs, dep).await?
             } else {
-                let mut errs = crate::db::DatabaseModel::new()
-                    .package_data()
-                    .idx_model(&dep)
-                    .expect(db)
+                let mut errs = receipts
+                    .dependency_errors
+                    .get(db, dep)
                     .await?
-                    .installed()
-                    .expect(db)
-                    .await?
-                    .status()
-                    .dependency_errors()
-                    .get_mut(db)
-                    .await?;
+                    .ok_or_else(not_found)?;
                 errs.0.remove(id);
-                errs.save(db).await?;
+                receipts.dependency_errors.set(db, errs, dep).await?
             }
         }
     }
@@ -97,10 +121,29 @@ pub async fn cleanup(ctx: &RpcContext, id: &PackageId, version: &Version) -> Res
         .await
         .apply(|res| errors.handle(res));
     errors.extend(
-        futures::future::join_all(images.into_iter().flatten().map(|image| async {
-            let image = image; // move into future
-            ctx.docker.remove_image(&image.id, None, None).await
-        }))
+        futures::future::join_all(
+            images
+                .into_iter()
+                .flatten()
+                .flat_map(|image| image.repo_tags)
+                .filter(|tag| {
+                    tag.starts_with(&format!("start9/{}/", id))
+                        && tag.ends_with(&format!(":{}", version))
+                })
+                .map(|tag| async {
+                    let tag = tag; // move into future
+                    ctx.docker
+                        .remove_image(
+                            &tag,
+                            Some(RemoveImageOptions {
+                                force: true,
+                                noprune: false,
+                            }),
+                            None,
+                        )
+                        .await
+                }),
+        )
         .await,
     );
     let pkg_archive_dir = ctx
@@ -113,13 +156,15 @@ pub async fn cleanup(ctx: &RpcContext, id: &PackageId, version: &Version) -> Res
             .await
             .apply(|res| errors.handle(res));
     }
-    let docker_path = ctx
-        .datadir
-        .join(PKG_DOCKER_DIR)
-        .join(id)
-        .join(version.as_str());
-    if tokio::fs::metadata(&docker_path).await.is_ok() {
-        tokio::fs::remove_dir_all(&docker_path)
+    let assets_path = asset_dir(&ctx.datadir, id, version);
+    if tokio::fs::metadata(&assets_path).await.is_ok() {
+        tokio::fs::remove_dir_all(&assets_path)
+            .await
+            .apply(|res| errors.handle(res));
+    }
+    let scripts_path = script_dir(&ctx.datadir, id, version);
+    if tokio::fs::metadata(&scripts_path).await.is_ok() {
+        tokio::fs::remove_dir_all(&scripts_path)
             .await
             .apply(|res| errors.handle(res));
     }
@@ -127,24 +172,50 @@ pub async fn cleanup(ctx: &RpcContext, id: &PackageId, version: &Version) -> Res
     errors.into_result()
 }
 
-#[instrument(skip(ctx, db))]
+pub struct CleanupFailedReceipts {
+    package_data_entry: LockReceipt<PackageDataEntry, String>,
+    package_entries: LockReceipt<AllPackageData, ()>,
+}
+
+impl CleanupFailedReceipts {
+    pub async fn new<'a>(db: &'a mut impl DbHandle) -> Result<Self, Error> {
+        let mut locks = Vec::new();
+
+        let setup = Self::setup(&mut locks);
+        Ok(setup(&db.lock_all(locks).await?)?)
+    }
+
+    pub fn setup(locks: &mut Vec<LockTargetId>) -> impl FnOnce(&Verifier) -> Result<Self, Error> {
+        let package_data_entry = crate::db::DatabaseModel::new()
+            .package_data()
+            .star()
+            .make_locker(LockType::Write)
+            .add_to_keys(locks);
+        let package_entries = crate::db::DatabaseModel::new()
+            .package_data()
+            .make_locker(LockType::Write)
+            .add_to_keys(locks);
+        move |skeleton_key| {
+            Ok(Self {
+                package_data_entry: package_data_entry.verify(skeleton_key).unwrap(),
+                package_entries: package_entries.verify(skeleton_key).unwrap(),
+            })
+        }
+    }
+}
+
+#[instrument(skip(ctx, db, receipts))]
 pub async fn cleanup_failed<Db: DbHandle>(
     ctx: &RpcContext,
     db: &mut Db,
     id: &PackageId,
+    receipts: &CleanupFailedReceipts,
 ) -> Result<(), Error> {
-    crate::db::DatabaseModel::new()
-        .package_data()
-        .lock(db, LockType::Write)
-        .await?;
-    let pde = crate::db::DatabaseModel::new()
-        .package_data()
-        .idx_model(id)
-        .expect(db)
+    let pde = receipts
+        .package_data_entry
+        .get(db, id)
         .await?
-        .get(db, true)
-        .await?
-        .into_owned();
+        .ok_or_else(not_found)?;
     if let Some(manifest) = match &pde {
         PackageDataEntry::Installing { manifest, .. }
         | PackageDataEntry::Restoring { manifest, .. } => Some(manifest),
@@ -173,26 +244,25 @@ pub async fn cleanup_failed<Db: DbHandle>(
 
     match pde {
         PackageDataEntry::Installing { .. } | PackageDataEntry::Restoring { .. } => {
-            crate::db::DatabaseModel::new()
-                .package_data()
-                .remove(db, id)
-                .await?;
+            let mut entries = receipts.package_entries.get(db).await?;
+            entries.0.remove(id);
+            receipts.package_entries.set(db, entries).await?;
         }
         PackageDataEntry::Updating {
             installed,
             static_files,
             ..
         } => {
-            crate::db::DatabaseModel::new()
-                .package_data()
-                .idx_model(id)
-                .put(
+            receipts
+                .package_data_entry
+                .set(
                     db,
-                    &PackageDataEntry::Installed {
+                    PackageDataEntry::Installed {
                         manifest: installed.manifest.clone(),
                         installed,
                         static_files,
                     },
+                    id,
                 )
                 .await?;
         }
@@ -202,38 +272,74 @@ pub async fn cleanup_failed<Db: DbHandle>(
     Ok(())
 }
 
-#[instrument(skip(db, current_dependencies))]
-pub async fn remove_from_current_dependents_lists<
-    'a,
-    Db: DbHandle,
-    I: IntoIterator<Item = &'a PackageId>,
->(
+#[instrument(skip(db, current_dependencies, current_dependent_receipt))]
+pub async fn remove_from_current_dependents_lists<'a, Db: DbHandle>(
     db: &mut Db,
     id: &'a PackageId,
-    current_dependencies: I,
+    current_dependencies: &'a CurrentDependencies,
+    current_dependent_receipt: &LockReceipt<CurrentDependents, String>,
 ) -> Result<(), Error> {
-    for dep in current_dependencies.into_iter().chain(std::iter::once(id)) {
-        if let Some(current_dependents) = crate::db::DatabaseModel::new()
-            .package_data()
-            .idx_model(dep)
-            .and_then(|m| m.installed())
-            .map::<_, BTreeMap<PackageId, CurrentDependencyInfo>>(|m| m.current_dependents())
-            .check(db)
-            .await?
-        {
-            if current_dependents
-                .clone()
-                .idx_model(id)
-                .exists(db, true)
-                .await?
-            {
-                current_dependents.remove(db, id).await?
+    for dep in current_dependencies.0.keys().chain(std::iter::once(id)) {
+        if let Some(mut current_dependents) = current_dependent_receipt.get(db, dep).await? {
+            if current_dependents.0.remove(id).is_some() {
+                current_dependent_receipt
+                    .set(db, current_dependents, dep)
+                    .await?;
             }
         }
     }
     Ok(())
 }
+pub struct UninstallReceipts {
+    config: ConfigReceipts,
+    removing: LockReceipt<InstalledPackageDataEntry, ()>,
+    packages: LockReceipt<AllPackageData, ()>,
+    current_dependents: LockReceipt<CurrentDependents, String>,
+    update_depenency_receipts: UpdateDependencyReceipts,
+}
+impl UninstallReceipts {
+    pub async fn new<'a>(db: &'a mut impl DbHandle, id: &PackageId) -> Result<Self, Error> {
+        let mut locks = Vec::new();
 
+        let setup = Self::setup(&mut locks, id);
+        Ok(setup(&db.lock_all(locks).await?)?)
+    }
+
+    pub fn setup(
+        locks: &mut Vec<LockTargetId>,
+        id: &PackageId,
+    ) -> impl FnOnce(&Verifier) -> Result<Self, Error> {
+        let config = ConfigReceipts::setup(locks);
+        let removing = crate::db::DatabaseModel::new()
+            .package_data()
+            .idx_model(id)
+            .and_then(|pde| pde.removing())
+            .make_locker(LockType::Write)
+            .add_to_keys(locks);
+
+        let current_dependents = crate::db::DatabaseModel::new()
+            .package_data()
+            .star()
+            .installed()
+            .map(|x| x.current_dependents())
+            .make_locker(LockType::Write)
+            .add_to_keys(locks);
+        let packages = crate::db::DatabaseModel::new()
+            .package_data()
+            .make_locker(LockType::Write)
+            .add_to_keys(locks);
+        let update_depenency_receipts = UpdateDependencyReceipts::setup(locks);
+        move |skeleton_key| {
+            Ok(Self {
+                config: config(skeleton_key)?,
+                removing: removing.verify(skeleton_key)?,
+                current_dependents: current_dependents.verify(skeleton_key)?,
+                update_depenency_receipts: update_depenency_receipts(skeleton_key)?,
+                packages: packages.verify(skeleton_key)?,
+            })
+        }
+    }
+}
 #[instrument(skip(ctx, secrets, db))]
 pub async fn uninstall<Ex>(
     ctx: &RpcContext,
@@ -242,69 +348,109 @@ pub async fn uninstall<Ex>(
     id: &PackageId,
 ) -> Result<(), Error>
 where
-    for<'a> &'a mut Ex: Executor<'a, Database = Sqlite>,
+    for<'a> &'a mut Ex: Executor<'a, Database = Postgres>,
 {
     let mut tx = db.begin().await?;
     crate::db::DatabaseModel::new()
         .package_data()
+        .idx_model(&id)
         .lock(&mut tx, LockType::Write)
         .await?;
-    let entry = crate::db::DatabaseModel::new()
-        .package_data()
-        .idx_model(id)
-        .and_then(|pde| pde.removing())
-        .get(&mut tx, true)
-        .await?
-        .into_owned()
-        .ok_or_else(|| {
-            Error::new(
-                eyre!("Package not in removing state: {}", id),
-                crate::ErrorKind::NotFound,
-            )
-        })?;
+    let receipts = UninstallReceipts::new(&mut tx, id).await?;
+    let entry = receipts.removing.get(&mut tx).await?;
     cleanup(ctx, &entry.manifest.id, &entry.manifest.version).await?;
 
-    crate::db::DatabaseModel::new()
-        .package_data()
-        .remove(&mut tx, id)
-        .await?;
-
+    let packages = {
+        let mut packages = receipts.packages.get(&mut tx).await?;
+        packages.0.remove(id);
+        packages
+    };
+    let dependents_paths: Vec<PathBuf> = entry
+        .current_dependents
+        .0
+        .keys()
+        .flat_map(|x| packages.0.get(x))
+        .flat_map(|x| x.manifest_borrow().volumes.values())
+        .flat_map(|x| x.pointer_path(&ctx.datadir))
+        .collect();
+    receipts.packages.set(&mut tx, packages).await?;
     // once we have removed the package entry, we can change all the dependent pointers to null
-    reconfigure_dependents_with_live_pointers(ctx, &mut tx, &entry).await?;
+    reconfigure_dependents_with_live_pointers(ctx, &mut tx, &receipts.config, &entry).await?;
 
     remove_from_current_dependents_lists(
         &mut tx,
         &entry.manifest.id,
-        entry.current_dependencies.keys(),
+        &entry.current_dependencies,
+        &receipts.current_dependents,
     )
     .await?;
     update_dependency_errors_of_dependents(
         ctx,
         &mut tx,
         &entry.manifest.id,
-        entry.current_dependents.keys(),
+        &entry.current_dependents,
+        &receipts.update_depenency_receipts,
     )
     .await?;
     let volumes = ctx
         .datadir
         .join(crate::volume::PKG_VOLUME_DIR)
         .join(&entry.manifest.id);
-    if tokio::fs::metadata(&volumes).await.is_ok() {
-        tokio::fs::remove_dir_all(&volumes).await?;
-    }
-    tx.commit(None).await?;
+
+    tracing::debug!("Cleaning up {:?} at {:?}", volumes, dependents_paths);
+    cleanup_folder(volumes, Arc::new(dependents_paths)).await;
     remove_tor_keys(secrets, &entry.manifest.id).await?;
+    tx.commit().await?;
     Ok(())
 }
 
 #[instrument(skip(secrets))]
 pub async fn remove_tor_keys<Ex>(secrets: &mut Ex, id: &PackageId) -> Result<(), Error>
 where
-    for<'a> &'a mut Ex: Executor<'a, Database = Sqlite>,
+    for<'a> &'a mut Ex: Executor<'a, Database = Postgres>,
 {
     let id_str = id.as_str();
-    sqlx::query!("DELETE FROM tor WHERE package = ?", id_str)
+    sqlx::query!("DELETE FROM tor WHERE package = $1", id_str)
         .execute(secrets)
         .await?;
     Ok(())
 }
+
+/// Needed to remove, without removing the folders that are mounted in the other docker containers
+pub fn cleanup_folder(
+    path: PathBuf,
+    dependents_volumes: Arc<Vec<PathBuf>>,
+) -> futures::future::BoxFuture<'static, ()> {
+    Box::pin(async move {
+        let meta_data = match tokio::fs::metadata(&path).await {
+            Ok(a) => a,
+            Err(_e) => {
+                return;
+            }
+        };
+        if !meta_data.is_dir() {
+            tracing::error!("is_not dir, remove {:?}", path);
+            let _ = tokio::fs::remove_file(&path).await;
+            return;
+        }
+        if !dependents_volumes
+            .iter()
+            .any(|v| v.starts_with(&path) || v == &path)
+        {
+            tracing::error!("No parents, remove {:?}", path);
+            let _ = tokio::fs::remove_dir_all(&path).await;
+            return;
+        }
+        let mut read_dir = match tokio::fs::read_dir(&path).await {
+            Ok(a) => a,
+            Err(_e) => {
+                return;
+            }
+        };
+        tracing::error!("Parents, recurse {:?}", path);
+        while let Some(entry) = read_dir.next_entry().await.ok().flatten() {
+            let entry_path = entry.path();
+            cleanup_folder(entry_path, dependents_volumes.clone()).await;
+        }
+    })
+}

backend/src/install/mod.rs

@@ -1,23 +1,23 @@
-use std::collections::{BTreeMap, BTreeSet};
+use std::collections::BTreeMap;
 use std::io::SeekFrom;
 use std::marker::PhantomData;
 use std::path::{Path, PathBuf};
 use std::process::Stdio;
 use std::sync::atomic::Ordering;
 use std::sync::Arc;
-use std::time::{Duration, Instant};
+use std::time::Duration;
 
 use color_eyre::eyre::eyre;
 use emver::VersionRange;
-use futures::future::{self, BoxFuture};
-use futures::{stream, FutureExt, StreamExt, TryStreamExt};
+use futures::future::BoxFuture;
+use futures::{FutureExt, StreamExt, TryStreamExt};
 use http::header::CONTENT_LENGTH;
 use http::{Request, Response, StatusCode};
 use hyper::Body;
 use patch_db::{DbHandle, LockType};
 use reqwest::Url;
+use rpc_toolkit::command;
 use rpc_toolkit::yajrc::RpcError;
-use rpc_toolkit::{command, Context};
 use tokio::fs::{File, OpenOptions};
 use tokio::io::{AsyncRead, AsyncSeek, AsyncSeekExt};
 use tokio::process::Command;
@@ -25,16 +25,17 @@ use tokio_stream::wrappers::ReadDirStream;
 use tracing::instrument;
 
 use self::cleanup::{cleanup_failed, remove_from_current_dependents_lists};
+use crate::config::ConfigReceipts;
 use crate::context::{CliContext, RpcContext};
 use crate::core::rpc_continuations::{RequestGuid, RpcContinuation};
 use crate::db::model::{
-    CurrentDependencyInfo, InstalledPackageDataEntry, PackageDataEntry, RecoveredPackageInfo,
-    StaticDependencyInfo, StaticFiles,
+    CurrentDependencies, CurrentDependencyInfo, CurrentDependents, InstalledPackageDataEntry,
+    PackageDataEntry, StaticDependencyInfo, StaticFiles,
 };
-use crate::db::util::WithRevision;
 use crate::dependencies::{
     add_dependent_to_current_dependents_lists, break_all_dependents_transitive,
-    reconfigure_dependents_with_live_pointers, BreakageRes, DependencyError, DependencyErrors,
+    reconfigure_dependents_with_live_pointers, BreakTransitiveReceipts, BreakageRes,
+    DependencyError, DependencyErrors,
 };
 use crate::install::cleanup::{cleanup, update_dependency_errors_of_dependents};
 use crate::install::progress::{InstallProgress, InstallProgressTracker};
@@ -43,27 +44,26 @@ use crate::s9pk::manifest::{Manifest, PackageId};
 use crate::s9pk::reader::S9pkReader;
 use crate::status::{MainStatus, Status};
 use crate::util::io::{copy_and_shutdown, response_to_reader};
-use crate::util::serde::{display_serializable, IoFormat, Port};
+use crate::util::serde::{display_serializable, Port};
 use crate::util::{display_none, AsyncFileExt, Version};
 use crate::version::{Current, VersionT};
-use crate::volume::asset_dir;
+use crate::volume::{asset_dir, script_dir};
 use crate::{Error, ErrorKind, ResultExt};
 
 pub mod cleanup;
 pub mod progress;
 pub mod update;
 
-pub const PKG_ARCHIVE_DIR: &'static str = "package-data/archive";
-pub const PKG_PUBLIC_DIR: &'static str = "package-data/public";
-pub const PKG_DOCKER_DIR: &'static str = "package-data/docker";
-pub const PKG_WASM_DIR: &'static str = "package-data/wasm";
+pub const PKG_ARCHIVE_DIR: &str = "package-data/archive";
+pub const PKG_PUBLIC_DIR: &str = "package-data/public";
+pub const PKG_WASM_DIR: &str = "package-data/wasm";
 
 #[command(display(display_serializable))]
 pub async fn list(#[context] ctx: RpcContext) -> Result<Vec<(PackageId, Version)>, Error> {
     let mut hdl = ctx.db.handle();
     let package_data = crate::db::DatabaseModel::new()
         .package_data()
-        .get(&mut hdl, true)
+        .get(&mut hdl)
         .await?;
 
     Ok(package_data
@@ -113,19 +113,20 @@ impl std::fmt::Display for MinMax {
 
 #[command(
     custom_cli(cli_install(async, context(CliContext))),
-    display(display_none)
+    display(display_none),
+    metadata(sync_db = true)
 )]
 #[instrument(skip(ctx))]
 pub async fn install(
     #[context] ctx: RpcContext,
     #[arg] id: String,
-    #[arg(short = "m", long = "marketplace-url", rename = "marketplace-url")]
+    #[arg(short = 'm', long = "marketplace-url", rename = "marketplace-url")]
     marketplace_url: Option<Url>,
-    #[arg(short = "v", long = "version-spec", rename = "version-spec")] version_spec: Option<
+    #[arg(short = 'v', long = "version-spec", rename = "version-spec")] version_spec: Option<
         String,
     >,
     #[arg(long = "version-priority", rename = "version-priority")] version_priority: Option<MinMax>,
-) -> Result<WithRevision<()>, Error> {
+) -> Result<(), Error> {
     let version_str = match &version_spec {
         None => "*",
         Some(v) => &*v,
@@ -141,7 +142,7 @@ pub async fn install(
         version,
         version_priority,
         Current::new().compat(),
-        platforms::TARGET_ARCH,
+        &*crate::ARCH,
     ))
     .await
     .with_kind(crate::ErrorKind::Registry)?
@@ -157,7 +158,7 @@ pub async fn install(
         man.version,
         version_priority,
         Current::new().compat(),
-        platforms::TARGET_ARCH,
+        &*crate::ARCH,
     ))
     .await
     .with_kind(crate::ErrorKind::Registry)?
@@ -189,9 +190,10 @@ pub async fn install(
                         id,
                         man.version,
                         Current::new().compat(),
-                        platforms::TARGET_ARCH,
+                        &*crate::ARCH,
                     ))
-                    .await?,
+                    .await?
+                    .error_for_status()?,
                 ),
                 &mut File::create(public_dir_path.join("LICENSE.md")).await?,
             )
@@ -207,9 +209,10 @@ pub async fn install(
                         id,
                         man.version,
                         Current::new().compat(),
-                        platforms::TARGET_ARCH,
+                        &*crate::ARCH,
                     ))
-                    .await?,
+                    .await?
+                    .error_for_status()?,
                 ),
                 &mut File::create(public_dir_path.join("INSTRUCTIONS.md")).await?,
             )
@@ -225,9 +228,10 @@ pub async fn install(
                         id,
                         man.version,
                         Current::new().compat(),
-                        platforms::TARGET_ARCH,
+                        &*crate::ARCH,
                     ))
-                    .await?,
+                    .await?
+                    .error_for_status()?,
                 ),
                 &mut File::create(public_dir_path.join(format!("icon.{}", icon_type))).await?,
             )
@@ -282,7 +286,7 @@ pub async fn install(
         }
     }
     pde.save(&mut tx).await?;
-    let res = tx.commit(None).await?;
+    tx.commit().await?;
     drop(db_handle);
 
     tokio::spawn(async move {
@@ -318,10 +322,7 @@ pub async fn install(
         }
     });
 
-    Ok(WithRevision {
-        revision: res,
-        response: (),
-    })
+    Ok(())
 }
 
 #[command(rpc_only, display(display_none))]
@@ -329,9 +330,37 @@ pub async fn install(
 pub async fn sideload(
     #[context] ctx: RpcContext,
     #[arg] manifest: Manifest,
+    #[arg] icon: Option<String>,
 ) -> Result<RequestGuid, Error> {
     let new_ctx = ctx.clone();
     let guid = RequestGuid::new();
+    if let Some(icon) = icon {
+        use tokio::io::AsyncWriteExt;
+
+        let public_dir_path = ctx
+            .datadir
+            .join(PKG_PUBLIC_DIR)
+            .join(&manifest.id)
+            .join(manifest.version.as_str());
+        tokio::fs::create_dir_all(&public_dir_path).await?;
+
+        let invalid_data_url =
+            || Error::new(eyre!("Invalid Icon Data URL"), ErrorKind::InvalidRequest);
+        let data = icon
+            .strip_prefix(&format!(
+                "data:image/{};base64,",
+                manifest.assets.icon_type()
+            ))
+            .ok_or_else(&invalid_data_url)?;
+        let mut icon_file =
+            File::create(public_dir_path.join(format!("icon.{}", manifest.assets.icon_type())))
+                .await?;
+        icon_file
+            .write_all(&base64::decode(data).with_kind(ErrorKind::InvalidRequest)?)
+            .await?;
+        icon_file.sync_all().await?;
+    }
+
     let handler = Box::new(|req: Request<Body>| {
         async move {
             let content_length = match req.headers().get(CONTENT_LENGTH).map(|a| a.to_str()) {
@@ -365,13 +394,13 @@ pub async fn sideload(
             match pde.take() {
                 Some(PackageDataEntry::Installed {
                     installed,
-                    manifest,
                     static_files,
+                    ..
                 }) => {
                     *pde = Some(PackageDataEntry::Updating {
                         install_progress: progress.clone(),
                         installed,
-                        manifest,
+                        manifest: manifest.clone(),
                         static_files,
                     })
                 }
@@ -394,7 +423,7 @@ pub async fn sideload(
                 }
             }
             pde.save(&mut tx).await?;
-            tx.commit(None).await?;
+            tx.commit().await?;
 
             if let Err(e) = download_install_s9pk(
                 &new_ctx,
@@ -445,23 +474,11 @@ pub async fn sideload(
         }
         .boxed()
     });
-    let cont = RpcContinuation {
-        created_at: Instant::now(), // TODO
-        handler: handler,
-    };
-    // gc the map
-    let mut guard = ctx.rpc_stream_continuations.lock().await;
-    let garbage_collected = std::mem::take(&mut *guard)
-        .into_iter()
-        .filter(|(_, v)| v.created_at.elapsed() < Duration::from_secs(30))
-        .collect::<BTreeMap<RequestGuid, RpcContinuation>>();
-    *guard = garbage_collected;
-    drop(guard);
-    // insert the new continuation
-    ctx.rpc_stream_continuations
-        .lock()
-        .await
-        .insert(guid.clone(), cont);
+    ctx.add_continuation(
+        guid.clone(),
+        RpcContinuation::rest(handler, Duration::from_secs(30)),
+    )
+    .await;
     Ok(guid)
 }
 
@@ -477,14 +494,21 @@ async fn cli_install(
         let path = PathBuf::from(target);
 
         // inspect manifest no verify
-        let manifest = crate::inspect::manifest(path.clone(), true, Some(IoFormat::Json)).await?;
+        let mut reader = S9pkReader::open(&path, false).await?;
+        let manifest = reader.manifest().await?;
+        let icon = reader.icon().await?.to_vec().await?;
+        let icon_str = format!(
+            "data:image/{};base64,{}",
+            manifest.assets.icon_type(),
+            base64::encode(&icon)
+        );
 
         // rpc call remote sideload
         tracing::debug!("calling package.sideload");
         let guid = rpc_toolkit::command_helpers::call_remote(
             ctx.clone(),
             "package.sideload",
-            serde_json::json!({ "manifest": manifest }),
+            serde_json::json!({ "manifest": manifest, "icon": icon_str }),
             PhantomData::<RequestGuid>,
         )
         .await?
@@ -495,14 +519,9 @@ async fn cli_install(
         let file = tokio::fs::File::open(path).await?;
         let content_length = file.metadata().await?.len();
         let body = Body::wrap_stream(tokio_util::io::ReaderStream::new(file));
-        let client = reqwest::Client::new();
-        let res = client
-            .post(format!(
-                "{}://{}/rest/rpc/{}",
-                ctx.protocol(),
-                ctx.host(),
-                guid
-            ))
+        let res = ctx
+            .client
+            .post(format!("{}rest/rpc/{}", ctx.base_url, guid,))
             .header(CONTENT_LENGTH, content_length)
             .body(body)
             .send()
@@ -536,7 +555,7 @@ async fn cli_install(
             ctx,
             "package.install",
             params,
-            PhantomData::<WithRevision<()>>,
+            PhantomData::<()>,
         )
         .await?
         .result?;
@@ -547,7 +566,8 @@ async fn cli_install(
 
 #[command(
     subcommands(self(uninstall_impl(async)), uninstall_dry),
-    display(display_none)
+    display(display_none),
+    metadata(sync_db = true)
 )]
 pub async fn uninstall(#[arg] id: PackageId) -> Result<PackageId, Error> {
     Ok(id)
@@ -562,8 +582,15 @@ pub async fn uninstall_dry(
     let mut db = ctx.db.handle();
     let mut tx = db.begin().await?;
     let mut breakages = BTreeMap::new();
-    break_all_dependents_transitive(&mut tx, &id, DependencyError::NotInstalled, &mut breakages)
-        .await?;
+    let receipts = BreakTransitiveReceipts::new(&mut tx).await?;
+    break_all_dependents_transitive(
+        &mut tx,
+        &id,
+        DependencyError::NotInstalled,
+        &mut breakages,
+        &receipts,
+    )
+    .await?;
 
     tx.abort().await?;
 
@@ -571,9 +598,14 @@ pub async fn uninstall_dry(
 }
 
 #[instrument(skip(ctx))]
-pub async fn uninstall_impl(ctx: RpcContext, id: PackageId) -> Result<WithRevision<()>, Error> {
+pub async fn uninstall_impl(ctx: RpcContext, id: PackageId) -> Result<(), Error> {
     let mut handle = ctx.db.handle();
     let mut tx = handle.begin().await?;
+    crate::db::DatabaseModel::new()
+        .package_data()
+        .idx_model(&id)
+        .lock(&mut tx, LockType::Write)
+        .await?;
 
     let mut pde = crate::db::DatabaseModel::new()
         .package_data()
@@ -599,7 +631,7 @@ pub async fn uninstall_impl(ctx: RpcContext, id: PackageId) -> Result<WithRevisi
         removing: installed,
     });
     pde.save(&mut tx).await?;
-    let res = tx.commit(None).await?;
+    tx.commit().await?;
     drop(handle);
 
     tokio::spawn(async move {
@@ -636,46 +668,36 @@ pub async fn uninstall_impl(ctx: RpcContext, id: PackageId) -> Result<WithRevisi
         }
     });
 
-    Ok(WithRevision {
-        revision: res,
-        response: (),
-    })
+    Ok(())
 }
 
-#[command(rename = "delete-recovered", display(display_none))]
-pub async fn delete_recovered(
-    #[context] ctx: RpcContext,
-    #[arg] id: PackageId,
-) -> Result<WithRevision<()>, Error> {
-    let mut handle = ctx.db.handle();
-    let mut tx = handle.begin().await?;
-    let mut sql_tx = ctx.secret_store.begin().await?;
+pub struct DownloadInstallReceipts {
+    package_receipts: crate::db::package::PackageReceipts,
+    manifest_receipts: crate::db::package::ManifestReceipts,
+}
 
-    let mut recovered_packages = crate::db::DatabaseModel::new()
-        .recovered_packages()
-        .get_mut(&mut tx)
-        .await?;
-    recovered_packages.remove(&id).ok_or_else(|| {
-        Error::new(
-            eyre!("{} not in recovered-packages", id),
-            crate::ErrorKind::NotFound,
-        )
-    })?;
-    recovered_packages.save(&mut tx).await?;
+impl DownloadInstallReceipts {
+    pub async fn new<'a>(db: &'a mut impl DbHandle, id: &PackageId) -> Result<Self, Error> {
+        let mut locks = Vec::new();
 
-    let volumes = ctx.datadir.join(crate::volume::PKG_VOLUME_DIR).join(&id);
-    if tokio::fs::metadata(&volumes).await.is_ok() {
-        tokio::fs::remove_dir_all(&volumes).await?;
+        let setup = Self::setup(&mut locks, id);
+        Ok(setup(&db.lock_all(locks).await?)?)
     }
-    cleanup::remove_tor_keys(&mut sql_tx, &id).await?;
 
-    let res = tx.commit(None).await?;
-    sql_tx.commit().await?;
+    pub fn setup(
+        locks: &mut Vec<patch_db::LockTargetId>,
+        id: &PackageId,
+    ) -> impl FnOnce(&patch_db::Verifier) -> Result<Self, Error> {
+        let package_receipts = crate::db::package::PackageReceipts::setup(locks);
+        let manifest_receipts = crate::db::package::ManifestReceipts::setup(locks, id);
 
-    Ok(WithRevision {
-        revision: res,
-        response: (),
-    })
+        move |skeleton_key| {
+            Ok(Self {
+                package_receipts: package_receipts(skeleton_key)?,
+                manifest_receipts: manifest_receipts(skeleton_key)?,
+            })
+        }
+    }
 }
 
 #[instrument(skip(ctx, temp_manifest, s9pk))]
@@ -688,18 +710,31 @@ pub async fn download_install_s9pk(
 ) -> Result<(), Error> {
     let pkg_id = &temp_manifest.id;
     let version = &temp_manifest.version;
+    let mut previous_state: Option<MainStatus> = None;
 
     if let Err(e) = async {
+        if crate::db::DatabaseModel::new()
+            .package_data()
+            .idx_model(&pkg_id)
+            .and_then(|x| x.installed())
+            .exists(&mut ctx.db.handle())
+            .await
+            .unwrap_or(false)
+        {
+            previous_state = crate::control::stop_impl(ctx.clone(), pkg_id.clone())
+                .await
+                .ok();
+        }
         let mut db_handle = ctx.db.handle();
         let mut tx = db_handle.begin().await?;
+        let receipts = DownloadInstallReceipts::new(&mut tx, &pkg_id).await?;
         // Build set of existing manifests
         let mut manifests = Vec::new();
-        for pkg in crate::db::package::get_packages(&mut tx).await? {
-            match crate::db::package::get_manifest(&mut tx, &pkg).await? {
-                Some(m) => {
-                    manifests.push(m);
-                }
-                None => {}
+        for pkg in crate::db::package::get_packages(&mut tx, &receipts.package_receipts).await? {
+            if let Some(m) =
+                crate::db::package::get_manifest(&mut tx, &pkg, &receipts.manifest_receipts).await?
+            {
+                manifests.push(m);
             }
         }
         // Build map of current port -> ssl mappings
@@ -712,7 +747,7 @@ pub async fn download_install_s9pk(
                 for (p, lan) in cfg {
                     if p.0 == 80 && lan.ssl || p.0 == 443 && !lan.ssl {
                         return Err(Error::new(
-                            eyre!("SSL Conflict with EmbassyOS"),
+                            eyre!("SSL Conflict with embassyOS"),
                             ErrorKind::LanPortConflict,
                         ));
                     }
@@ -732,6 +767,7 @@ pub async fn download_install_s9pk(
                 }
             }
         }
+        drop(receipts);
         tx.save().await?;
         drop(db_handle);
 
@@ -790,23 +826,55 @@ pub async fn download_install_s9pk(
     }
     .await
     {
+        if previous_state.map(|x| x.running()).unwrap_or(false) {
+            crate::control::start(ctx.clone(), pkg_id.clone()).await?;
+        }
+
         let mut handle = ctx.db.handle();
         let mut tx = handle.begin().await?;
+        let receipts = cleanup::CleanupFailedReceipts::new(&mut tx).await?;
 
-        if let Err(e) = cleanup_failed(&ctx, &mut tx, pkg_id).await {
+        if let Err(e) = cleanup_failed(&ctx, &mut tx, pkg_id, &receipts).await {
             tracing::error!("Failed to clean up {}@{}: {}", pkg_id, version, e);
             tracing::debug!("{:?}", e);
         } else {
-            tx.commit(None).await?;
+            tx.commit().await?;
         }
         Err(e)
     } else {
+        if previous_state.map(|x| x.running()).unwrap_or(false) {
+            crate::control::start(ctx.clone(), pkg_id.clone()).await?;
+        }
         Ok(())
     }
 }
 
+pub struct InstallS9Receipts {
+    config: ConfigReceipts,
+}
+
+impl InstallS9Receipts {
+    pub async fn new<'a>(db: &'a mut impl DbHandle) -> Result<Self, Error> {
+        let mut locks = Vec::new();
+
+        let setup = Self::setup(&mut locks);
+        Ok(setup(&db.lock_all(locks).await?)?)
+    }
+
+    pub fn setup(
+        locks: &mut Vec<patch_db::LockTargetId>,
+    ) -> impl FnOnce(&patch_db::Verifier) -> Result<Self, Error> {
+        let config = ConfigReceipts::setup(locks);
+        move |skeleton_key| {
+            Ok(Self {
+                config: config(skeleton_key)?,
+            })
+        }
+    }
+}
+
 #[instrument(skip(ctx, rdr))]
-pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin>(
+pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin + Send + Sync>(
     ctx: &RpcContext,
     pkg_id: &PackageId,
     version: &Version,
@@ -836,7 +904,7 @@ pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin>(
             .package_data()
             .idx_model(dep)
             .map::<_, Manifest>(|pde| pde.manifest())
-            .get(&mut ctx.db.handle(), false)
+            .get(&mut ctx.db.handle())
             .await?
             .into_owned()
         {
@@ -848,7 +916,7 @@ pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin>(
                 dep,
                 info.version,
                 Current::new().compat(),
-                platforms::TARGET_ARCH,
+                &*crate::ARCH,
             ))
             .await
             .with_kind(crate::ErrorKind::Registry)?
@@ -883,7 +951,7 @@ pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin>(
                         dep,
                         info.version,
                         Current::new().compat(),
-                        platforms::TARGET_ARCH,
+                        &*crate::ARCH,
                     ))
                     .await
                     .with_kind(crate::ErrorKind::Registry)?;
@@ -970,44 +1038,11 @@ pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin>(
     tracing::info!("Install {}@{}: Unpacking Docker Images", pkg_id, version);
     progress
         .track_read_during(progress_model.clone(), &ctx.db, || async {
-            let image_tar_dir = ctx
-                .datadir
-                .join(PKG_DOCKER_DIR)
-                .join(pkg_id)
-                .join(version.as_str());
-            if tokio::fs::metadata(&image_tar_dir).await.is_err() {
-                tokio::fs::create_dir_all(&image_tar_dir)
-                    .await
-                    .with_ctx(|_| {
-                        (
-                            crate::ErrorKind::Filesystem,
-                            image_tar_dir.display().to_string(),
-                        )
-                    })?;
-            }
-            let image_tar_path = image_tar_dir.join("image.tar");
-            let mut tee = Command::new("tee")
-                .arg(&image_tar_path)
-                .stdin(Stdio::piped())
-                .stdout(Stdio::piped())
-                .spawn()?;
             let mut load = Command::new("docker")
                 .arg("load")
                 .stdin(Stdio::piped())
                 .stderr(Stdio::piped())
                 .spawn()?;
-            let tee_in = tee.stdin.take().ok_or_else(|| {
-                Error::new(
-                    eyre!("Could not write to stdin of tee"),
-                    crate::ErrorKind::Docker,
-                )
-            })?;
-            let mut tee_out = tee.stdout.take().ok_or_else(|| {
-                Error::new(
-                    eyre!("Could not read from stdout of tee"),
-                    crate::ErrorKind::Docker,
-                )
-            })?;
             let load_in = load.stdin.take().ok_or_else(|| {
                 Error::new(
                     eyre!("Could not write to stdin of docker load"),
@@ -1015,10 +1050,7 @@ pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin>(
                 )
             })?;
             let mut docker_rdr = rdr.docker_images().await?;
-            tokio::try_join!(
-                copy_and_shutdown(&mut docker_rdr, tee_in),
-                copy_and_shutdown(&mut tee_out, load_in),
-            )?;
+            copy_and_shutdown(&mut docker_rdr, load_in).await?;
             let res = load.wait_with_output().await?;
             if !res.status.success() {
                 Err(Error::new(
@@ -1046,6 +1078,18 @@ pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin>(
             let mut tar = tokio_tar::Archive::new(rdr.assets().await?);
             tar.unpack(asset_dir).await?;
 
+            let script_dir = script_dir(&ctx.datadir, pkg_id, version);
+            if tokio::fs::metadata(&script_dir).await.is_err() {
+                tokio::fs::create_dir_all(&script_dir).await?;
+            }
+            if let Some(mut hdl) = rdr.scripts().await? {
+                tokio::io::copy(
+                    &mut hdl,
+                    &mut File::create(script_dir.join("embassy.js")).await?,
+                )
+                .await?;
+            }
+
             Ok(())
         })
         .await?;
@@ -1082,23 +1126,25 @@ pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin>(
     tracing::info!("Install {}@{}: Created manager", pkg_id, version);
 
     let static_files = StaticFiles::local(pkg_id, version, manifest.assets.icon_type());
-    let current_dependencies: BTreeMap<_, _> = manifest
-        .dependencies
-        .0
-        .iter()
-        .filter_map(|(id, info)| {
-            if info.requirement.required() {
-                Some((id.clone(), CurrentDependencyInfo::default()))
-            } else {
-                None
-            }
-        })
-        .collect();
+    let current_dependencies: CurrentDependencies = CurrentDependencies(
+        manifest
+            .dependencies
+            .0
+            .iter()
+            .filter_map(|(id, info)| {
+                if info.requirement.required() {
+                    Some((id.clone(), CurrentDependencyInfo::default()))
+                } else {
+                    None
+                }
+            })
+            .collect(),
+    );
     let current_dependents = {
         let mut deps = BTreeMap::new();
         for package in crate::db::DatabaseModel::new()
             .package_data()
-            .keys(&mut tx, true)
+            .keys(&mut tx)
             .await?
         {
             // update dependency_info on dependents
@@ -1132,14 +1178,14 @@ pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin>(
                 .await?
                 .installed()
                 .and_then(|i| i.current_dependencies().idx_model(pkg_id))
-                .get(&mut tx, true)
+                .get(&mut tx)
                 .await?
                 .to_owned()
             {
                 deps.insert(package, dep);
             }
         }
-        deps
+        CurrentDependents(deps)
     };
     let mut pde = model
         .clone()
@@ -1183,6 +1229,8 @@ pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin>(
         },
     );
     pde.save(&mut tx).await?;
+    let receipts = InstallS9Receipts::new(&mut tx).await?;
+    // UpdateDependencyReceipts
     let mut dep_errs = model
         .expect(&mut tx)
         .await?
@@ -1193,38 +1241,45 @@ pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin>(
         .dependency_errors()
         .get_mut(&mut tx)
         .await?;
-    *dep_errs = DependencyErrors::init(ctx, &mut tx, &manifest, &current_dependencies).await?;
+    *dep_errs = DependencyErrors::init(
+        ctx,
+        &mut tx,
+        &manifest,
+        &current_dependencies,
+        &receipts.config.try_heal_receipts,
+    )
+    .await?;
     dep_errs.save(&mut tx).await?;
 
     if let PackageDataEntry::Updating {
-        installed: prev,
-        manifest: prev_manifest,
-        ..
+        installed: prev, ..
     } = prev
     {
         let prev_is_configured = prev.status.configured;
-        let prev_migration = prev_manifest
+        let prev_migration = prev
+            .manifest
             .migrations
             .to(
                 ctx,
                 version,
                 pkg_id,
-                &prev_manifest.version,
-                &prev_manifest.volumes,
+                &prev.manifest.version,
+                &prev.manifest.volumes,
             )
             .map(futures::future::Either::Left);
         let migration = manifest
             .migrations
             .from(
+                &manifest.containers,
                 ctx,
-                &prev_manifest.version,
+                &prev.manifest.version,
                 pkg_id,
                 version,
                 &manifest.volumes,
             )
             .map(futures::future::Either::Right);
 
-        let viable_migration = if prev_manifest.version > manifest.version {
+        let viable_migration = if prev.manifest.version > manifest.version {
             prev_migration.or(migration)
         } else {
             migration.or(prev_migration)
@@ -1245,8 +1300,26 @@ pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin>(
                 false,
                 &mut BTreeMap::new(),
                 &mut BTreeMap::new(),
+                &receipts.config,
             )
             .await?;
+        } else {
+            remove_from_current_dependents_lists(
+                &mut tx,
+                pkg_id,
+                &prev.current_dependencies,
+                &receipts.config.current_dependents,
+            )
+            .await?; // remove previous
+            add_dependent_to_current_dependents_lists(
+                &mut tx,
+                pkg_id,
+                &current_dependencies,
+                &receipts.config.current_dependents,
+            )
+            .await?; // add new
+        }
+        if configured || manifest.config.is_none() {
             let mut main_status = crate::db::DatabaseModel::new()
                 .package_data()
                 .idx_model(pkg_id)
@@ -1262,17 +1335,16 @@ pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin>(
             *main_status = prev.status.main;
             main_status.save(&mut tx).await?;
         }
-        remove_from_current_dependents_lists(&mut tx, pkg_id, prev.current_dependencies.keys())
-            .await?; // remove previous
-        add_dependent_to_current_dependents_lists(&mut tx, pkg_id, &current_dependencies).await?; // add new
         update_dependency_errors_of_dependents(
             ctx,
             &mut tx,
             pkg_id,
-            current_dependents
-                .keys()
-                .chain(prev.current_dependents.keys())
-                .collect::<BTreeSet<_>>(),
+            &CurrentDependents({
+                let mut current_dependents = current_dependents.0.clone();
+                current_dependents.append(&mut prev.current_dependents.0.clone());
+                current_dependents
+            }),
+            &receipts.config.update_dependency_receipts,
         )
         .await?;
         if &prev.manifest.version != version {
@@ -1284,88 +1356,55 @@ pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin>(
             .restore(
                 ctx,
                 &mut tx,
-                &mut sql_tx,
                 pkg_id,
                 version,
                 &manifest.interfaces,
                 &manifest.volumes,
             )
             .await?;
-        add_dependent_to_current_dependents_lists(&mut tx, pkg_id, &current_dependencies).await?;
-        update_dependency_errors_of_dependents(ctx, &mut tx, pkg_id, current_dependents.keys())
-            .await?;
-    } else if let Some(recovered) = {
-        // solve taxonomy escalation
-        crate::db::DatabaseModel::new()
-            .recovered_packages()
-            .lock(&mut tx, LockType::Write)
-            .await?;
-        crate::db::DatabaseModel::new()
-            .recovered_packages()
-            .idx_model(pkg_id)
-            .get(&mut tx, true)
-            .await?
-            .into_owned()
-    } {
-        handle_recovered_package(recovered, manifest, ctx, pkg_id, version, &mut tx).await?;
-        add_dependent_to_current_dependents_lists(&mut tx, pkg_id, &current_dependencies).await?;
-        update_dependency_errors_of_dependents(ctx, &mut tx, pkg_id, current_dependents.keys())
-            .await?;
-    } else {
-        add_dependent_to_current_dependents_lists(&mut tx, pkg_id, &current_dependencies).await?;
-        update_dependency_errors_of_dependents(ctx, &mut tx, pkg_id, current_dependents.keys())
-            .await?;
-    }
-
-    crate::db::DatabaseModel::new()
-        .recovered_packages()
-        .remove(&mut tx, pkg_id)
-        .await?;
-
-    if let Some(installed) = pde.installed() {
-        reconfigure_dependents_with_live_pointers(ctx, &mut tx, installed).await?;
-    }
-
-    sql_tx.commit().await?;
-    tx.commit(None).await?;
-
-    tracing::info!("Install {}@{}: Complete", pkg_id, version);
-
-    Ok(())
-}
-
-#[instrument(skip(ctx, tx))]
-async fn handle_recovered_package(
-    recovered: RecoveredPackageInfo,
-    manifest: Manifest,
-    ctx: &RpcContext,
-    pkg_id: &PackageId,
-    version: &Version,
-    tx: &mut patch_db::Transaction<&mut patch_db::PatchDbHandle>,
-) -> Result<(), Error> {
-    let configured = if let Some(migration) =
-        manifest
-            .migrations
-            .from(ctx, &recovered.version, pkg_id, version, &manifest.volumes)
-    {
-        migration.await?.configured
-    } else {
-        false
-    };
-    if configured && manifest.config.is_some() {
-        crate::config::configure(
-            ctx,
-            tx,
+        add_dependent_to_current_dependents_lists(
+            &mut tx,
             pkg_id,
-            None,
-            &None,
-            false,
-            &mut BTreeMap::new(),
-            &mut BTreeMap::new(),
+            &current_dependencies,
+            &receipts.config.current_dependents,
+        )
+        .await?;
+        update_dependency_errors_of_dependents(
+            ctx,
+            &mut tx,
+            pkg_id,
+            &current_dependents,
+            &receipts.config.update_dependency_receipts,
+        )
+        .await?;
+    } else {
+        add_dependent_to_current_dependents_lists(
+            &mut tx,
+            pkg_id,
+            &current_dependencies,
+            &receipts.config.current_dependents,
+        )
+        .await?;
+        update_dependency_errors_of_dependents(
+            ctx,
+            &mut tx,
+            pkg_id,
+            &current_dependents,
+            &receipts.config.update_dependency_receipts,
         )
         .await?;
     }
 
+    if let Some(installed) = pde.installed() {
+        reconfigure_dependents_with_live_pointers(ctx, &mut tx, &receipts.config, installed)
+            .await?;
+    }
+
+    sql_tx.commit().await?;
+    tx.commit().await?;
+
+    tracing::info!("Install {}@{}: Complete", pkg_id, version);
+
     Ok(())
 }
 
@@ -1383,7 +1422,9 @@ pub fn load_images<'a, P: AsRef<Path> + 'a + Send + Sync>(
                 .try_for_each(|entry| async move {
                     let m = entry.metadata().await?;
                     if m.is_file() {
-                        if entry.path().extension().and_then(|ext| ext.to_str()) == Some("tar") {
+                        let path = entry.path();
+                        let ext = path.extension().and_then(|ext| ext.to_str());
+                        if ext == Some("tar") || ext == Some("s9pk") {
                             let mut load = Command::new("docker")
                                 .arg("load")
                                 .stdin(Stdio::piped())
@@ -1395,8 +1436,24 @@ pub fn load_images<'a, P: AsRef<Path> + 'a + Send + Sync>(
                                     crate::ErrorKind::Docker,
                                 )
                             })?;
-                            let mut docker_rdr = File::open(&entry.path()).await?;
-                            copy_and_shutdown(&mut docker_rdr, load_in).await?;
+                            match ext {
+                                Some("tar") => {
+                                    copy_and_shutdown(&mut File::open(&path).await?, load_in)
+                                        .await?
+                                }
+                                Some("s9pk") => {
+                                    copy_and_shutdown(
+                                        &mut S9pkReader::open(&path, false)
+                                            .await?
+                                            .docker_images()
+                                            .await?,
+                                        load_in,
+                                    )
+                                    .await?
+                                }
+                                _ => unreachable!(),
+                            };
+
                             let res = load.wait_with_output().await?;
                             if !res.status.success() {
                                 Err(Error::new(

backend/src/install/progress.rs

@@ -44,10 +44,14 @@ impl InstallProgress {
         mut db: Db,
     ) -> Result<(), Error> {
         while !self.download_complete.load(Ordering::SeqCst) {
-            model.put(&mut db, &self).await?;
+            let mut tx = db.begin().await?;
+            model.put(&mut tx, &self).await?;
+            tx.save().await?;
             tokio::time::sleep(Duration::from_secs(1)).await;
         }
-        model.put(&mut db, &self).await?;
+        let mut tx = db.begin().await?;
+        model.put(&mut tx, &self).await?;
+        tx.save().await?;
         Ok(())
     }
     pub async fn track_download_during<
@@ -74,10 +78,14 @@ impl InstallProgress {
         complete: Arc<AtomicBool>,
     ) -> Result<(), Error> {
         while !complete.load(Ordering::SeqCst) {
-            model.put(&mut db, &self).await?;
+            let mut tx = db.begin().await?;
+            model.put(&mut tx, &self).await?;
+            tx.save().await?;
             tokio::time::sleep(Duration::from_secs(1)).await;
         }
-        model.put(&mut db, &self).await?;
+        let mut tx = db.begin().await?;
+        model.put(&mut tx, &self).await?;
+        tx.save().await?;
         Ok(())
     }
     pub async fn track_read_during<
@@ -105,6 +113,7 @@ impl InstallProgress {
 }
 
 #[pin_project::pin_project]
+#[derive(Debug)]
 pub struct InstallProgressTracker<RW> {
     #[pin]
     inner: RW,

backend/src/install/update.rs

@@ -1,20 +1,66 @@
 use std::collections::BTreeMap;
 
-use patch_db::{DbHandle, LockType};
+use patch_db::{DbHandle, LockReceipt, LockTargetId, LockType, Verifier};
 use rpc_toolkit::command;
+use tracing::instrument;
 
+use crate::config::not_found;
 use crate::context::RpcContext;
-use crate::dependencies::{break_transitive, BreakageRes, DependencyError};
+use crate::db::model::CurrentDependents;
+use crate::dependencies::{
+    break_transitive, BreakTransitiveReceipts, BreakageRes, DependencyError,
+};
 use crate::s9pk::manifest::PackageId;
 use crate::util::serde::display_serializable;
 use crate::util::Version;
 use crate::Error;
 
+pub struct UpdateReceipts {
+    break_receipts: BreakTransitiveReceipts,
+    current_dependents: LockReceipt<CurrentDependents, String>,
+    dependency: LockReceipt<crate::dependencies::DepInfo, (String, String)>,
+}
+
+impl UpdateReceipts {
+    pub async fn new<'a>(db: &'a mut impl DbHandle) -> Result<Self, Error> {
+        let mut locks = Vec::new();
+
+        let setup = Self::setup(&mut locks);
+        Ok(setup(&db.lock_all(locks).await?)?)
+    }
+
+    pub fn setup(locks: &mut Vec<LockTargetId>) -> impl FnOnce(&Verifier) -> Result<Self, Error> {
+        let break_receipts = BreakTransitiveReceipts::setup(locks);
+        let current_dependents = crate::db::DatabaseModel::new()
+            .package_data()
+            .star()
+            .installed()
+            .map(|x| x.current_dependents())
+            .make_locker(LockType::Write)
+            .add_to_keys(locks);
+        let dependency = crate::db::DatabaseModel::new()
+            .package_data()
+            .star()
+            .installed()
+            .map(|x| x.manifest().dependencies().star())
+            .make_locker(LockType::Write)
+            .add_to_keys(locks);
+        move |skeleton_key| {
+            Ok(Self {
+                break_receipts: break_receipts(skeleton_key)?,
+                current_dependents: current_dependents.verify(skeleton_key)?,
+                dependency: dependency.verify(skeleton_key)?,
+            })
+        }
+    }
+}
+
 #[command(subcommands(dry))]
 pub async fn update() -> Result<(), Error> {
     Ok(())
 }
 
+#[instrument(skip(ctx))]
 #[command(display(display_serializable))]
 pub async fn dry(
     #[context] ctx: RpcContext,
@@ -24,49 +70,34 @@ pub async fn dry(
     let mut db = ctx.db.handle();
     let mut tx = db.begin().await?;
     let mut breakages = BTreeMap::new();
-    crate::db::DatabaseModel::new()
-        .package_data()
-        .lock(&mut tx, LockType::Read)
-        .await?;
-    for dependent in crate::db::DatabaseModel::new()
-        .package_data()
-        .idx_model(&id)
-        .and_then(|m| m.installed())
-        .expect(&mut tx)
-        .await?
-        .current_dependents()
-        .keys(&mut tx, true)
+    let receipts = UpdateReceipts::new(&mut tx).await?;
+
+    for dependent in receipts
+        .current_dependents
+        .get(&mut tx, &id)
         .await?
+        .ok_or_else(not_found)?
+        .0
+        .keys()
         .into_iter()
-        .filter(|dependent| &id != dependent)
+        .filter(|dependent| &&id != dependent)
     {
-        let version_req = crate::db::DatabaseModel::new()
-            .package_data()
-            .idx_model(&dependent)
-            .and_then(|m| m.installed())
-            .expect(&mut tx)
-            .await?
-            .manifest()
-            .dependencies()
-            .idx_model(&id)
-            .expect(&mut tx)
-            .await?
-            .get(&mut tx, true)
-            .await?
-            .into_owned()
-            .version;
-        if !version.satisfies(&version_req) {
-            break_transitive(
-                &mut tx,
-                &dependent,
-                &id,
-                DependencyError::IncorrectVersion {
-                    expected: version_req,
-                    received: version.clone(),
-                },
-                &mut breakages,
-            )
-            .await?;
+        if let Some(dep_info) = receipts.dependency.get(&mut tx, (&dependent, &id)).await? {
+            let version_req = dep_info.version;
+            if !version.satisfies(&version_req) {
+                break_transitive(
+                    &mut tx,
+                    &dependent,
+                    &id,
+                    DependencyError::IncorrectVersion {
+                        expected: version_req,
+                        received: version.clone(),
+                    },
+                    &mut breakages,
+                    &receipts.break_receipts,
+                )
+                .await?;
+            }
         }
     }
     tx.abort().await?;

backend/src/lib.rs

@@ -1,10 +1,19 @@
-pub const CONFIG_PATH: &str = "/etc/embassy/config.yaml";
-#[cfg(not(feature = "beta"))]
-pub const DEFAULT_MARKETPLACE: &str = "https://marketplace.start9.com";
-#[cfg(feature = "beta")]
-pub const DEFAULT_MARKETPLACE: &str = "https://beta-registry-0-3.start9labs.com";
+#![recursion_limit = "256"]
+
+pub const DEFAULT_MARKETPLACE: &str = "https://registry.start9.com";
+// pub const COMMUNITY_MARKETPLACE: &str = "https://community-registry.start9.com";
 pub const BUFFER_SIZE: usize = 1024;
 pub const HOST_IP: [u8; 4] = [172, 18, 0, 1];
+pub const TARGET: &str = current_platform::CURRENT_PLATFORM;
+lazy_static::lazy_static! {
+    pub static ref ARCH: &'static str = {
+        let (arch, _) = TARGET.split_once("-").unwrap();
+        arch
+    };
+    pub static ref IS_RASPBERRY_PI: bool = {
+        *ARCH == "aarch64"
+    };
+}
 
 pub mod action;
 pub mod auth;
@@ -31,13 +40,14 @@ pub mod middleware;
 pub mod migration;
 pub mod net;
 pub mod notifications;
+pub mod os_install;
+pub mod procedure;
 pub mod properties;
 pub mod s9pk;
 pub mod setup;
 pub mod shutdown;
 pub mod sound;
 pub mod ssh;
-pub mod static_server;
 pub mod status;
 pub mod system;
 pub mod update;
@@ -77,6 +87,7 @@ pub fn main_api() -> Result<(), RpcError> {
 
 #[command(subcommands(
     system::logs,
+    system::kernel_logs,
     system::metrics,
     shutdown::shutdown,
     shutdown::restart,
@@ -92,12 +103,12 @@ pub fn server() -> Result<(), RpcError> {
     install::install,
     install::sideload,
     install::uninstall,
-    install::delete_recovered,
     install::list,
     install::update::update,
     config::config,
     control::start,
     control::stop,
+    control::restart,
     logs::logs,
     properties::properties,
     dependencies::dependency,
@@ -127,3 +138,8 @@ pub fn diagnostic_api() -> Result<(), RpcError> {
 pub fn setup_api() -> Result<(), RpcError> {
     Ok(())
 }
+
+#[command(subcommands(version::git_info, echo, os_install::install))]
+pub fn install_api() -> Result<(), RpcError> {
+    Ok(())
+}

backend/src/logs.rs

@@ -1,22 +1,119 @@
+use std::future::Future;
+use std::marker::PhantomData;
+use std::ops::{Deref, DerefMut};
 use std::process::Stdio;
 use std::time::{Duration, UNIX_EPOCH};
 
 use chrono::{DateTime, Utc};
-use clap::ArgMatches;
 use color_eyre::eyre::eyre;
-use futures::TryStreamExt;
+use futures::stream::BoxStream;
+use futures::{FutureExt, SinkExt, Stream, StreamExt, TryStreamExt};
+use hyper::upgrade::Upgraded;
+use hyper::Error as HyperError;
 use rpc_toolkit::command;
+use rpc_toolkit::yajrc::RpcError;
 use serde::{Deserialize, Serialize};
 use tokio::io::{AsyncBufReadExt, BufReader};
-use tokio::process::Command;
+use tokio::process::{Child, Command};
+use tokio::task::JoinError;
 use tokio_stream::wrappers::LinesStream;
+use tokio_tungstenite::tungstenite::protocol::frame::coding::CloseCode;
+use tokio_tungstenite::tungstenite::protocol::CloseFrame;
+use tokio_tungstenite::tungstenite::Message;
+use tokio_tungstenite::WebSocketStream;
 use tracing::instrument;
 
-use crate::action::docker::DockerAction;
+use crate::context::{CliContext, RpcContext};
+use crate::core::rpc_continuations::{RequestGuid, RpcContinuation};
 use crate::error::ResultExt;
+use crate::procedure::docker::DockerProcedure;
 use crate::s9pk::manifest::PackageId;
+use crate::util::display_none;
 use crate::util::serde::Reversible;
-use crate::Error;
+use crate::{Error, ErrorKind};
+
+#[pin_project::pin_project]
+struct LogStream {
+    _child: Child,
+    #[pin]
+    entries: BoxStream<'static, Result<JournalctlEntry, Error>>,
+}
+impl Deref for LogStream {
+    type Target = BoxStream<'static, Result<JournalctlEntry, Error>>;
+    fn deref(&self) -> &Self::Target {
+        &self.entries
+    }
+}
+impl DerefMut for LogStream {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.entries
+    }
+}
+impl Stream for LogStream {
+    type Item = Result<JournalctlEntry, Error>;
+    fn poll_next(
+        self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Option<Self::Item>> {
+        let this = self.project();
+        Stream::poll_next(this.entries, cx)
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        self.entries.size_hint()
+    }
+}
+
+#[instrument(skip(logs, ws_fut))]
+async fn ws_handler<
+    WSFut: Future<Output = Result<Result<WebSocketStream<Upgraded>, HyperError>, JoinError>>,
+>(
+    first_entry: Option<LogEntry>,
+    mut logs: LogStream,
+    ws_fut: WSFut,
+) -> Result<(), Error> {
+    let mut stream = ws_fut
+        .await
+        .with_kind(crate::ErrorKind::Network)?
+        .with_kind(crate::ErrorKind::Unknown)?;
+
+    if let Some(first_entry) = first_entry {
+        stream
+            .send(Message::Text(
+                serde_json::to_string(&first_entry).with_kind(ErrorKind::Serialization)?,
+            ))
+            .await
+            .with_kind(ErrorKind::Network)?;
+    }
+
+    let mut ws_closed = false;
+    while let Some(entry) = tokio::select! {
+        a = logs.try_next() => Some(a?),
+        a = stream.try_next() => { a.with_kind(crate::ErrorKind::Network)?; ws_closed = true; None }
+    } {
+        if let Some(entry) = entry {
+            let (_, log_entry) = entry.log_entry()?;
+            stream
+                .send(Message::Text(
+                    serde_json::to_string(&log_entry).with_kind(ErrorKind::Serialization)?,
+                ))
+                .await
+                .with_kind(ErrorKind::Network)?;
+        }
+    }
+
+    if !ws_closed {
+        stream
+            .close(Some(CloseFrame {
+                code: CloseCode::Normal,
+                reason: "Log Stream Finished".into(),
+            }))
+            .await
+            .with_kind(ErrorKind::Network)?;
+    }
+
+    Ok(())
+}
 
 #[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]
 #[serde(rename_all = "kebab-case")]
@@ -25,6 +122,12 @@ pub struct LogResponse {
     start_cursor: Option<String>,
     end_cursor: Option<String>,
 }
+#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]
+#[serde(rename_all = "kebab-case")]
+pub struct LogFollowResponse {
+    start_cursor: Option<String>,
+    guid: RequestGuid,
+}
 
 #[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]
 pub struct LogEntry {
@@ -106,70 +209,182 @@ fn deserialize_string_or_utf8_array<'de, D: serde::de::Deserializer<'de>>(
 
 #[derive(Debug)]
 pub enum LogSource {
+    Kernel,
     Service(&'static str),
     Container(PackageId),
 }
 
-pub fn display_logs(all: LogResponse, _: &ArgMatches<'_>) {
-    for entry in all.entries.iter() {
-        println!("{}", entry);
-    }
-}
-
-#[command(display(display_logs))]
+#[command(
+    custom_cli(cli_logs(async, context(CliContext))),
+    subcommands(self(logs_nofollow(async)), logs_follow),
+    display(display_none)
+)]
 pub async fn logs(
     #[arg] id: PackageId,
-    #[arg] limit: Option<usize>,
-    #[arg] cursor: Option<String>,
-    #[arg] before_flag: Option<bool>,
+    #[arg(short = 'l', long = "limit")] limit: Option<usize>,
+    #[arg(short = 'c', long = "cursor")] cursor: Option<String>,
+    #[arg(short = 'B', long = "before", default)] before: bool,
+    #[arg(short = 'f', long = "follow", default)] follow: bool,
+) -> Result<(PackageId, Option<usize>, Option<String>, bool, bool), Error> {
+    Ok((id, limit, cursor, before, follow))
+}
+pub async fn cli_logs(
+    ctx: CliContext,
+    (id, limit, cursor, before, follow): (PackageId, Option<usize>, Option<String>, bool, bool),
+) -> Result<(), RpcError> {
+    if follow {
+        if cursor.is_some() {
+            return Err(RpcError::from(Error::new(
+                eyre!("The argument '--cursor <cursor>' cannot be used with '--follow'"),
+                crate::ErrorKind::InvalidRequest,
+            )));
+        }
+        if before {
+            return Err(RpcError::from(Error::new(
+                eyre!("The argument '--before' cannot be used with '--follow'"),
+                crate::ErrorKind::InvalidRequest,
+            )));
+        }
+        cli_logs_generic_follow(ctx, "package.logs.follow", Some(id), limit).await
+    } else {
+        cli_logs_generic_nofollow(ctx, "package.logs", Some(id), limit, cursor, before).await
+    }
+}
+pub async fn logs_nofollow(
+    _ctx: (),
+    (id, limit, cursor, before, _): (PackageId, Option<usize>, Option<String>, bool, bool),
 ) -> Result<LogResponse, Error> {
-    Ok(fetch_logs(
-        LogSource::Container(id),
-        limit,
-        cursor,
-        before_flag.unwrap_or(false),
-    )
-    .await?)
+    fetch_logs(LogSource::Container(id), limit, cursor, before).await
+}
+#[command(rpc_only, rename = "follow", display(display_none))]
+pub async fn logs_follow(
+    #[context] ctx: RpcContext,
+    #[parent_data] (id, limit, _, _, _): (PackageId, Option<usize>, Option<String>, bool, bool),
+) -> Result<LogFollowResponse, Error> {
+    follow_logs(ctx, LogSource::Container(id), limit).await
 }
 
-#[instrument]
-pub async fn fetch_logs(
-    id: LogSource,
+pub async fn cli_logs_generic_nofollow(
+    ctx: CliContext,
+    method: &str,
+    id: Option<PackageId>,
     limit: Option<usize>,
     cursor: Option<String>,
-    before_flag: bool,
-) -> Result<LogResponse, Error> {
-    let limit = limit.unwrap_or(50);
-    let limit_formatted = format!("-n{}", limit);
+    before: bool,
+) -> Result<(), RpcError> {
+    let res = rpc_toolkit::command_helpers::call_remote(
+        ctx.clone(),
+        method,
+        serde_json::json!({
+            "id": id,
+            "limit": limit,
+            "cursor": cursor,
+            "before": before,
+        }),
+        PhantomData::<LogResponse>,
+    )
+    .await?
+    .result?;
 
-    let mut args = vec!["--output=json", "--output-fields=MESSAGE", &limit_formatted];
-    let id_formatted = match id {
-        LogSource::Service(id) => {
-            args.push("-u");
-            id.to_owned()
-        }
-        LogSource::Container(id) => {
-            format!("CONTAINER_NAME={}", DockerAction::container_name(&id, None))
+    for entry in res.entries.iter() {
+        println!("{}", entry);
+    }
+
+    Ok(())
+}
+
+pub async fn cli_logs_generic_follow(
+    ctx: CliContext,
+    method: &str,
+    id: Option<PackageId>,
+    limit: Option<usize>,
+) -> Result<(), RpcError> {
+    let res = rpc_toolkit::command_helpers::call_remote(
+        ctx.clone(),
+        method,
+        serde_json::json!({
+            "id": id,
+            "limit": limit,
+        }),
+        PhantomData::<LogFollowResponse>,
+    )
+    .await?
+    .result?;
+
+    let mut base_url = ctx.base_url.clone();
+    let ws_scheme = match base_url.scheme() {
+        "https" => "wss",
+        "http" => "ws",
+        _ => {
+            return Err(Error::new(
+                eyre!("Cannot parse scheme from base URL"),
+                crate::ErrorKind::ParseUrl,
+            )
+            .into())
         }
     };
-    args.push(&id_formatted);
-
-    let cursor_formatted = format!("--after-cursor={}", cursor.clone().unwrap_or("".to_owned()));
-    let mut get_prev_logs_and_reverse = false;
-    if cursor.is_some() {
-        args.push(&cursor_formatted);
-        if before_flag {
-            get_prev_logs_and_reverse = true;
+    base_url.set_scheme(ws_scheme).or_else(|_| {
+        Err(Error::new(
+            eyre!("Cannot set URL scheme"),
+            crate::ErrorKind::ParseUrl,
+        ))
+    })?;
+    let (mut stream, _) =
+                // base_url is "http://127.0.0.1/", with a trailing slash, so we don't put a leading slash in this path:
+                tokio_tungstenite::connect_async(format!("{}ws/rpc/{}", base_url, res.guid)).await?;
+    while let Some(log) = stream.try_next().await? {
+        match log {
+            Message::Text(log) => {
+                println!("{}", serde_json::from_str::<LogEntry>(&log)?);
+            }
+            _ => (),
         }
     }
-    if get_prev_logs_and_reverse {
-        args.push("--reverse");
+
+    Ok(())
+}
+
+async fn journalctl(
+    id: LogSource,
+    limit: usize,
+    cursor: Option<&str>,
+    before: bool,
+    follow: bool,
+) -> Result<LogStream, Error> {
+    let mut cmd = Command::new("journalctl");
+    cmd.kill_on_drop(true);
+
+    cmd.arg("--output=json");
+    cmd.arg("--output-fields=MESSAGE");
+    cmd.arg(format!("-n{}", limit));
+    match id {
+        LogSource::Kernel => {
+            cmd.arg("-k");
+        }
+        LogSource::Service(id) => {
+            cmd.arg("-u");
+            cmd.arg(id);
+        }
+        LogSource::Container(id) => {
+            cmd.arg(format!(
+                "CONTAINER_NAME={}",
+                DockerProcedure::container_name(&id, None)
+            ));
+        }
+    };
+
+    let cursor_formatted = format!("--after-cursor={}", cursor.clone().unwrap_or(""));
+    if cursor.is_some() {
+        cmd.arg(&cursor_formatted);
+        if before {
+            cmd.arg("--reverse");
+        }
+    }
+    if follow {
+        cmd.arg("--follow");
     }
 
-    let mut child = Command::new("journalctl")
-        .args(args)
-        .stdout(Stdio::piped())
-        .spawn()?;
+    let mut child = cmd.stdout(Stdio::piped()).spawn()?;
     let out = BufReader::new(
         child
             .stdout
@@ -179,7 +394,7 @@ pub async fn fetch_logs(
 
     let journalctl_entries = LinesStream::new(out.lines());
 
-    let mut deserialized_entries = journalctl_entries
+    let deserialized_entries = journalctl_entries
         .map_err(|e| Error::new(e, crate::ErrorKind::Journald))
         .and_then(|s| {
             futures::future::ready(
@@ -188,16 +403,37 @@ pub async fn fetch_logs(
             )
         });
 
+    Ok(LogStream {
+        _child: child,
+        entries: deserialized_entries.boxed(),
+    })
+}
+
+#[instrument]
+pub async fn fetch_logs(
+    id: LogSource,
+    limit: Option<usize>,
+    cursor: Option<String>,
+    before: bool,
+) -> Result<LogResponse, Error> {
+    let limit = limit.unwrap_or(50);
+    let mut stream = journalctl(id, limit, cursor.as_deref(), before, false).await?;
+
     let mut entries = Vec::with_capacity(limit);
     let mut start_cursor = None;
 
-    if let Some(first) = deserialized_entries.try_next().await? {
+    if let Some(first) = tokio::time::timeout(Duration::from_secs(1), stream.try_next())
+        .await
+        .ok()
+        .transpose()?
+        .flatten()
+    {
         let (cursor, entry) = first.log_entry()?;
         start_cursor = Some(cursor);
         entries.push(entry);
     }
 
-    let (mut end_cursor, entries) = deserialized_entries
+    let (mut end_cursor, entries) = stream
         .try_fold(
             (start_cursor.clone(), entries),
             |(_, mut acc), entry| async move {
@@ -209,7 +445,7 @@ pub async fn fetch_logs(
         .await?;
     let mut entries = Reversible::new(entries);
     // reverse again so output is always in increasing chronological order
-    if get_prev_logs_and_reverse {
+    if cursor.is_some() && before {
         entries.reverse();
         std::mem::swap(&mut start_cursor, &mut end_cursor);
     }
@@ -220,21 +456,81 @@ pub async fn fetch_logs(
     })
 }
 
-#[tokio::test]
-pub async fn test_logs() {
-    let response = fetch_logs(
-        // change `tor.service` to an actual journald unit on your machine
-        // LogSource::Service("tor.service"),
-        // first run `docker run --name=hello-world.embassy --log-driver=journald hello-world`
-        LogSource::Container("hello-world".parse().unwrap()),
-        // Some(5),
-        None,
-        None,
-        // Some("s=1b8c418e28534400856c27b211dd94fd;i=5a7;b=97571c13a1284f87bc0639b5cff5acbe;m=740e916;t=5ca073eea3445;x=f45bc233ca328348".to_owned()),
-        false,
+#[instrument(skip(ctx))]
+pub async fn follow_logs(
+    ctx: RpcContext,
+    id: LogSource,
+    limit: Option<usize>,
+) -> Result<LogFollowResponse, Error> {
+    let limit = limit.unwrap_or(50);
+    let mut stream = journalctl(id, limit, None, false, true).await?;
+
+    let mut start_cursor = None;
+    let mut first_entry = None;
+
+    if let Some(first) = tokio::time::timeout(Duration::from_secs(1), stream.try_next())
+        .await
+        .ok()
+        .transpose()?
+        .flatten()
+    {
+        let (cursor, entry) = first.log_entry()?;
+        start_cursor = Some(cursor);
+        first_entry = Some(entry);
+    }
+
+    let guid = RequestGuid::new();
+    ctx.add_continuation(
+        guid.clone(),
+        RpcContinuation::ws(
+            Box::new(move |ws_fut| ws_handler(first_entry, stream, ws_fut).boxed()),
+            Duration::from_secs(30),
+        ),
     )
-    .await
-    .unwrap();
-    let serialized = serde_json::to_string_pretty(&response).unwrap();
-    println!("{}", serialized);
+    .await;
+    Ok(LogFollowResponse { start_cursor, guid })
 }
+
+// #[tokio::test]
+// pub async fn test_logs() {
+//     let response = fetch_logs(
+//         // change `tor.service` to an actual journald unit on your machine
+//         // LogSource::Service("tor.service"),
+//         // first run `docker run --name=hello-world.embassy --log-driver=journald hello-world`
+//         LogSource::Container("hello-world".parse().unwrap()),
+//         // Some(5),
+//         None,
+//         None,
+//         // Some("s=1b8c418e28534400856c27b211dd94fd;i=5a7;b=97571c13a1284f87bc0639b5cff5acbe;m=740e916;t=5ca073eea3445;x=f45bc233ca328348".to_owned()),
+//         false,
+//         true,
+//     )
+//     .await
+//     .unwrap();
+//     let serialized = serde_json::to_string_pretty(&response).unwrap();
+//     println!("{}", serialized);
+// }
+
+// #[tokio::test]
+// pub async fn test_logs() {
+//     let mut cmd = Command::new("journalctl");
+//     cmd.kill_on_drop(true);
+
+//     cmd.arg("-f");
+//     cmd.arg("CONTAINER_NAME=hello-world.embassy");
+
+//     let mut child = cmd.stdout(Stdio::piped()).spawn().unwrap();
+//     let out = BufReader::new(
+//         child
+//             .stdout
+//             .take()
+//             .ok_or_else(|| Error::new(eyre!("No stdout available"), crate::ErrorKind::Journald))
+//             .unwrap(),
+//     );
+
+//     let mut journalctl_entries = LinesStream::new(out.lines());
+
+//     while let Some(line) = journalctl_entries.try_next().await.unwrap() {
+//         dbg!(line);
+//     }
+// }